From 51ffb3cc106889c9aea5836a09754e53fa9ba5d1 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sat, 5 Oct 2019 21:01:52 -0700 Subject: [PATCH 01/19] Added c2 package & tests for initial c2 --- c2/http.go | 36 ++++++++++ c2/mocks/io.gen.go | 48 +++++++++++++ c2/mocks/mocks.go | 3 + c2/server.go | 42 ++++++++++++ c2/server_test.go | 37 ++++++++++ c2/task.go | 48 +++++++++++++ c2/task_test.go | 57 ++++++++++++++++ go.mod | 2 +- go.sum | 167 +++++++++++++++++++++++++++++++++++++++++++++ 9 files changed, 439 insertions(+), 1 deletion(-) create mode 100644 c2/http.go create mode 100644 c2/mocks/io.gen.go create mode 100644 c2/mocks/mocks.go create mode 100644 c2/server.go create mode 100644 c2/server_test.go create mode 100644 c2/task.go create mode 100644 c2/task_test.go diff --git a/c2/http.go b/c2/http.go new file mode 100644 index 00000000..63799af8 --- /dev/null +++ b/c2/http.go @@ -0,0 +1,36 @@ +package c2 + +import ( + "encoding/json" + "io/ioutil" + "net/http" + + "github.com/kcarretto/paragon/transport" + "go.uber.org/zap" +) + +// HTTP handles agent messages sent from http. +func (srv *Server) HTTP(w http.ResponseWriter, req *http.Request) { + data, err := ioutil.ReadAll(req.Body) + if err != nil { + srv.Logger.Error("Failed to read agent message", zap.Error(err)) + return + } + + srv.Logger.Debug("Received agent message", zap.String("agent_msg", string(data))) + + var msg transport.Response + if err := json.Unmarshal(data, &msg); err != nil { + srv.Logger.Error("Failed to unmarshal agent message", zap.Error(err)) + return + } + + if err = srv.HandleMessage(w, msg); err != nil { + srv.Logger.Error("Agent communication failed", zap.Error(err)) + return + } + + // TODO: Add agent metadata + srv.Logger.Info("Replied to agent message") + +} diff --git a/c2/mocks/io.gen.go b/c2/mocks/io.gen.go new file mode 100644 index 00000000..abfc2f27 --- /dev/null +++ b/c2/mocks/io.gen.go @@ -0,0 +1,48 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: io (interfaces: Writer) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockWriter is a mock of Writer interface +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// Write mocks base method +func (m *MockWriter) Write(arg0 []byte) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Write indicates an expected call of Write +func (mr *MockWriterMockRecorder) Write(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriter)(nil).Write), arg0) +} diff --git a/c2/mocks/mocks.go b/c2/mocks/mocks.go new file mode 100644 index 00000000..6a6b6a90 --- /dev/null +++ b/c2/mocks/mocks.go @@ -0,0 +1,3 @@ +package mocks + +//go:generate mockgen -destination=io.gen.go -package=mocks io Writer diff --git a/c2/server.go b/c2/server.go new file mode 100644 index 00000000..386090c7 --- /dev/null +++ b/c2/server.go @@ -0,0 +1,42 @@ +package c2 + +import ( + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/kcarretto/paragon/transport" + + "go.uber.org/zap" + "gocloud.dev/pubsub" +) + +// Server handles agent messages and replies with new tasks for the agent to execute. +type Server struct { + Logger *zap.Logger + AgentOutput *pubsub.Topic + + mu sync.RWMutex + tasks []queuedTask +} + +// HandleMessage received from the agent, and write a reply to the provided writer. +func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { + // TODO: Get available tasks. srv.GetTasks(msg.Metadata) + tasks := srv.GetTasks(msg.Metadata) + reply := transport.Payload{ + Tasks: tasks, + } + + data, err := json.Marshal(reply) + if err != nil { + return fmt.Errorf("failed to marshal server reply message: %w", err) + } + + if _, err = w.Write(data); err != nil { + return fmt.Errorf("failed to send reply message to agent: %w", err) + } + + return nil +} diff --git a/c2/server_test.go b/c2/server_test.go new file mode 100644 index 00000000..d9ba37dd --- /dev/null +++ b/c2/server_test.go @@ -0,0 +1,37 @@ +package c2_test + +import ( + "encoding/json" + "testing" + + "github.com/golang/mock/gomock" + + "github.com/kcarretto/paragon/c2" + "github.com/kcarretto/paragon/c2/mocks" + "github.com/kcarretto/paragon/transport" + "github.com/stretchr/testify/require" +) + +func TestHandleMessage(t *testing.T) { + expectedTask := transport.Task{ID: "HelloThere"} + expectedReply := transport.Payload{ + Tasks: []transport.Task{expectedTask}, + } + expected, err := json.Marshal(expectedReply) + require.NoError(t, err) + + // Prepare mock + ctrl := gomock.NewController(t) + defer ctrl.Finish() + replyWriter := mocks.NewMockWriter(ctrl) + replyWriter.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { + require.Equal(t, string(expected), string(p)) + return len(p), nil + }) + + srv := &c2.Server{} + srv.QueueTask(expectedTask, func(agent transport.Metadata) bool { return true }) + + err = srv.HandleMessage(replyWriter, transport.Response{}) + require.NoError(t, err) +} diff --git a/c2/task.go b/c2/task.go new file mode 100644 index 00000000..ff4a7486 --- /dev/null +++ b/c2/task.go @@ -0,0 +1,48 @@ +package c2 + +import ( + "github.com/kcarretto/paragon/transport" +) + +type queuedTask struct { + task transport.Task + filter func(transport.Metadata) bool +} + +// QueueTask prepares a task to be sent to the first agent that reports metadata matching the filter. +func (srv *Server) QueueTask(task transport.Task, filter func(transport.Metadata) bool) { + srv.mu.Lock() + srv.tasks = append(srv.tasks, queuedTask{ + task, + filter, + }) + srv.mu.Unlock() +} + +// GetTasks for an agent based on the metadata it reported and the current tasks in the server queue. +func (srv *Server) GetTasks(agent transport.Metadata) []transport.Task { + srv.mu.Lock() + n := 0 + tasks := make([]transport.Task, 0, len(srv.tasks)) + for _, t := range srv.tasks { + if t.filter == nil || t.filter(agent) { + tasks = append(tasks, t.task) + } else { + srv.tasks[n] = t + n++ + } + } + srv.tasks = srv.tasks[:n] + srv.mu.Unlock() + + return tasks +} + +// TaskCount returns the total number of tasks left in the queue. +func (srv *Server) TaskCount() int { + srv.mu.RLock() + count := len(srv.tasks) + srv.mu.RUnlock() + + return count +} diff --git a/c2/task_test.go b/c2/task_test.go new file mode 100644 index 00000000..dd577446 --- /dev/null +++ b/c2/task_test.go @@ -0,0 +1,57 @@ +package c2_test + +import ( + "testing" + + "github.com/kcarretto/paragon/c2" + "github.com/kcarretto/paragon/transport" + "github.com/stretchr/testify/require" +) + +func TestQueueTask(t *testing.T) { + expectedTask := transport.Task{ID: "TestID1"} + + srv := &c2.Server{} + srv.QueueTask( + expectedTask, + func(a transport.Metadata) bool { + return true + }, + ) + + require.Equal(t, 1, srv.TaskCount()) + + tasks := srv.GetTasks(transport.Metadata{}) + require.Equal(t, 1, len(tasks)) + require.Equal(t, expectedTask, tasks[0]) + + require.Equal(t, 0, srv.TaskCount()) +} + +func TestFilteredTask(t *testing.T) { + expectedTask := transport.Task{ID: "TestID1"} + unexpectedTask := transport.Task{ID: "Nope"} + + srv := &c2.Server{} + srv.QueueTask( + expectedTask, + func(transport.Metadata) bool { + return true + }, + ) + require.Equal(t, 1, srv.TaskCount()) + + srv.QueueTask( + unexpectedTask, + func(transport.Metadata) bool { + return false + }, + ) + require.Equal(t, 2, srv.TaskCount()) + + tasks := srv.GetTasks(transport.Metadata{}) + require.Equal(t, 1, len(tasks)) + require.Equal(t, expectedTask, tasks[0]) + + require.Equal(t, 1, srv.TaskCount()) +} diff --git a/go.mod b/go.mod index 1dad0aaf..5ded6a79 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/mock v1.3.1 - github.com/kr/pretty v0.1.0 // indirect github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.3.0 github.com/shirou/gopsutil v2.18.12+incompatible @@ -16,6 +15,7 @@ require ( go.uber.org/atomic v1.4.0 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.10.0 + gocloud.dev v0.17.0 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index cbd2f006..0268640a 100644 --- a/go.sum +++ b/go.sum @@ -1,29 +1,113 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +cloud.google.com/go v0.38.0/go.mod h1:990N+gfupTy94rShfmMCWGDn0LpTmnzTp2qbd1dvSRU= +cloud.google.com/go v0.39.0 h1:UgQP9na6OTfp4dsAiz/eFpFA1C6tPdH5wiRdi19tuMw= +cloud.google.com/go v0.39.0/go.mod h1:rVLT6fkc8chs9sfPtFc1SBH6em7n+ZoXaG+87tDISts= +contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= +contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= +contrib.go.opencensus.io/exporter/stackdriver v0.12.1/go.mod h1:iwB6wGarfphGGe/e5CWqyUk/cLzKnWsOKPVW3no6OTw= +contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0WkU1aRorQCFfRxvqjUUPMbF3fE= +contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= +github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= +github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= +github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= +github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= +github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/GoogleCloudPlatform/cloudsql-proxy v0.0.0-20190605020000-c4ba1fdf4d36/go.mod h1:aJ4qN3TfrelA6NZ6AXsXRfmEVaYin3EDbSPJrKS8OXo= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= +github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= +github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/devigned/tab v0.1.1/go.mod h1:XG9mPq0dFghrYvoBF3xdRrJzSTX1b7IQrvaL9mzjeJY= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/fortytw2/leaktest v1.2.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-ini/ini v1.25.4/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= +github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= +github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= +github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= +github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no= +github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= +github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s= +github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= +github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= +github.com/googleapis/gax-go/v2 v2.0.4 h1:hU4mGcQI4DaAYW+IbTun+2qEZVFxK0ySjQLTbS0VQKc= +github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.2/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= +github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= +github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/profile v1.3.0 h1:OQIvuDgm00gWVWGTf4m4mCt6W1/0YqU7Ntg0mySWgaI= github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= +github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0= +go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= +go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= +go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.starlark.net v0.0.0-20190717023158-fc7a7f44baed h1:4jRCa7xxEE+8+CDxrq64JNGsuui8/Y1/enYVey/vsjo= go.starlark.net v0.0.0-20190717023158-fc7a7f44baed/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= @@ -32,16 +116,99 @@ go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= +gocloud.dev v0.17.0 h1:UuDiCphYsiNhRNLtgHVL/eZheQeCt00hL3XjDfbt820= +gocloud.dev v0.17.0/go.mod h1:tIHTRdR1V5dlD8sTkzYdTGizBJ314BDykJ8KmadEXwo= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= +golang.org/x/lint v0.0.0-20190301231843-5614ed5bae6f/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190409202823-959b441ac422/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE= +golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190620070143-6f217b454f45/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= +golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/api v0.4.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.5.0/go.mod h1:8k5glujaEP+g9n7WNsDg8QP6cUVNI86fCNMcbazEtwE= +google.golang.org/api v0.6.0 h1:2tJEkRfnZL5g1GeBUlITh/rqT5HG3sFcoVCUUxmgJ2g= +google.golang.org/api v0.6.0/go.mod h1:btoxGiFvQNVUZQ8W08zLtrVS08CNpINPEfxXxgJL1Q4= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190508193815-b515fa19cec8/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= +google.golang.org/genproto v0.0.0-20190530194941-fb225487d101/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601 h1:9VBRTdmgQxbs6HE0sUnMrSWNePppAJU07NYvX5dIB04= +google.golang.org/genproto v0.0.0-20190620144150-6af8c5fc6601/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= +google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= +google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= +google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= +google.golang.org/grpc v1.21.1 h1:j6XxA85m/6txkUCHvzlV5f+HBNl/1r5cZ2A/3IEFOO8= +google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= +gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +pack.ag/amqp v0.11.2/go.mod h1:4/cbmt4EJXSKlG6LCfWHoqmN0uFdy5i/+YFz+fTfhV4= From 906f85d5bad9e42f3b1b1f155fd450637c702042 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sun, 6 Oct 2019 00:19:58 -0700 Subject: [PATCH 02/19] Added http test --- c2/http.go | 4 ++-- c2/http_test.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ c2/server.go | 3 ++- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 c2/http_test.go diff --git a/c2/http.go b/c2/http.go index 63799af8..ee5cb123 100644 --- a/c2/http.go +++ b/c2/http.go @@ -9,8 +9,8 @@ import ( "go.uber.org/zap" ) -// HTTP handles agent messages sent from http. -func (srv *Server) HTTP(w http.ResponseWriter, req *http.Request) { +// ServeHTTP implements http.Handler to handle agent messages sent via http. +func (srv *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { data, err := ioutil.ReadAll(req.Body) if err != nil { srv.Logger.Error("Failed to read agent message", zap.Error(err)) diff --git a/c2/http_test.go b/c2/http_test.go new file mode 100644 index 00000000..033f07dc --- /dev/null +++ b/c2/http_test.go @@ -0,0 +1,50 @@ +package c2_test + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/kcarretto/paragon/c2" + "github.com/kcarretto/paragon/transport" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestHTTP(t *testing.T) { + expectedTask := transport.Task{ + ID: "ABC", + } + unexpectedTask := transport.Task{ + ID: "XYZ", + } + + msg := transport.Response{} + data, err := json.Marshal(msg) + require.NoError(t, err) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBuffer(data)) + + srv := &c2.Server{ + Logger: zap.NewNop(), + } + srv.QueueTask(expectedTask, func(transport.Metadata) bool { return true }) + srv.QueueTask(unexpectedTask, func(transport.Metadata) bool { return false }) + + srv.ServeHTTP(w, req) + + body, err := ioutil.ReadAll(w.Result().Body) + defer w.Result().Body.Close() + require.NoError(t, err) + + var reply transport.Payload + err = json.Unmarshal(body, &reply) + require.NoError(t, err) + require.Equal(t, 1, len(reply.Tasks)) + require.Equal(t, expectedTask, reply.Tasks[0]) +} diff --git a/c2/server.go b/c2/server.go index 386090c7..fb7603cd 100644 --- a/c2/server.go +++ b/c2/server.go @@ -23,7 +23,8 @@ type Server struct { // HandleMessage received from the agent, and write a reply to the provided writer. func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { - // TODO: Get available tasks. srv.GetTasks(msg.Metadata) + // TODO: Handle results + tasks := srv.GetTasks(msg.Metadata) reply := transport.Payload{ Tasks: tasks, From f3101d962cb45b89ae48a9c63d9e30038e3904f2 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sun, 6 Oct 2019 00:30:52 -0700 Subject: [PATCH 03/19] Updated go mod --- go.mod | 1 + go.sum | 3 +++ 2 files changed, 4 insertions(+) diff --git a/go.mod b/go.mod index 5ded6a79..920e2aed 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.10.0 gocloud.dev v0.17.0 + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect ) diff --git a/go.sum b/go.sum index 0268640a..e0b053ce 100644 --- a/go.sum +++ b/go.sum @@ -138,6 +138,8 @@ golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b h1:lkjdUzSyJ5P1+eal9fxXX9Xg2BTfswsonKUse48C0uE= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190402181905-9f3314589c9a/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -173,6 +175,7 @@ golang.org/x/tools v0.0.0-20190312170243-e65039ee4138/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190422233926-fe54fb35175b/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= golang.org/x/tools v0.0.0-20190506145303-2d16b83fe98c/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b h1:mSUCVIwDx4hfXJfWsOPfdzEHxzb2Xjl6BQ8YgPnazQA= golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 0a7e6000b74f33d8cd789155ad8706033d61e9a3 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sun, 6 Oct 2019 01:01:16 -0700 Subject: [PATCH 04/19] Publish task results if topic is set --- c2/http.go | 12 ++++++------ c2/http_test.go | 2 +- c2/server.go | 33 +++++++++++++++++++++++++++++---- c2/server_test.go | 8 ++++++-- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/c2/http.go b/c2/http.go index ee5cb123..9c286b41 100644 --- a/c2/http.go +++ b/c2/http.go @@ -13,24 +13,24 @@ import ( func (srv *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { data, err := ioutil.ReadAll(req.Body) if err != nil { - srv.Logger.Error("Failed to read agent message", zap.Error(err)) + srv.Log.Error("Failed to read agent message", zap.Error(err)) return } - srv.Logger.Debug("Received agent message", zap.String("agent_msg", string(data))) + srv.Log.Debug("Received agent message", zap.String("agent_msg", string(data))) var msg transport.Response if err := json.Unmarshal(data, &msg); err != nil { - srv.Logger.Error("Failed to unmarshal agent message", zap.Error(err)) + srv.Log.Error("Failed to unmarshal agent message", zap.Error(err)) return } - if err = srv.HandleMessage(w, msg); err != nil { - srv.Logger.Error("Agent communication failed", zap.Error(err)) + if err = srv.HandleMessage(req.Context(), w, msg); err != nil { + srv.Log.Error("Agent communication failed", zap.Error(err)) return } // TODO: Add agent metadata - srv.Logger.Info("Replied to agent message") + srv.Log.Info("Replied to agent message") } diff --git a/c2/http_test.go b/c2/http_test.go index 033f07dc..4fb448f2 100644 --- a/c2/http_test.go +++ b/c2/http_test.go @@ -31,7 +31,7 @@ func TestHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBuffer(data)) srv := &c2.Server{ - Logger: zap.NewNop(), + Log: zap.NewNop(), } srv.QueueTask(expectedTask, func(transport.Metadata) bool { return true }) srv.QueueTask(unexpectedTask, func(transport.Metadata) bool { return false }) diff --git a/c2/server.go b/c2/server.go index fb7603cd..b51a2516 100644 --- a/c2/server.go +++ b/c2/server.go @@ -1,6 +1,7 @@ package c2 import ( + "context" "encoding/json" "fmt" "io" @@ -14,16 +15,16 @@ import ( // Server handles agent messages and replies with new tasks for the agent to execute. type Server struct { - Logger *zap.Logger - AgentOutput *pubsub.Topic + Log *zap.Logger + TaskResults *pubsub.Topic mu sync.RWMutex tasks []queuedTask } // HandleMessage received from the agent, and write a reply to the provided writer. -func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { - // TODO: Handle results +func (srv *Server) HandleMessage(ctx context.Context, w io.Writer, msg transport.Response) error { + srv.publishResults(ctx, msg) tasks := srv.GetTasks(msg.Metadata) reply := transport.Payload{ @@ -41,3 +42,27 @@ func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { return nil } + +func (srv *Server) publishResults(ctx context.Context, msg transport.Response) { + if srv.TaskResults == nil { + srv.Log.Warn("No topic set for task results") + return + } + + for _, result := range msg.Results { + // TODO: Send all results, or only ones with data? + body, err := json.Marshal(result) + if err != nil { + srv.Log.Error("Failed to marshal result to json", zap.Error(err)) + } + event := &pubsub.Message{ + Body: body, + // TODO: Add agent metadata + } + go func() { + if err = srv.TaskResults.Send(ctx, event); err != nil { + srv.Log.Error("Failed to publish task result", zap.Error(err)) + } + }() + } +} diff --git a/c2/server_test.go b/c2/server_test.go index d9ba37dd..06b53824 100644 --- a/c2/server_test.go +++ b/c2/server_test.go @@ -1,10 +1,12 @@ package c2_test import ( + "context" "encoding/json" "testing" "github.com/golang/mock/gomock" + "go.uber.org/zap" "github.com/kcarretto/paragon/c2" "github.com/kcarretto/paragon/c2/mocks" @@ -29,9 +31,11 @@ func TestHandleMessage(t *testing.T) { return len(p), nil }) - srv := &c2.Server{} + srv := &c2.Server{ + Log: zap.NewNop(), + } srv.QueueTask(expectedTask, func(agent transport.Metadata) bool { return true }) - err = srv.HandleMessage(replyWriter, transport.Response{}) + err = srv.HandleMessage(context.Background(), replyWriter, transport.Response{}) require.NoError(t, err) } From 67fe226b8641da98308e605c54b75014a853cd0a Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Fri, 11 Oct 2019 01:11:03 -0700 Subject: [PATCH 05/19] Added protobuf spec for codec and events --- api/codec/agent.pb.go | 263 +++++++++++++++++++++ api/codec/agent.proto | 31 +++ api/codec/codec.go | 12 + api/codec/server.pb.go | 129 ++++++++++ api/codec/server.proto | 14 ++ api/events/events.go | 10 + api/events/task.pb.go | 242 +++++++++++++++++++ api/events/task.proto | 38 +++ api/vendor/google/protobuf/timestamp.proto | 138 +++++++++++ 9 files changed, 877 insertions(+) create mode 100644 api/codec/agent.pb.go create mode 100644 api/codec/agent.proto create mode 100644 api/codec/codec.go create mode 100644 api/codec/server.pb.go create mode 100644 api/codec/server.proto create mode 100644 api/events/events.go create mode 100644 api/events/task.pb.go create mode 100644 api/events/task.proto create mode 100644 api/vendor/google/protobuf/timestamp.proto diff --git a/api/codec/agent.pb.go b/api/codec/agent.pb.go new file mode 100644 index 00000000..d66eb29c --- /dev/null +++ b/api/codec/agent.pb.go @@ -0,0 +1,263 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: agent.proto + +package codec + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type AgentMessage struct { + Metadata *AgentMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + Results []*Result `protobuf:"bytes,2,rep,name=results,proto3" json:"results,omitempty"` + Logs []string `protobuf:"bytes,3,rep,name=logs,proto3" json:"logs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AgentMessage) Reset() { *m = AgentMessage{} } +func (m *AgentMessage) String() string { return proto.CompactTextString(m) } +func (*AgentMessage) ProtoMessage() {} +func (*AgentMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_56ede974c0020f77, []int{0} +} + +func (m *AgentMessage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_AgentMessage.Unmarshal(m, b) +} +func (m *AgentMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_AgentMessage.Marshal(b, m, deterministic) +} +func (m *AgentMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_AgentMessage.Merge(m, src) +} +func (m *AgentMessage) XXX_Size() int { + return xxx_messageInfo_AgentMessage.Size(m) +} +func (m *AgentMessage) XXX_DiscardUnknown() { + xxx_messageInfo_AgentMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_AgentMessage proto.InternalMessageInfo + +func (m *AgentMessage) GetMetadata() *AgentMetadata { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *AgentMessage) GetResults() []*Result { + if m != nil { + return m.Results + } + return nil +} + +func (m *AgentMessage) GetLogs() []string { + if m != nil { + return m.Logs + } + return nil +} + +type AgentMetadata struct { + AgentID string `protobuf:"bytes,1,opt,name=agentID,proto3" json:"agentID,omitempty"` + MachineUUID string `protobuf:"bytes,2,opt,name=machineUUID,proto3" json:"machineUUID,omitempty"` + SessionID string `protobuf:"bytes,3,opt,name=sessionID,proto3" json:"sessionID,omitempty"` + Hostname string `protobuf:"bytes,4,opt,name=hostname,proto3" json:"hostname,omitempty"` + PrimaryIP string `protobuf:"bytes,5,opt,name=primaryIP,proto3" json:"primaryIP,omitempty"` + PrimaryMAC string `protobuf:"bytes,6,opt,name=primaryMAC,proto3" json:"primaryMAC,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AgentMetadata) Reset() { *m = AgentMetadata{} } +func (m *AgentMetadata) String() string { return proto.CompactTextString(m) } +func (*AgentMetadata) ProtoMessage() {} +func (*AgentMetadata) Descriptor() ([]byte, []int) { + return fileDescriptor_56ede974c0020f77, []int{1} +} + +func (m *AgentMetadata) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_AgentMetadata.Unmarshal(m, b) +} +func (m *AgentMetadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_AgentMetadata.Marshal(b, m, deterministic) +} +func (m *AgentMetadata) XXX_Merge(src proto.Message) { + xxx_messageInfo_AgentMetadata.Merge(m, src) +} +func (m *AgentMetadata) XXX_Size() int { + return xxx_messageInfo_AgentMetadata.Size(m) +} +func (m *AgentMetadata) XXX_DiscardUnknown() { + xxx_messageInfo_AgentMetadata.DiscardUnknown(m) +} + +var xxx_messageInfo_AgentMetadata proto.InternalMessageInfo + +func (m *AgentMetadata) GetAgentID() string { + if m != nil { + return m.AgentID + } + return "" +} + +func (m *AgentMetadata) GetMachineUUID() string { + if m != nil { + return m.MachineUUID + } + return "" +} + +func (m *AgentMetadata) GetSessionID() string { + if m != nil { + return m.SessionID + } + return "" +} + +func (m *AgentMetadata) GetHostname() string { + if m != nil { + return m.Hostname + } + return "" +} + +func (m *AgentMetadata) GetPrimaryIP() string { + if m != nil { + return m.PrimaryIP + } + return "" +} + +func (m *AgentMetadata) GetPrimaryMAC() string { + if m != nil { + return m.PrimaryMAC + } + return "" +} + +type Result struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Output []string `protobuf:"bytes,2,rep,name=output,proto3" json:"output,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + ExecStartTime *timestamp.Timestamp `protobuf:"bytes,4,opt,name=execStartTime,proto3" json:"execStartTime,omitempty"` + ExecStopTime *timestamp.Timestamp `protobuf:"bytes,5,opt,name=execStopTime,proto3" json:"execStopTime,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Result) Reset() { *m = Result{} } +func (m *Result) String() string { return proto.CompactTextString(m) } +func (*Result) ProtoMessage() {} +func (*Result) Descriptor() ([]byte, []int) { + return fileDescriptor_56ede974c0020f77, []int{2} +} + +func (m *Result) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Result.Unmarshal(m, b) +} +func (m *Result) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Result.Marshal(b, m, deterministic) +} +func (m *Result) XXX_Merge(src proto.Message) { + xxx_messageInfo_Result.Merge(m, src) +} +func (m *Result) XXX_Size() int { + return xxx_messageInfo_Result.Size(m) +} +func (m *Result) XXX_DiscardUnknown() { + xxx_messageInfo_Result.DiscardUnknown(m) +} + +var xxx_messageInfo_Result proto.InternalMessageInfo + +func (m *Result) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *Result) GetOutput() []string { + if m != nil { + return m.Output + } + return nil +} + +func (m *Result) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +func (m *Result) GetExecStartTime() *timestamp.Timestamp { + if m != nil { + return m.ExecStartTime + } + return nil +} + +func (m *Result) GetExecStopTime() *timestamp.Timestamp { + if m != nil { + return m.ExecStopTime + } + return nil +} + +func init() { + proto.RegisterType((*AgentMessage)(nil), "codec.AgentMessage") + proto.RegisterType((*AgentMetadata)(nil), "codec.AgentMetadata") + proto.RegisterType((*Result)(nil), "codec.Result") +} + +func init() { proto.RegisterFile("agent.proto", fileDescriptor_56ede974c0020f77) } + +var fileDescriptor_56ede974c0020f77 = []byte{ + // 374 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0x31, 0x4f, 0x83, 0x40, + 0x18, 0x0d, 0x50, 0x68, 0xf9, 0x68, 0x1d, 0x2e, 0x8d, 0xb9, 0x34, 0x46, 0x49, 0x07, 0x65, 0x02, + 0x53, 0x77, 0x63, 0xb5, 0x0b, 0x43, 0x13, 0x83, 0xed, 0xe2, 0x76, 0x85, 0x93, 0x5e, 0x2c, 0x1c, + 0xb9, 0x3b, 0x12, 0x1d, 0xfc, 0x75, 0x2e, 0xfe, 0x2c, 0xd3, 0x03, 0x6a, 0x3b, 0xb9, 0xf1, 0xde, + 0xfb, 0xde, 0xe3, 0xfb, 0xee, 0x81, 0x47, 0x72, 0x5a, 0xaa, 0xb0, 0x12, 0x5c, 0x71, 0x64, 0xa7, + 0x3c, 0xa3, 0xe9, 0xe4, 0x2a, 0xe7, 0x3c, 0xdf, 0xd1, 0x48, 0x93, 0x9b, 0xfa, 0x2d, 0x52, 0xac, + 0xa0, 0x52, 0x91, 0xa2, 0x6a, 0xe6, 0xa6, 0x5f, 0x30, 0x9c, 0xef, 0x6d, 0x4b, 0x2a, 0x25, 0xc9, + 0x29, 0xba, 0x85, 0x41, 0x41, 0x15, 0xc9, 0x88, 0x22, 0xd8, 0xf0, 0x8d, 0xc0, 0x9b, 0x8d, 0x43, + 0x1d, 0x15, 0xb6, 0x63, 0x8d, 0x96, 0x1c, 0xa6, 0xd0, 0x0d, 0xf4, 0x05, 0x95, 0xf5, 0x4e, 0x49, + 0x6c, 0xfa, 0x56, 0xe0, 0xcd, 0x46, 0xad, 0x21, 0xd1, 0x6c, 0xd2, 0xa9, 0x08, 0x41, 0x6f, 0xc7, + 0x73, 0x89, 0x2d, 0xdf, 0x0a, 0xdc, 0x44, 0x7f, 0x4f, 0xbf, 0x0d, 0x18, 0x9d, 0x04, 0x23, 0x0c, + 0x7d, 0x7d, 0x47, 0xbc, 0xd0, 0xff, 0x77, 0x93, 0x0e, 0x22, 0x1f, 0xbc, 0x82, 0xa4, 0x5b, 0x56, + 0xd2, 0xf5, 0x3a, 0x5e, 0x60, 0x53, 0xab, 0xc7, 0x14, 0xba, 0x00, 0x57, 0x52, 0x29, 0x19, 0x2f, + 0xe3, 0x05, 0xb6, 0xb4, 0xfe, 0x47, 0xa0, 0x09, 0x0c, 0xb6, 0x5c, 0xaa, 0x92, 0x14, 0x14, 0xf7, + 0xb4, 0x78, 0xc0, 0x7b, 0x67, 0x25, 0x58, 0x41, 0xc4, 0x67, 0xfc, 0x8c, 0xed, 0xc6, 0x79, 0x20, + 0xd0, 0x25, 0x40, 0x0b, 0x96, 0xf3, 0x27, 0xec, 0x68, 0xf9, 0x88, 0x99, 0xfe, 0x18, 0xe0, 0x34, + 0xd7, 0xa2, 0x33, 0x30, 0x59, 0xd6, 0x6e, 0x6e, 0xb2, 0x0c, 0x9d, 0x83, 0xc3, 0x6b, 0x55, 0xd5, + 0x4a, 0x3f, 0x8e, 0x9b, 0xb4, 0x08, 0x8d, 0xc1, 0xa6, 0x42, 0x70, 0xd1, 0xae, 0xd9, 0x00, 0xf4, + 0x00, 0x23, 0xfa, 0x41, 0xd3, 0x17, 0x45, 0x84, 0x5a, 0xb1, 0x76, 0x4f, 0x6f, 0x36, 0x09, 0x9b, + 0x1a, 0xc3, 0xae, 0xc6, 0x70, 0xd5, 0xd5, 0x98, 0x9c, 0x1a, 0xd0, 0x3d, 0x0c, 0x1b, 0x82, 0x57, + 0x3a, 0xc0, 0xfe, 0x37, 0xe0, 0x64, 0xfe, 0x31, 0x78, 0xbd, 0xce, 0x99, 0xda, 0xd6, 0x9b, 0x30, + 0xe5, 0x45, 0xf4, 0x9e, 0x12, 0x21, 0xa8, 0x52, 0x3c, 0xaa, 0x88, 0x20, 0x39, 0x2f, 0x23, 0x52, + 0xb1, 0x48, 0xd7, 0xbb, 0x71, 0x74, 0xd6, 0xdd, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x43, 0xc2, + 0xed, 0x4a, 0x77, 0x02, 0x00, 0x00, +} diff --git a/api/codec/agent.proto b/api/codec/agent.proto new file mode 100644 index 00000000..dcc113c0 --- /dev/null +++ b/api/codec/agent.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package codec; + +option go_package = "github.com/kcarretto/paragon/api/codec"; + +import "google/protobuf/timestamp.proto"; + + +message AgentMessage { + AgentMetadata metadata = 1; + repeated Result results = 2; + repeated string logs = 3; +} + +message AgentMetadata { + string agentID = 1; + string machineUUID = 2; + string sessionID = 3; + string hostname = 4; + string primaryIP = 5; + string primaryMAC = 6; +} + +message Result { + string id = 1; + repeated string output = 2; + string error = 3; + google.protobuf.Timestamp execStartTime = 4; + google.protobuf.Timestamp execStopTime = 5; +} \ No newline at end of file diff --git a/api/codec/codec.go b/api/codec/codec.go new file mode 100644 index 00000000..5871186b --- /dev/null +++ b/api/codec/codec.go @@ -0,0 +1,12 @@ +// Package codec defines a standardized format for messages sent between the server and agents. +// It does not mandate an encoding mechanism for transporting such data. Any format for serializing +// messages sent between the agent and the server (i.e. protobuf or JSON) is acceptable so long as +// both the agent and the server utilize the same format. +package codec + +// VERSION describes the currently built version of codec. +const VERSION = "0.0.1" + +// TODO: Add python support via --python_out=. + +//go:generate protoc -I=../vendor/ -I=../../ -I=. --go_out=paths=source_relative:. agent.proto server.proto diff --git a/api/codec/server.pb.go b/api/codec/server.pb.go new file mode 100644 index 00000000..9028081e --- /dev/null +++ b/api/codec/server.pb.go @@ -0,0 +1,129 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: server.proto + +package codec + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type ServerMessage struct { + Tasks []*Task `protobuf:"bytes,1,rep,name=tasks,proto3" json:"tasks,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ServerMessage) Reset() { *m = ServerMessage{} } +func (m *ServerMessage) String() string { return proto.CompactTextString(m) } +func (*ServerMessage) ProtoMessage() {} +func (*ServerMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_ad098daeda4239f7, []int{0} +} + +func (m *ServerMessage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServerMessage.Unmarshal(m, b) +} +func (m *ServerMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServerMessage.Marshal(b, m, deterministic) +} +func (m *ServerMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServerMessage.Merge(m, src) +} +func (m *ServerMessage) XXX_Size() int { + return xxx_messageInfo_ServerMessage.Size(m) +} +func (m *ServerMessage) XXX_DiscardUnknown() { + xxx_messageInfo_ServerMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_ServerMessage proto.InternalMessageInfo + +func (m *ServerMessage) GetTasks() []*Task { + if m != nil { + return m.Tasks + } + return nil +} + +type Task struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Task) Reset() { *m = Task{} } +func (m *Task) String() string { return proto.CompactTextString(m) } +func (*Task) ProtoMessage() {} +func (*Task) Descriptor() ([]byte, []int) { + return fileDescriptor_ad098daeda4239f7, []int{1} +} + +func (m *Task) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Task.Unmarshal(m, b) +} +func (m *Task) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Task.Marshal(b, m, deterministic) +} +func (m *Task) XXX_Merge(src proto.Message) { + xxx_messageInfo_Task.Merge(m, src) +} +func (m *Task) XXX_Size() int { + return xxx_messageInfo_Task.Size(m) +} +func (m *Task) XXX_DiscardUnknown() { + xxx_messageInfo_Task.DiscardUnknown(m) +} + +var xxx_messageInfo_Task proto.InternalMessageInfo + +func (m *Task) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *Task) GetContent() string { + if m != nil { + return m.Content + } + return "" +} + +func init() { + proto.RegisterType((*ServerMessage)(nil), "codec.ServerMessage") + proto.RegisterType((*Task)(nil), "codec.Task") +} + +func init() { proto.RegisterFile("server.proto", fileDescriptor_ad098daeda4239f7) } + +var fileDescriptor_ad098daeda4239f7 = []byte{ + // 168 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x34, 0x8e, 0xb1, 0xce, 0x82, 0x30, + 0x14, 0x85, 0x03, 0xff, 0x8f, 0xc6, 0xa2, 0x0e, 0x9d, 0x3a, 0x22, 0x83, 0x61, 0x6a, 0x0d, 0xbe, + 0x81, 0xbb, 0x0b, 0x3a, 0xb9, 0x5d, 0xca, 0x0d, 0x36, 0x44, 0x6e, 0xd3, 0x5e, 0x7d, 0x7e, 0x63, + 0x8d, 0xe3, 0xf7, 0x9d, 0x93, 0x9c, 0x23, 0xd6, 0x11, 0xc3, 0x0b, 0x83, 0xf6, 0x81, 0x98, 0x64, + 0x61, 0x69, 0x40, 0x5b, 0xb7, 0x62, 0x73, 0x49, 0xfa, 0x8c, 0x31, 0xc2, 0x88, 0x72, 0x27, 0x0a, + 0x86, 0x38, 0x45, 0x95, 0x55, 0x7f, 0x4d, 0xd9, 0x96, 0x3a, 0xf5, 0xf4, 0x15, 0xe2, 0xd4, 0x7d, + 0x93, 0xfa, 0x20, 0xfe, 0x3f, 0x28, 0xb7, 0x22, 0x77, 0x83, 0xca, 0xaa, 0xac, 0x59, 0x75, 0xb9, + 0x1b, 0xa4, 0x12, 0x4b, 0x4b, 0x33, 0xe3, 0xcc, 0x2a, 0x4f, 0xf2, 0x87, 0xa7, 0xe6, 0xb6, 0x1f, + 0x1d, 0xdf, 0x9f, 0xbd, 0xb6, 0xf4, 0x30, 0x93, 0x85, 0x10, 0x90, 0x99, 0x8c, 0x87, 0x00, 0x23, + 0xcd, 0x06, 0xbc, 0x33, 0x69, 0xa7, 0x5f, 0xa4, 0x77, 0xc7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xee, 0x57, 0xdc, 0xf8, 0xad, 0x00, 0x00, 0x00, +} diff --git a/api/codec/server.proto b/api/codec/server.proto new file mode 100644 index 00000000..9675be55 --- /dev/null +++ b/api/codec/server.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package codec; + +option go_package = "github.com/kcarretto/paragon/api/codec"; + +message ServerMessage { + repeated Task tasks = 1; +} + +message Task { + string id = 1; + string content = 2; +} \ No newline at end of file diff --git a/api/events/events.go b/api/events/events.go new file mode 100644 index 00000000..ac683805 --- /dev/null +++ b/api/events/events.go @@ -0,0 +1,10 @@ +// Package events defines a standardized format for paragon events that are published and may be +// subscribed to. +package events + +// VERSION describes the currently built version of events. +const VERSION = "0.0.1" + +// TODO: Add python support via --python_out=. + +//go:generate protoc -I=../../ -I=. --go_out=paths=source_relative:. task.proto diff --git a/api/events/task.pb.go b/api/events/task.pb.go new file mode 100644 index 00000000..b4ff7060 --- /dev/null +++ b/api/events/task.pb.go @@ -0,0 +1,242 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: task.proto + +package events + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + codec "github.com/kcarretto/paragon/api/codec" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type TaskQueued struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` + Filter *codec.AgentMetadata `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TaskQueued) Reset() { *m = TaskQueued{} } +func (m *TaskQueued) String() string { return proto.CompactTextString(m) } +func (*TaskQueued) ProtoMessage() {} +func (*TaskQueued) Descriptor() ([]byte, []int) { + return fileDescriptor_ce5d8dd45b4a91ff, []int{0} +} + +func (m *TaskQueued) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TaskQueued.Unmarshal(m, b) +} +func (m *TaskQueued) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TaskQueued.Marshal(b, m, deterministic) +} +func (m *TaskQueued) XXX_Merge(src proto.Message) { + xxx_messageInfo_TaskQueued.Merge(m, src) +} +func (m *TaskQueued) XXX_Size() int { + return xxx_messageInfo_TaskQueued.Size(m) +} +func (m *TaskQueued) XXX_DiscardUnknown() { + xxx_messageInfo_TaskQueued.DiscardUnknown(m) +} + +var xxx_messageInfo_TaskQueued proto.InternalMessageInfo + +func (m *TaskQueued) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *TaskQueued) GetContent() string { + if m != nil { + return m.Content + } + return "" +} + +func (m *TaskQueued) GetFilter() *codec.AgentMetadata { + if m != nil { + return m.Filter + } + return nil +} + +type TaskClaimed struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Agent *codec.AgentMetadata `protobuf:"bytes,2,opt,name=agent,proto3" json:"agent,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TaskClaimed) Reset() { *m = TaskClaimed{} } +func (m *TaskClaimed) String() string { return proto.CompactTextString(m) } +func (*TaskClaimed) ProtoMessage() {} +func (*TaskClaimed) Descriptor() ([]byte, []int) { + return fileDescriptor_ce5d8dd45b4a91ff, []int{1} +} + +func (m *TaskClaimed) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TaskClaimed.Unmarshal(m, b) +} +func (m *TaskClaimed) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TaskClaimed.Marshal(b, m, deterministic) +} +func (m *TaskClaimed) XXX_Merge(src proto.Message) { + xxx_messageInfo_TaskClaimed.Merge(m, src) +} +func (m *TaskClaimed) XXX_Size() int { + return xxx_messageInfo_TaskClaimed.Size(m) +} +func (m *TaskClaimed) XXX_DiscardUnknown() { + xxx_messageInfo_TaskClaimed.DiscardUnknown(m) +} + +var xxx_messageInfo_TaskClaimed proto.InternalMessageInfo + +func (m *TaskClaimed) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *TaskClaimed) GetAgent() *codec.AgentMetadata { + if m != nil { + return m.Agent + } + return nil +} + +type TaskExecuted struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Output string `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + ExecStartTime int64 `protobuf:"varint,4,opt,name=execStartTime,proto3" json:"execStartTime,omitempty"` + ExecStopTime int64 `protobuf:"varint,5,opt,name=execStopTime,proto3" json:"execStopTime,omitempty"` + RecvTime int64 `protobuf:"varint,6,opt,name=recvTime,proto3" json:"recvTime,omitempty"` + Agent *codec.AgentMetadata `protobuf:"bytes,7,opt,name=agent,proto3" json:"agent,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TaskExecuted) Reset() { *m = TaskExecuted{} } +func (m *TaskExecuted) String() string { return proto.CompactTextString(m) } +func (*TaskExecuted) ProtoMessage() {} +func (*TaskExecuted) Descriptor() ([]byte, []int) { + return fileDescriptor_ce5d8dd45b4a91ff, []int{2} +} + +func (m *TaskExecuted) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TaskExecuted.Unmarshal(m, b) +} +func (m *TaskExecuted) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TaskExecuted.Marshal(b, m, deterministic) +} +func (m *TaskExecuted) XXX_Merge(src proto.Message) { + xxx_messageInfo_TaskExecuted.Merge(m, src) +} +func (m *TaskExecuted) XXX_Size() int { + return xxx_messageInfo_TaskExecuted.Size(m) +} +func (m *TaskExecuted) XXX_DiscardUnknown() { + xxx_messageInfo_TaskExecuted.DiscardUnknown(m) +} + +var xxx_messageInfo_TaskExecuted proto.InternalMessageInfo + +func (m *TaskExecuted) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *TaskExecuted) GetOutput() string { + if m != nil { + return m.Output + } + return "" +} + +func (m *TaskExecuted) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +func (m *TaskExecuted) GetExecStartTime() int64 { + if m != nil { + return m.ExecStartTime + } + return 0 +} + +func (m *TaskExecuted) GetExecStopTime() int64 { + if m != nil { + return m.ExecStopTime + } + return 0 +} + +func (m *TaskExecuted) GetRecvTime() int64 { + if m != nil { + return m.RecvTime + } + return 0 +} + +func (m *TaskExecuted) GetAgent() *codec.AgentMetadata { + if m != nil { + return m.Agent + } + return nil +} + +func init() { + proto.RegisterType((*TaskQueued)(nil), "events.TaskQueued") + proto.RegisterType((*TaskClaimed)(nil), "events.TaskClaimed") + proto.RegisterType((*TaskExecuted)(nil), "events.TaskExecuted") +} + +func init() { proto.RegisterFile("task.proto", fileDescriptor_ce5d8dd45b4a91ff) } + +var fileDescriptor_ce5d8dd45b4a91ff = []byte{ + // 296 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0xc1, 0x4e, 0xc2, 0x40, + 0x14, 0x4c, 0x41, 0x8a, 0x3c, 0xd0, 0xc3, 0x06, 0x4d, 0xc3, 0x89, 0x34, 0x26, 0xa2, 0x31, 0x6d, + 0xa2, 0x5f, 0xa0, 0xc6, 0x83, 0x07, 0x0f, 0x56, 0x4e, 0xde, 0x1e, 0xbb, 0x4f, 0xdc, 0x00, 0xdd, + 0x66, 0xfb, 0x96, 0xf0, 0xb5, 0x7e, 0x8b, 0xe9, 0x2e, 0x68, 0x48, 0x30, 0x1e, 0x67, 0xde, 0x64, + 0x66, 0x5e, 0x06, 0x80, 0xb1, 0x5e, 0x64, 0x95, 0x35, 0x6c, 0x44, 0x4c, 0x6b, 0x2a, 0xb9, 0x1e, + 0x9d, 0x61, 0xa5, 0x73, 0x69, 0x14, 0xc9, 0x1c, 0xe7, 0x54, 0x72, 0x38, 0xa7, 0x0a, 0x60, 0x8a, + 0xf5, 0xe2, 0xd5, 0x91, 0x23, 0x25, 0x4e, 0xa1, 0xa5, 0x55, 0x12, 0x8d, 0xa3, 0x49, 0xaf, 0x68, + 0x69, 0x25, 0x12, 0xe8, 0x4a, 0x53, 0x32, 0x95, 0x9c, 0xb4, 0x3c, 0xb9, 0x83, 0xe2, 0x06, 0xe2, + 0x0f, 0xbd, 0x64, 0xb2, 0x49, 0x7b, 0x1c, 0x4d, 0xfa, 0xb7, 0xc3, 0xcc, 0x7b, 0x67, 0xf7, 0x8d, + 0xf7, 0x0b, 0x31, 0x2a, 0x64, 0x2c, 0xb6, 0x9a, 0xf4, 0x19, 0xfa, 0x4d, 0xca, 0xe3, 0x12, 0xf5, + 0xea, 0x40, 0xcc, 0x35, 0x74, 0x7c, 0x27, 0x1f, 0xf2, 0x97, 0x57, 0x90, 0xa4, 0x5f, 0x11, 0x0c, + 0x1a, 0xaf, 0xa7, 0x0d, 0x49, 0xc7, 0x07, 0xcc, 0xce, 0x21, 0x36, 0x8e, 0x2b, 0xb7, 0xab, 0xbc, + 0x45, 0x62, 0x08, 0x1d, 0xb2, 0xd6, 0x84, 0xc2, 0xbd, 0x22, 0x00, 0x71, 0x01, 0x27, 0xb4, 0x21, + 0xf9, 0xc6, 0x68, 0x79, 0xaa, 0x57, 0x94, 0x1c, 0x8d, 0xa3, 0x49, 0xbb, 0xd8, 0x27, 0x45, 0x0a, + 0x83, 0x40, 0x98, 0xca, 0x8b, 0x3a, 0x5e, 0xb4, 0xc7, 0x89, 0x11, 0x1c, 0x5b, 0x92, 0x6b, 0x7f, + 0x8f, 0xfd, 0xfd, 0x07, 0xff, 0x3e, 0xd8, 0xfd, 0xf7, 0xc1, 0x87, 0xab, 0xf7, 0xcb, 0xb9, 0xe6, + 0x4f, 0x37, 0xcb, 0xa4, 0x59, 0xe5, 0x0b, 0x89, 0xd6, 0x12, 0xb3, 0xc9, 0x2b, 0xb4, 0x38, 0x37, + 0x65, 0xde, 0xec, 0x18, 0x36, 0x9d, 0xc5, 0x7e, 0xc3, 0xbb, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x5e, 0x6f, 0x75, 0xbf, 0xf0, 0x01, 0x00, 0x00, +} diff --git a/api/events/task.proto b/api/events/task.proto new file mode 100644 index 00000000..d31bcf4c --- /dev/null +++ b/api/events/task.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +package events; + +option go_package = "github.com/kcarretto/paragon/api/events"; + +import "api/codec/agent.proto"; + +// message AgentMetadata { +// string agentID = 1; +// string machineUUID = 2; +// string sessionID = 3; +// string hostname = 4; +// string primaryIP = 5; +// string primaryMAC = 6; +// } + +message TaskQueued { + string id = 1; + string content = 2; + codec.AgentMetadata filter = 3; +} + +message TaskClaimed { + string id = 1; + codec.AgentMetadata agent = 2; +} + +message TaskExecuted { + string id = 1; + string output = 2; + string error = 3; + int64 execStartTime = 4; + int64 execStopTime = 5; + int64 recvTime = 6; + + codec.AgentMetadata agent = 7; +} diff --git a/api/vendor/google/protobuf/timestamp.proto b/api/vendor/google/protobuf/timestamp.proto new file mode 100644 index 00000000..7279266f --- /dev/null +++ b/api/vendor/google/protobuf/timestamp.proto @@ -0,0 +1,138 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "github.com/golang/protobuf/ptypes/timestamp"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} \ No newline at end of file From 4957fd2e0b89d51a5c4bc3c0889c88f1cdddd045 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Fri, 11 Oct 2019 20:21:55 -0700 Subject: [PATCH 06/19] C2 work --- c2/server.go | 22 +++++-- c2/task.go | 145 ++++++++++++++++++++++++++++++++++++++++++- cmd/c2/main.go | 127 +++++++++++++++++++++++-------------- cmd/c2/pubsub_gcp.go | 40 ++++++++++++ cmd/c2/pubsub_mem.go | 21 +++++++ go.mod | 1 + go.sum | 10 +++ 7 files changed, 312 insertions(+), 54 deletions(-) create mode 100644 cmd/c2/pubsub_gcp.go create mode 100644 cmd/c2/pubsub_mem.go diff --git a/c2/server.go b/c2/server.go index b51a2516..bffe7eb9 100644 --- a/c2/server.go +++ b/c2/server.go @@ -6,9 +6,12 @@ import ( "fmt" "io" "sync" + "time" + "github.com/kcarretto/paragon/api/events" "github.com/kcarretto/paragon/transport" + "github.com/golang/protobuf/proto" "go.uber.org/zap" "gocloud.dev/pubsub" ) @@ -19,7 +22,7 @@ type Server struct { TaskResults *pubsub.Topic mu sync.RWMutex - tasks []queuedTask + tasks []queueEntry } // HandleMessage received from the agent, and write a reply to the provided writer. @@ -50,17 +53,26 @@ func (srv *Server) publishResults(ctx context.Context, msg transport.Response) { } for _, result := range msg.Results { + event := events.TaskExecuted{ + Id: result.ID, + Output: string(result.Output), + Error: result.Error, + ExecStartTime: result.ExecStartTime.Unix(), + ExecStopTime: result.ExecStopTime.Unix(), + RecvTime: time.Now().Unix(), + } + // TODO: Send all results, or only ones with data? - body, err := json.Marshal(result) + body, err := proto.Marshal(&event) if err != nil { srv.Log.Error("Failed to marshal result to json", zap.Error(err)) } - event := &pubsub.Message{ + msg := &pubsub.Message{ Body: body, - // TODO: Add agent metadata + // TODO: Add c2 metadata } go func() { - if err = srv.TaskResults.Send(ctx, event); err != nil { + if err = srv.TaskResults.Send(ctx, msg); err != nil { srv.Log.Error("Failed to publish task result", zap.Error(err)) } }() diff --git a/c2/task.go b/c2/task.go index ff4a7486..539e9e20 100644 --- a/c2/task.go +++ b/c2/task.go @@ -1,24 +1,55 @@ package c2 import ( + "context" + + "github.com/golang/protobuf/proto" + "github.com/kcarretto/paragon/api/events" "github.com/kcarretto/paragon/transport" + + "go.uber.org/zap" + "gocloud.dev/pubsub" ) -type queuedTask struct { +type queueEntry struct { task transport.Task filter func(transport.Metadata) bool } +// TODO: Convert underlying struct to map + // QueueTask prepares a task to be sent to the first agent that reports metadata matching the filter. func (srv *Server) QueueTask(task transport.Task, filter func(transport.Metadata) bool) { srv.mu.Lock() - srv.tasks = append(srv.tasks, queuedTask{ + srv.tasks = append(srv.tasks, queueEntry{ task, filter, }) srv.mu.Unlock() } +// removeTask removes the first task from the queue with a matching ID. If no matching task is found +// in the queue, removeTask is a no-op. +func (srv *Server) removeTask(id string) { + srv.mu.RLock() + var index = -1 + for i, entry := range srv.tasks { + if entry.task.ID == id { + index = i + break + } + } + srv.mu.RUnlock() + + if index < 0 { + return + } + + // Remove element from the queue + srv.tasks = append(srv.tasks[:index], srv.tasks[index+1:]...) + +} + // GetTasks for an agent based on the metadata it reported and the current tasks in the server queue. func (srv *Server) GetTasks(agent transport.Metadata) []transport.Task { srv.mu.Lock() @@ -46,3 +77,113 @@ func (srv *Server) TaskCount() int { return count } + +// ListenForQueuedTasks receives TaskQueued events and queues the corresponding task. +func (srv *Server) ListenForQueuedTasks(ctx context.Context, subscription *pubsub.Subscription) { + for { + select { + case <-ctx.Done(): + return + default: + msg, err := subscription.Receive(ctx) + if err != nil { + srv.Log.Error("Subscription failed", zap.Error(err)) + return + } + + var event events.TaskQueued + if err = proto.Unmarshal(msg.Body, &event); err != nil { + srv.Log.Error("Failed to unmarshal TaskQueued event", zap.Error(err)) + } + + criteria := transport.Metadata{ + AgentID: event.Filter.GetAgentID(), + MachineUUID: event.Filter.GetMachineUUID(), + SessionID: event.Filter.GetSessionID(), + Hostname: event.Filter.GetHostname(), + PrimaryIP: event.Filter.GetPrimaryIP(), + PrimaryMAC: event.Filter.GetPrimaryMAC(), + } + + task := transport.Task{ + ID: event.GetId(), + Content: []byte(event.GetContent()), + } + + srv.QueueTask(task, claimFilter(criteria)) + } + } +} + +// ListenForClaimedTasks receives TaskClaimed events and dequeues the corresponding task. +func (srv *Server) ListenForClaimedTasks(ctx context.Context, subscription *pubsub.Subscription) { + for { + select { + case <-ctx.Done(): + return + default: + msg, err := subscription.Receive(ctx) + if err != nil { + srv.Log.Error("Subscription failed", zap.Error(err)) + return + } + + var event events.TaskClaimed + if err = proto.Unmarshal(msg.Body, &event); err != nil { + srv.Log.Error("Failed to unmarshal TaskClaimed event", zap.Error(err)) + } + + srv.removeTask(event.GetId()) + } + } +} + +// claimFilter returns a filter that implements a simple algorithm for task claiming. +func claimFilter(criteria transport.Metadata) func(agent transport.Metadata) bool { + return func(agent transport.Metadata) bool { + // True if session ids match. + if id := criteria.SessionID; id != "" { + if id == agent.SessionID { + return true + } + } + + // Restrict if agentID criteria is set. + if agentID := criteria.AgentID; agentID != "" { + if agentID != agent.AgentID { + return false + } + } + + // True if machine uuids match. + if uuid := criteria.MachineUUID; uuid != "" { + if uuid == agent.MachineUUID { + return true + } + } + + // True if MAC address matches + if mac := criteria.PrimaryMAC; mac != "" { + if mac == agent.PrimaryMAC { + return true + } + } + + // True if IP address matches + if addr := criteria.PrimaryIP; addr != "" { + if addr == agent.PrimaryIP { + return true + } + } + + // True if hostname matches + if hostname := criteria.Hostname; hostname != "" { + if hostname == agent.Hostname { + return true + } + } + + // False otherwise + return false + } +} diff --git a/cmd/c2/main.go b/cmd/c2/main.go index b0e191f1..f9888c6f 100644 --- a/cmd/c2/main.go +++ b/cmd/c2/main.go @@ -1,58 +1,91 @@ package main import ( - "encoding/json" - "fmt" - "io/ioutil" - "math/rand" + "context" "net/http" + "os" - "github.com/kcarretto/paragon/transport" + "github.com/kcarretto/paragon/c2" + "go.uber.org/zap" ) func main() { - router := http.NewServeMux() - router.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - data, err := ioutil.ReadAll(req.Body) - if err != nil { - panic(err) - } - - var agentResponse transport.Response - if err := json.Unmarshal(data, &agentResponse); err != nil { - panic(err) - } - - fmt.Printf("\n\nReceived request: %+v\n", agentResponse) - - var tasks []transport.Task - if rand.Intn(2) == 0 { - task := transport.Task{ - ID: "task1", - Content: []byte(`print("Task from C2 ;)")`), - } - tasks = append(tasks, task) - } - - resp := transport.Payload{ - Tasks: tasks, - } - - respData, err := json.Marshal(resp) - if err != nil { - panic(err) - } - - if _, err = w.Write(respData); err != nil { - panic(err) - } - - fmt.Println("Responded to agent") - - }) - - fmt.Println("Running C2. Serving on 127.0.0.1:8080") - if err := http.ListenAndServe("127.0.0.1:8080", router); err != nil { + ctx := context.Background() + + logger, err := zap.NewDevelopment() + if err != nil { panic(err) } + + httpAddr := os.Getenv("HTTP_ADDR") + if httpAddr == "" { + httpAddr = "127.0.0.1:8080" + } + + resultTopic, err := openTopic(ctx, "tasks.result_received") + if err != nil { + logger.Panic("Failed to open pubsub topic", zap.Error(err)) + } + defer resultTopic.Shutdown(ctx) + + queueTopic, err := openSubscription(ctx, "tasks.queued") + if err != nil { + logger.Panic("Failed to subscribe to pubsub topic", zap.Error(err)) + } + defer queueTopic.Shutdown(ctx) + + srv := &c2.Server{ + Log: logger, + TaskResults: resultTopic, + } + + go func() { + + }() + + logger.Info("Started C2 server", zap.String("http_addr", httpAddr)) + if err := http.ListenAndServe(httpAddr, srv); err != nil { + logger.Panic("Failed to serve HTTP", zap.Error(err)) + } + + // router := http.NewServeMux() + // router.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + // data, err := ioutil.ReadAll(req.Body) + // if err != nil { + // panic(err) + // } + + // var agentResponse transport.Response + // if err := json.Unmarshal(data, &agentResponse); err != nil { + // panic(err) + // } + + // fmt.Printf("\n\nReceived request: %+v\n", agentResponse) + + // var tasks []transport.Task + // if rand.Intn(2) == 0 { + // task := transport.Task{ + // ID: "task1", + // Content: []byte(`print("Task from C2 ;)")`), + // } + // tasks = append(tasks, task) + // } + + // resp := transport.Payload{ + // Tasks: tasks, + // } + + // respData, err := json.Marshal(resp) + // if err != nil { + // panic(err) + // } + + // if _, err = w.Write(respData); err != nil { + // panic(err) + // } + + // fmt.Println("Responded to agent") + + // }) + } diff --git a/cmd/c2/pubsub_gcp.go b/cmd/c2/pubsub_gcp.go new file mode 100644 index 00000000..ffab2371 --- /dev/null +++ b/cmd/c2/pubsub_gcp.go @@ -0,0 +1,40 @@ +// +build gcp + +package main + +import ( + "context" + "fmt" + "os" + + "gocloud.dev/pubsub" + _ "gocloud.dev/pubsub/gcppubsub" +) + +func getURI(topic string) (string, error) { + project := os.Getenv("GCP_PROJECT") + if project == "" { + return "", fmt.Errorf("must set GCP_PROJECT environment variable to use GCP pubsub") + } + + uri := fmt.Sprintf("gcppubsub://projects/%s/topics/%s", project, topic) + return uri, nil +} + +func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { + uri, err := getURI(topic) + if err != nil { + return nil, err + } + + return pubsub.OpenTopic(ctx, uri) +} + +func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { + uri, err := getURI(topic) + if err != nil { + return nil, err + } + + return pubsub.OpenSubscription(ctx, uri) +} diff --git a/cmd/c2/pubsub_mem.go b/cmd/c2/pubsub_mem.go new file mode 100644 index 00000000..652c964a --- /dev/null +++ b/cmd/c2/pubsub_mem.go @@ -0,0 +1,21 @@ +// +build !gcp + +package main + +import ( + "context" + "fmt" + + "gocloud.dev/pubsub" + _ "gocloud.dev/pubsub/mempubsub" +) + +func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { + uri := fmt.Sprintf("mem://%s", topic) + return pubsub.OpenTopic(ctx, uri) +} + +func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { + uri := fmt.Sprintf("mem://%s", topic) + return pubsub.OpenSubscription(ctx, uri) +} diff --git a/go.mod b/go.mod index 920e2aed..c9c55637 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/mock v1.3.1 + github.com/golang/protobuf v1.3.2 github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.3.0 github.com/shirou/gopsutil v2.18.12+incompatible diff --git a/go.sum b/go.sum index e0b053ce..3985652a 100644 --- a/go.sum +++ b/go.sum @@ -10,10 +10,12 @@ contrib.go.opencensus.io/integrations/ocsql v0.1.4/go.mod h1:8DsSdjz3F+APR+0z0Wk contrib.go.opencensus.io/resource v0.1.1/go.mod h1:F361eGI91LCmW1I/Saf+rX0+OFcigGlFvXwEGEnkRLA= github.com/Azure/azure-amqp-common-go/v2 v2.1.0/go.mod h1:R8rea+gJRuJR6QxTir/XuEd+YuKoUiazDC/N96FiDEU= github.com/Azure/azure-pipeline-go v0.1.8/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= +github.com/Azure/azure-pipeline-go v0.1.9 h1:u7JFb9fFTE6Y/j8ae2VK33ePrRqJqoCM/IWkQdAZ+rg= github.com/Azure/azure-pipeline-go v0.1.9/go.mod h1:XA1kFWRVhSK+KNFiOhfv83Fv8L9achrP7OxIzeTn1Yg= github.com/Azure/azure-sdk-for-go v29.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-sdk-for-go v30.1.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/azure-service-bus-go v0.9.1/go.mod h1:yzBx6/BUGfjfeqbRZny9AQIbIe3AcV9WZbAdpkoXOa0= +github.com/Azure/azure-storage-blob-go v0.6.0 h1:SEATKb3LIHcaSIX+E6/K4kJpwfuozFEsmt5rS56N6CE= github.com/Azure/azure-storage-blob-go v0.6.0/go.mod h1:oGfmITT1V6x//CswqY2gtAHND+xIP64/qL7a5QJix0Y= github.com/Azure/go-autorest v12.0.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= @@ -22,6 +24,7 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUW github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/aws/aws-sdk-go v1.15.27/go.mod h1:mFuSZ37Z9YOHbQEwBWztmVzqXrEkub65tZoCYDt7FT0= github.com/aws/aws-sdk-go v1.19.18/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aws/aws-sdk-go v1.19.45 h1:jAxmC8qqa7mW531FDgM8Ahbqlb3zmiHgTpJU6fY3vJ0= github.com/aws/aws-sdk-go v1.19.45/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -50,17 +53,23 @@ github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFU github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-replayers/grpcreplay v0.1.0 h1:eNb1y9rZFmY4ax45uEEECSa8fsxGRU+8Bil52ASAwic= github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= +github.com/google/go-replayers/httpreplay v0.1.0 h1:AX7FUb4BjrrzNvblr/OlgwrmFiep6soj5K2QSDW7BGk= github.com/google/go-replayers/httpreplay v0.1.0/go.mod h1:YKZViNhiGgqdBlUbI2MwGpq4pXxNmhJLPHQ7cv2b5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible h1:xmapqc1AyLoB+ddYT6r04bD9lIjlOqGaREovi0SzFaE= github.com/google/martian v2.1.1-0.20190517191504-25dcb96d9e51+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/wire v0.3.0 h1:imGQZGEVEHpje5056+K+cgdO72p0LQv2xIIFXNGUf60= github.com/google/wire v0.3.0/go.mod h1:i1DMg/Lu8Sz5yYl25iOdmc5CT5qusaa+zmRWs16741s= github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= @@ -72,6 +81,7 @@ github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= From eeb278b2ac0500377447580781baeb14f6dac29a Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Fri, 11 Oct 2019 21:21:59 -0700 Subject: [PATCH 07/19] Working on further simplification --- agent/agent.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 agent/agent.go diff --git a/agent/agent.go b/agent/agent.go new file mode 100644 index 00000000..260e6463 --- /dev/null +++ b/agent/agent.go @@ -0,0 +1,67 @@ +package agent + +import ( + "context" + "fmt" + + "github.com/kcarretto/paragon/api/codec" +) + +type AgentMessage codec.AgentMessage +type ServerMessage codec.ServerMessage + +var ErrNoTransports = fmt.Errorf("all available transports failed to send message") + +type ServerMessageWriter interface{} +type AgentMessageWriter interface{} + +type Sender interface { + Send(ServerMessageWriter, AgentMessage) error +} + +type Receiver interface { + Receive(AgentMessageWriter, ServerMessage) +} + +type Agent struct { + Sender + Receiver + + Transports []Sender +} + +func (agent Agent) Send(w ServerMessageWriter, msg AgentMessage) error { + if agent.Sender != nil { + return agent.Sender.Send(w, msg) + } + + for _, transport := range agent.Transports { + if err := transport.Send(w, msg); err != nil { + // TODO: Error Handler + continue + } + + return nil + } + + return ErrNoTransports +} + +func (agent Agent) Run(ctx context.Context) error { + + var agentMsg AgentMessage + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + var srvMsg ServerMessage + if err := agent.Send(&srvMsg, agentMsg); err != nil { + return err + } + + agent.Receive(&agentMsg, srvMsg) + } + } +} From 206818dc4ad78fbbfdceaf100f2e0ff68629c564 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Tue, 15 Oct 2019 19:54:23 -0700 Subject: [PATCH 08/19] Added message writers & tests --- agent/agent.gen_test.go | 83 +++++++++++++++++++++++++++++ agent/agent.go | 54 +++++++++++-------- agent/agent_test.go | 115 ++++++++++++++++++++++++++++++++++++++++ agent/errors.go | 6 +++ agent/io.gen_test.go | 100 ++++++++++++++++++++++++++++++++++ agent/message.go | 51 ++++++++++++++++++ agent/message_test.go | 45 ++++++++++++++++ agent/mocks_test.go | 4 ++ agent/transport.go | 36 +++++++++++++ go.mod | 2 +- 10 files changed, 474 insertions(+), 22 deletions(-) create mode 100644 agent/agent.gen_test.go create mode 100644 agent/agent_test.go create mode 100644 agent/errors.go create mode 100644 agent/io.gen_test.go create mode 100644 agent/message.go create mode 100644 agent/message_test.go create mode 100644 agent/mocks_test.go create mode 100644 agent/transport.go diff --git a/agent/agent.gen_test.go b/agent/agent.gen_test.go new file mode 100644 index 00000000..61f74096 --- /dev/null +++ b/agent/agent.gen_test.go @@ -0,0 +1,83 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kcarretto/paragon/agent (interfaces: Sender,Receiver) + +// Package agent_test is a generated GoMock package. +package agent_test + +import ( + gomock "github.com/golang/mock/gomock" + agent "github.com/kcarretto/paragon/agent" + reflect "reflect" +) + +// MockSender is a mock of Sender interface +type MockSender struct { + ctrl *gomock.Controller + recorder *MockSenderMockRecorder +} + +// MockSenderMockRecorder is the mock recorder for MockSender +type MockSenderMockRecorder struct { + mock *MockSender +} + +// NewMockSender creates a new mock instance +func NewMockSender(ctrl *gomock.Controller) *MockSender { + mock := &MockSender{ctrl: ctrl} + mock.recorder = &MockSenderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSender) EXPECT() *MockSenderMockRecorder { + return m.recorder +} + +// Send mocks base method +func (m *MockSender) Send(arg0 agent.ServerMessageWriter, arg1 agent.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send +func (mr *MockSenderMockRecorder) Send(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSender)(nil).Send), arg0, arg1) +} + +// MockReceiver is a mock of Receiver interface +type MockReceiver struct { + ctrl *gomock.Controller + recorder *MockReceiverMockRecorder +} + +// MockReceiverMockRecorder is the mock recorder for MockReceiver +type MockReceiverMockRecorder struct { + mock *MockReceiver +} + +// NewMockReceiver creates a new mock instance +func NewMockReceiver(ctrl *gomock.Controller) *MockReceiver { + mock := &MockReceiver{ctrl: ctrl} + mock.recorder = &MockReceiverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockReceiver) EXPECT() *MockReceiverMockRecorder { + return m.recorder +} + +// Receive mocks base method +func (m *MockReceiver) Receive(arg0 agent.MessageWriter, arg1 agent.ServerMessage) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Receive", arg0, arg1) +} + +// Receive indicates an expected call of Receive +func (mr *MockReceiverMockRecorder) Receive(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Receive", reflect.TypeOf((*MockReceiver)(nil).Receive), arg0, arg1) +} diff --git a/agent/agent.go b/agent/agent.go index 260e6463..f3824251 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2,61 +2,73 @@ package agent import ( "context" - "fmt" + "time" - "github.com/kcarretto/paragon/api/codec" + "go.uber.org/zap" ) -type AgentMessage codec.AgentMessage -type ServerMessage codec.ServerMessage - -var ErrNoTransports = fmt.Errorf("all available transports failed to send message") - -type ServerMessageWriter interface{} -type AgentMessageWriter interface{} - +// A Sender is responsible for transporting messages to a server. type Sender interface { - Send(ServerMessageWriter, AgentMessage) error + Send(ServerMessageWriter, Message) error } +// A Receiver is responsible for handling messages sent by a server. type Receiver interface { - Receive(AgentMessageWriter, ServerMessage) + Receive(MessageWriter, ServerMessage) } +// An Agent communicates with server(s) using the configured transport. type Agent struct { - Sender Receiver + Log *zap.Logger - Transports []Sender + Transports []Transport + + MaxIdleTime time.Duration + lastSend time.Time } -func (agent Agent) Send(w ServerMessageWriter, msg AgentMessage) error { - if agent.Sender != nil { - return agent.Sender.Send(w, msg) +// Send messages to a server using the configured transports. Returns ErrNoTransports if all fail or +// if none are configured. +func (agent Agent) Send(w ServerMessageWriter, msg Message) error { + // Don't send empty messages unless it has been at least MaxIdleTime since the last send. + if msg.IsEmpty() && time.Since(agent.lastSend) <= agent.MaxIdleTime { + return nil } + // Attempt to send using available transports. for _, transport := range agent.Transports { if err := transport.Send(w, msg); err != nil { - // TODO: Error Handler + transport.Log.Error( + "Failed to send message using transport", + zap.Error(err), + zap.Reflect("transport", transport), + ) continue } + // When send is successful, update the timestamp + agent.lastSend = time.Now() + + // Sleep the transport's interval on success + transport.Sleep() + return nil } return ErrNoTransports } +// Run the agent, sending agent messages to a server using configured transports. func (agent Agent) Run(ctx context.Context) error { - - var agentMsg AgentMessage - for { select { case <-ctx.Done(): return ctx.Err() default: + var agentMsg Message var srvMsg ServerMessage + if err := agent.Send(&srvMsg, agentMsg); err != nil { return err } diff --git a/agent/agent_test.go b/agent/agent_test.go new file mode 100644 index 00000000..bb1d8be8 --- /dev/null +++ b/agent/agent_test.go @@ -0,0 +1,115 @@ +package agent_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kcarretto/paragon/agent" + + gomock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestSend(t *testing.T) { + logger, err := zap.NewDevelopment() + require.NoError(t, err) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + badSender := NewMockSender(ctrl) + sender := NewMockSender(ctrl) + unusedSender := NewMockSender(ctrl) + + srvMsg := agent.ServerMessage{} + agentMsg := agent.Message{} + badSender.EXPECT().Send(&srvMsg, agentMsg).Return(errors.New("oops ^_^")) + sender.EXPECT().Send(&srvMsg, agentMsg).Return(nil) + + testAgent := &agent.Agent{ + Log: logger, + Transports: []agent.Transport{ + agent.Transport{ + Sender: badSender, + Log: logger.Named("transport.bad"), + Name: "Bad Sender", + }, + agent.Transport{ + Sender: sender, + Log: logger.Named("transport.good"), + Name: "Good Sender", + }, + agent.Transport{ + Sender: unusedSender, + Log: logger.Named("transport.should_not_see"), + Name: "Unusued Sender", + }, + }, + } + err = testAgent.Send(&srvMsg, agentMsg) + require.NoError(t, err) +} + +func TestAgentRun(t *testing.T) { + logger, err := zap.NewDevelopment() + require.NoError(t, err) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + sender := NewMockSender(ctrl) + recv := NewMockReceiver(ctrl) + + sender.EXPECT().Send(gomock.Not(gomock.Nil()), agent.Message{}).Return(nil).AnyTimes() + recv.EXPECT().Receive(gomock.Not(gomock.Nil()), agent.ServerMessage{}).AnyTimes() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + testAgent := &agent.Agent{ + Log: logger, + Receiver: recv, + Transports: []agent.Transport{ + agent.Transport{ + Sender: sender, + Log: logger.Named("transport"), + Name: "Test Sender", + }, + }, + } + err = testAgent.Run(ctx) + require.True(t, errors.Is(err, context.DeadlineExceeded), "Unexpected error: %v", err) +} + +func TestRunErrNoTransport(t *testing.T) { + logger, err := zap.NewDevelopment() + require.NoError(t, err) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + srvMsg := agent.ServerMessage{} + agentMsg := agent.Message{} + badSender := NewMockSender(ctrl) + badSender.EXPECT().Send(&srvMsg, agentMsg).Return(errors.New("oops ^_^")) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + testAgent := &agent.Agent{ + Log: logger, + Transports: []agent.Transport{ + agent.Transport{ + Sender: badSender, + Log: logger.Named("transport.bad"), + Name: "Bad Sender", + }, + }, + } + + err = testAgent.Run(ctx) + require.True(t, errors.Is(err, agent.ErrNoTransports), "Unexpected error: %v", err) +} diff --git a/agent/errors.go b/agent/errors.go new file mode 100644 index 00000000..10e28d1e --- /dev/null +++ b/agent/errors.go @@ -0,0 +1,6 @@ +package agent + +import "fmt" + +// ErrNoTransports occurs if all transports fail to send a message or if none are available. +var ErrNoTransports = fmt.Errorf("all available transports failed to send message") diff --git a/agent/io.gen_test.go b/agent/io.gen_test.go new file mode 100644 index 00000000..77c3ee65 --- /dev/null +++ b/agent/io.gen_test.go @@ -0,0 +1,100 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: io (interfaces: Writer,WriteCloser) + +// Package agent_test is a generated GoMock package. +package agent_test + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockWriter is a mock of Writer interface +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// Write mocks base method +func (m *MockWriter) Write(arg0 []byte) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Write indicates an expected call of Write +func (mr *MockWriterMockRecorder) Write(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriter)(nil).Write), arg0) +} + +// MockWriteCloser is a mock of WriteCloser interface +type MockWriteCloser struct { + ctrl *gomock.Controller + recorder *MockWriteCloserMockRecorder +} + +// MockWriteCloserMockRecorder is the mock recorder for MockWriteCloser +type MockWriteCloserMockRecorder struct { + mock *MockWriteCloser +} + +// NewMockWriteCloser creates a new mock instance +func NewMockWriteCloser(ctrl *gomock.Controller) *MockWriteCloser { + mock := &MockWriteCloser{ctrl: ctrl} + mock.recorder = &MockWriteCloserMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWriteCloser) EXPECT() *MockWriteCloserMockRecorder { + return m.recorder +} + +// Close mocks base method +func (m *MockWriteCloser) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close +func (mr *MockWriteCloserMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWriteCloser)(nil).Close)) +} + +// Write mocks base method +func (m *MockWriteCloser) Write(arg0 []byte) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Write indicates an expected call of Write +func (mr *MockWriteCloserMockRecorder) Write(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriteCloser)(nil).Write), arg0) +} diff --git a/agent/message.go b/agent/message.go new file mode 100644 index 00000000..f716e6d3 --- /dev/null +++ b/agent/message.go @@ -0,0 +1,51 @@ +package agent + +import ( + "io" + + "github.com/kcarretto/paragon/api/codec" +) + +// MessageWriter is responsible for writing output to be collected as a message from the agent. +type MessageWriter interface { + io.Writer + io.StringWriter + WriteResult(*codec.Result) +} + +// ServerMessageWriter is responsible for writing output to be collected as a message from the server. +type ServerMessageWriter interface { + WriteServerMessage(*ServerMessage) +} + +// Message is an alias for codec.AgentMessage with some extended functionality. +type Message codec.AgentMessage + +// Write log output to be included in a message to a server. +func (msg *Message) Write(output []byte) (int, error) { + return msg.WriteString(string(output)) +} + +// WriteString writes log output to be included in a message to a server. +func (msg *Message) WriteString(output string) (int, error) { + msg.Logs = append(msg.Logs, output) + return len(output), nil +} + +// WriteResult writes execution output to be included in a message to a server. +func (msg *Message) WriteResult(result *codec.Result) { + msg.Results = append(msg.Results, result) +} + +// IsEmpty checks if the message has no significant contents +func (msg Message) IsEmpty() bool { + return len(msg.Results) <= 0 && len(msg.Logs) <= 0 +} + +// ServerMessage is an alias for codec.ServerMessage with some extended functionality. +type ServerMessage codec.ServerMessage + +// WriteServerMessage replaces this message with the provided message. +func (msg *ServerMessage) WriteServerMessage(srvMsg *ServerMessage) { + msg.Tasks = append(msg.Tasks, srvMsg.Tasks...) +} diff --git a/agent/message_test.go b/agent/message_test.go new file mode 100644 index 00000000..f2c90b4d --- /dev/null +++ b/agent/message_test.go @@ -0,0 +1,45 @@ +package agent_test + +import ( + "testing" + + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" + "github.com/stretchr/testify/require" +) + +func TestMessageWrite(t *testing.T) { + expected := "test message" + + msg := &agent.Message{} + require.True(t, msg.IsEmpty()) + + n, err := msg.Write([]byte(expected)) + require.NoError(t, err) + require.Equal(t, len(expected), n) + require.Equal(t, 1, len(msg.Logs)) + require.Equal(t, expected, msg.Logs[0]) +} + +func TestMessageWriteResult(t *testing.T) { + expected := &codec.Result{} + + msg := &agent.Message{} + msg.WriteResult(expected) + require.Equal(t, 1, len(msg.Results)) + require.Equal(t, expected, msg.Results[0]) +} + +func TestServerMessageWrite(t *testing.T) { + expectedTask := &codec.Task{} + expected := &agent.ServerMessage{} + + msg := &agent.ServerMessage{ + Tasks: []*codec.Task{expectedTask}, + } + expected.WriteServerMessage(msg) + + require.NotNil(t, expected) + require.Equal(t, 1, len(expected.Tasks)) + require.Equal(t, expectedTask, expected.Tasks[0]) +} diff --git a/agent/mocks_test.go b/agent/mocks_test.go new file mode 100644 index 00000000..0ef5bd2c --- /dev/null +++ b/agent/mocks_test.go @@ -0,0 +1,4 @@ +package agent_test + +//go:generate mockgen -destination=io.gen_test.go -package=agent_test io Writer,WriteCloser +//go:generate mockgen -destination=agent.gen_test.go -package=agent_test github.com/kcarretto/paragon/agent Sender,Receiver diff --git a/agent/transport.go b/agent/transport.go new file mode 100644 index 00000000..7d2e9773 --- /dev/null +++ b/agent/transport.go @@ -0,0 +1,36 @@ +package agent + +import ( + "math/rand" + "time" + + "go.uber.org/zap" +) + +// A Transport wraps an implementation of Sender with metadata used for operation. +type Transport struct { + Sender + Log *zap.Logger + + Name string + Interval time.Duration + Jitter time.Duration +} + +// Sleep for the transport's configured interval & jitter duration. If Jitter is non-zero, a random +// duration <= Jitter is added to the transports interval. +func (t Transport) Sleep() { + // Random duration added to sleep delay [0, Jitter) + var jitter time.Duration + if max := t.Jitter.Nanoseconds(); max > 0 { + jitter = time.Duration(rand.Int63n(max)) + } + + delay := t.Interval + jitter + + // Ratelimit log messages + if delay > 1*time.Second { + t.Log.Debug("Sleeping for transport interval + jitter", zap.Duration("delay", delay)) + } + time.Sleep(delay) +} diff --git a/go.mod b/go.mod index c9c55637..1189c654 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kcarretto/paragon -go 1.12 +go 1.13 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect From b1533dd01fa5355f46c8598ad2f4c6d9aa64b585 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Wed, 16 Oct 2019 17:49:16 -0700 Subject: [PATCH 09/19] Updated to fix debug http server --- Dockerfile | 3 +- agent/agent.go | 5 +- agent/agent_test.go | 5 +- {transport => agent}/debug/auth.go | 0 {transport => agent}/debug/html.go | 7 +- {transport => agent}/debug/http.go | 71 +++++++------ agent/debug/sender.go | 75 +++++++++++++ {transport => agent}/http/http.go | 0 api/codec/codec.go | 35 +++++++ cmd/agent/debug.go | 23 ++-- cmd/agent/dev.go | 26 +++-- cmd/agent/main.go | 19 +++- cmd/agent/prod.go | 5 +- cmd/agent/receiver.go | 48 +++++++++ cmd/agent/run.go | 148 +++++++++++++------------- transport/buffer.go | 162 ----------------------------- transport/buffer_test.go | 127 ---------------------- transport/codec.go | 108 ------------------- transport/debug/transport.go | 76 -------------- transport/errors.go | 6 -- transport/mocks/codec.gen.go | 87 ---------------- transport/mocks/io.gen.go | 100 ------------------ transport/mocks/mocks.go | 4 - transport/receiver.go | 50 --------- transport/receiver_test.go | 35 ------- transport/registry.go | 113 -------------------- transport/registry_test.go | 52 --------- transport/sender.go | 33 ------ transport/transport.go | 54 ---------- transport/transport_test.go | 30 ------ 30 files changed, 326 insertions(+), 1181 deletions(-) rename {transport => agent}/debug/auth.go (100%) rename {transport => agent}/debug/html.go (96%) rename {transport => agent}/debug/http.go (58%) create mode 100644 agent/debug/sender.go rename {transport => agent}/http/http.go (100%) create mode 100644 cmd/agent/receiver.go delete mode 100644 transport/buffer.go delete mode 100644 transport/buffer_test.go delete mode 100644 transport/codec.go delete mode 100644 transport/debug/transport.go delete mode 100644 transport/errors.go delete mode 100644 transport/mocks/codec.gen.go delete mode 100644 transport/mocks/io.gen.go delete mode 100644 transport/mocks/mocks.go delete mode 100644 transport/receiver.go delete mode 100644 transport/receiver_test.go delete mode 100644 transport/registry.go delete mode 100644 transport/registry_test.go delete mode 100644 transport/sender.go delete mode 100644 transport/transport.go delete mode 100644 transport/transport_test.go diff --git a/Dockerfile b/Dockerfile index a300817e..d8aba7c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,8 @@ RUN apk add alpine-sdk git \ # Debug Build FROM base as build COPY ./cmd /app/cmd -COPY ./transport /app/transport +COPY ./api /app/api +COPY ./agent /app/agent COPY ./script /app/script RUN go build -tags=debug -o ./build/agent ./cmd/agent diff --git a/agent/agent.go b/agent/agent.go index f3824251..43bf24c1 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -61,18 +61,21 @@ func (agent Agent) Send(w ServerMessageWriter, msg Message) error { // Run the agent, sending agent messages to a server using configured transports. func (agent Agent) Run(ctx context.Context) error { + agent.Log.Debug("Starting agent execution") + var agentMsg Message + for { select { case <-ctx.Done(): return ctx.Err() default: - var agentMsg Message var srvMsg ServerMessage if err := agent.Send(&srvMsg, agentMsg); err != nil { return err } + agentMsg = Message{} agent.Receive(&agentMsg, srvMsg) } } diff --git a/agent/agent_test.go b/agent/agent_test.go index bb1d8be8..9757d8d5 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -70,8 +70,9 @@ func TestAgentRun(t *testing.T) { defer cancel() testAgent := &agent.Agent{ - Log: logger, - Receiver: recv, + Log: logger, + Receiver: recv, + MaxIdleTime: time.Second * 1, Transports: []agent.Transport{ agent.Transport{ Sender: sender, diff --git a/transport/debug/auth.go b/agent/debug/auth.go similarity index 100% rename from transport/debug/auth.go rename to agent/debug/auth.go diff --git a/transport/debug/html.go b/agent/debug/html.go similarity index 96% rename from transport/debug/html.go rename to agent/debug/html.go index d0e3fc25..1136f5b0 100644 --- a/transport/debug/html.go +++ b/agent/debug/html.go @@ -32,7 +32,7 @@ const debugHTML = ` "showMethod": "fadeIn", "hideMethod": "fadeOut" } - + function addToResults(result) { $('#resultsList').prepend( $('
  • ').append( @@ -45,7 +45,7 @@ const debugHTML = ` } function queueScript(){ let script = { - "content": btoa($("#scriptTextArea").val()) + "content": $("#scriptTextArea").val() } $.ajax({ url: '/queue', @@ -70,7 +70,8 @@ const debugHTML = ` if (response.results != null){ response.results.map(function(result){ if (result.output != null){ - addToResults(atob(result.output)); + output = result.output.join("\n") + addToResults(output); } else if (result.error != "") { addToResults("error: "+result.error) } else { diff --git a/transport/debug/http.go b/agent/debug/http.go similarity index 58% rename from transport/debug/http.go rename to agent/debug/http.go index ecc396bc..9ee17643 100644 --- a/transport/debug/http.go +++ b/agent/debug/http.go @@ -8,7 +8,8 @@ import ( "os" "strconv" - "github.com/kcarretto/paragon/transport" + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" "go.uber.org/zap" ) @@ -20,46 +21,39 @@ func use(h http.HandlerFunc, middleware ...func(http.HandlerFunc) http.HandlerFu return h } -func (t *Transport) handleIndex(w http.ResponseWriter, req *http.Request) { +func (transport *Sender) handleIndex(w http.ResponseWriter, req *http.Request) { w.Write([]byte(debugHTML)) return } -func (t *Transport) handleQueue(w http.ResponseWriter, req *http.Request) { +// handleQueue handles an http request to queue a task. +func (transport *Sender) handleQueue(w http.ResponseWriter, req *http.Request) { + // Read request data data, err := ioutil.ReadAll(req.Body) if err != nil { - t.Logger.Error("Failed to read request body", zap.Error(err)) + transport.Log.Error("Failed to read request body", zap.Error(err)) msg := fmt.Sprintf("failed to read request body: %s", err.Error()) http.Error(w, msg, http.StatusBadRequest) return } + transport.Log.Debug("Request to queue task", zap.String("task", string(data))) - t.Logger.Debug("Request to queue task", zap.String("task", string(data))) - var task transport.Task + // Unmarshal request into task + var task codec.Task if err := json.Unmarshal(data, &task); err != nil { - t.Logger.Error("Failed to parse request body", zap.Error(err)) + transport.Log.Error("Failed to parse request body", zap.Error(err)) msg := fmt.Sprintf("failed to parse request body to json: %s", err.Error()) http.Error(w, msg, http.StatusBadRequest) return } - serverMsg, err := json.Marshal( - transport.Payload{ - Tasks: []transport.Task{task}, - }, - ) - if err != nil { - t.Logger.Error("Failed to marshal server message", zap.Error(err)) - msg := fmt.Sprintf("failed to marshal server message: %s", err.Error()) - http.Error(w, msg, http.StatusInternalServerError) - return - } - - t.WritePayload(serverMsg) + // Queue the task + transport.QueueTask(&task) } -func (t *Transport) handleResponses(w http.ResponseWriter, req *http.Request) { +// handleMessages handles an http request to view agent messages +func (transport *Sender) handleMessages(w http.ResponseWriter, req *http.Request) { params := req.URL.Query() offsetStr := params.Get("offset") @@ -90,17 +84,17 @@ func (t *Transport) handleResponses(w http.ResponseWriter, req *http.Request) { limit = 1 } - messages := []transport.Response{} - for i := offset; i < len(t.messages) && i <= offset+limit; i++ { - messages = append(messages, t.messages[i]) + messages := []agent.Message{} + for i := offset; i < len(transport.messages) && i <= offset+limit; i++ { + messages = append(messages, transport.messages[i]) } - t.Logger.Debug( + transport.Log.Debug( "Retrieving responses", zap.Int("offset", offset), zap.Int("limit", limit), zap.Int("sent_messages", len(messages)), - zap.Int("total_messages", len(t.messages)), + zap.Int("total_messages", len(transport.messages)), ) respData, err := json.Marshal(messages) @@ -111,14 +105,13 @@ func (t *Transport) handleResponses(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") if _, err = w.Write(respData); err != nil { - t.Logger.Error("Failed to write response data to client", zap.Error(err)) + transport.Log.Error("Failed to write response data to client", zap.Error(err)) http.Error(w, fmt.Sprintf("failed to write response data to client: %s", err.Error()), http.StatusInternalServerError) } - } // listenAndServe configures and runs an http server for agent debugging. -func (t *Transport) listenAndServe() { +func (transport *Sender) listenAndServe() { router := http.NewServeMux() var middleware []func(http.HandlerFunc) http.HandlerFunc @@ -132,16 +125,22 @@ func (t *Transport) listenAndServe() { middleware = append(middleware, prepareBasicAuth(username, password)) } - router.HandleFunc("/", use(t.handleIndex, middleware...)) - router.HandleFunc("/queue", use(t.handleQueue, middleware...)) - router.HandleFunc("/messages", use(t.handleResponses, middleware...)) - http.Handle("/", router) - httpAddr := os.Getenv("DEBUG_API_ADDR") if httpAddr == "" { httpAddr = "127.0.0.1:8080" } - t.Logger.Info("HTTP DEBUG ENABLED", zap.String("http_addr", httpAddr)) - http.ListenAndServe(httpAddr, router) + router.HandleFunc("/", use(transport.handleIndex, middleware...)) + router.HandleFunc("/queue", use(transport.handleQueue, middleware...)) + router.HandleFunc("/messages", use(transport.handleMessages, middleware...)) + transport.srv = &http.Server{ + Addr: httpAddr, + Handler: router, + } + + transport.Log.Info("HTTP DEBUG ENABLED", zap.String("http_addr", httpAddr)) + if err := transport.srv.ListenAndServe(); err != nil { + transport.Log.Error("HTTP Debug Server encountered an error while closing", zap.Error(err)) + } + transport.Log.Info("HTTP Debug Server Closed") } diff --git a/agent/debug/sender.go b/agent/debug/sender.go new file mode 100644 index 00000000..d528f0cc --- /dev/null +++ b/agent/debug/sender.go @@ -0,0 +1,75 @@ +package debug + +import ( + "context" + "net/http" + "sync" + + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" + "go.uber.org/zap" +) + +// Sender is for debugging purposes, it manages a local http server to interact with an agent. +type Sender struct { + Log *zap.Logger + + srv *http.Server + active bool + wg sync.WaitGroup + messages []agent.Message + tasks []*codec.Task +} + +// Send appends an agent message to the result array, and writes any queued tasks to the provided +// writer. +func (transport *Sender) Send(w agent.ServerMessageWriter, msg agent.Message) error { + transport.ensureActive() + + transport.messages = append(transport.messages, msg) + + tasks := make([]*codec.Task, len(transport.tasks)) + copy(tasks, transport.tasks) + transport.tasks = transport.tasks[:0] + + w.WriteServerMessage(&agent.ServerMessage{ + Tasks: tasks, + }) + + return nil +} + +// QueueTask buffers a task for execution. +func (transport *Sender) QueueTask(task *codec.Task) { + transport.tasks = append(transport.tasks, task) +} + +// Close stops the goroutine responsible for running the debug http server if it is active. +func (transport *Sender) Close() (err error) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if transport.srv != nil { + err = transport.srv.Shutdown(ctx) + } + + transport.wg.Wait() + transport.active = false + + return err +} + +// ensureActive starts a goroutine to consume stdin if the transport is not yet active. +func (transport *Sender) ensureActive() { + if transport.active { + return + } + + transport.wg.Add(1) + go func() { + defer transport.wg.Done() + transport.listenAndServe() + }() + + transport.active = true +} diff --git a/transport/http/http.go b/agent/http/http.go similarity index 100% rename from transport/http/http.go rename to agent/http/http.go diff --git a/api/codec/codec.go b/api/codec/codec.go index 5871186b..af6467f6 100644 --- a/api/codec/codec.go +++ b/api/codec/codec.go @@ -4,9 +4,44 @@ // both the agent and the server utilize the same format. package codec +import ( + "time" + + timestamp "github.com/golang/protobuf/ptypes/timestamp" +) + // VERSION describes the currently built version of codec. const VERSION = "0.0.1" // TODO: Add python support via --python_out=. //go:generate protoc -I=../vendor/ -I=../../ -I=. --go_out=paths=source_relative:. agent.proto server.proto + +// Start recording results for task execution. +func (m *Result) Start() { + start := time.Now() + + m.ExecStartTime = ×tamp.Timestamp{ + Seconds: start.Unix(), + Nanos: int32(start.Nanosecond()), + } +} + +// Write appends task execution output to the result. +func (m *Result) Write(p []byte) (int, error) { + m.Output = append(m.Output, string(p)) + return len(p), nil +} + +// CloseWithError marks the end of task execution, and provides any error the task resulted in. +func (m *Result) CloseWithError(err error) { + stop := time.Now() + m.ExecStopTime = ×tamp.Timestamp{ + Seconds: stop.Unix(), + Nanos: int32(stop.Nanosecond()), + } + if err != nil { + m.Error = err.Error() + } + +} diff --git a/cmd/agent/debug.go b/cmd/agent/debug.go index ee46c4fb..8111cc07 100644 --- a/cmd/agent/debug.go +++ b/cmd/agent/debug.go @@ -4,9 +4,10 @@ package main import ( "io" + "time" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/debug" + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/agent/debug" "go.uber.org/zap" ) @@ -30,12 +31,16 @@ func getLogger() *zap.Logger { func configureLogger(logger *zap.Logger, buf io.Writer) {} -func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { - registry.Add(transport.New( - "debug", - &debug.Transport{ - Logger: logger, - PayloadWriter: receiver, +func getTransports(logger *zap.Logger) []agent.Transport { + return []agent.Transport{ + agent.Transport{ + Name: "Debug", + Log: logger.Named("debug"), + Interval: 5 * time.Second, + Jitter: 1 * time.Second, + Sender: &debug.Sender{ + Log: logger.Named("debug"), + }, }, - )) + } } diff --git a/cmd/agent/dev.go b/cmd/agent/dev.go index 395b412f..795300f4 100644 --- a/cmd/agent/dev.go +++ b/cmd/agent/dev.go @@ -5,8 +5,8 @@ package main import ( "io" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/http" + "github.com/kcarretto/paragon/agent" + // "github.com/kcarretto/paragon/agent/http" "go.uber.org/zap" ) @@ -30,13 +30,17 @@ func getLogger() *zap.Logger { func configureLogger(logger *zap.Logger, buf io.Writer) {} -func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { - registry.Add(transport.New( - "http", - http.Transport{ - PayloadWriter: receiver, - Logger: logger.Named("http"), - URL: "http://127.0.0.1:8080", - }, - )) +func getTransports(logger *zap.Logger) (transports []agent.Transport) { + // registry.Add() + return } +// func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { +// registry.Add(transport.New( +// "http", +// http.Transport{ +// PayloadWriter: receiver, +// Logger: logger.Named("http"), +// URL: "http://127.0.0.1:8080", +// }, +// )) +// } diff --git a/cmd/agent/main.go b/cmd/agent/main.go index aa4f91e8..13176305 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -6,8 +6,8 @@ import ( "os/signal" "sync" "syscall" - "time" + "github.com/kcarretto/paragon/agent" "github.com/pkg/profile" "go.uber.org/zap" ) @@ -21,6 +21,16 @@ func runLoop() { // Initialize logger logger := getLogger() + // Initialize Agent + paragon := &agent.Agent{ + Receiver: Receiver{ + ctx, + logger.Named("agent.exec"), + }, + Log: logger.Named("agent"), + Transports: getTransports(logger.Named("agent.transport")), + } + // Handle panic defer func() { if err := recover(); err != nil { @@ -31,10 +41,10 @@ func runLoop() { // Run agent var wg sync.WaitGroup wg.Add(1) - go func(logger *zap.Logger) { + go func() { defer wg.Done() - run(ctx, logger) - }(logger.With(zap.Time("agent_loop_start", time.Now()))) + paragon.Run(ctx) + }() // Listen for interupts sigint := make(chan os.Signal, 1) @@ -59,7 +69,6 @@ func runLoop() { // After interupt, wait for threads to finish cancel() wg.Wait() - } func main() { diff --git a/cmd/agent/prod.go b/cmd/agent/prod.go index a20334e1..67ff2bcf 100644 --- a/cmd/agent/prod.go +++ b/cmd/agent/prod.go @@ -6,7 +6,7 @@ import ( "fmt" "io" - "github.com/kcarretto/paragon/transport" + "github.com/kcarretto/paragon/agent" "go.uber.org/zap" ) @@ -31,6 +31,7 @@ func getLogger() *zap.Logger { func configureLogger(logger *zap.Logger, buf io.Writer) {} -func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { +func getTransports(logger *zap.Logger) (transports []agent.Transport) { // registry.Add() + return } diff --git a/cmd/agent/receiver.go b/cmd/agent/receiver.go new file mode 100644 index 00000000..aa389af2 --- /dev/null +++ b/cmd/agent/receiver.go @@ -0,0 +1,48 @@ +package main + +import ( + "bytes" + "context" + + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" + "github.com/kcarretto/paragon/script" + "github.com/kcarretto/paragon/script/stdlib" + "go.uber.org/zap" +) + +type Receiver struct { + context.Context + + Log *zap.Logger +} + +func (r Receiver) Receive(w agent.MessageWriter, msg agent.ServerMessage) { + r.Log.Debug("Received new payload from server", + zap.Int("num_tasks", len(msg.Tasks)), + zap.Reflect("payload", msg), + ) + + for _, task := range msg.Tasks { + result := &codec.Result{ + Id: task.GetId(), + } + result.Start() + code := script.New( + task.GetId(), + bytes.NewBufferString(task.Content), + script.WithOutput(result), + stdlib.Load(), + ) // TODO: Add libraries, set output + + err := code.Exec(r) + if err != nil { + r.Log.Error("failed to execute script", zap.Error(err), zap.String("task_id", task.GetId())) + } + + r.Log.Debug("completed script execution", zap.String("task", task.String())) + + result.CloseWithError(err) + w.WriteResult(result) + } +} diff --git a/cmd/agent/run.go b/cmd/agent/run.go index 63e855eb..7758ac85 100644 --- a/cmd/agent/run.go +++ b/cmd/agent/run.go @@ -1,89 +1,89 @@ package main -import ( - "bytes" - "context" - "fmt" - "time" +// import ( +// "bytes" +// "context" +// "fmt" +// "time" - "github.com/kcarretto/paragon/script" - "github.com/kcarretto/paragon/script/stdlib" - "github.com/kcarretto/paragon/transport" - "go.uber.org/zap" -) +// "github.com/kcarretto/paragon/script" +// "github.com/kcarretto/paragon/script/stdlib" +// "github.com/kcarretto/paragon/agent" +// "go.uber.org/zap" +// ) -func run(ctx context.Context, logger *zap.Logger) { - // Initialize buffer - buffer := &transport.Buffer{ - Encoder: transport.NewDefaultEncoder(), - Metadata: transport.Metadata{}, - MaxIdleTime: time.Second * 5, - } +// func run(ctx context.Context, logger *zap.Logger) { +// // Initialize buffer +// buffer := &transport.Buffer{ +// Encoder: transport.NewDefaultEncoder(), +// Metadata: transport.Metadata{}, +// MaxIdleTime: time.Second * 5, +// } - // Configure the logger - configureLogger(logger, buffer) +// // Configure the logger +// configureLogger(logger, buffer) - // Initialize registry - registry := &transport.Registry{} +// // Initialize registry +// registry := &transport.Registry{} - // Handle payloads from the server - hLogger := logger.Named("scripts") - handler := func(w transport.ResultWriter, payload transport.Payload, err error) { - if err != nil { - hLogger.Error("Payload decode error", zap.Error(err)) - return - } +// // Handle payloads from the server +// hLogger := logger.Named("scripts") +// handler := func(w transport.ResultWriter, payload transport.Payload, err error) { +// if err != nil { +// hLogger.Error("Payload decode error", zap.Error(err)) +// return +// } - hLogger.Debug("Executing new payload from server", - zap.Int("num_tasks", len(payload.Tasks)), - zap.Reflect("payload", payload), - ) +// hLogger.Debug("Executing new payload from server", +// zap.Int("num_tasks", len(payload.Tasks)), +// zap.Reflect("payload", payload), +// ) - for _, task := range payload.Tasks { - output := transport.NewResult(task) - code := script.New( - task.ID, - bytes.NewBuffer(task.Content), - script.WithOutput(output), - stdlib.Load(), - ) // TODO: Add libraries, set output - fmt.Println(code.Libraries) - err = code.Exec(ctx) - if err != nil { - hLogger.Error("failed to execute script", zap.Error(err), zap.String("task_id", task.ID)) - } +// for _, task := range payload.Tasks { +// output := transport.NewResult(task) +// code := script.New( +// task.ID, +// bytes.NewBuffer(task.Content), +// script.WithOutput(output), +// stdlib.Load(), +// ) // TODO: Add libraries, set output +// fmt.Println(code.Libraries) +// err = code.Exec(ctx) +// if err != nil { +// hLogger.Error("failed to execute script", zap.Error(err), zap.String("task_id", task.ID)) +// } - hLogger.Debug("completed script execution", zap.String("task_id", task.ID)) +// hLogger.Debug("completed script execution", zap.String("task_id", task.ID)) - output.CloseWithError(err) - w.WriteResult(output) - } - } +// output.CloseWithError(err) +// w.WriteResult(output) +// } +// } - // Initialize Receiver - receiver := &transport.Receiver{ - ResultWriter: buffer, - Decoder: transport.NewDefaultDecoder(), - Handler: transport.PayloadHandlerFn(handler), - } +// // Initialize Receiver +// receiver := &transport.Receiver{ +// ResultWriter: buffer, +// Decoder: transport.NewDefaultDecoder(), +// Handler: transport.PayloadHandlerFn(handler), +// } - // Register Transports - addTransports(logger.Named("transport"), receiver, registry) +// // Register Transports +// addTransports(logger.Named("transport"), receiver, registry) - // Initialize MultiSender - sender := transport.MultiSender{ - Transports: registry, - OnError: func(t transport.Transport, err error) { - logger.Named("transport").Named(t.Name).Error("Failed to transport data", zap.Error(err)) - }, - } +// // Initialize MultiSender +// sender := transport.MultiSender{ +// Transports: registry, +// OnError: func(t transport.Transport, err error) { +// logger.Named("transport").Named(t.Name).Error("Failed to transport data", zap.Error(err)) +// }, +// } - // Flush buffer using multi sender - for { - if _, err := sender.Send(buffer); err != nil { - // TODO: Handle encode error - // TODO: Handle ErrNoTransports - } - time.Sleep(time.Millisecond * 50) - } -} +// // Flush buffer using multi sender +// for { +// if _, err := sender.Send(buffer); err != nil { +// // TODO: Handle encode error +// // TODO: Handle ErrNoTransports +// } +// time.Sleep(time.Millisecond * 50) +// } +// } diff --git a/transport/buffer.go b/transport/buffer.go deleted file mode 100644 index eaaadd2e..00000000 --- a/transport/buffer.go +++ /dev/null @@ -1,162 +0,0 @@ -package transport - -import ( - "bytes" - "fmt" - "io" - "sync" - "time" -) - -// A TimestampWriterTo extends io.WriterTo with a method that returns the timestamp of when the -// contents were last successfully written. -type TimestampWriterTo interface { - io.WriterTo - Timestamp() time.Time -} - -// Buffer is a write safe io.Writer & io.WriterTo that is used to buffer output and safely copy it -// to a transport writer. If copying fails, buffer does not lose data. -type Buffer struct { - Encoder - - Metadata Metadata - MaxIdleTime time.Duration - - mu sync.RWMutex - output *bytes.Buffer - results []*Result - timestamp time.Time -} - -// WriteResult writes structured task execution output to the response. -func (b *Buffer) WriteResult(result *Result) { - if result == nil { - return - } - - b.mu.Lock() - b.results = append(b.results, result) - b.mu.Unlock() -} - -// Write buffers the provided data until it is consumed by a transport. It is safe for -// concurrent use. -func (b *Buffer) Write(data []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if b.output == nil { - b.output = bytes.NewBuffer(data) - return len(data), nil - } - - return b.output.Write(data) -} - -// WriteTo implements io.WriterTo, allowing Buffer to be used by io.Copy. It will write it's -// contents to the provided writer. It will preserve any data in the case of error. -func (b *Buffer) WriteTo(w io.Writer) (int64, error) { - output := b.copyOutput() - results := b.copyResults() - - if len(output) <= 0 && len(results) <= 0 && time.Since(b.Timestamp()) <= b.MaxIdleTime { - return 0, nil - } - - resp := Response{ - Metadata: b.Metadata, - Log: output, - Results: results, - } - data, err := b.Encode(resp) - if err != nil { - b.restore(results, output) - return 0, fmt.Errorf("failed to encode response data: %w", err) - } - - n, err := w.Write(data) - if err != nil { - b.restore(results, output) - return 0, fmt.Errorf("failed to transport response data: %w", err) - } - - // Update timestamp - b.timestamp = time.Now() - - return int64(n), nil -} - -// Sync is a nop to implement zapcore.WriteSyncer -func (b *Buffer) Sync() error { - return nil -} - -// Timestamp returns the time that the buffer was last successfully written to a transport. -func (b *Buffer) Timestamp() time.Time { - b.mu.RLock() - t := b.timestamp - b.mu.RUnlock() - return t -} - -// copyResults returns a copy of the current result buffer and truncates it. -func (b *Buffer) copyResults() (results []*Result) { - b.mu.Lock() - defer b.mu.Unlock() - - if len(b.results) <= 0 { - return - } - - results = make([]*Result, len(b.results)) - copy(results, b.results) - b.results = b.results[len(b.results):] - - return -} - -// copyOutput returns a copy of the current output buffer and truncates it. -func (b *Buffer) copyOutput() (data []byte) { - b.mu.Lock() - defer b.mu.Unlock() - - if b.output == nil { - b.output = &bytes.Buffer{} - } - - if b.output.Len() <= 0 { - return - } - - data = make([]byte, b.output.Len()) - copy(data, b.output.Bytes()) - b.output.Reset() - - return -} - -// restore the provided state to the buffer. -func (b *Buffer) restore(results []*Result, data []byte) error { - // Restore results buffer - for _, result := range results { - b.WriteResult(result) - } - - // Restore output buffer - size := len(data) - n, err := b.Write(data) - if err == nil && n != size { - err = fmt.Errorf("failed to restore full output buffer") - } - - return err -} - -// NewBuffer Initializes and returns a new transport buffer. -func NewBuffer(data []byte) *Buffer { - return &Buffer{ - Encoder: NewDefaultEncoder(), - output: bytes.NewBuffer(data), - } -} diff --git a/transport/buffer_test.go b/transport/buffer_test.go deleted file mode 100644 index 83e8cf74..00000000 --- a/transport/buffer_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package transport_test - -import ( - "testing" - - "github.com/golang/mock/gomock" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/mocks" - "github.com/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestBufferSync(t *testing.T) { - buffer := transport.Buffer{} - require.NoError(t, buffer.Sync()) -} - -func TestBufferWrite(t *testing.T) { - expected := []byte("Some test data") - - buffer := transport.Buffer{} - - n, err := buffer.Write(expected) - require.NoError(t, err) - require.Equal(t, len(expected), n) -} - -func TestBufferWriteTo(t *testing.T) { - // Prepare test data - expectedOutput := []byte("Some test output data") - expectedResult := transport.NewResult(transport.Task{ - ID: "SomeTestTask", - }) - _, err := expectedResult.Write([]byte("some test result data")) - require.NoError(t, err) - - response := transport.Response{ - transport.Metadata{}, - []*transport.Result{ - expectedResult, - }, - expectedOutput, - } - encoder := transport.NewDefaultEncoder() - expected, err := encoder.Encode(response) - require.NoError(t, err) - - // Prepare mock - ctrl := gomock.NewController(t) - defer ctrl.Finish() - dst := mocks.NewMockWriter(ctrl) - dst.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { - require.Equal(t, string(expected), string(p)) - return len(p), nil - }) - - // Initialize buffer and write expected data - buffer := transport.NewBuffer(expectedOutput) - - // Write result to buffer - buffer.WriteResult(expectedResult) - - // Write buffer to mock writer - num, err := buffer.WriteTo(dst) - require.NoError(t, err) - require.Equal(t, int64(len(expected)), num) - - // Ensure buffer timestamp was updated - require.NotZero(t, buffer.Timestamp(), "Buffer should update timestamp after successful write") -} - -func TestBufferWriteToError(t *testing.T) { - // Prepare test data - expectedOutput := []byte("Some test output data") - expectedResult := transport.NewResult(transport.Task{ - ID: "SomeTestTask", - }) - _, err := expectedResult.Write([]byte("some test result data")) - require.NoError(t, err) - - response := transport.Response{ - transport.Metadata{}, - []*transport.Result{ - expectedResult, - }, - expectedOutput, - } - encoder := transport.NewDefaultEncoder() - expected, err := encoder.Encode(response) - require.NoError(t, err) - - // Prepare mock - ctrl := gomock.NewController(t) - defer ctrl.Finish() - dst := mocks.NewMockWriter(ctrl) - gomock.InOrder( - dst.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { - require.Equal(t, string(expected), string(p)) - return 4, errors.New("uh oh") - }), - dst.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { - require.Equal(t, string(expected), string(p)) - return len(p), nil - }), - ) - - // Initialize buffer and write expected data - buffer := transport.NewBuffer(expectedOutput) - - // Write result to buffer - buffer.WriteResult(expectedResult) - - // Write buffer to mock writer (will error with "uh oh") - _, err = buffer.WriteTo(dst) - require.Error(t, err) - - // Ensure buffer timestamp was not updated - require.Zero(t, buffer.Timestamp(), "Buffer should not update timestamp after failed write") - - // Write buffer to mock writer again, ensure no data was lost - num, err := buffer.WriteTo(dst) - require.NoError(t, err) - require.Equal(t, int64(len(expected)), num) - - // Ensure buffer timestamp was updated - require.NotZero(t, buffer.Timestamp(), "Buffer should update timestamp after successful write") -} diff --git a/transport/codec.go b/transport/codec.go deleted file mode 100644 index 7216bbd1..00000000 --- a/transport/codec.go +++ /dev/null @@ -1,108 +0,0 @@ -package transport - -import ( - "encoding/json" - "time" -) - -// An Encoder is responsible for marshaling a response to bytes. -type Encoder interface { - Encode(Response) ([]byte, error) -} - -// EncoderFn is a function that implements Encoder. -type EncoderFn func(Response) ([]byte, error) - -// Encode wraps fn to provide an implementation of Encoder. -func (fn EncoderFn) Encode(resp Response) ([]byte, error) { - return fn(resp) -} - -// NewDefaultEncoder returns a JSON Encoder. -func NewDefaultEncoder() Encoder { - return EncoderFn(func(resp Response) ([]byte, error) { - return json.Marshal(resp) - }) -} - -// A Decoder is responsible for unmarshaling received data into a payload struct. -type Decoder interface { - Decode([]byte) (Payload, error) -} - -// DecoderFn is a function that implements Decoder. -type DecoderFn func([]byte) (Payload, error) - -// Decode wraps fn to provide an implementation of Decoder. -func (fn DecoderFn) Decode(data []byte) (Payload, error) { - return fn(data) -} - -// NewDefaultDecoder returns a JSON Decoder. -func NewDefaultDecoder() Decoder { - return DecoderFn(func(data []byte) (payload Payload, err error) { - err = json.Unmarshal(data, &payload) - return - }) -} - -// Payload holds structured information received from a server. -type Payload struct { - Tasks []Task -} - -// Response holds structured information that will be transported to a server. -type Response struct { - Metadata Metadata - Results []*Result `json:"results"` - Log []byte `json:"log"` -} - -// Metadata holds agent metadata useful for identifying an agent. -type Metadata struct{} - -// Task stores instructions to execute and metadata. -type Task struct { - ID string `json:"id"` - Content []byte `json:"content"` -} - -// Result stores task execution output and metadata. -type Result struct { - ID string `json:"id"` - Output []byte `json:"output"` - Error string `json:"error"` - - ExecStartTime time.Time `json:"exec_start_time"` - ExecStopTime time.Time `json:"exec_stop_time"` -} - -// Write implements io.Writer by appending output to the current result buffer. It is safe for -// concurrent use and always returns len(p), nil. -func (r *Result) Write(p []byte) (int, error) { - r.Output = append(r.Output, p...) - - return len(p), nil -} - -// Close implements io.Closer by wrapping CloseWithError to indicate no error. Always returns nil. -func (r *Result) Close() error { - r.CloseWithError(nil) - return nil -} - -// CloseWithError sets the result error and execution stop time. -func (r *Result) CloseWithError(err error) { - if err != nil { - r.Error = err.Error() - } - r.ExecStopTime = time.Now() -} - -// NewResult initializes and returns a new Result to store execution output for the provided task. -func NewResult(task Task) *Result { - return &Result{ - ID: task.ID, - ExecStartTime: time.Now(), - } -} diff --git a/transport/debug/transport.go b/transport/debug/transport.go deleted file mode 100644 index c8a8ba11..00000000 --- a/transport/debug/transport.go +++ /dev/null @@ -1,76 +0,0 @@ -package debug - -import ( - "encoding/json" - "io" - "os" - "sync" - - "github.com/kcarretto/paragon/transport" - "go.uber.org/zap" -) - -// Transport is for debugging purposes, it manages a local http server to interact with an agent. -type Transport struct { - io.Reader - transport.PayloadWriter - Logger *zap.Logger - - wg sync.WaitGroup - input chan []byte - active bool - messages []transport.Response -} - -// Write response data to an in memory buffer that is queried by the debug http server. -func (t *Transport) Write(data []byte) (int, error) { - t.ensureActive() - - var resp transport.Response - if err := json.Unmarshal(data, &resp); err != nil { - t.Logger.DPanic("Invalid response JSON written", zap.String("data", string(data)), zap.Error(err)) - } - - t.messages = append(t.messages, resp) - return len(data), nil -} - -// Close stops the goroutine responsible for running the debug http server if it is active. -func (t *Transport) Close() error { - if t.input != nil { - close(t.input) - } - t.wg.Wait() - - t.active = true - - return nil -} - -// ensureActive starts a goroutine to consume stdin if the transport is not yet active. -func (t *Transport) ensureActive() { - if t.active { - return - } - - t.wg.Add(1) - go func() { - defer t.wg.Done() - t.listenAndServe() - }() - - t.active = true -} - -// New initializes and returns a local transport, which must be closed. -func New(receiver transport.PayloadWriter) *Transport { - - input := make(chan []byte, 1) - t := Transport{ - Reader: os.Stdin, - PayloadWriter: receiver, - input: input, - } - - return &t -} diff --git a/transport/errors.go b/transport/errors.go deleted file mode 100644 index 2688784f..00000000 --- a/transport/errors.go +++ /dev/null @@ -1,6 +0,0 @@ -package transport - -import "errors" - -// ErrNoTransportAvailable occurs when all available transports fail to report output, or if no transports were configured. -var ErrNoTransportAvailable = errors.New("no transport available") diff --git a/transport/mocks/codec.gen.go b/transport/mocks/codec.gen.go deleted file mode 100644 index 7952fd51..00000000 --- a/transport/mocks/codec.gen.go +++ /dev/null @@ -1,87 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kcarretto/paragon/transport (interfaces: Encoder,Decoder) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - gomock "github.com/golang/mock/gomock" - transport "github.com/kcarretto/paragon/transport" - reflect "reflect" -) - -// MockEncoder is a mock of Encoder interface -type MockEncoder struct { - ctrl *gomock.Controller - recorder *MockEncoderMockRecorder -} - -// MockEncoderMockRecorder is the mock recorder for MockEncoder -type MockEncoderMockRecorder struct { - mock *MockEncoder -} - -// NewMockEncoder creates a new mock instance -func NewMockEncoder(ctrl *gomock.Controller) *MockEncoder { - mock := &MockEncoder{ctrl: ctrl} - mock.recorder = &MockEncoderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockEncoder) EXPECT() *MockEncoderMockRecorder { - return m.recorder -} - -// Encode mocks base method -func (m *MockEncoder) Encode(arg0 transport.Response) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Encode", arg0) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Encode indicates an expected call of Encode -func (mr *MockEncoderMockRecorder) Encode(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Encode", reflect.TypeOf((*MockEncoder)(nil).Encode), arg0) -} - -// MockDecoder is a mock of Decoder interface -type MockDecoder struct { - ctrl *gomock.Controller - recorder *MockDecoderMockRecorder -} - -// MockDecoderMockRecorder is the mock recorder for MockDecoder -type MockDecoderMockRecorder struct { - mock *MockDecoder -} - -// NewMockDecoder creates a new mock instance -func NewMockDecoder(ctrl *gomock.Controller) *MockDecoder { - mock := &MockDecoder{ctrl: ctrl} - mock.recorder = &MockDecoderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockDecoder) EXPECT() *MockDecoderMockRecorder { - return m.recorder -} - -// Decode mocks base method -func (m *MockDecoder) Decode(arg0 []byte) (transport.Payload, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Decode", arg0) - ret0, _ := ret[0].(transport.Payload) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Decode indicates an expected call of Decode -func (mr *MockDecoderMockRecorder) Decode(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Decode", reflect.TypeOf((*MockDecoder)(nil).Decode), arg0) -} diff --git a/transport/mocks/io.gen.go b/transport/mocks/io.gen.go deleted file mode 100644 index 3e1e67b0..00000000 --- a/transport/mocks/io.gen.go +++ /dev/null @@ -1,100 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: io (interfaces: Writer,WriteCloser) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockWriter is a mock of Writer interface -type MockWriter struct { - ctrl *gomock.Controller - recorder *MockWriterMockRecorder -} - -// MockWriterMockRecorder is the mock recorder for MockWriter -type MockWriterMockRecorder struct { - mock *MockWriter -} - -// NewMockWriter creates a new mock instance -func NewMockWriter(ctrl *gomock.Controller) *MockWriter { - mock := &MockWriter{ctrl: ctrl} - mock.recorder = &MockWriterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockWriter) EXPECT() *MockWriterMockRecorder { - return m.recorder -} - -// Write mocks base method -func (m *MockWriter) Write(arg0 []byte) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Write", arg0) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Write indicates an expected call of Write -func (mr *MockWriterMockRecorder) Write(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriter)(nil).Write), arg0) -} - -// MockWriteCloser is a mock of WriteCloser interface -type MockWriteCloser struct { - ctrl *gomock.Controller - recorder *MockWriteCloserMockRecorder -} - -// MockWriteCloserMockRecorder is the mock recorder for MockWriteCloser -type MockWriteCloserMockRecorder struct { - mock *MockWriteCloser -} - -// NewMockWriteCloser creates a new mock instance -func NewMockWriteCloser(ctrl *gomock.Controller) *MockWriteCloser { - mock := &MockWriteCloser{ctrl: ctrl} - mock.recorder = &MockWriteCloserMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockWriteCloser) EXPECT() *MockWriteCloserMockRecorder { - return m.recorder -} - -// Close mocks base method -func (m *MockWriteCloser) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close -func (mr *MockWriteCloserMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWriteCloser)(nil).Close)) -} - -// Write mocks base method -func (m *MockWriteCloser) Write(arg0 []byte) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Write", arg0) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Write indicates an expected call of Write -func (mr *MockWriteCloserMockRecorder) Write(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriteCloser)(nil).Write), arg0) -} diff --git a/transport/mocks/mocks.go b/transport/mocks/mocks.go deleted file mode 100644 index c6a6ef94..00000000 --- a/transport/mocks/mocks.go +++ /dev/null @@ -1,4 +0,0 @@ -package mocks - -//go:generate mockgen -destination=io.gen.go -package=mocks io Writer,WriteCloser -//go:generate mockgen -destination=codec.gen.go -package=mocks github.com/kcarretto/paragon/transport Encoder,Decoder diff --git a/transport/receiver.go b/transport/receiver.go deleted file mode 100644 index 2bf192ec..00000000 --- a/transport/receiver.go +++ /dev/null @@ -1,50 +0,0 @@ -package transport - -import ( - "io" - "sync" -) - -// ResultWriter buffers result output that will be transported to the server. -type ResultWriter interface { - io.Writer - WriteResult(*Result) -} - -// A PayloadWriter sends payloads from the server to a consumer. -type PayloadWriter interface { - WritePayload([]byte) - // WriteError(error) -} - -// A PayloadHandler is responsible for consuming and acting upon payloads received from the server. -type PayloadHandler interface { - HandlePayload(ResultWriter, Payload, error) -} - -// PayloadHandlerFn is a function that implements PayloadHandler. -type PayloadHandlerFn func(ResultWriter, Payload, error) - -// HandlePayload wraps fn to provide an implementation of PayloadHandler. -func (fn PayloadHandlerFn) HandlePayload(w ResultWriter, data Payload, err error) { - fn(w, data, err) -} - -// A Receiver consumes payloads from the server. -type Receiver struct { - mu sync.Mutex - - ResultWriter - Decoder - Handler func(ResultWriter, Payload, error) -} - -// WritePayload implements PayloadWriter, passing data to the handler for consumption. It is safe -// for concurrent use by multiple transports. -func (recv *Receiver) WritePayload(data []byte) { - recv.mu.Lock() - defer recv.mu.Unlock() - - payload, err := recv.Decode(data) - recv.Handler(recv.ResultWriter, payload, err) -} diff --git a/transport/receiver_test.go b/transport/receiver_test.go deleted file mode 100644 index 3ff6570c..00000000 --- a/transport/receiver_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package transport_test - -import ( - "testing" - - "github.com/golang/mock/gomock" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/mocks" - "github.com/stretchr/testify/require" -) - -func TestReceiver(t *testing.T) { - expectedData := []byte("here is a whole bunch of payload data") - expectedPayload := transport.Payload{} - - // Prepare mock controller - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // Prepare mocks - decoder := mocks.NewMockDecoder(ctrl) - decoder.EXPECT().Decode(expectedData).Return(expectedPayload, nil) - - recv := transport.Receiver{ - Decoder: transport.DecoderFn(func(data []byte) (transport.Payload, error) { - return decoder.Decode(data) - }), - Handler: transport.PayloadHandlerFn(func(w transport.ResultWriter, payload transport.Payload, err error) { - require.Equal(t, expectedPayload, payload) - require.NoError(t, err) - }), - } - - recv.WritePayload(expectedData) -} diff --git a/transport/registry.go b/transport/registry.go deleted file mode 100644 index 29fe3517..00000000 --- a/transport/registry.go +++ /dev/null @@ -1,113 +0,0 @@ -package transport - -import ( - "errors" - "io" - "sort" - "sync" -) - -// A Registry tracks transports and associated metadata. -type Registry struct { - SortBy func(t1, t2 Transport) bool - - mu sync.RWMutex - transports map[string]Transport - cache []Transport -} - -func (reg *Registry) set(transport Transport) { - reg.mu.Lock() - if reg.transports == nil { - reg.transports = map[string]Transport{} - } - reg.transports[transport.Name] = transport - reg.cache = nil - reg.mu.Unlock() -} - -// Add a transport to the registry. If a transport with the same name is already registered, this -// method is a no-op. Use Update() to modify existing transports. -func (reg *Registry) Add(transport Transport) { - if _, ok := reg.transports[transport.Name]; ok { - return - } - - reg.set(transport) -} - -// Get returns a registered transport with the given name. -func (reg *Registry) Get(name string) (Transport, error) { - reg.mu.RLock() - transport, ok := reg.transports[name] - reg.mu.RUnlock() - - if !ok { - return Transport{}, errors.New("transport not registered") - } - - return transport, nil -} - -// Update the transport with the provided name by applying options to it. -func (reg *Registry) Update(name string, options ...Option) error { - transport, err := reg.Get(name) - if err != nil { - return err - } - - for _, opt := range options { - opt(&transport) - } - - reg.set(transport) - - return nil -} - -// List transports, sorted by the provided SortBy method or by priority if no method was provided. -func (reg *Registry) List() []Transport { - if reg.cache != nil { - return reg.cache - } - - if reg.SortBy == nil { - reg.SortBy = func(t1, t2 Transport) bool { - return t1.Priority < t2.Priority - } - } - - transports := make([]Transport, 0, len(reg.transports)) - for _, transport := range reg.transports { - transports = append(transports, transport) - } - - sort.Slice(transports, func(i, j int) bool { - return reg.SortBy(transports[i], transports[j]) - }) - - reg.cache = transports - - return reg.cache -} - -// Close a transport by name. Close is a no-op if the transport is not an io.Closer. -func (reg *Registry) Close(name string) error { - transport, err := reg.Get(name) - if err != nil { - return err - } - - if closer, ok := transport.Writer.(io.Closer); ok { - return closer.Close() - } - - return nil -} - -// CloseAll attempts to close all transports. -func (reg *Registry) CloseAll() { - for name := range reg.transports { - reg.Close(name) - } -} diff --git a/transport/registry_test.go b/transport/registry_test.go deleted file mode 100644 index c56771a4..00000000 --- a/transport/registry_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package transport_test - -import ( - "testing" - "time" - - "github.com/golang/mock/gomock" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/mocks" - "github.com/stretchr/testify/require" -) - -func TestRegistry(t *testing.T) { - // Prepare mock controller - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // Prepare mocks - writer1 := mocks.NewMockWriteCloser(ctrl) - writer2 := mocks.NewMockWriteCloser(ctrl) - gomock.InOrder( - writer1.EXPECT().Close(), - ) - - // Add transports to the registry - reg := transport.Registry{} - reg.Add(transport.New("RegTest1", writer1, transport.SetInterval(time.Second*100))) - reg.Add(transport.New("RegTest2", writer2)) - - // Get a transport - tp1, err := reg.Get("RegTest1") - require.NoError(t, err) - require.Equal(t, time.Second*100, tp1.Interval) - - // Update the transport - err = reg.Update("RegTest2", transport.SetInterval(time.Second*23), transport.SetPriority(-15)) - require.NoError(t, err) - tp2, err := reg.Get("RegTest2") - require.NoError(t, err) - require.Equal(t, time.Second*23, tp2.Interval) - require.Equal(t, -15, tp2.Priority) - - // List transports - lst := reg.List() - require.Equal(t, 2, len(lst)) - require.Equal(t, "RegTest2", lst[0].Name, "Transport list in unexpected order") - require.Equal(t, "RegTest1", lst[1].Name, "Transport list in unexpected order") - - // Close the first transport - err = reg.Close(tp1.Name) - require.NoError(t, err) -} diff --git a/transport/sender.go b/transport/sender.go deleted file mode 100644 index a0b3de03..00000000 --- a/transport/sender.go +++ /dev/null @@ -1,33 +0,0 @@ -package transport - -import ( - "io" - "time" -) - -// MultiSender is used to send a response buffer to the server using transports from the registry. -type MultiSender struct { - Transports *Registry - OnError func(Transport, error) -} - -// Send the response buffer to the server using available transports. If a transport fails to send -// the respond, OnError will be invoked if it is set. Transports will be attempted in the order -// defined by the registry sorting mechanism. If all transports are exhausted, ErrNoTransportAvailable -// will be returned. -func (sender MultiSender) Send(buffer TimestampWriterTo) (int64, error) { - for _, transport := range sender.Transports.List() { - delay := transport.Interval - time.Since(buffer.Timestamp()) - time.Sleep(delay) - - n, err := buffer.WriteTo(transport) - if err != nil && err != io.EOF && sender.OnError != nil { - sender.OnError(transport, err) - continue - } - - return n, nil - } - - return 0, ErrNoTransportAvailable -} diff --git a/transport/transport.go b/transport/transport.go deleted file mode 100644 index d1997d75..00000000 --- a/transport/transport.go +++ /dev/null @@ -1,54 +0,0 @@ -package transport - -import ( - "io" - "time" -) - -// An Option enables additional configuration of a transport. -type Option func(*Transport) - -// Transport wraps an underlying io.Writer with metadata. -type Transport struct { - io.Writer - - Name string - Priority int - Interval time.Duration - Jitter time.Duration -} - -// New creates and initializes a new Transport. -func New(name string, writer io.Writer, options ...Option) Transport { - transport := Transport{ - Writer: writer, - Name: name, - } - - for _, opt := range options { - opt(&transport) - } - - return transport -} - -// SetPriority metadata for the transport. -func SetPriority(priority int) Option { - return func(transport *Transport) { - transport.Priority = priority - } -} - -// SetInterval metadata for the transport. -func SetInterval(interval time.Duration) Option { - return func(transport *Transport) { - transport.Interval = interval - } -} - -// SetJitter metadata for the transport. -func SetJitter(jitter time.Duration) Option { - return func(transport *Transport) { - transport.Jitter = jitter - } -} diff --git a/transport/transport_test.go b/transport/transport_test.go deleted file mode 100644 index 2ecd0296..00000000 --- a/transport/transport_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package transport_test - -import ( - "io/ioutil" - "testing" - "time" - - "github.com/kcarretto/paragon/transport" - "github.com/stretchr/testify/require" -) - -func TestNew(t *testing.T) { - name := "MyTransport" - interval := time.Hour * 5 - jitter := time.Nanosecond * 10 - priority := 1337 - - transport := transport.New( - name, - ioutil.Discard, - transport.SetInterval(interval), - transport.SetJitter(jitter), - transport.SetPriority(priority), - ) - - require.Equal(t, name, transport.Name) - require.Equal(t, interval, transport.Interval) - require.Equal(t, jitter, transport.Jitter) - require.Equal(t, priority, transport.Priority) -} From a05e9b4760fd57778d6b1e9cd5a59d21ff526be8 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sat, 5 Oct 2019 21:01:52 -0700 Subject: [PATCH 10/19] Added c2 package & tests for initial c2 --- c2/http.go | 36 ++++++++++++++++++++++++++++ c2/mocks/io.gen.go | 48 +++++++++++++++++++++++++++++++++++++ c2/mocks/mocks.go | 3 +++ c2/server.go | 42 ++++++++++++++++++++++++++++++++ c2/server_test.go | 37 ++++++++++++++++++++++++++++ c2/task.go | 48 +++++++++++++++++++++++++++++++++++++ c2/task_test.go | 57 +++++++++++++++++++++++++++++++++++++++++++ go.mod | 2 +- go.sum | 60 ---------------------------------------------- 9 files changed, 272 insertions(+), 61 deletions(-) create mode 100644 c2/http.go create mode 100644 c2/mocks/io.gen.go create mode 100644 c2/mocks/mocks.go create mode 100644 c2/server.go create mode 100644 c2/server_test.go create mode 100644 c2/task.go create mode 100644 c2/task_test.go delete mode 100644 go.sum diff --git a/c2/http.go b/c2/http.go new file mode 100644 index 00000000..63799af8 --- /dev/null +++ b/c2/http.go @@ -0,0 +1,36 @@ +package c2 + +import ( + "encoding/json" + "io/ioutil" + "net/http" + + "github.com/kcarretto/paragon/transport" + "go.uber.org/zap" +) + +// HTTP handles agent messages sent from http. +func (srv *Server) HTTP(w http.ResponseWriter, req *http.Request) { + data, err := ioutil.ReadAll(req.Body) + if err != nil { + srv.Logger.Error("Failed to read agent message", zap.Error(err)) + return + } + + srv.Logger.Debug("Received agent message", zap.String("agent_msg", string(data))) + + var msg transport.Response + if err := json.Unmarshal(data, &msg); err != nil { + srv.Logger.Error("Failed to unmarshal agent message", zap.Error(err)) + return + } + + if err = srv.HandleMessage(w, msg); err != nil { + srv.Logger.Error("Agent communication failed", zap.Error(err)) + return + } + + // TODO: Add agent metadata + srv.Logger.Info("Replied to agent message") + +} diff --git a/c2/mocks/io.gen.go b/c2/mocks/io.gen.go new file mode 100644 index 00000000..abfc2f27 --- /dev/null +++ b/c2/mocks/io.gen.go @@ -0,0 +1,48 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: io (interfaces: Writer) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockWriter is a mock of Writer interface +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// Write mocks base method +func (m *MockWriter) Write(arg0 []byte) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Write indicates an expected call of Write +func (mr *MockWriterMockRecorder) Write(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriter)(nil).Write), arg0) +} diff --git a/c2/mocks/mocks.go b/c2/mocks/mocks.go new file mode 100644 index 00000000..6a6b6a90 --- /dev/null +++ b/c2/mocks/mocks.go @@ -0,0 +1,3 @@ +package mocks + +//go:generate mockgen -destination=io.gen.go -package=mocks io Writer diff --git a/c2/server.go b/c2/server.go new file mode 100644 index 00000000..386090c7 --- /dev/null +++ b/c2/server.go @@ -0,0 +1,42 @@ +package c2 + +import ( + "encoding/json" + "fmt" + "io" + "sync" + + "github.com/kcarretto/paragon/transport" + + "go.uber.org/zap" + "gocloud.dev/pubsub" +) + +// Server handles agent messages and replies with new tasks for the agent to execute. +type Server struct { + Logger *zap.Logger + AgentOutput *pubsub.Topic + + mu sync.RWMutex + tasks []queuedTask +} + +// HandleMessage received from the agent, and write a reply to the provided writer. +func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { + // TODO: Get available tasks. srv.GetTasks(msg.Metadata) + tasks := srv.GetTasks(msg.Metadata) + reply := transport.Payload{ + Tasks: tasks, + } + + data, err := json.Marshal(reply) + if err != nil { + return fmt.Errorf("failed to marshal server reply message: %w", err) + } + + if _, err = w.Write(data); err != nil { + return fmt.Errorf("failed to send reply message to agent: %w", err) + } + + return nil +} diff --git a/c2/server_test.go b/c2/server_test.go new file mode 100644 index 00000000..d9ba37dd --- /dev/null +++ b/c2/server_test.go @@ -0,0 +1,37 @@ +package c2_test + +import ( + "encoding/json" + "testing" + + "github.com/golang/mock/gomock" + + "github.com/kcarretto/paragon/c2" + "github.com/kcarretto/paragon/c2/mocks" + "github.com/kcarretto/paragon/transport" + "github.com/stretchr/testify/require" +) + +func TestHandleMessage(t *testing.T) { + expectedTask := transport.Task{ID: "HelloThere"} + expectedReply := transport.Payload{ + Tasks: []transport.Task{expectedTask}, + } + expected, err := json.Marshal(expectedReply) + require.NoError(t, err) + + // Prepare mock + ctrl := gomock.NewController(t) + defer ctrl.Finish() + replyWriter := mocks.NewMockWriter(ctrl) + replyWriter.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { + require.Equal(t, string(expected), string(p)) + return len(p), nil + }) + + srv := &c2.Server{} + srv.QueueTask(expectedTask, func(agent transport.Metadata) bool { return true }) + + err = srv.HandleMessage(replyWriter, transport.Response{}) + require.NoError(t, err) +} diff --git a/c2/task.go b/c2/task.go new file mode 100644 index 00000000..ff4a7486 --- /dev/null +++ b/c2/task.go @@ -0,0 +1,48 @@ +package c2 + +import ( + "github.com/kcarretto/paragon/transport" +) + +type queuedTask struct { + task transport.Task + filter func(transport.Metadata) bool +} + +// QueueTask prepares a task to be sent to the first agent that reports metadata matching the filter. +func (srv *Server) QueueTask(task transport.Task, filter func(transport.Metadata) bool) { + srv.mu.Lock() + srv.tasks = append(srv.tasks, queuedTask{ + task, + filter, + }) + srv.mu.Unlock() +} + +// GetTasks for an agent based on the metadata it reported and the current tasks in the server queue. +func (srv *Server) GetTasks(agent transport.Metadata) []transport.Task { + srv.mu.Lock() + n := 0 + tasks := make([]transport.Task, 0, len(srv.tasks)) + for _, t := range srv.tasks { + if t.filter == nil || t.filter(agent) { + tasks = append(tasks, t.task) + } else { + srv.tasks[n] = t + n++ + } + } + srv.tasks = srv.tasks[:n] + srv.mu.Unlock() + + return tasks +} + +// TaskCount returns the total number of tasks left in the queue. +func (srv *Server) TaskCount() int { + srv.mu.RLock() + count := len(srv.tasks) + srv.mu.RUnlock() + + return count +} diff --git a/c2/task_test.go b/c2/task_test.go new file mode 100644 index 00000000..dd577446 --- /dev/null +++ b/c2/task_test.go @@ -0,0 +1,57 @@ +package c2_test + +import ( + "testing" + + "github.com/kcarretto/paragon/c2" + "github.com/kcarretto/paragon/transport" + "github.com/stretchr/testify/require" +) + +func TestQueueTask(t *testing.T) { + expectedTask := transport.Task{ID: "TestID1"} + + srv := &c2.Server{} + srv.QueueTask( + expectedTask, + func(a transport.Metadata) bool { + return true + }, + ) + + require.Equal(t, 1, srv.TaskCount()) + + tasks := srv.GetTasks(transport.Metadata{}) + require.Equal(t, 1, len(tasks)) + require.Equal(t, expectedTask, tasks[0]) + + require.Equal(t, 0, srv.TaskCount()) +} + +func TestFilteredTask(t *testing.T) { + expectedTask := transport.Task{ID: "TestID1"} + unexpectedTask := transport.Task{ID: "Nope"} + + srv := &c2.Server{} + srv.QueueTask( + expectedTask, + func(transport.Metadata) bool { + return true + }, + ) + require.Equal(t, 1, srv.TaskCount()) + + srv.QueueTask( + unexpectedTask, + func(transport.Metadata) bool { + return false + }, + ) + require.Equal(t, 2, srv.TaskCount()) + + tasks := srv.GetTasks(transport.Metadata{}) + require.Equal(t, 1, len(tasks)) + require.Equal(t, expectedTask, tasks[0]) + + require.Equal(t, 1, srv.TaskCount()) +} diff --git a/go.mod b/go.mod index 980497b8..0a7e5d3d 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,6 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/mock v1.3.1 - github.com/kr/pretty v0.1.0 // indirect github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.3.0 github.com/shirou/gopsutil v2.18.12+incompatible @@ -18,6 +17,7 @@ require ( go.uber.org/atomic v1.4.0 // indirect go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.10.0 + gocloud.dev v0.17.0 golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect google.golang.org/appengine v1.6.5 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect diff --git a/go.sum b/go.sum deleted file mode 100644 index 067fea18..00000000 --- a/go.sum +++ /dev/null @@ -1,60 +0,0 @@ -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d h1:G0m3OIz70MZUWq3EgK3CesDbo8upS2Vm9/P3FtgI+Jk= -github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= -github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/go-ole/go-ole v1.2.4 h1:nNBDSCOigTSiarFpYE9J/KtEA1IOW4CNeqT9TQDqCxI= -github.com/go-ole/go-ole v1.2.4/go.mod h1:XCwSNxSkXRo4vlyPy93sltvi/qJq0jqQhjqQNIwKuxM= -github.com/golang/mock v1.3.1 h1:qGJ6qTW+x6xX/my+8YUVl4WNpX9B7+/l2tRsHGZ7f2s= -github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= -github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= -github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= -github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/profile v1.3.0 h1:OQIvuDgm00gWVWGTf4m4mCt6W1/0YqU7Ntg0mySWgaI= -github.com/pkg/profile v1.3.0/go.mod h1:hJw3o1OdXxsrSjjVksARp5W95eeEaEfptyVZyv6JUPA= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/shirou/gopsutil v2.18.12+incompatible h1:1eaJvGomDnH74/5cF4CTmTbLHAriGFsTZppLXDX93OM= -github.com/shirou/gopsutil v2.18.12+incompatible/go.mod h1:5b4v6he4MtMOwMlS0TUMTu2PcXUg8+E1lC7eC3UO/RA= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 h1:udFKJ0aHUL60LboW/A+DfgoHVedieIzIXE8uylPue0U= -github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4/go.mod h1:qsXQc7+bwAM3Q1u/4XEfrquwF8Lw7D7y5cD8CuHnfIc= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749 h1:bUGsEnyNbVPw06Bs80sCeARAlK8lhwqGyi6UT8ymuGk= -github.com/shurcooL/httpfs v0.0.0-20190707220628-8d4bc4ba7749/go.mod h1:ZY1cvUeJuFPAdZ/B6v7RHavJWZn2YPVFQ1OSXhCGOkg= -github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd h1:ug7PpSOB5RBPK1Kg6qskGBoP3Vnj/aNYFTznWvlkGo0= -github.com/shurcooL/vfsgen v0.0.0-20181202132449-6a9ea43bcacd/go.mod h1:TrYk7fJVaAttu97ZZKrO9UbRa8izdowaMIZcxYMbVaw= -github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= -github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -go.starlark.net v0.0.0-20190717023158-fc7a7f44baed h1:4jRCa7xxEE+8+CDxrq64JNGsuui8/Y1/enYVey/vsjo= -go.starlark.net v0.0.0-20190717023158-fc7a7f44baed/go.mod h1:c1/X6cHgvdXj6pUlmWKMkuqRnW4K8x2vwt6JAaaircg= -go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= -go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= -go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= -go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= -go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= -go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= -golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65 h1:+rhAzEzT3f4JtomfC371qB+0Ola2caSKcY69NUBZrRQ= -golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= -golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 h1:HyfiK1WMnHj5FXFXatD+Qs1A/xC2Run6RzeW1SyHxpc= -golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= -golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262 h1:qsl9y/CJx34tuA7QCPNp86JNJe4spst6Ff8MjvPUdPg= -golang.org/x/tools v0.0.0-20190425150028-36563e24a262/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q= -google.golang.org/appengine v1.6.5 h1:tycE03LOZYQNhDpS27tcQdAzLCVMaj7QT2SXxebnpCM= -google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= -gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From fb69177c94917f506e9a4f700bbcea476024da39 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sun, 6 Oct 2019 00:19:58 -0700 Subject: [PATCH 11/19] Added http test --- c2/http.go | 4 ++-- c2/http_test.go | 50 +++++++++++++++++++++++++++++++++++++++++++++++++ c2/server.go | 3 ++- 3 files changed, 54 insertions(+), 3 deletions(-) create mode 100644 c2/http_test.go diff --git a/c2/http.go b/c2/http.go index 63799af8..ee5cb123 100644 --- a/c2/http.go +++ b/c2/http.go @@ -9,8 +9,8 @@ import ( "go.uber.org/zap" ) -// HTTP handles agent messages sent from http. -func (srv *Server) HTTP(w http.ResponseWriter, req *http.Request) { +// ServeHTTP implements http.Handler to handle agent messages sent via http. +func (srv *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { data, err := ioutil.ReadAll(req.Body) if err != nil { srv.Logger.Error("Failed to read agent message", zap.Error(err)) diff --git a/c2/http_test.go b/c2/http_test.go new file mode 100644 index 00000000..033f07dc --- /dev/null +++ b/c2/http_test.go @@ -0,0 +1,50 @@ +package c2_test + +import ( + "bytes" + "encoding/json" + "io/ioutil" + "net/http" + "net/http/httptest" + "testing" + + "github.com/kcarretto/paragon/c2" + "github.com/kcarretto/paragon/transport" + + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestHTTP(t *testing.T) { + expectedTask := transport.Task{ + ID: "ABC", + } + unexpectedTask := transport.Task{ + ID: "XYZ", + } + + msg := transport.Response{} + data, err := json.Marshal(msg) + require.NoError(t, err) + + w := httptest.NewRecorder() + req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBuffer(data)) + + srv := &c2.Server{ + Logger: zap.NewNop(), + } + srv.QueueTask(expectedTask, func(transport.Metadata) bool { return true }) + srv.QueueTask(unexpectedTask, func(transport.Metadata) bool { return false }) + + srv.ServeHTTP(w, req) + + body, err := ioutil.ReadAll(w.Result().Body) + defer w.Result().Body.Close() + require.NoError(t, err) + + var reply transport.Payload + err = json.Unmarshal(body, &reply) + require.NoError(t, err) + require.Equal(t, 1, len(reply.Tasks)) + require.Equal(t, expectedTask, reply.Tasks[0]) +} diff --git a/c2/server.go b/c2/server.go index 386090c7..fb7603cd 100644 --- a/c2/server.go +++ b/c2/server.go @@ -23,7 +23,8 @@ type Server struct { // HandleMessage received from the agent, and write a reply to the provided writer. func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { - // TODO: Get available tasks. srv.GetTasks(msg.Metadata) + // TODO: Handle results + tasks := srv.GetTasks(msg.Metadata) reply := transport.Payload{ Tasks: tasks, From 6a6b5c4f03c4737c432f810af111144cb47042c9 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sun, 6 Oct 2019 00:30:52 -0700 Subject: [PATCH 12/19] Updated go mod --- go.mod | 1 + 1 file changed, 1 insertion(+) diff --git a/go.mod b/go.mod index 0a7e5d3d..49815bda 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( go.uber.org/multierr v1.1.0 // indirect go.uber.org/zap v1.10.0 gocloud.dev v0.17.0 + golang.org/x/net v0.0.0-20190620200207-3b0461eec859 // indirect golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0 // indirect google.golang.org/appengine v1.6.5 gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 // indirect From ca38c8a92f5117b2f7922836e390e9354035fca8 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Sun, 6 Oct 2019 01:01:16 -0700 Subject: [PATCH 13/19] Publish task results if topic is set --- c2/http.go | 12 ++++++------ c2/http_test.go | 2 +- c2/server.go | 33 +++++++++++++++++++++++++++++---- c2/server_test.go | 8 ++++++-- 4 files changed, 42 insertions(+), 13 deletions(-) diff --git a/c2/http.go b/c2/http.go index ee5cb123..9c286b41 100644 --- a/c2/http.go +++ b/c2/http.go @@ -13,24 +13,24 @@ import ( func (srv *Server) ServeHTTP(w http.ResponseWriter, req *http.Request) { data, err := ioutil.ReadAll(req.Body) if err != nil { - srv.Logger.Error("Failed to read agent message", zap.Error(err)) + srv.Log.Error("Failed to read agent message", zap.Error(err)) return } - srv.Logger.Debug("Received agent message", zap.String("agent_msg", string(data))) + srv.Log.Debug("Received agent message", zap.String("agent_msg", string(data))) var msg transport.Response if err := json.Unmarshal(data, &msg); err != nil { - srv.Logger.Error("Failed to unmarshal agent message", zap.Error(err)) + srv.Log.Error("Failed to unmarshal agent message", zap.Error(err)) return } - if err = srv.HandleMessage(w, msg); err != nil { - srv.Logger.Error("Agent communication failed", zap.Error(err)) + if err = srv.HandleMessage(req.Context(), w, msg); err != nil { + srv.Log.Error("Agent communication failed", zap.Error(err)) return } // TODO: Add agent metadata - srv.Logger.Info("Replied to agent message") + srv.Log.Info("Replied to agent message") } diff --git a/c2/http_test.go b/c2/http_test.go index 033f07dc..4fb448f2 100644 --- a/c2/http_test.go +++ b/c2/http_test.go @@ -31,7 +31,7 @@ func TestHTTP(t *testing.T) { req := httptest.NewRequest(http.MethodPost, "/", bytes.NewBuffer(data)) srv := &c2.Server{ - Logger: zap.NewNop(), + Log: zap.NewNop(), } srv.QueueTask(expectedTask, func(transport.Metadata) bool { return true }) srv.QueueTask(unexpectedTask, func(transport.Metadata) bool { return false }) diff --git a/c2/server.go b/c2/server.go index fb7603cd..b51a2516 100644 --- a/c2/server.go +++ b/c2/server.go @@ -1,6 +1,7 @@ package c2 import ( + "context" "encoding/json" "fmt" "io" @@ -14,16 +15,16 @@ import ( // Server handles agent messages and replies with new tasks for the agent to execute. type Server struct { - Logger *zap.Logger - AgentOutput *pubsub.Topic + Log *zap.Logger + TaskResults *pubsub.Topic mu sync.RWMutex tasks []queuedTask } // HandleMessage received from the agent, and write a reply to the provided writer. -func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { - // TODO: Handle results +func (srv *Server) HandleMessage(ctx context.Context, w io.Writer, msg transport.Response) error { + srv.publishResults(ctx, msg) tasks := srv.GetTasks(msg.Metadata) reply := transport.Payload{ @@ -41,3 +42,27 @@ func (srv *Server) HandleMessage(w io.Writer, msg transport.Response) error { return nil } + +func (srv *Server) publishResults(ctx context.Context, msg transport.Response) { + if srv.TaskResults == nil { + srv.Log.Warn("No topic set for task results") + return + } + + for _, result := range msg.Results { + // TODO: Send all results, or only ones with data? + body, err := json.Marshal(result) + if err != nil { + srv.Log.Error("Failed to marshal result to json", zap.Error(err)) + } + event := &pubsub.Message{ + Body: body, + // TODO: Add agent metadata + } + go func() { + if err = srv.TaskResults.Send(ctx, event); err != nil { + srv.Log.Error("Failed to publish task result", zap.Error(err)) + } + }() + } +} diff --git a/c2/server_test.go b/c2/server_test.go index d9ba37dd..06b53824 100644 --- a/c2/server_test.go +++ b/c2/server_test.go @@ -1,10 +1,12 @@ package c2_test import ( + "context" "encoding/json" "testing" "github.com/golang/mock/gomock" + "go.uber.org/zap" "github.com/kcarretto/paragon/c2" "github.com/kcarretto/paragon/c2/mocks" @@ -29,9 +31,11 @@ func TestHandleMessage(t *testing.T) { return len(p), nil }) - srv := &c2.Server{} + srv := &c2.Server{ + Log: zap.NewNop(), + } srv.QueueTask(expectedTask, func(agent transport.Metadata) bool { return true }) - err = srv.HandleMessage(replyWriter, transport.Response{}) + err = srv.HandleMessage(context.Background(), replyWriter, transport.Response{}) require.NoError(t, err) } From e11cacfc8e2f3ea29fe2892a05de9869c93ae909 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Fri, 11 Oct 2019 01:11:03 -0700 Subject: [PATCH 14/19] Added protobuf spec for codec and events --- api/codec/agent.pb.go | 263 +++++++++++++++++++++ api/codec/agent.proto | 31 +++ api/codec/codec.go | 12 + api/codec/server.pb.go | 129 ++++++++++ api/codec/server.proto | 14 ++ api/events/events.go | 10 + api/events/task.pb.go | 242 +++++++++++++++++++ api/events/task.proto | 38 +++ api/vendor/google/protobuf/timestamp.proto | 138 +++++++++++ 9 files changed, 877 insertions(+) create mode 100644 api/codec/agent.pb.go create mode 100644 api/codec/agent.proto create mode 100644 api/codec/codec.go create mode 100644 api/codec/server.pb.go create mode 100644 api/codec/server.proto create mode 100644 api/events/events.go create mode 100644 api/events/task.pb.go create mode 100644 api/events/task.proto create mode 100644 api/vendor/google/protobuf/timestamp.proto diff --git a/api/codec/agent.pb.go b/api/codec/agent.pb.go new file mode 100644 index 00000000..d66eb29c --- /dev/null +++ b/api/codec/agent.pb.go @@ -0,0 +1,263 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: agent.proto + +package codec + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + timestamp "github.com/golang/protobuf/ptypes/timestamp" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type AgentMessage struct { + Metadata *AgentMetadata `protobuf:"bytes,1,opt,name=metadata,proto3" json:"metadata,omitempty"` + Results []*Result `protobuf:"bytes,2,rep,name=results,proto3" json:"results,omitempty"` + Logs []string `protobuf:"bytes,3,rep,name=logs,proto3" json:"logs,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AgentMessage) Reset() { *m = AgentMessage{} } +func (m *AgentMessage) String() string { return proto.CompactTextString(m) } +func (*AgentMessage) ProtoMessage() {} +func (*AgentMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_56ede974c0020f77, []int{0} +} + +func (m *AgentMessage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_AgentMessage.Unmarshal(m, b) +} +func (m *AgentMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_AgentMessage.Marshal(b, m, deterministic) +} +func (m *AgentMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_AgentMessage.Merge(m, src) +} +func (m *AgentMessage) XXX_Size() int { + return xxx_messageInfo_AgentMessage.Size(m) +} +func (m *AgentMessage) XXX_DiscardUnknown() { + xxx_messageInfo_AgentMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_AgentMessage proto.InternalMessageInfo + +func (m *AgentMessage) GetMetadata() *AgentMetadata { + if m != nil { + return m.Metadata + } + return nil +} + +func (m *AgentMessage) GetResults() []*Result { + if m != nil { + return m.Results + } + return nil +} + +func (m *AgentMessage) GetLogs() []string { + if m != nil { + return m.Logs + } + return nil +} + +type AgentMetadata struct { + AgentID string `protobuf:"bytes,1,opt,name=agentID,proto3" json:"agentID,omitempty"` + MachineUUID string `protobuf:"bytes,2,opt,name=machineUUID,proto3" json:"machineUUID,omitempty"` + SessionID string `protobuf:"bytes,3,opt,name=sessionID,proto3" json:"sessionID,omitempty"` + Hostname string `protobuf:"bytes,4,opt,name=hostname,proto3" json:"hostname,omitempty"` + PrimaryIP string `protobuf:"bytes,5,opt,name=primaryIP,proto3" json:"primaryIP,omitempty"` + PrimaryMAC string `protobuf:"bytes,6,opt,name=primaryMAC,proto3" json:"primaryMAC,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *AgentMetadata) Reset() { *m = AgentMetadata{} } +func (m *AgentMetadata) String() string { return proto.CompactTextString(m) } +func (*AgentMetadata) ProtoMessage() {} +func (*AgentMetadata) Descriptor() ([]byte, []int) { + return fileDescriptor_56ede974c0020f77, []int{1} +} + +func (m *AgentMetadata) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_AgentMetadata.Unmarshal(m, b) +} +func (m *AgentMetadata) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_AgentMetadata.Marshal(b, m, deterministic) +} +func (m *AgentMetadata) XXX_Merge(src proto.Message) { + xxx_messageInfo_AgentMetadata.Merge(m, src) +} +func (m *AgentMetadata) XXX_Size() int { + return xxx_messageInfo_AgentMetadata.Size(m) +} +func (m *AgentMetadata) XXX_DiscardUnknown() { + xxx_messageInfo_AgentMetadata.DiscardUnknown(m) +} + +var xxx_messageInfo_AgentMetadata proto.InternalMessageInfo + +func (m *AgentMetadata) GetAgentID() string { + if m != nil { + return m.AgentID + } + return "" +} + +func (m *AgentMetadata) GetMachineUUID() string { + if m != nil { + return m.MachineUUID + } + return "" +} + +func (m *AgentMetadata) GetSessionID() string { + if m != nil { + return m.SessionID + } + return "" +} + +func (m *AgentMetadata) GetHostname() string { + if m != nil { + return m.Hostname + } + return "" +} + +func (m *AgentMetadata) GetPrimaryIP() string { + if m != nil { + return m.PrimaryIP + } + return "" +} + +func (m *AgentMetadata) GetPrimaryMAC() string { + if m != nil { + return m.PrimaryMAC + } + return "" +} + +type Result struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Output []string `protobuf:"bytes,2,rep,name=output,proto3" json:"output,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + ExecStartTime *timestamp.Timestamp `protobuf:"bytes,4,opt,name=execStartTime,proto3" json:"execStartTime,omitempty"` + ExecStopTime *timestamp.Timestamp `protobuf:"bytes,5,opt,name=execStopTime,proto3" json:"execStopTime,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Result) Reset() { *m = Result{} } +func (m *Result) String() string { return proto.CompactTextString(m) } +func (*Result) ProtoMessage() {} +func (*Result) Descriptor() ([]byte, []int) { + return fileDescriptor_56ede974c0020f77, []int{2} +} + +func (m *Result) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Result.Unmarshal(m, b) +} +func (m *Result) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Result.Marshal(b, m, deterministic) +} +func (m *Result) XXX_Merge(src proto.Message) { + xxx_messageInfo_Result.Merge(m, src) +} +func (m *Result) XXX_Size() int { + return xxx_messageInfo_Result.Size(m) +} +func (m *Result) XXX_DiscardUnknown() { + xxx_messageInfo_Result.DiscardUnknown(m) +} + +var xxx_messageInfo_Result proto.InternalMessageInfo + +func (m *Result) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *Result) GetOutput() []string { + if m != nil { + return m.Output + } + return nil +} + +func (m *Result) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +func (m *Result) GetExecStartTime() *timestamp.Timestamp { + if m != nil { + return m.ExecStartTime + } + return nil +} + +func (m *Result) GetExecStopTime() *timestamp.Timestamp { + if m != nil { + return m.ExecStopTime + } + return nil +} + +func init() { + proto.RegisterType((*AgentMessage)(nil), "codec.AgentMessage") + proto.RegisterType((*AgentMetadata)(nil), "codec.AgentMetadata") + proto.RegisterType((*Result)(nil), "codec.Result") +} + +func init() { proto.RegisterFile("agent.proto", fileDescriptor_56ede974c0020f77) } + +var fileDescriptor_56ede974c0020f77 = []byte{ + // 374 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0x31, 0x4f, 0x83, 0x40, + 0x18, 0x0d, 0x50, 0x68, 0xf9, 0x68, 0x1d, 0x2e, 0x8d, 0xb9, 0x34, 0x46, 0x49, 0x07, 0x65, 0x02, + 0x53, 0x77, 0x63, 0xb5, 0x0b, 0x43, 0x13, 0x83, 0xed, 0xe2, 0x76, 0x85, 0x93, 0x5e, 0x2c, 0x1c, + 0xb9, 0x3b, 0x12, 0x1d, 0xfc, 0x75, 0x2e, 0xfe, 0x2c, 0xd3, 0x03, 0x6a, 0x3b, 0xb9, 0xf1, 0xde, + 0xfb, 0xde, 0xe3, 0xfb, 0xee, 0x81, 0x47, 0x72, 0x5a, 0xaa, 0xb0, 0x12, 0x5c, 0x71, 0x64, 0xa7, + 0x3c, 0xa3, 0xe9, 0xe4, 0x2a, 0xe7, 0x3c, 0xdf, 0xd1, 0x48, 0x93, 0x9b, 0xfa, 0x2d, 0x52, 0xac, + 0xa0, 0x52, 0x91, 0xa2, 0x6a, 0xe6, 0xa6, 0x5f, 0x30, 0x9c, 0xef, 0x6d, 0x4b, 0x2a, 0x25, 0xc9, + 0x29, 0xba, 0x85, 0x41, 0x41, 0x15, 0xc9, 0x88, 0x22, 0xd8, 0xf0, 0x8d, 0xc0, 0x9b, 0x8d, 0x43, + 0x1d, 0x15, 0xb6, 0x63, 0x8d, 0x96, 0x1c, 0xa6, 0xd0, 0x0d, 0xf4, 0x05, 0x95, 0xf5, 0x4e, 0x49, + 0x6c, 0xfa, 0x56, 0xe0, 0xcd, 0x46, 0xad, 0x21, 0xd1, 0x6c, 0xd2, 0xa9, 0x08, 0x41, 0x6f, 0xc7, + 0x73, 0x89, 0x2d, 0xdf, 0x0a, 0xdc, 0x44, 0x7f, 0x4f, 0xbf, 0x0d, 0x18, 0x9d, 0x04, 0x23, 0x0c, + 0x7d, 0x7d, 0x47, 0xbc, 0xd0, 0xff, 0x77, 0x93, 0x0e, 0x22, 0x1f, 0xbc, 0x82, 0xa4, 0x5b, 0x56, + 0xd2, 0xf5, 0x3a, 0x5e, 0x60, 0x53, 0xab, 0xc7, 0x14, 0xba, 0x00, 0x57, 0x52, 0x29, 0x19, 0x2f, + 0xe3, 0x05, 0xb6, 0xb4, 0xfe, 0x47, 0xa0, 0x09, 0x0c, 0xb6, 0x5c, 0xaa, 0x92, 0x14, 0x14, 0xf7, + 0xb4, 0x78, 0xc0, 0x7b, 0x67, 0x25, 0x58, 0x41, 0xc4, 0x67, 0xfc, 0x8c, 0xed, 0xc6, 0x79, 0x20, + 0xd0, 0x25, 0x40, 0x0b, 0x96, 0xf3, 0x27, 0xec, 0x68, 0xf9, 0x88, 0x99, 0xfe, 0x18, 0xe0, 0x34, + 0xd7, 0xa2, 0x33, 0x30, 0x59, 0xd6, 0x6e, 0x6e, 0xb2, 0x0c, 0x9d, 0x83, 0xc3, 0x6b, 0x55, 0xd5, + 0x4a, 0x3f, 0x8e, 0x9b, 0xb4, 0x08, 0x8d, 0xc1, 0xa6, 0x42, 0x70, 0xd1, 0xae, 0xd9, 0x00, 0xf4, + 0x00, 0x23, 0xfa, 0x41, 0xd3, 0x17, 0x45, 0x84, 0x5a, 0xb1, 0x76, 0x4f, 0x6f, 0x36, 0x09, 0x9b, + 0x1a, 0xc3, 0xae, 0xc6, 0x70, 0xd5, 0xd5, 0x98, 0x9c, 0x1a, 0xd0, 0x3d, 0x0c, 0x1b, 0x82, 0x57, + 0x3a, 0xc0, 0xfe, 0x37, 0xe0, 0x64, 0xfe, 0x31, 0x78, 0xbd, 0xce, 0x99, 0xda, 0xd6, 0x9b, 0x30, + 0xe5, 0x45, 0xf4, 0x9e, 0x12, 0x21, 0xa8, 0x52, 0x3c, 0xaa, 0x88, 0x20, 0x39, 0x2f, 0x23, 0x52, + 0xb1, 0x48, 0xd7, 0xbb, 0x71, 0x74, 0xd6, 0xdd, 0x6f, 0x00, 0x00, 0x00, 0xff, 0xff, 0x43, 0xc2, + 0xed, 0x4a, 0x77, 0x02, 0x00, 0x00, +} diff --git a/api/codec/agent.proto b/api/codec/agent.proto new file mode 100644 index 00000000..dcc113c0 --- /dev/null +++ b/api/codec/agent.proto @@ -0,0 +1,31 @@ +syntax = "proto3"; + +package codec; + +option go_package = "github.com/kcarretto/paragon/api/codec"; + +import "google/protobuf/timestamp.proto"; + + +message AgentMessage { + AgentMetadata metadata = 1; + repeated Result results = 2; + repeated string logs = 3; +} + +message AgentMetadata { + string agentID = 1; + string machineUUID = 2; + string sessionID = 3; + string hostname = 4; + string primaryIP = 5; + string primaryMAC = 6; +} + +message Result { + string id = 1; + repeated string output = 2; + string error = 3; + google.protobuf.Timestamp execStartTime = 4; + google.protobuf.Timestamp execStopTime = 5; +} \ No newline at end of file diff --git a/api/codec/codec.go b/api/codec/codec.go new file mode 100644 index 00000000..5871186b --- /dev/null +++ b/api/codec/codec.go @@ -0,0 +1,12 @@ +// Package codec defines a standardized format for messages sent between the server and agents. +// It does not mandate an encoding mechanism for transporting such data. Any format for serializing +// messages sent between the agent and the server (i.e. protobuf or JSON) is acceptable so long as +// both the agent and the server utilize the same format. +package codec + +// VERSION describes the currently built version of codec. +const VERSION = "0.0.1" + +// TODO: Add python support via --python_out=. + +//go:generate protoc -I=../vendor/ -I=../../ -I=. --go_out=paths=source_relative:. agent.proto server.proto diff --git a/api/codec/server.pb.go b/api/codec/server.pb.go new file mode 100644 index 00000000..9028081e --- /dev/null +++ b/api/codec/server.pb.go @@ -0,0 +1,129 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: server.proto + +package codec + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type ServerMessage struct { + Tasks []*Task `protobuf:"bytes,1,rep,name=tasks,proto3" json:"tasks,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ServerMessage) Reset() { *m = ServerMessage{} } +func (m *ServerMessage) String() string { return proto.CompactTextString(m) } +func (*ServerMessage) ProtoMessage() {} +func (*ServerMessage) Descriptor() ([]byte, []int) { + return fileDescriptor_ad098daeda4239f7, []int{0} +} + +func (m *ServerMessage) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ServerMessage.Unmarshal(m, b) +} +func (m *ServerMessage) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ServerMessage.Marshal(b, m, deterministic) +} +func (m *ServerMessage) XXX_Merge(src proto.Message) { + xxx_messageInfo_ServerMessage.Merge(m, src) +} +func (m *ServerMessage) XXX_Size() int { + return xxx_messageInfo_ServerMessage.Size(m) +} +func (m *ServerMessage) XXX_DiscardUnknown() { + xxx_messageInfo_ServerMessage.DiscardUnknown(m) +} + +var xxx_messageInfo_ServerMessage proto.InternalMessageInfo + +func (m *ServerMessage) GetTasks() []*Task { + if m != nil { + return m.Tasks + } + return nil +} + +type Task struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *Task) Reset() { *m = Task{} } +func (m *Task) String() string { return proto.CompactTextString(m) } +func (*Task) ProtoMessage() {} +func (*Task) Descriptor() ([]byte, []int) { + return fileDescriptor_ad098daeda4239f7, []int{1} +} + +func (m *Task) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_Task.Unmarshal(m, b) +} +func (m *Task) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_Task.Marshal(b, m, deterministic) +} +func (m *Task) XXX_Merge(src proto.Message) { + xxx_messageInfo_Task.Merge(m, src) +} +func (m *Task) XXX_Size() int { + return xxx_messageInfo_Task.Size(m) +} +func (m *Task) XXX_DiscardUnknown() { + xxx_messageInfo_Task.DiscardUnknown(m) +} + +var xxx_messageInfo_Task proto.InternalMessageInfo + +func (m *Task) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *Task) GetContent() string { + if m != nil { + return m.Content + } + return "" +} + +func init() { + proto.RegisterType((*ServerMessage)(nil), "codec.ServerMessage") + proto.RegisterType((*Task)(nil), "codec.Task") +} + +func init() { proto.RegisterFile("server.proto", fileDescriptor_ad098daeda4239f7) } + +var fileDescriptor_ad098daeda4239f7 = []byte{ + // 168 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x34, 0x8e, 0xb1, 0xce, 0x82, 0x30, + 0x14, 0x85, 0x03, 0xff, 0x8f, 0xc6, 0xa2, 0x0e, 0x9d, 0x3a, 0x22, 0x83, 0x61, 0x6a, 0x0d, 0xbe, + 0x81, 0xbb, 0x0b, 0x3a, 0xb9, 0x5d, 0xca, 0x0d, 0x36, 0x44, 0x6e, 0xd3, 0x5e, 0x7d, 0x7e, 0x63, + 0x8d, 0xe3, 0xf7, 0x9d, 0x93, 0x9c, 0x23, 0xd6, 0x11, 0xc3, 0x0b, 0x83, 0xf6, 0x81, 0x98, 0x64, + 0x61, 0x69, 0x40, 0x5b, 0xb7, 0x62, 0x73, 0x49, 0xfa, 0x8c, 0x31, 0xc2, 0x88, 0x72, 0x27, 0x0a, + 0x86, 0x38, 0x45, 0x95, 0x55, 0x7f, 0x4d, 0xd9, 0x96, 0x3a, 0xf5, 0xf4, 0x15, 0xe2, 0xd4, 0x7d, + 0x93, 0xfa, 0x20, 0xfe, 0x3f, 0x28, 0xb7, 0x22, 0x77, 0x83, 0xca, 0xaa, 0xac, 0x59, 0x75, 0xb9, + 0x1b, 0xa4, 0x12, 0x4b, 0x4b, 0x33, 0xe3, 0xcc, 0x2a, 0x4f, 0xf2, 0x87, 0xa7, 0xe6, 0xb6, 0x1f, + 0x1d, 0xdf, 0x9f, 0xbd, 0xb6, 0xf4, 0x30, 0x93, 0x85, 0x10, 0x90, 0x99, 0x8c, 0x87, 0x00, 0x23, + 0xcd, 0x06, 0xbc, 0x33, 0x69, 0xa7, 0x5f, 0xa4, 0x77, 0xc7, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, + 0xee, 0x57, 0xdc, 0xf8, 0xad, 0x00, 0x00, 0x00, +} diff --git a/api/codec/server.proto b/api/codec/server.proto new file mode 100644 index 00000000..9675be55 --- /dev/null +++ b/api/codec/server.proto @@ -0,0 +1,14 @@ +syntax = "proto3"; + +package codec; + +option go_package = "github.com/kcarretto/paragon/api/codec"; + +message ServerMessage { + repeated Task tasks = 1; +} + +message Task { + string id = 1; + string content = 2; +} \ No newline at end of file diff --git a/api/events/events.go b/api/events/events.go new file mode 100644 index 00000000..ac683805 --- /dev/null +++ b/api/events/events.go @@ -0,0 +1,10 @@ +// Package events defines a standardized format for paragon events that are published and may be +// subscribed to. +package events + +// VERSION describes the currently built version of events. +const VERSION = "0.0.1" + +// TODO: Add python support via --python_out=. + +//go:generate protoc -I=../../ -I=. --go_out=paths=source_relative:. task.proto diff --git a/api/events/task.pb.go b/api/events/task.pb.go new file mode 100644 index 00000000..b4ff7060 --- /dev/null +++ b/api/events/task.pb.go @@ -0,0 +1,242 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// source: task.proto + +package events + +import ( + fmt "fmt" + proto "github.com/golang/protobuf/proto" + codec "github.com/kcarretto/paragon/api/codec" + math "math" +) + +// Reference imports to suppress errors if they are not otherwise used. +var _ = proto.Marshal +var _ = fmt.Errorf +var _ = math.Inf + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the proto package it is being compiled against. +// A compilation error at this line likely means your copy of the +// proto package needs to be updated. +const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package + +type TaskQueued struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Content string `protobuf:"bytes,2,opt,name=content,proto3" json:"content,omitempty"` + Filter *codec.AgentMetadata `protobuf:"bytes,3,opt,name=filter,proto3" json:"filter,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TaskQueued) Reset() { *m = TaskQueued{} } +func (m *TaskQueued) String() string { return proto.CompactTextString(m) } +func (*TaskQueued) ProtoMessage() {} +func (*TaskQueued) Descriptor() ([]byte, []int) { + return fileDescriptor_ce5d8dd45b4a91ff, []int{0} +} + +func (m *TaskQueued) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TaskQueued.Unmarshal(m, b) +} +func (m *TaskQueued) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TaskQueued.Marshal(b, m, deterministic) +} +func (m *TaskQueued) XXX_Merge(src proto.Message) { + xxx_messageInfo_TaskQueued.Merge(m, src) +} +func (m *TaskQueued) XXX_Size() int { + return xxx_messageInfo_TaskQueued.Size(m) +} +func (m *TaskQueued) XXX_DiscardUnknown() { + xxx_messageInfo_TaskQueued.DiscardUnknown(m) +} + +var xxx_messageInfo_TaskQueued proto.InternalMessageInfo + +func (m *TaskQueued) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *TaskQueued) GetContent() string { + if m != nil { + return m.Content + } + return "" +} + +func (m *TaskQueued) GetFilter() *codec.AgentMetadata { + if m != nil { + return m.Filter + } + return nil +} + +type TaskClaimed struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Agent *codec.AgentMetadata `protobuf:"bytes,2,opt,name=agent,proto3" json:"agent,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TaskClaimed) Reset() { *m = TaskClaimed{} } +func (m *TaskClaimed) String() string { return proto.CompactTextString(m) } +func (*TaskClaimed) ProtoMessage() {} +func (*TaskClaimed) Descriptor() ([]byte, []int) { + return fileDescriptor_ce5d8dd45b4a91ff, []int{1} +} + +func (m *TaskClaimed) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TaskClaimed.Unmarshal(m, b) +} +func (m *TaskClaimed) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TaskClaimed.Marshal(b, m, deterministic) +} +func (m *TaskClaimed) XXX_Merge(src proto.Message) { + xxx_messageInfo_TaskClaimed.Merge(m, src) +} +func (m *TaskClaimed) XXX_Size() int { + return xxx_messageInfo_TaskClaimed.Size(m) +} +func (m *TaskClaimed) XXX_DiscardUnknown() { + xxx_messageInfo_TaskClaimed.DiscardUnknown(m) +} + +var xxx_messageInfo_TaskClaimed proto.InternalMessageInfo + +func (m *TaskClaimed) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *TaskClaimed) GetAgent() *codec.AgentMetadata { + if m != nil { + return m.Agent + } + return nil +} + +type TaskExecuted struct { + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + Output string `protobuf:"bytes,2,opt,name=output,proto3" json:"output,omitempty"` + Error string `protobuf:"bytes,3,opt,name=error,proto3" json:"error,omitempty"` + ExecStartTime int64 `protobuf:"varint,4,opt,name=execStartTime,proto3" json:"execStartTime,omitempty"` + ExecStopTime int64 `protobuf:"varint,5,opt,name=execStopTime,proto3" json:"execStopTime,omitempty"` + RecvTime int64 `protobuf:"varint,6,opt,name=recvTime,proto3" json:"recvTime,omitempty"` + Agent *codec.AgentMetadata `protobuf:"bytes,7,opt,name=agent,proto3" json:"agent,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TaskExecuted) Reset() { *m = TaskExecuted{} } +func (m *TaskExecuted) String() string { return proto.CompactTextString(m) } +func (*TaskExecuted) ProtoMessage() {} +func (*TaskExecuted) Descriptor() ([]byte, []int) { + return fileDescriptor_ce5d8dd45b4a91ff, []int{2} +} + +func (m *TaskExecuted) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TaskExecuted.Unmarshal(m, b) +} +func (m *TaskExecuted) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TaskExecuted.Marshal(b, m, deterministic) +} +func (m *TaskExecuted) XXX_Merge(src proto.Message) { + xxx_messageInfo_TaskExecuted.Merge(m, src) +} +func (m *TaskExecuted) XXX_Size() int { + return xxx_messageInfo_TaskExecuted.Size(m) +} +func (m *TaskExecuted) XXX_DiscardUnknown() { + xxx_messageInfo_TaskExecuted.DiscardUnknown(m) +} + +var xxx_messageInfo_TaskExecuted proto.InternalMessageInfo + +func (m *TaskExecuted) GetId() string { + if m != nil { + return m.Id + } + return "" +} + +func (m *TaskExecuted) GetOutput() string { + if m != nil { + return m.Output + } + return "" +} + +func (m *TaskExecuted) GetError() string { + if m != nil { + return m.Error + } + return "" +} + +func (m *TaskExecuted) GetExecStartTime() int64 { + if m != nil { + return m.ExecStartTime + } + return 0 +} + +func (m *TaskExecuted) GetExecStopTime() int64 { + if m != nil { + return m.ExecStopTime + } + return 0 +} + +func (m *TaskExecuted) GetRecvTime() int64 { + if m != nil { + return m.RecvTime + } + return 0 +} + +func (m *TaskExecuted) GetAgent() *codec.AgentMetadata { + if m != nil { + return m.Agent + } + return nil +} + +func init() { + proto.RegisterType((*TaskQueued)(nil), "events.TaskQueued") + proto.RegisterType((*TaskClaimed)(nil), "events.TaskClaimed") + proto.RegisterType((*TaskExecuted)(nil), "events.TaskExecuted") +} + +func init() { proto.RegisterFile("task.proto", fileDescriptor_ce5d8dd45b4a91ff) } + +var fileDescriptor_ce5d8dd45b4a91ff = []byte{ + // 296 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x51, 0xc1, 0x4e, 0xc2, 0x40, + 0x14, 0x4c, 0x41, 0x8a, 0x3c, 0xd0, 0xc3, 0x06, 0x4d, 0xc3, 0x89, 0x34, 0x26, 0xa2, 0x31, 0x6d, + 0xa2, 0x5f, 0xa0, 0xc6, 0x83, 0x07, 0x0f, 0x56, 0x4e, 0xde, 0x1e, 0xbb, 0x4f, 0xdc, 0x00, 0xdd, + 0x66, 0xfb, 0x96, 0xf0, 0xb5, 0x7e, 0x8b, 0xe9, 0x2e, 0x68, 0x48, 0x30, 0x1e, 0x67, 0xde, 0x64, + 0x66, 0x5e, 0x06, 0x80, 0xb1, 0x5e, 0x64, 0x95, 0x35, 0x6c, 0x44, 0x4c, 0x6b, 0x2a, 0xb9, 0x1e, + 0x9d, 0x61, 0xa5, 0x73, 0x69, 0x14, 0xc9, 0x1c, 0xe7, 0x54, 0x72, 0x38, 0xa7, 0x0a, 0x60, 0x8a, + 0xf5, 0xe2, 0xd5, 0x91, 0x23, 0x25, 0x4e, 0xa1, 0xa5, 0x55, 0x12, 0x8d, 0xa3, 0x49, 0xaf, 0x68, + 0x69, 0x25, 0x12, 0xe8, 0x4a, 0x53, 0x32, 0x95, 0x9c, 0xb4, 0x3c, 0xb9, 0x83, 0xe2, 0x06, 0xe2, + 0x0f, 0xbd, 0x64, 0xb2, 0x49, 0x7b, 0x1c, 0x4d, 0xfa, 0xb7, 0xc3, 0xcc, 0x7b, 0x67, 0xf7, 0x8d, + 0xf7, 0x0b, 0x31, 0x2a, 0x64, 0x2c, 0xb6, 0x9a, 0xf4, 0x19, 0xfa, 0x4d, 0xca, 0xe3, 0x12, 0xf5, + 0xea, 0x40, 0xcc, 0x35, 0x74, 0x7c, 0x27, 0x1f, 0xf2, 0x97, 0x57, 0x90, 0xa4, 0x5f, 0x11, 0x0c, + 0x1a, 0xaf, 0xa7, 0x0d, 0x49, 0xc7, 0x07, 0xcc, 0xce, 0x21, 0x36, 0x8e, 0x2b, 0xb7, 0xab, 0xbc, + 0x45, 0x62, 0x08, 0x1d, 0xb2, 0xd6, 0x84, 0xc2, 0xbd, 0x22, 0x00, 0x71, 0x01, 0x27, 0xb4, 0x21, + 0xf9, 0xc6, 0x68, 0x79, 0xaa, 0x57, 0x94, 0x1c, 0x8d, 0xa3, 0x49, 0xbb, 0xd8, 0x27, 0x45, 0x0a, + 0x83, 0x40, 0x98, 0xca, 0x8b, 0x3a, 0x5e, 0xb4, 0xc7, 0x89, 0x11, 0x1c, 0x5b, 0x92, 0x6b, 0x7f, + 0x8f, 0xfd, 0xfd, 0x07, 0xff, 0x3e, 0xd8, 0xfd, 0xf7, 0xc1, 0x87, 0xab, 0xf7, 0xcb, 0xb9, 0xe6, + 0x4f, 0x37, 0xcb, 0xa4, 0x59, 0xe5, 0x0b, 0x89, 0xd6, 0x12, 0xb3, 0xc9, 0x2b, 0xb4, 0x38, 0x37, + 0x65, 0xde, 0xec, 0x18, 0x36, 0x9d, 0xc5, 0x7e, 0xc3, 0xbb, 0xef, 0x00, 0x00, 0x00, 0xff, 0xff, + 0x5e, 0x6f, 0x75, 0xbf, 0xf0, 0x01, 0x00, 0x00, +} diff --git a/api/events/task.proto b/api/events/task.proto new file mode 100644 index 00000000..d31bcf4c --- /dev/null +++ b/api/events/task.proto @@ -0,0 +1,38 @@ +syntax = "proto3"; + +package events; + +option go_package = "github.com/kcarretto/paragon/api/events"; + +import "api/codec/agent.proto"; + +// message AgentMetadata { +// string agentID = 1; +// string machineUUID = 2; +// string sessionID = 3; +// string hostname = 4; +// string primaryIP = 5; +// string primaryMAC = 6; +// } + +message TaskQueued { + string id = 1; + string content = 2; + codec.AgentMetadata filter = 3; +} + +message TaskClaimed { + string id = 1; + codec.AgentMetadata agent = 2; +} + +message TaskExecuted { + string id = 1; + string output = 2; + string error = 3; + int64 execStartTime = 4; + int64 execStopTime = 5; + int64 recvTime = 6; + + codec.AgentMetadata agent = 7; +} diff --git a/api/vendor/google/protobuf/timestamp.proto b/api/vendor/google/protobuf/timestamp.proto new file mode 100644 index 00000000..7279266f --- /dev/null +++ b/api/vendor/google/protobuf/timestamp.proto @@ -0,0 +1,138 @@ +// Protocol Buffers - Google's data interchange format +// Copyright 2008 Google Inc. All rights reserved. +// https://developers.google.com/protocol-buffers/ +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following disclaimer +// in the documentation and/or other materials provided with the +// distribution. +// * Neither the name of Google Inc. nor the names of its +// contributors may be used to endorse or promote products derived from +// this software without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +syntax = "proto3"; + +package google.protobuf; + +option csharp_namespace = "Google.Protobuf.WellKnownTypes"; +option cc_enable_arenas = true; +option go_package = "github.com/golang/protobuf/ptypes/timestamp"; +option java_package = "com.google.protobuf"; +option java_outer_classname = "TimestampProto"; +option java_multiple_files = true; +option objc_class_prefix = "GPB"; + +// A Timestamp represents a point in time independent of any time zone or local +// calendar, encoded as a count of seconds and fractions of seconds at +// nanosecond resolution. The count is relative to an epoch at UTC midnight on +// January 1, 1970, in the proleptic Gregorian calendar which extends the +// Gregorian calendar backwards to year one. +// +// All minutes are 60 seconds long. Leap seconds are "smeared" so that no leap +// second table is needed for interpretation, using a [24-hour linear +// smear](https://developers.google.com/time/smear). +// +// The range is from 0001-01-01T00:00:00Z to 9999-12-31T23:59:59.999999999Z. By +// restricting to that range, we ensure that we can convert to and from [RFC +// 3339](https://www.ietf.org/rfc/rfc3339.txt) date strings. +// +// # Examples +// +// Example 1: Compute Timestamp from POSIX `time()`. +// +// Timestamp timestamp; +// timestamp.set_seconds(time(NULL)); +// timestamp.set_nanos(0); +// +// Example 2: Compute Timestamp from POSIX `gettimeofday()`. +// +// struct timeval tv; +// gettimeofday(&tv, NULL); +// +// Timestamp timestamp; +// timestamp.set_seconds(tv.tv_sec); +// timestamp.set_nanos(tv.tv_usec * 1000); +// +// Example 3: Compute Timestamp from Win32 `GetSystemTimeAsFileTime()`. +// +// FILETIME ft; +// GetSystemTimeAsFileTime(&ft); +// UINT64 ticks = (((UINT64)ft.dwHighDateTime) << 32) | ft.dwLowDateTime; +// +// // A Windows tick is 100 nanoseconds. Windows epoch 1601-01-01T00:00:00Z +// // is 11644473600 seconds before Unix epoch 1970-01-01T00:00:00Z. +// Timestamp timestamp; +// timestamp.set_seconds((INT64) ((ticks / 10000000) - 11644473600LL)); +// timestamp.set_nanos((INT32) ((ticks % 10000000) * 100)); +// +// Example 4: Compute Timestamp from Java `System.currentTimeMillis()`. +// +// long millis = System.currentTimeMillis(); +// +// Timestamp timestamp = Timestamp.newBuilder().setSeconds(millis / 1000) +// .setNanos((int) ((millis % 1000) * 1000000)).build(); +// +// +// Example 5: Compute Timestamp from current time in Python. +// +// timestamp = Timestamp() +// timestamp.GetCurrentTime() +// +// # JSON Mapping +// +// In JSON format, the Timestamp type is encoded as a string in the +// [RFC 3339](https://www.ietf.org/rfc/rfc3339.txt) format. That is, the +// format is "{year}-{month}-{day}T{hour}:{min}:{sec}[.{frac_sec}]Z" +// where {year} is always expressed using four digits while {month}, {day}, +// {hour}, {min}, and {sec} are zero-padded to two digits each. The fractional +// seconds, which can go up to 9 digits (i.e. up to 1 nanosecond resolution), +// are optional. The "Z" suffix indicates the timezone ("UTC"); the timezone +// is required. A proto3 JSON serializer should always use UTC (as indicated by +// "Z") when printing the Timestamp type and a proto3 JSON parser should be +// able to accept both UTC and other timezones (as indicated by an offset). +// +// For example, "2017-01-15T01:30:15.01Z" encodes 15.01 seconds past +// 01:30 UTC on January 15, 2017. +// +// In JavaScript, one can convert a Date object to this format using the +// standard +// [toISOString()](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Date/toISOString) +// method. In Python, a standard `datetime.datetime` object can be converted +// to this format using +// [`strftime`](https://docs.python.org/2/library/time.html#time.strftime) with +// the time format spec '%Y-%m-%dT%H:%M:%S.%fZ'. Likewise, in Java, one can use +// the Joda Time's [`ISODateTimeFormat.dateTime()`]( +// http://www.joda.org/joda-time/apidocs/org/joda/time/format/ISODateTimeFormat.html#dateTime%2D%2D +// ) to obtain a formatter capable of generating timestamps in this format. +// +// +message Timestamp { + // Represents seconds of UTC time since Unix epoch + // 1970-01-01T00:00:00Z. Must be from 0001-01-01T00:00:00Z to + // 9999-12-31T23:59:59Z inclusive. + int64 seconds = 1; + + // Non-negative fractions of a second at nanosecond resolution. Negative + // second values with fractions must still have non-negative nanos values + // that count forward in time. Must be from 0 to 999,999,999 + // inclusive. + int32 nanos = 2; +} \ No newline at end of file From a59fa099bc636a12d071441cc8dd8cc27069d295 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Fri, 11 Oct 2019 20:21:55 -0700 Subject: [PATCH 15/19] C2 work --- c2/server.go | 22 +++++-- c2/task.go | 145 ++++++++++++++++++++++++++++++++++++++++++- cmd/c2/main.go | 127 +++++++++++++++++++++++-------------- cmd/c2/pubsub_gcp.go | 40 ++++++++++++ cmd/c2/pubsub_mem.go | 21 +++++++ go.mod | 1 + 6 files changed, 302 insertions(+), 54 deletions(-) create mode 100644 cmd/c2/pubsub_gcp.go create mode 100644 cmd/c2/pubsub_mem.go diff --git a/c2/server.go b/c2/server.go index b51a2516..bffe7eb9 100644 --- a/c2/server.go +++ b/c2/server.go @@ -6,9 +6,12 @@ import ( "fmt" "io" "sync" + "time" + "github.com/kcarretto/paragon/api/events" "github.com/kcarretto/paragon/transport" + "github.com/golang/protobuf/proto" "go.uber.org/zap" "gocloud.dev/pubsub" ) @@ -19,7 +22,7 @@ type Server struct { TaskResults *pubsub.Topic mu sync.RWMutex - tasks []queuedTask + tasks []queueEntry } // HandleMessage received from the agent, and write a reply to the provided writer. @@ -50,17 +53,26 @@ func (srv *Server) publishResults(ctx context.Context, msg transport.Response) { } for _, result := range msg.Results { + event := events.TaskExecuted{ + Id: result.ID, + Output: string(result.Output), + Error: result.Error, + ExecStartTime: result.ExecStartTime.Unix(), + ExecStopTime: result.ExecStopTime.Unix(), + RecvTime: time.Now().Unix(), + } + // TODO: Send all results, or only ones with data? - body, err := json.Marshal(result) + body, err := proto.Marshal(&event) if err != nil { srv.Log.Error("Failed to marshal result to json", zap.Error(err)) } - event := &pubsub.Message{ + msg := &pubsub.Message{ Body: body, - // TODO: Add agent metadata + // TODO: Add c2 metadata } go func() { - if err = srv.TaskResults.Send(ctx, event); err != nil { + if err = srv.TaskResults.Send(ctx, msg); err != nil { srv.Log.Error("Failed to publish task result", zap.Error(err)) } }() diff --git a/c2/task.go b/c2/task.go index ff4a7486..539e9e20 100644 --- a/c2/task.go +++ b/c2/task.go @@ -1,24 +1,55 @@ package c2 import ( + "context" + + "github.com/golang/protobuf/proto" + "github.com/kcarretto/paragon/api/events" "github.com/kcarretto/paragon/transport" + + "go.uber.org/zap" + "gocloud.dev/pubsub" ) -type queuedTask struct { +type queueEntry struct { task transport.Task filter func(transport.Metadata) bool } +// TODO: Convert underlying struct to map + // QueueTask prepares a task to be sent to the first agent that reports metadata matching the filter. func (srv *Server) QueueTask(task transport.Task, filter func(transport.Metadata) bool) { srv.mu.Lock() - srv.tasks = append(srv.tasks, queuedTask{ + srv.tasks = append(srv.tasks, queueEntry{ task, filter, }) srv.mu.Unlock() } +// removeTask removes the first task from the queue with a matching ID. If no matching task is found +// in the queue, removeTask is a no-op. +func (srv *Server) removeTask(id string) { + srv.mu.RLock() + var index = -1 + for i, entry := range srv.tasks { + if entry.task.ID == id { + index = i + break + } + } + srv.mu.RUnlock() + + if index < 0 { + return + } + + // Remove element from the queue + srv.tasks = append(srv.tasks[:index], srv.tasks[index+1:]...) + +} + // GetTasks for an agent based on the metadata it reported and the current tasks in the server queue. func (srv *Server) GetTasks(agent transport.Metadata) []transport.Task { srv.mu.Lock() @@ -46,3 +77,113 @@ func (srv *Server) TaskCount() int { return count } + +// ListenForQueuedTasks receives TaskQueued events and queues the corresponding task. +func (srv *Server) ListenForQueuedTasks(ctx context.Context, subscription *pubsub.Subscription) { + for { + select { + case <-ctx.Done(): + return + default: + msg, err := subscription.Receive(ctx) + if err != nil { + srv.Log.Error("Subscription failed", zap.Error(err)) + return + } + + var event events.TaskQueued + if err = proto.Unmarshal(msg.Body, &event); err != nil { + srv.Log.Error("Failed to unmarshal TaskQueued event", zap.Error(err)) + } + + criteria := transport.Metadata{ + AgentID: event.Filter.GetAgentID(), + MachineUUID: event.Filter.GetMachineUUID(), + SessionID: event.Filter.GetSessionID(), + Hostname: event.Filter.GetHostname(), + PrimaryIP: event.Filter.GetPrimaryIP(), + PrimaryMAC: event.Filter.GetPrimaryMAC(), + } + + task := transport.Task{ + ID: event.GetId(), + Content: []byte(event.GetContent()), + } + + srv.QueueTask(task, claimFilter(criteria)) + } + } +} + +// ListenForClaimedTasks receives TaskClaimed events and dequeues the corresponding task. +func (srv *Server) ListenForClaimedTasks(ctx context.Context, subscription *pubsub.Subscription) { + for { + select { + case <-ctx.Done(): + return + default: + msg, err := subscription.Receive(ctx) + if err != nil { + srv.Log.Error("Subscription failed", zap.Error(err)) + return + } + + var event events.TaskClaimed + if err = proto.Unmarshal(msg.Body, &event); err != nil { + srv.Log.Error("Failed to unmarshal TaskClaimed event", zap.Error(err)) + } + + srv.removeTask(event.GetId()) + } + } +} + +// claimFilter returns a filter that implements a simple algorithm for task claiming. +func claimFilter(criteria transport.Metadata) func(agent transport.Metadata) bool { + return func(agent transport.Metadata) bool { + // True if session ids match. + if id := criteria.SessionID; id != "" { + if id == agent.SessionID { + return true + } + } + + // Restrict if agentID criteria is set. + if agentID := criteria.AgentID; agentID != "" { + if agentID != agent.AgentID { + return false + } + } + + // True if machine uuids match. + if uuid := criteria.MachineUUID; uuid != "" { + if uuid == agent.MachineUUID { + return true + } + } + + // True if MAC address matches + if mac := criteria.PrimaryMAC; mac != "" { + if mac == agent.PrimaryMAC { + return true + } + } + + // True if IP address matches + if addr := criteria.PrimaryIP; addr != "" { + if addr == agent.PrimaryIP { + return true + } + } + + // True if hostname matches + if hostname := criteria.Hostname; hostname != "" { + if hostname == agent.Hostname { + return true + } + } + + // False otherwise + return false + } +} diff --git a/cmd/c2/main.go b/cmd/c2/main.go index b0e191f1..f9888c6f 100644 --- a/cmd/c2/main.go +++ b/cmd/c2/main.go @@ -1,58 +1,91 @@ package main import ( - "encoding/json" - "fmt" - "io/ioutil" - "math/rand" + "context" "net/http" + "os" - "github.com/kcarretto/paragon/transport" + "github.com/kcarretto/paragon/c2" + "go.uber.org/zap" ) func main() { - router := http.NewServeMux() - router.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { - data, err := ioutil.ReadAll(req.Body) - if err != nil { - panic(err) - } - - var agentResponse transport.Response - if err := json.Unmarshal(data, &agentResponse); err != nil { - panic(err) - } - - fmt.Printf("\n\nReceived request: %+v\n", agentResponse) - - var tasks []transport.Task - if rand.Intn(2) == 0 { - task := transport.Task{ - ID: "task1", - Content: []byte(`print("Task from C2 ;)")`), - } - tasks = append(tasks, task) - } - - resp := transport.Payload{ - Tasks: tasks, - } - - respData, err := json.Marshal(resp) - if err != nil { - panic(err) - } - - if _, err = w.Write(respData); err != nil { - panic(err) - } - - fmt.Println("Responded to agent") - - }) - - fmt.Println("Running C2. Serving on 127.0.0.1:8080") - if err := http.ListenAndServe("127.0.0.1:8080", router); err != nil { + ctx := context.Background() + + logger, err := zap.NewDevelopment() + if err != nil { panic(err) } + + httpAddr := os.Getenv("HTTP_ADDR") + if httpAddr == "" { + httpAddr = "127.0.0.1:8080" + } + + resultTopic, err := openTopic(ctx, "tasks.result_received") + if err != nil { + logger.Panic("Failed to open pubsub topic", zap.Error(err)) + } + defer resultTopic.Shutdown(ctx) + + queueTopic, err := openSubscription(ctx, "tasks.queued") + if err != nil { + logger.Panic("Failed to subscribe to pubsub topic", zap.Error(err)) + } + defer queueTopic.Shutdown(ctx) + + srv := &c2.Server{ + Log: logger, + TaskResults: resultTopic, + } + + go func() { + + }() + + logger.Info("Started C2 server", zap.String("http_addr", httpAddr)) + if err := http.ListenAndServe(httpAddr, srv); err != nil { + logger.Panic("Failed to serve HTTP", zap.Error(err)) + } + + // router := http.NewServeMux() + // router.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { + // data, err := ioutil.ReadAll(req.Body) + // if err != nil { + // panic(err) + // } + + // var agentResponse transport.Response + // if err := json.Unmarshal(data, &agentResponse); err != nil { + // panic(err) + // } + + // fmt.Printf("\n\nReceived request: %+v\n", agentResponse) + + // var tasks []transport.Task + // if rand.Intn(2) == 0 { + // task := transport.Task{ + // ID: "task1", + // Content: []byte(`print("Task from C2 ;)")`), + // } + // tasks = append(tasks, task) + // } + + // resp := transport.Payload{ + // Tasks: tasks, + // } + + // respData, err := json.Marshal(resp) + // if err != nil { + // panic(err) + // } + + // if _, err = w.Write(respData); err != nil { + // panic(err) + // } + + // fmt.Println("Responded to agent") + + // }) + } diff --git a/cmd/c2/pubsub_gcp.go b/cmd/c2/pubsub_gcp.go new file mode 100644 index 00000000..ffab2371 --- /dev/null +++ b/cmd/c2/pubsub_gcp.go @@ -0,0 +1,40 @@ +// +build gcp + +package main + +import ( + "context" + "fmt" + "os" + + "gocloud.dev/pubsub" + _ "gocloud.dev/pubsub/gcppubsub" +) + +func getURI(topic string) (string, error) { + project := os.Getenv("GCP_PROJECT") + if project == "" { + return "", fmt.Errorf("must set GCP_PROJECT environment variable to use GCP pubsub") + } + + uri := fmt.Sprintf("gcppubsub://projects/%s/topics/%s", project, topic) + return uri, nil +} + +func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { + uri, err := getURI(topic) + if err != nil { + return nil, err + } + + return pubsub.OpenTopic(ctx, uri) +} + +func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { + uri, err := getURI(topic) + if err != nil { + return nil, err + } + + return pubsub.OpenSubscription(ctx, uri) +} diff --git a/cmd/c2/pubsub_mem.go b/cmd/c2/pubsub_mem.go new file mode 100644 index 00000000..652c964a --- /dev/null +++ b/cmd/c2/pubsub_mem.go @@ -0,0 +1,21 @@ +// +build !gcp + +package main + +import ( + "context" + "fmt" + + "gocloud.dev/pubsub" + _ "gocloud.dev/pubsub/mempubsub" +) + +func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { + uri := fmt.Sprintf("mem://%s", topic) + return pubsub.OpenTopic(ctx, uri) +} + +func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { + uri := fmt.Sprintf("mem://%s", topic) + return pubsub.OpenSubscription(ctx, uri) +} diff --git a/go.mod b/go.mod index 49815bda..691b3870 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect github.com/go-ole/go-ole v1.2.4 // indirect github.com/golang/mock v1.3.1 + github.com/golang/protobuf v1.3.2 github.com/pkg/errors v0.8.1 github.com/pkg/profile v1.3.0 github.com/shirou/gopsutil v2.18.12+incompatible From b5c5b95317519534e806e24c8871cca91c22ec28 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Fri, 11 Oct 2019 21:21:59 -0700 Subject: [PATCH 16/19] Working on further simplification --- agent/agent.go | 67 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 agent/agent.go diff --git a/agent/agent.go b/agent/agent.go new file mode 100644 index 00000000..260e6463 --- /dev/null +++ b/agent/agent.go @@ -0,0 +1,67 @@ +package agent + +import ( + "context" + "fmt" + + "github.com/kcarretto/paragon/api/codec" +) + +type AgentMessage codec.AgentMessage +type ServerMessage codec.ServerMessage + +var ErrNoTransports = fmt.Errorf("all available transports failed to send message") + +type ServerMessageWriter interface{} +type AgentMessageWriter interface{} + +type Sender interface { + Send(ServerMessageWriter, AgentMessage) error +} + +type Receiver interface { + Receive(AgentMessageWriter, ServerMessage) +} + +type Agent struct { + Sender + Receiver + + Transports []Sender +} + +func (agent Agent) Send(w ServerMessageWriter, msg AgentMessage) error { + if agent.Sender != nil { + return agent.Sender.Send(w, msg) + } + + for _, transport := range agent.Transports { + if err := transport.Send(w, msg); err != nil { + // TODO: Error Handler + continue + } + + return nil + } + + return ErrNoTransports +} + +func (agent Agent) Run(ctx context.Context) error { + + var agentMsg AgentMessage + + for { + select { + case <-ctx.Done(): + return ctx.Err() + default: + var srvMsg ServerMessage + if err := agent.Send(&srvMsg, agentMsg); err != nil { + return err + } + + agent.Receive(&agentMsg, srvMsg) + } + } +} From ab2380ca94fef93f29a038c6ac4dc79779df91a1 Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Tue, 15 Oct 2019 19:54:23 -0700 Subject: [PATCH 17/19] Added message writers & tests --- agent/agent.gen_test.go | 83 +++++++++++++++++++++++++++++ agent/agent.go | 54 +++++++++++-------- agent/agent_test.go | 115 ++++++++++++++++++++++++++++++++++++++++ agent/errors.go | 6 +++ agent/io.gen_test.go | 100 ++++++++++++++++++++++++++++++++++ agent/message.go | 51 ++++++++++++++++++ agent/message_test.go | 45 ++++++++++++++++ agent/mocks_test.go | 4 ++ agent/transport.go | 36 +++++++++++++ go.mod | 2 +- 10 files changed, 474 insertions(+), 22 deletions(-) create mode 100644 agent/agent.gen_test.go create mode 100644 agent/agent_test.go create mode 100644 agent/errors.go create mode 100644 agent/io.gen_test.go create mode 100644 agent/message.go create mode 100644 agent/message_test.go create mode 100644 agent/mocks_test.go create mode 100644 agent/transport.go diff --git a/agent/agent.gen_test.go b/agent/agent.gen_test.go new file mode 100644 index 00000000..61f74096 --- /dev/null +++ b/agent/agent.gen_test.go @@ -0,0 +1,83 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/kcarretto/paragon/agent (interfaces: Sender,Receiver) + +// Package agent_test is a generated GoMock package. +package agent_test + +import ( + gomock "github.com/golang/mock/gomock" + agent "github.com/kcarretto/paragon/agent" + reflect "reflect" +) + +// MockSender is a mock of Sender interface +type MockSender struct { + ctrl *gomock.Controller + recorder *MockSenderMockRecorder +} + +// MockSenderMockRecorder is the mock recorder for MockSender +type MockSenderMockRecorder struct { + mock *MockSender +} + +// NewMockSender creates a new mock instance +func NewMockSender(ctrl *gomock.Controller) *MockSender { + mock := &MockSender{ctrl: ctrl} + mock.recorder = &MockSenderMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockSender) EXPECT() *MockSenderMockRecorder { + return m.recorder +} + +// Send mocks base method +func (m *MockSender) Send(arg0 agent.ServerMessageWriter, arg1 agent.Message) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Send", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// Send indicates an expected call of Send +func (mr *MockSenderMockRecorder) Send(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Send", reflect.TypeOf((*MockSender)(nil).Send), arg0, arg1) +} + +// MockReceiver is a mock of Receiver interface +type MockReceiver struct { + ctrl *gomock.Controller + recorder *MockReceiverMockRecorder +} + +// MockReceiverMockRecorder is the mock recorder for MockReceiver +type MockReceiverMockRecorder struct { + mock *MockReceiver +} + +// NewMockReceiver creates a new mock instance +func NewMockReceiver(ctrl *gomock.Controller) *MockReceiver { + mock := &MockReceiver{ctrl: ctrl} + mock.recorder = &MockReceiverMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockReceiver) EXPECT() *MockReceiverMockRecorder { + return m.recorder +} + +// Receive mocks base method +func (m *MockReceiver) Receive(arg0 agent.MessageWriter, arg1 agent.ServerMessage) { + m.ctrl.T.Helper() + m.ctrl.Call(m, "Receive", arg0, arg1) +} + +// Receive indicates an expected call of Receive +func (mr *MockReceiverMockRecorder) Receive(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Receive", reflect.TypeOf((*MockReceiver)(nil).Receive), arg0, arg1) +} diff --git a/agent/agent.go b/agent/agent.go index 260e6463..f3824251 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -2,61 +2,73 @@ package agent import ( "context" - "fmt" + "time" - "github.com/kcarretto/paragon/api/codec" + "go.uber.org/zap" ) -type AgentMessage codec.AgentMessage -type ServerMessage codec.ServerMessage - -var ErrNoTransports = fmt.Errorf("all available transports failed to send message") - -type ServerMessageWriter interface{} -type AgentMessageWriter interface{} - +// A Sender is responsible for transporting messages to a server. type Sender interface { - Send(ServerMessageWriter, AgentMessage) error + Send(ServerMessageWriter, Message) error } +// A Receiver is responsible for handling messages sent by a server. type Receiver interface { - Receive(AgentMessageWriter, ServerMessage) + Receive(MessageWriter, ServerMessage) } +// An Agent communicates with server(s) using the configured transport. type Agent struct { - Sender Receiver + Log *zap.Logger - Transports []Sender + Transports []Transport + + MaxIdleTime time.Duration + lastSend time.Time } -func (agent Agent) Send(w ServerMessageWriter, msg AgentMessage) error { - if agent.Sender != nil { - return agent.Sender.Send(w, msg) +// Send messages to a server using the configured transports. Returns ErrNoTransports if all fail or +// if none are configured. +func (agent Agent) Send(w ServerMessageWriter, msg Message) error { + // Don't send empty messages unless it has been at least MaxIdleTime since the last send. + if msg.IsEmpty() && time.Since(agent.lastSend) <= agent.MaxIdleTime { + return nil } + // Attempt to send using available transports. for _, transport := range agent.Transports { if err := transport.Send(w, msg); err != nil { - // TODO: Error Handler + transport.Log.Error( + "Failed to send message using transport", + zap.Error(err), + zap.Reflect("transport", transport), + ) continue } + // When send is successful, update the timestamp + agent.lastSend = time.Now() + + // Sleep the transport's interval on success + transport.Sleep() + return nil } return ErrNoTransports } +// Run the agent, sending agent messages to a server using configured transports. func (agent Agent) Run(ctx context.Context) error { - - var agentMsg AgentMessage - for { select { case <-ctx.Done(): return ctx.Err() default: + var agentMsg Message var srvMsg ServerMessage + if err := agent.Send(&srvMsg, agentMsg); err != nil { return err } diff --git a/agent/agent_test.go b/agent/agent_test.go new file mode 100644 index 00000000..bb1d8be8 --- /dev/null +++ b/agent/agent_test.go @@ -0,0 +1,115 @@ +package agent_test + +import ( + "context" + "errors" + "testing" + "time" + + "github.com/kcarretto/paragon/agent" + + gomock "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + "go.uber.org/zap" +) + +func TestSend(t *testing.T) { + logger, err := zap.NewDevelopment() + require.NoError(t, err) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + badSender := NewMockSender(ctrl) + sender := NewMockSender(ctrl) + unusedSender := NewMockSender(ctrl) + + srvMsg := agent.ServerMessage{} + agentMsg := agent.Message{} + badSender.EXPECT().Send(&srvMsg, agentMsg).Return(errors.New("oops ^_^")) + sender.EXPECT().Send(&srvMsg, agentMsg).Return(nil) + + testAgent := &agent.Agent{ + Log: logger, + Transports: []agent.Transport{ + agent.Transport{ + Sender: badSender, + Log: logger.Named("transport.bad"), + Name: "Bad Sender", + }, + agent.Transport{ + Sender: sender, + Log: logger.Named("transport.good"), + Name: "Good Sender", + }, + agent.Transport{ + Sender: unusedSender, + Log: logger.Named("transport.should_not_see"), + Name: "Unusued Sender", + }, + }, + } + err = testAgent.Send(&srvMsg, agentMsg) + require.NoError(t, err) +} + +func TestAgentRun(t *testing.T) { + logger, err := zap.NewDevelopment() + require.NoError(t, err) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + sender := NewMockSender(ctrl) + recv := NewMockReceiver(ctrl) + + sender.EXPECT().Send(gomock.Not(gomock.Nil()), agent.Message{}).Return(nil).AnyTimes() + recv.EXPECT().Receive(gomock.Not(gomock.Nil()), agent.ServerMessage{}).AnyTimes() + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + testAgent := &agent.Agent{ + Log: logger, + Receiver: recv, + Transports: []agent.Transport{ + agent.Transport{ + Sender: sender, + Log: logger.Named("transport"), + Name: "Test Sender", + }, + }, + } + err = testAgent.Run(ctx) + require.True(t, errors.Is(err, context.DeadlineExceeded), "Unexpected error: %v", err) +} + +func TestRunErrNoTransport(t *testing.T) { + logger, err := zap.NewDevelopment() + require.NoError(t, err) + + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + srvMsg := agent.ServerMessage{} + agentMsg := agent.Message{} + badSender := NewMockSender(ctrl) + badSender.EXPECT().Send(&srvMsg, agentMsg).Return(errors.New("oops ^_^")) + + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Millisecond) + defer cancel() + + testAgent := &agent.Agent{ + Log: logger, + Transports: []agent.Transport{ + agent.Transport{ + Sender: badSender, + Log: logger.Named("transport.bad"), + Name: "Bad Sender", + }, + }, + } + + err = testAgent.Run(ctx) + require.True(t, errors.Is(err, agent.ErrNoTransports), "Unexpected error: %v", err) +} diff --git a/agent/errors.go b/agent/errors.go new file mode 100644 index 00000000..10e28d1e --- /dev/null +++ b/agent/errors.go @@ -0,0 +1,6 @@ +package agent + +import "fmt" + +// ErrNoTransports occurs if all transports fail to send a message or if none are available. +var ErrNoTransports = fmt.Errorf("all available transports failed to send message") diff --git a/agent/io.gen_test.go b/agent/io.gen_test.go new file mode 100644 index 00000000..77c3ee65 --- /dev/null +++ b/agent/io.gen_test.go @@ -0,0 +1,100 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: io (interfaces: Writer,WriteCloser) + +// Package agent_test is a generated GoMock package. +package agent_test + +import ( + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockWriter is a mock of Writer interface +type MockWriter struct { + ctrl *gomock.Controller + recorder *MockWriterMockRecorder +} + +// MockWriterMockRecorder is the mock recorder for MockWriter +type MockWriterMockRecorder struct { + mock *MockWriter +} + +// NewMockWriter creates a new mock instance +func NewMockWriter(ctrl *gomock.Controller) *MockWriter { + mock := &MockWriter{ctrl: ctrl} + mock.recorder = &MockWriterMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWriter) EXPECT() *MockWriterMockRecorder { + return m.recorder +} + +// Write mocks base method +func (m *MockWriter) Write(arg0 []byte) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Write indicates an expected call of Write +func (mr *MockWriterMockRecorder) Write(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriter)(nil).Write), arg0) +} + +// MockWriteCloser is a mock of WriteCloser interface +type MockWriteCloser struct { + ctrl *gomock.Controller + recorder *MockWriteCloserMockRecorder +} + +// MockWriteCloserMockRecorder is the mock recorder for MockWriteCloser +type MockWriteCloserMockRecorder struct { + mock *MockWriteCloser +} + +// NewMockWriteCloser creates a new mock instance +func NewMockWriteCloser(ctrl *gomock.Controller) *MockWriteCloser { + mock := &MockWriteCloser{ctrl: ctrl} + mock.recorder = &MockWriteCloserMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockWriteCloser) EXPECT() *MockWriteCloserMockRecorder { + return m.recorder +} + +// Close mocks base method +func (m *MockWriteCloser) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close +func (mr *MockWriteCloserMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWriteCloser)(nil).Close)) +} + +// Write mocks base method +func (m *MockWriteCloser) Write(arg0 []byte) (int, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Write", arg0) + ret0, _ := ret[0].(int) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Write indicates an expected call of Write +func (mr *MockWriteCloserMockRecorder) Write(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriteCloser)(nil).Write), arg0) +} diff --git a/agent/message.go b/agent/message.go new file mode 100644 index 00000000..f716e6d3 --- /dev/null +++ b/agent/message.go @@ -0,0 +1,51 @@ +package agent + +import ( + "io" + + "github.com/kcarretto/paragon/api/codec" +) + +// MessageWriter is responsible for writing output to be collected as a message from the agent. +type MessageWriter interface { + io.Writer + io.StringWriter + WriteResult(*codec.Result) +} + +// ServerMessageWriter is responsible for writing output to be collected as a message from the server. +type ServerMessageWriter interface { + WriteServerMessage(*ServerMessage) +} + +// Message is an alias for codec.AgentMessage with some extended functionality. +type Message codec.AgentMessage + +// Write log output to be included in a message to a server. +func (msg *Message) Write(output []byte) (int, error) { + return msg.WriteString(string(output)) +} + +// WriteString writes log output to be included in a message to a server. +func (msg *Message) WriteString(output string) (int, error) { + msg.Logs = append(msg.Logs, output) + return len(output), nil +} + +// WriteResult writes execution output to be included in a message to a server. +func (msg *Message) WriteResult(result *codec.Result) { + msg.Results = append(msg.Results, result) +} + +// IsEmpty checks if the message has no significant contents +func (msg Message) IsEmpty() bool { + return len(msg.Results) <= 0 && len(msg.Logs) <= 0 +} + +// ServerMessage is an alias for codec.ServerMessage with some extended functionality. +type ServerMessage codec.ServerMessage + +// WriteServerMessage replaces this message with the provided message. +func (msg *ServerMessage) WriteServerMessage(srvMsg *ServerMessage) { + msg.Tasks = append(msg.Tasks, srvMsg.Tasks...) +} diff --git a/agent/message_test.go b/agent/message_test.go new file mode 100644 index 00000000..f2c90b4d --- /dev/null +++ b/agent/message_test.go @@ -0,0 +1,45 @@ +package agent_test + +import ( + "testing" + + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" + "github.com/stretchr/testify/require" +) + +func TestMessageWrite(t *testing.T) { + expected := "test message" + + msg := &agent.Message{} + require.True(t, msg.IsEmpty()) + + n, err := msg.Write([]byte(expected)) + require.NoError(t, err) + require.Equal(t, len(expected), n) + require.Equal(t, 1, len(msg.Logs)) + require.Equal(t, expected, msg.Logs[0]) +} + +func TestMessageWriteResult(t *testing.T) { + expected := &codec.Result{} + + msg := &agent.Message{} + msg.WriteResult(expected) + require.Equal(t, 1, len(msg.Results)) + require.Equal(t, expected, msg.Results[0]) +} + +func TestServerMessageWrite(t *testing.T) { + expectedTask := &codec.Task{} + expected := &agent.ServerMessage{} + + msg := &agent.ServerMessage{ + Tasks: []*codec.Task{expectedTask}, + } + expected.WriteServerMessage(msg) + + require.NotNil(t, expected) + require.Equal(t, 1, len(expected.Tasks)) + require.Equal(t, expectedTask, expected.Tasks[0]) +} diff --git a/agent/mocks_test.go b/agent/mocks_test.go new file mode 100644 index 00000000..0ef5bd2c --- /dev/null +++ b/agent/mocks_test.go @@ -0,0 +1,4 @@ +package agent_test + +//go:generate mockgen -destination=io.gen_test.go -package=agent_test io Writer,WriteCloser +//go:generate mockgen -destination=agent.gen_test.go -package=agent_test github.com/kcarretto/paragon/agent Sender,Receiver diff --git a/agent/transport.go b/agent/transport.go new file mode 100644 index 00000000..7d2e9773 --- /dev/null +++ b/agent/transport.go @@ -0,0 +1,36 @@ +package agent + +import ( + "math/rand" + "time" + + "go.uber.org/zap" +) + +// A Transport wraps an implementation of Sender with metadata used for operation. +type Transport struct { + Sender + Log *zap.Logger + + Name string + Interval time.Duration + Jitter time.Duration +} + +// Sleep for the transport's configured interval & jitter duration. If Jitter is non-zero, a random +// duration <= Jitter is added to the transports interval. +func (t Transport) Sleep() { + // Random duration added to sleep delay [0, Jitter) + var jitter time.Duration + if max := t.Jitter.Nanoseconds(); max > 0 { + jitter = time.Duration(rand.Int63n(max)) + } + + delay := t.Interval + jitter + + // Ratelimit log messages + if delay > 1*time.Second { + t.Log.Debug("Sleeping for transport interval + jitter", zap.Duration("delay", delay)) + } + time.Sleep(delay) +} diff --git a/go.mod b/go.mod index 691b3870..78a403d3 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/kcarretto/paragon -go 1.12 +go 1.13 require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect From 9d106a4a888ed1a4f2e4a093affc1937c503779e Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Wed, 16 Oct 2019 17:49:16 -0700 Subject: [PATCH 18/19] Updated to fix debug http server --- Dockerfile | 3 +- agent/agent.go | 5 +- agent/agent_test.go | 5 +- {transport => agent}/debug/auth.go | 0 {transport => agent}/debug/html.go | 7 +- {transport => agent}/debug/http.go | 71 +++++++------ agent/debug/sender.go | 75 +++++++++++++ {transport => agent}/http/http.go | 0 api/codec/codec.go | 35 +++++++ cmd/agent/debug.go | 23 ++-- cmd/agent/dev.go | 26 +++-- cmd/agent/main.go | 19 +++- cmd/agent/prod.go | 5 +- cmd/agent/receiver.go | 48 +++++++++ cmd/agent/run.go | 148 +++++++++++++------------- transport/buffer.go | 162 ----------------------------- transport/buffer_test.go | 127 ---------------------- transport/codec.go | 108 ------------------- transport/debug/transport.go | 76 -------------- transport/errors.go | 6 -- transport/mocks/codec.gen.go | 87 ---------------- transport/mocks/io.gen.go | 100 ------------------ transport/mocks/mocks.go | 4 - transport/receiver.go | 50 --------- transport/receiver_test.go | 35 ------- transport/registry.go | 113 -------------------- transport/registry_test.go | 52 --------- transport/sender.go | 33 ------ transport/transport.go | 54 ---------- transport/transport_test.go | 30 ------ 30 files changed, 326 insertions(+), 1181 deletions(-) rename {transport => agent}/debug/auth.go (100%) rename {transport => agent}/debug/html.go (96%) rename {transport => agent}/debug/http.go (58%) create mode 100644 agent/debug/sender.go rename {transport => agent}/http/http.go (100%) create mode 100644 cmd/agent/receiver.go delete mode 100644 transport/buffer.go delete mode 100644 transport/buffer_test.go delete mode 100644 transport/codec.go delete mode 100644 transport/debug/transport.go delete mode 100644 transport/errors.go delete mode 100644 transport/mocks/codec.gen.go delete mode 100644 transport/mocks/io.gen.go delete mode 100644 transport/mocks/mocks.go delete mode 100644 transport/receiver.go delete mode 100644 transport/receiver_test.go delete mode 100644 transport/registry.go delete mode 100644 transport/registry_test.go delete mode 100644 transport/sender.go delete mode 100644 transport/transport.go delete mode 100644 transport/transport_test.go diff --git a/Dockerfile b/Dockerfile index a300817e..d8aba7c4 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,8 @@ RUN apk add alpine-sdk git \ # Debug Build FROM base as build COPY ./cmd /app/cmd -COPY ./transport /app/transport +COPY ./api /app/api +COPY ./agent /app/agent COPY ./script /app/script RUN go build -tags=debug -o ./build/agent ./cmd/agent diff --git a/agent/agent.go b/agent/agent.go index f3824251..43bf24c1 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -61,18 +61,21 @@ func (agent Agent) Send(w ServerMessageWriter, msg Message) error { // Run the agent, sending agent messages to a server using configured transports. func (agent Agent) Run(ctx context.Context) error { + agent.Log.Debug("Starting agent execution") + var agentMsg Message + for { select { case <-ctx.Done(): return ctx.Err() default: - var agentMsg Message var srvMsg ServerMessage if err := agent.Send(&srvMsg, agentMsg); err != nil { return err } + agentMsg = Message{} agent.Receive(&agentMsg, srvMsg) } } diff --git a/agent/agent_test.go b/agent/agent_test.go index bb1d8be8..9757d8d5 100644 --- a/agent/agent_test.go +++ b/agent/agent_test.go @@ -70,8 +70,9 @@ func TestAgentRun(t *testing.T) { defer cancel() testAgent := &agent.Agent{ - Log: logger, - Receiver: recv, + Log: logger, + Receiver: recv, + MaxIdleTime: time.Second * 1, Transports: []agent.Transport{ agent.Transport{ Sender: sender, diff --git a/transport/debug/auth.go b/agent/debug/auth.go similarity index 100% rename from transport/debug/auth.go rename to agent/debug/auth.go diff --git a/transport/debug/html.go b/agent/debug/html.go similarity index 96% rename from transport/debug/html.go rename to agent/debug/html.go index d0e3fc25..1136f5b0 100644 --- a/transport/debug/html.go +++ b/agent/debug/html.go @@ -32,7 +32,7 @@ const debugHTML = ` "showMethod": "fadeIn", "hideMethod": "fadeOut" } - + function addToResults(result) { $('#resultsList').prepend( $('
  • ').append( @@ -45,7 +45,7 @@ const debugHTML = ` } function queueScript(){ let script = { - "content": btoa($("#scriptTextArea").val()) + "content": $("#scriptTextArea").val() } $.ajax({ url: '/queue', @@ -70,7 +70,8 @@ const debugHTML = ` if (response.results != null){ response.results.map(function(result){ if (result.output != null){ - addToResults(atob(result.output)); + output = result.output.join("\n") + addToResults(output); } else if (result.error != "") { addToResults("error: "+result.error) } else { diff --git a/transport/debug/http.go b/agent/debug/http.go similarity index 58% rename from transport/debug/http.go rename to agent/debug/http.go index ecc396bc..9ee17643 100644 --- a/transport/debug/http.go +++ b/agent/debug/http.go @@ -8,7 +8,8 @@ import ( "os" "strconv" - "github.com/kcarretto/paragon/transport" + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" "go.uber.org/zap" ) @@ -20,46 +21,39 @@ func use(h http.HandlerFunc, middleware ...func(http.HandlerFunc) http.HandlerFu return h } -func (t *Transport) handleIndex(w http.ResponseWriter, req *http.Request) { +func (transport *Sender) handleIndex(w http.ResponseWriter, req *http.Request) { w.Write([]byte(debugHTML)) return } -func (t *Transport) handleQueue(w http.ResponseWriter, req *http.Request) { +// handleQueue handles an http request to queue a task. +func (transport *Sender) handleQueue(w http.ResponseWriter, req *http.Request) { + // Read request data data, err := ioutil.ReadAll(req.Body) if err != nil { - t.Logger.Error("Failed to read request body", zap.Error(err)) + transport.Log.Error("Failed to read request body", zap.Error(err)) msg := fmt.Sprintf("failed to read request body: %s", err.Error()) http.Error(w, msg, http.StatusBadRequest) return } + transport.Log.Debug("Request to queue task", zap.String("task", string(data))) - t.Logger.Debug("Request to queue task", zap.String("task", string(data))) - var task transport.Task + // Unmarshal request into task + var task codec.Task if err := json.Unmarshal(data, &task); err != nil { - t.Logger.Error("Failed to parse request body", zap.Error(err)) + transport.Log.Error("Failed to parse request body", zap.Error(err)) msg := fmt.Sprintf("failed to parse request body to json: %s", err.Error()) http.Error(w, msg, http.StatusBadRequest) return } - serverMsg, err := json.Marshal( - transport.Payload{ - Tasks: []transport.Task{task}, - }, - ) - if err != nil { - t.Logger.Error("Failed to marshal server message", zap.Error(err)) - msg := fmt.Sprintf("failed to marshal server message: %s", err.Error()) - http.Error(w, msg, http.StatusInternalServerError) - return - } - - t.WritePayload(serverMsg) + // Queue the task + transport.QueueTask(&task) } -func (t *Transport) handleResponses(w http.ResponseWriter, req *http.Request) { +// handleMessages handles an http request to view agent messages +func (transport *Sender) handleMessages(w http.ResponseWriter, req *http.Request) { params := req.URL.Query() offsetStr := params.Get("offset") @@ -90,17 +84,17 @@ func (t *Transport) handleResponses(w http.ResponseWriter, req *http.Request) { limit = 1 } - messages := []transport.Response{} - for i := offset; i < len(t.messages) && i <= offset+limit; i++ { - messages = append(messages, t.messages[i]) + messages := []agent.Message{} + for i := offset; i < len(transport.messages) && i <= offset+limit; i++ { + messages = append(messages, transport.messages[i]) } - t.Logger.Debug( + transport.Log.Debug( "Retrieving responses", zap.Int("offset", offset), zap.Int("limit", limit), zap.Int("sent_messages", len(messages)), - zap.Int("total_messages", len(t.messages)), + zap.Int("total_messages", len(transport.messages)), ) respData, err := json.Marshal(messages) @@ -111,14 +105,13 @@ func (t *Transport) handleResponses(w http.ResponseWriter, req *http.Request) { w.Header().Set("Content-Type", "application/json") if _, err = w.Write(respData); err != nil { - t.Logger.Error("Failed to write response data to client", zap.Error(err)) + transport.Log.Error("Failed to write response data to client", zap.Error(err)) http.Error(w, fmt.Sprintf("failed to write response data to client: %s", err.Error()), http.StatusInternalServerError) } - } // listenAndServe configures and runs an http server for agent debugging. -func (t *Transport) listenAndServe() { +func (transport *Sender) listenAndServe() { router := http.NewServeMux() var middleware []func(http.HandlerFunc) http.HandlerFunc @@ -132,16 +125,22 @@ func (t *Transport) listenAndServe() { middleware = append(middleware, prepareBasicAuth(username, password)) } - router.HandleFunc("/", use(t.handleIndex, middleware...)) - router.HandleFunc("/queue", use(t.handleQueue, middleware...)) - router.HandleFunc("/messages", use(t.handleResponses, middleware...)) - http.Handle("/", router) - httpAddr := os.Getenv("DEBUG_API_ADDR") if httpAddr == "" { httpAddr = "127.0.0.1:8080" } - t.Logger.Info("HTTP DEBUG ENABLED", zap.String("http_addr", httpAddr)) - http.ListenAndServe(httpAddr, router) + router.HandleFunc("/", use(transport.handleIndex, middleware...)) + router.HandleFunc("/queue", use(transport.handleQueue, middleware...)) + router.HandleFunc("/messages", use(transport.handleMessages, middleware...)) + transport.srv = &http.Server{ + Addr: httpAddr, + Handler: router, + } + + transport.Log.Info("HTTP DEBUG ENABLED", zap.String("http_addr", httpAddr)) + if err := transport.srv.ListenAndServe(); err != nil { + transport.Log.Error("HTTP Debug Server encountered an error while closing", zap.Error(err)) + } + transport.Log.Info("HTTP Debug Server Closed") } diff --git a/agent/debug/sender.go b/agent/debug/sender.go new file mode 100644 index 00000000..d528f0cc --- /dev/null +++ b/agent/debug/sender.go @@ -0,0 +1,75 @@ +package debug + +import ( + "context" + "net/http" + "sync" + + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" + "go.uber.org/zap" +) + +// Sender is for debugging purposes, it manages a local http server to interact with an agent. +type Sender struct { + Log *zap.Logger + + srv *http.Server + active bool + wg sync.WaitGroup + messages []agent.Message + tasks []*codec.Task +} + +// Send appends an agent message to the result array, and writes any queued tasks to the provided +// writer. +func (transport *Sender) Send(w agent.ServerMessageWriter, msg agent.Message) error { + transport.ensureActive() + + transport.messages = append(transport.messages, msg) + + tasks := make([]*codec.Task, len(transport.tasks)) + copy(tasks, transport.tasks) + transport.tasks = transport.tasks[:0] + + w.WriteServerMessage(&agent.ServerMessage{ + Tasks: tasks, + }) + + return nil +} + +// QueueTask buffers a task for execution. +func (transport *Sender) QueueTask(task *codec.Task) { + transport.tasks = append(transport.tasks, task) +} + +// Close stops the goroutine responsible for running the debug http server if it is active. +func (transport *Sender) Close() (err error) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() + + if transport.srv != nil { + err = transport.srv.Shutdown(ctx) + } + + transport.wg.Wait() + transport.active = false + + return err +} + +// ensureActive starts a goroutine to consume stdin if the transport is not yet active. +func (transport *Sender) ensureActive() { + if transport.active { + return + } + + transport.wg.Add(1) + go func() { + defer transport.wg.Done() + transport.listenAndServe() + }() + + transport.active = true +} diff --git a/transport/http/http.go b/agent/http/http.go similarity index 100% rename from transport/http/http.go rename to agent/http/http.go diff --git a/api/codec/codec.go b/api/codec/codec.go index 5871186b..af6467f6 100644 --- a/api/codec/codec.go +++ b/api/codec/codec.go @@ -4,9 +4,44 @@ // both the agent and the server utilize the same format. package codec +import ( + "time" + + timestamp "github.com/golang/protobuf/ptypes/timestamp" +) + // VERSION describes the currently built version of codec. const VERSION = "0.0.1" // TODO: Add python support via --python_out=. //go:generate protoc -I=../vendor/ -I=../../ -I=. --go_out=paths=source_relative:. agent.proto server.proto + +// Start recording results for task execution. +func (m *Result) Start() { + start := time.Now() + + m.ExecStartTime = ×tamp.Timestamp{ + Seconds: start.Unix(), + Nanos: int32(start.Nanosecond()), + } +} + +// Write appends task execution output to the result. +func (m *Result) Write(p []byte) (int, error) { + m.Output = append(m.Output, string(p)) + return len(p), nil +} + +// CloseWithError marks the end of task execution, and provides any error the task resulted in. +func (m *Result) CloseWithError(err error) { + stop := time.Now() + m.ExecStopTime = ×tamp.Timestamp{ + Seconds: stop.Unix(), + Nanos: int32(stop.Nanosecond()), + } + if err != nil { + m.Error = err.Error() + } + +} diff --git a/cmd/agent/debug.go b/cmd/agent/debug.go index ee46c4fb..8111cc07 100644 --- a/cmd/agent/debug.go +++ b/cmd/agent/debug.go @@ -4,9 +4,10 @@ package main import ( "io" + "time" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/debug" + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/agent/debug" "go.uber.org/zap" ) @@ -30,12 +31,16 @@ func getLogger() *zap.Logger { func configureLogger(logger *zap.Logger, buf io.Writer) {} -func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { - registry.Add(transport.New( - "debug", - &debug.Transport{ - Logger: logger, - PayloadWriter: receiver, +func getTransports(logger *zap.Logger) []agent.Transport { + return []agent.Transport{ + agent.Transport{ + Name: "Debug", + Log: logger.Named("debug"), + Interval: 5 * time.Second, + Jitter: 1 * time.Second, + Sender: &debug.Sender{ + Log: logger.Named("debug"), + }, }, - )) + } } diff --git a/cmd/agent/dev.go b/cmd/agent/dev.go index 395b412f..795300f4 100644 --- a/cmd/agent/dev.go +++ b/cmd/agent/dev.go @@ -5,8 +5,8 @@ package main import ( "io" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/http" + "github.com/kcarretto/paragon/agent" + // "github.com/kcarretto/paragon/agent/http" "go.uber.org/zap" ) @@ -30,13 +30,17 @@ func getLogger() *zap.Logger { func configureLogger(logger *zap.Logger, buf io.Writer) {} -func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { - registry.Add(transport.New( - "http", - http.Transport{ - PayloadWriter: receiver, - Logger: logger.Named("http"), - URL: "http://127.0.0.1:8080", - }, - )) +func getTransports(logger *zap.Logger) (transports []agent.Transport) { + // registry.Add() + return } +// func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { +// registry.Add(transport.New( +// "http", +// http.Transport{ +// PayloadWriter: receiver, +// Logger: logger.Named("http"), +// URL: "http://127.0.0.1:8080", +// }, +// )) +// } diff --git a/cmd/agent/main.go b/cmd/agent/main.go index aa4f91e8..13176305 100644 --- a/cmd/agent/main.go +++ b/cmd/agent/main.go @@ -6,8 +6,8 @@ import ( "os/signal" "sync" "syscall" - "time" + "github.com/kcarretto/paragon/agent" "github.com/pkg/profile" "go.uber.org/zap" ) @@ -21,6 +21,16 @@ func runLoop() { // Initialize logger logger := getLogger() + // Initialize Agent + paragon := &agent.Agent{ + Receiver: Receiver{ + ctx, + logger.Named("agent.exec"), + }, + Log: logger.Named("agent"), + Transports: getTransports(logger.Named("agent.transport")), + } + // Handle panic defer func() { if err := recover(); err != nil { @@ -31,10 +41,10 @@ func runLoop() { // Run agent var wg sync.WaitGroup wg.Add(1) - go func(logger *zap.Logger) { + go func() { defer wg.Done() - run(ctx, logger) - }(logger.With(zap.Time("agent_loop_start", time.Now()))) + paragon.Run(ctx) + }() // Listen for interupts sigint := make(chan os.Signal, 1) @@ -59,7 +69,6 @@ func runLoop() { // After interupt, wait for threads to finish cancel() wg.Wait() - } func main() { diff --git a/cmd/agent/prod.go b/cmd/agent/prod.go index a20334e1..67ff2bcf 100644 --- a/cmd/agent/prod.go +++ b/cmd/agent/prod.go @@ -6,7 +6,7 @@ import ( "fmt" "io" - "github.com/kcarretto/paragon/transport" + "github.com/kcarretto/paragon/agent" "go.uber.org/zap" ) @@ -31,6 +31,7 @@ func getLogger() *zap.Logger { func configureLogger(logger *zap.Logger, buf io.Writer) {} -func addTransports(logger *zap.Logger, receiver transport.PayloadWriter, registry *transport.Registry) { +func getTransports(logger *zap.Logger) (transports []agent.Transport) { // registry.Add() + return } diff --git a/cmd/agent/receiver.go b/cmd/agent/receiver.go new file mode 100644 index 00000000..aa389af2 --- /dev/null +++ b/cmd/agent/receiver.go @@ -0,0 +1,48 @@ +package main + +import ( + "bytes" + "context" + + "github.com/kcarretto/paragon/agent" + "github.com/kcarretto/paragon/api/codec" + "github.com/kcarretto/paragon/script" + "github.com/kcarretto/paragon/script/stdlib" + "go.uber.org/zap" +) + +type Receiver struct { + context.Context + + Log *zap.Logger +} + +func (r Receiver) Receive(w agent.MessageWriter, msg agent.ServerMessage) { + r.Log.Debug("Received new payload from server", + zap.Int("num_tasks", len(msg.Tasks)), + zap.Reflect("payload", msg), + ) + + for _, task := range msg.Tasks { + result := &codec.Result{ + Id: task.GetId(), + } + result.Start() + code := script.New( + task.GetId(), + bytes.NewBufferString(task.Content), + script.WithOutput(result), + stdlib.Load(), + ) // TODO: Add libraries, set output + + err := code.Exec(r) + if err != nil { + r.Log.Error("failed to execute script", zap.Error(err), zap.String("task_id", task.GetId())) + } + + r.Log.Debug("completed script execution", zap.String("task", task.String())) + + result.CloseWithError(err) + w.WriteResult(result) + } +} diff --git a/cmd/agent/run.go b/cmd/agent/run.go index 63e855eb..7758ac85 100644 --- a/cmd/agent/run.go +++ b/cmd/agent/run.go @@ -1,89 +1,89 @@ package main -import ( - "bytes" - "context" - "fmt" - "time" +// import ( +// "bytes" +// "context" +// "fmt" +// "time" - "github.com/kcarretto/paragon/script" - "github.com/kcarretto/paragon/script/stdlib" - "github.com/kcarretto/paragon/transport" - "go.uber.org/zap" -) +// "github.com/kcarretto/paragon/script" +// "github.com/kcarretto/paragon/script/stdlib" +// "github.com/kcarretto/paragon/agent" +// "go.uber.org/zap" +// ) -func run(ctx context.Context, logger *zap.Logger) { - // Initialize buffer - buffer := &transport.Buffer{ - Encoder: transport.NewDefaultEncoder(), - Metadata: transport.Metadata{}, - MaxIdleTime: time.Second * 5, - } +// func run(ctx context.Context, logger *zap.Logger) { +// // Initialize buffer +// buffer := &transport.Buffer{ +// Encoder: transport.NewDefaultEncoder(), +// Metadata: transport.Metadata{}, +// MaxIdleTime: time.Second * 5, +// } - // Configure the logger - configureLogger(logger, buffer) +// // Configure the logger +// configureLogger(logger, buffer) - // Initialize registry - registry := &transport.Registry{} +// // Initialize registry +// registry := &transport.Registry{} - // Handle payloads from the server - hLogger := logger.Named("scripts") - handler := func(w transport.ResultWriter, payload transport.Payload, err error) { - if err != nil { - hLogger.Error("Payload decode error", zap.Error(err)) - return - } +// // Handle payloads from the server +// hLogger := logger.Named("scripts") +// handler := func(w transport.ResultWriter, payload transport.Payload, err error) { +// if err != nil { +// hLogger.Error("Payload decode error", zap.Error(err)) +// return +// } - hLogger.Debug("Executing new payload from server", - zap.Int("num_tasks", len(payload.Tasks)), - zap.Reflect("payload", payload), - ) +// hLogger.Debug("Executing new payload from server", +// zap.Int("num_tasks", len(payload.Tasks)), +// zap.Reflect("payload", payload), +// ) - for _, task := range payload.Tasks { - output := transport.NewResult(task) - code := script.New( - task.ID, - bytes.NewBuffer(task.Content), - script.WithOutput(output), - stdlib.Load(), - ) // TODO: Add libraries, set output - fmt.Println(code.Libraries) - err = code.Exec(ctx) - if err != nil { - hLogger.Error("failed to execute script", zap.Error(err), zap.String("task_id", task.ID)) - } +// for _, task := range payload.Tasks { +// output := transport.NewResult(task) +// code := script.New( +// task.ID, +// bytes.NewBuffer(task.Content), +// script.WithOutput(output), +// stdlib.Load(), +// ) // TODO: Add libraries, set output +// fmt.Println(code.Libraries) +// err = code.Exec(ctx) +// if err != nil { +// hLogger.Error("failed to execute script", zap.Error(err), zap.String("task_id", task.ID)) +// } - hLogger.Debug("completed script execution", zap.String("task_id", task.ID)) +// hLogger.Debug("completed script execution", zap.String("task_id", task.ID)) - output.CloseWithError(err) - w.WriteResult(output) - } - } +// output.CloseWithError(err) +// w.WriteResult(output) +// } +// } - // Initialize Receiver - receiver := &transport.Receiver{ - ResultWriter: buffer, - Decoder: transport.NewDefaultDecoder(), - Handler: transport.PayloadHandlerFn(handler), - } +// // Initialize Receiver +// receiver := &transport.Receiver{ +// ResultWriter: buffer, +// Decoder: transport.NewDefaultDecoder(), +// Handler: transport.PayloadHandlerFn(handler), +// } - // Register Transports - addTransports(logger.Named("transport"), receiver, registry) +// // Register Transports +// addTransports(logger.Named("transport"), receiver, registry) - // Initialize MultiSender - sender := transport.MultiSender{ - Transports: registry, - OnError: func(t transport.Transport, err error) { - logger.Named("transport").Named(t.Name).Error("Failed to transport data", zap.Error(err)) - }, - } +// // Initialize MultiSender +// sender := transport.MultiSender{ +// Transports: registry, +// OnError: func(t transport.Transport, err error) { +// logger.Named("transport").Named(t.Name).Error("Failed to transport data", zap.Error(err)) +// }, +// } - // Flush buffer using multi sender - for { - if _, err := sender.Send(buffer); err != nil { - // TODO: Handle encode error - // TODO: Handle ErrNoTransports - } - time.Sleep(time.Millisecond * 50) - } -} +// // Flush buffer using multi sender +// for { +// if _, err := sender.Send(buffer); err != nil { +// // TODO: Handle encode error +// // TODO: Handle ErrNoTransports +// } +// time.Sleep(time.Millisecond * 50) +// } +// } diff --git a/transport/buffer.go b/transport/buffer.go deleted file mode 100644 index eaaadd2e..00000000 --- a/transport/buffer.go +++ /dev/null @@ -1,162 +0,0 @@ -package transport - -import ( - "bytes" - "fmt" - "io" - "sync" - "time" -) - -// A TimestampWriterTo extends io.WriterTo with a method that returns the timestamp of when the -// contents were last successfully written. -type TimestampWriterTo interface { - io.WriterTo - Timestamp() time.Time -} - -// Buffer is a write safe io.Writer & io.WriterTo that is used to buffer output and safely copy it -// to a transport writer. If copying fails, buffer does not lose data. -type Buffer struct { - Encoder - - Metadata Metadata - MaxIdleTime time.Duration - - mu sync.RWMutex - output *bytes.Buffer - results []*Result - timestamp time.Time -} - -// WriteResult writes structured task execution output to the response. -func (b *Buffer) WriteResult(result *Result) { - if result == nil { - return - } - - b.mu.Lock() - b.results = append(b.results, result) - b.mu.Unlock() -} - -// Write buffers the provided data until it is consumed by a transport. It is safe for -// concurrent use. -func (b *Buffer) Write(data []byte) (int, error) { - b.mu.Lock() - defer b.mu.Unlock() - - if b.output == nil { - b.output = bytes.NewBuffer(data) - return len(data), nil - } - - return b.output.Write(data) -} - -// WriteTo implements io.WriterTo, allowing Buffer to be used by io.Copy. It will write it's -// contents to the provided writer. It will preserve any data in the case of error. -func (b *Buffer) WriteTo(w io.Writer) (int64, error) { - output := b.copyOutput() - results := b.copyResults() - - if len(output) <= 0 && len(results) <= 0 && time.Since(b.Timestamp()) <= b.MaxIdleTime { - return 0, nil - } - - resp := Response{ - Metadata: b.Metadata, - Log: output, - Results: results, - } - data, err := b.Encode(resp) - if err != nil { - b.restore(results, output) - return 0, fmt.Errorf("failed to encode response data: %w", err) - } - - n, err := w.Write(data) - if err != nil { - b.restore(results, output) - return 0, fmt.Errorf("failed to transport response data: %w", err) - } - - // Update timestamp - b.timestamp = time.Now() - - return int64(n), nil -} - -// Sync is a nop to implement zapcore.WriteSyncer -func (b *Buffer) Sync() error { - return nil -} - -// Timestamp returns the time that the buffer was last successfully written to a transport. -func (b *Buffer) Timestamp() time.Time { - b.mu.RLock() - t := b.timestamp - b.mu.RUnlock() - return t -} - -// copyResults returns a copy of the current result buffer and truncates it. -func (b *Buffer) copyResults() (results []*Result) { - b.mu.Lock() - defer b.mu.Unlock() - - if len(b.results) <= 0 { - return - } - - results = make([]*Result, len(b.results)) - copy(results, b.results) - b.results = b.results[len(b.results):] - - return -} - -// copyOutput returns a copy of the current output buffer and truncates it. -func (b *Buffer) copyOutput() (data []byte) { - b.mu.Lock() - defer b.mu.Unlock() - - if b.output == nil { - b.output = &bytes.Buffer{} - } - - if b.output.Len() <= 0 { - return - } - - data = make([]byte, b.output.Len()) - copy(data, b.output.Bytes()) - b.output.Reset() - - return -} - -// restore the provided state to the buffer. -func (b *Buffer) restore(results []*Result, data []byte) error { - // Restore results buffer - for _, result := range results { - b.WriteResult(result) - } - - // Restore output buffer - size := len(data) - n, err := b.Write(data) - if err == nil && n != size { - err = fmt.Errorf("failed to restore full output buffer") - } - - return err -} - -// NewBuffer Initializes and returns a new transport buffer. -func NewBuffer(data []byte) *Buffer { - return &Buffer{ - Encoder: NewDefaultEncoder(), - output: bytes.NewBuffer(data), - } -} diff --git a/transport/buffer_test.go b/transport/buffer_test.go deleted file mode 100644 index 83e8cf74..00000000 --- a/transport/buffer_test.go +++ /dev/null @@ -1,127 +0,0 @@ -package transport_test - -import ( - "testing" - - "github.com/golang/mock/gomock" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/mocks" - "github.com/pkg/errors" - "github.com/stretchr/testify/require" -) - -func TestBufferSync(t *testing.T) { - buffer := transport.Buffer{} - require.NoError(t, buffer.Sync()) -} - -func TestBufferWrite(t *testing.T) { - expected := []byte("Some test data") - - buffer := transport.Buffer{} - - n, err := buffer.Write(expected) - require.NoError(t, err) - require.Equal(t, len(expected), n) -} - -func TestBufferWriteTo(t *testing.T) { - // Prepare test data - expectedOutput := []byte("Some test output data") - expectedResult := transport.NewResult(transport.Task{ - ID: "SomeTestTask", - }) - _, err := expectedResult.Write([]byte("some test result data")) - require.NoError(t, err) - - response := transport.Response{ - transport.Metadata{}, - []*transport.Result{ - expectedResult, - }, - expectedOutput, - } - encoder := transport.NewDefaultEncoder() - expected, err := encoder.Encode(response) - require.NoError(t, err) - - // Prepare mock - ctrl := gomock.NewController(t) - defer ctrl.Finish() - dst := mocks.NewMockWriter(ctrl) - dst.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { - require.Equal(t, string(expected), string(p)) - return len(p), nil - }) - - // Initialize buffer and write expected data - buffer := transport.NewBuffer(expectedOutput) - - // Write result to buffer - buffer.WriteResult(expectedResult) - - // Write buffer to mock writer - num, err := buffer.WriteTo(dst) - require.NoError(t, err) - require.Equal(t, int64(len(expected)), num) - - // Ensure buffer timestamp was updated - require.NotZero(t, buffer.Timestamp(), "Buffer should update timestamp after successful write") -} - -func TestBufferWriteToError(t *testing.T) { - // Prepare test data - expectedOutput := []byte("Some test output data") - expectedResult := transport.NewResult(transport.Task{ - ID: "SomeTestTask", - }) - _, err := expectedResult.Write([]byte("some test result data")) - require.NoError(t, err) - - response := transport.Response{ - transport.Metadata{}, - []*transport.Result{ - expectedResult, - }, - expectedOutput, - } - encoder := transport.NewDefaultEncoder() - expected, err := encoder.Encode(response) - require.NoError(t, err) - - // Prepare mock - ctrl := gomock.NewController(t) - defer ctrl.Finish() - dst := mocks.NewMockWriter(ctrl) - gomock.InOrder( - dst.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { - require.Equal(t, string(expected), string(p)) - return 4, errors.New("uh oh") - }), - dst.EXPECT().Write(gomock.Any()).DoAndReturn(func(p []byte) (int, error) { - require.Equal(t, string(expected), string(p)) - return len(p), nil - }), - ) - - // Initialize buffer and write expected data - buffer := transport.NewBuffer(expectedOutput) - - // Write result to buffer - buffer.WriteResult(expectedResult) - - // Write buffer to mock writer (will error with "uh oh") - _, err = buffer.WriteTo(dst) - require.Error(t, err) - - // Ensure buffer timestamp was not updated - require.Zero(t, buffer.Timestamp(), "Buffer should not update timestamp after failed write") - - // Write buffer to mock writer again, ensure no data was lost - num, err := buffer.WriteTo(dst) - require.NoError(t, err) - require.Equal(t, int64(len(expected)), num) - - // Ensure buffer timestamp was updated - require.NotZero(t, buffer.Timestamp(), "Buffer should update timestamp after successful write") -} diff --git a/transport/codec.go b/transport/codec.go deleted file mode 100644 index 7216bbd1..00000000 --- a/transport/codec.go +++ /dev/null @@ -1,108 +0,0 @@ -package transport - -import ( - "encoding/json" - "time" -) - -// An Encoder is responsible for marshaling a response to bytes. -type Encoder interface { - Encode(Response) ([]byte, error) -} - -// EncoderFn is a function that implements Encoder. -type EncoderFn func(Response) ([]byte, error) - -// Encode wraps fn to provide an implementation of Encoder. -func (fn EncoderFn) Encode(resp Response) ([]byte, error) { - return fn(resp) -} - -// NewDefaultEncoder returns a JSON Encoder. -func NewDefaultEncoder() Encoder { - return EncoderFn(func(resp Response) ([]byte, error) { - return json.Marshal(resp) - }) -} - -// A Decoder is responsible for unmarshaling received data into a payload struct. -type Decoder interface { - Decode([]byte) (Payload, error) -} - -// DecoderFn is a function that implements Decoder. -type DecoderFn func([]byte) (Payload, error) - -// Decode wraps fn to provide an implementation of Decoder. -func (fn DecoderFn) Decode(data []byte) (Payload, error) { - return fn(data) -} - -// NewDefaultDecoder returns a JSON Decoder. -func NewDefaultDecoder() Decoder { - return DecoderFn(func(data []byte) (payload Payload, err error) { - err = json.Unmarshal(data, &payload) - return - }) -} - -// Payload holds structured information received from a server. -type Payload struct { - Tasks []Task -} - -// Response holds structured information that will be transported to a server. -type Response struct { - Metadata Metadata - Results []*Result `json:"results"` - Log []byte `json:"log"` -} - -// Metadata holds agent metadata useful for identifying an agent. -type Metadata struct{} - -// Task stores instructions to execute and metadata. -type Task struct { - ID string `json:"id"` - Content []byte `json:"content"` -} - -// Result stores task execution output and metadata. -type Result struct { - ID string `json:"id"` - Output []byte `json:"output"` - Error string `json:"error"` - - ExecStartTime time.Time `json:"exec_start_time"` - ExecStopTime time.Time `json:"exec_stop_time"` -} - -// Write implements io.Writer by appending output to the current result buffer. It is safe for -// concurrent use and always returns len(p), nil. -func (r *Result) Write(p []byte) (int, error) { - r.Output = append(r.Output, p...) - - return len(p), nil -} - -// Close implements io.Closer by wrapping CloseWithError to indicate no error. Always returns nil. -func (r *Result) Close() error { - r.CloseWithError(nil) - return nil -} - -// CloseWithError sets the result error and execution stop time. -func (r *Result) CloseWithError(err error) { - if err != nil { - r.Error = err.Error() - } - r.ExecStopTime = time.Now() -} - -// NewResult initializes and returns a new Result to store execution output for the provided task. -func NewResult(task Task) *Result { - return &Result{ - ID: task.ID, - ExecStartTime: time.Now(), - } -} diff --git a/transport/debug/transport.go b/transport/debug/transport.go deleted file mode 100644 index c8a8ba11..00000000 --- a/transport/debug/transport.go +++ /dev/null @@ -1,76 +0,0 @@ -package debug - -import ( - "encoding/json" - "io" - "os" - "sync" - - "github.com/kcarretto/paragon/transport" - "go.uber.org/zap" -) - -// Transport is for debugging purposes, it manages a local http server to interact with an agent. -type Transport struct { - io.Reader - transport.PayloadWriter - Logger *zap.Logger - - wg sync.WaitGroup - input chan []byte - active bool - messages []transport.Response -} - -// Write response data to an in memory buffer that is queried by the debug http server. -func (t *Transport) Write(data []byte) (int, error) { - t.ensureActive() - - var resp transport.Response - if err := json.Unmarshal(data, &resp); err != nil { - t.Logger.DPanic("Invalid response JSON written", zap.String("data", string(data)), zap.Error(err)) - } - - t.messages = append(t.messages, resp) - return len(data), nil -} - -// Close stops the goroutine responsible for running the debug http server if it is active. -func (t *Transport) Close() error { - if t.input != nil { - close(t.input) - } - t.wg.Wait() - - t.active = true - - return nil -} - -// ensureActive starts a goroutine to consume stdin if the transport is not yet active. -func (t *Transport) ensureActive() { - if t.active { - return - } - - t.wg.Add(1) - go func() { - defer t.wg.Done() - t.listenAndServe() - }() - - t.active = true -} - -// New initializes and returns a local transport, which must be closed. -func New(receiver transport.PayloadWriter) *Transport { - - input := make(chan []byte, 1) - t := Transport{ - Reader: os.Stdin, - PayloadWriter: receiver, - input: input, - } - - return &t -} diff --git a/transport/errors.go b/transport/errors.go deleted file mode 100644 index 2688784f..00000000 --- a/transport/errors.go +++ /dev/null @@ -1,6 +0,0 @@ -package transport - -import "errors" - -// ErrNoTransportAvailable occurs when all available transports fail to report output, or if no transports were configured. -var ErrNoTransportAvailable = errors.New("no transport available") diff --git a/transport/mocks/codec.gen.go b/transport/mocks/codec.gen.go deleted file mode 100644 index 7952fd51..00000000 --- a/transport/mocks/codec.gen.go +++ /dev/null @@ -1,87 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: github.com/kcarretto/paragon/transport (interfaces: Encoder,Decoder) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - gomock "github.com/golang/mock/gomock" - transport "github.com/kcarretto/paragon/transport" - reflect "reflect" -) - -// MockEncoder is a mock of Encoder interface -type MockEncoder struct { - ctrl *gomock.Controller - recorder *MockEncoderMockRecorder -} - -// MockEncoderMockRecorder is the mock recorder for MockEncoder -type MockEncoderMockRecorder struct { - mock *MockEncoder -} - -// NewMockEncoder creates a new mock instance -func NewMockEncoder(ctrl *gomock.Controller) *MockEncoder { - mock := &MockEncoder{ctrl: ctrl} - mock.recorder = &MockEncoderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockEncoder) EXPECT() *MockEncoderMockRecorder { - return m.recorder -} - -// Encode mocks base method -func (m *MockEncoder) Encode(arg0 transport.Response) ([]byte, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Encode", arg0) - ret0, _ := ret[0].([]byte) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Encode indicates an expected call of Encode -func (mr *MockEncoderMockRecorder) Encode(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Encode", reflect.TypeOf((*MockEncoder)(nil).Encode), arg0) -} - -// MockDecoder is a mock of Decoder interface -type MockDecoder struct { - ctrl *gomock.Controller - recorder *MockDecoderMockRecorder -} - -// MockDecoderMockRecorder is the mock recorder for MockDecoder -type MockDecoderMockRecorder struct { - mock *MockDecoder -} - -// NewMockDecoder creates a new mock instance -func NewMockDecoder(ctrl *gomock.Controller) *MockDecoder { - mock := &MockDecoder{ctrl: ctrl} - mock.recorder = &MockDecoderMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockDecoder) EXPECT() *MockDecoderMockRecorder { - return m.recorder -} - -// Decode mocks base method -func (m *MockDecoder) Decode(arg0 []byte) (transport.Payload, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Decode", arg0) - ret0, _ := ret[0].(transport.Payload) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Decode indicates an expected call of Decode -func (mr *MockDecoderMockRecorder) Decode(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Decode", reflect.TypeOf((*MockDecoder)(nil).Decode), arg0) -} diff --git a/transport/mocks/io.gen.go b/transport/mocks/io.gen.go deleted file mode 100644 index 3e1e67b0..00000000 --- a/transport/mocks/io.gen.go +++ /dev/null @@ -1,100 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: io (interfaces: Writer,WriteCloser) - -// Package mocks is a generated GoMock package. -package mocks - -import ( - gomock "github.com/golang/mock/gomock" - reflect "reflect" -) - -// MockWriter is a mock of Writer interface -type MockWriter struct { - ctrl *gomock.Controller - recorder *MockWriterMockRecorder -} - -// MockWriterMockRecorder is the mock recorder for MockWriter -type MockWriterMockRecorder struct { - mock *MockWriter -} - -// NewMockWriter creates a new mock instance -func NewMockWriter(ctrl *gomock.Controller) *MockWriter { - mock := &MockWriter{ctrl: ctrl} - mock.recorder = &MockWriterMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockWriter) EXPECT() *MockWriterMockRecorder { - return m.recorder -} - -// Write mocks base method -func (m *MockWriter) Write(arg0 []byte) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Write", arg0) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Write indicates an expected call of Write -func (mr *MockWriterMockRecorder) Write(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriter)(nil).Write), arg0) -} - -// MockWriteCloser is a mock of WriteCloser interface -type MockWriteCloser struct { - ctrl *gomock.Controller - recorder *MockWriteCloserMockRecorder -} - -// MockWriteCloserMockRecorder is the mock recorder for MockWriteCloser -type MockWriteCloserMockRecorder struct { - mock *MockWriteCloser -} - -// NewMockWriteCloser creates a new mock instance -func NewMockWriteCloser(ctrl *gomock.Controller) *MockWriteCloser { - mock := &MockWriteCloser{ctrl: ctrl} - mock.recorder = &MockWriteCloserMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use -func (m *MockWriteCloser) EXPECT() *MockWriteCloserMockRecorder { - return m.recorder -} - -// Close mocks base method -func (m *MockWriteCloser) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close -func (mr *MockWriteCloserMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockWriteCloser)(nil).Close)) -} - -// Write mocks base method -func (m *MockWriteCloser) Write(arg0 []byte) (int, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Write", arg0) - ret0, _ := ret[0].(int) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Write indicates an expected call of Write -func (mr *MockWriteCloserMockRecorder) Write(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Write", reflect.TypeOf((*MockWriteCloser)(nil).Write), arg0) -} diff --git a/transport/mocks/mocks.go b/transport/mocks/mocks.go deleted file mode 100644 index c6a6ef94..00000000 --- a/transport/mocks/mocks.go +++ /dev/null @@ -1,4 +0,0 @@ -package mocks - -//go:generate mockgen -destination=io.gen.go -package=mocks io Writer,WriteCloser -//go:generate mockgen -destination=codec.gen.go -package=mocks github.com/kcarretto/paragon/transport Encoder,Decoder diff --git a/transport/receiver.go b/transport/receiver.go deleted file mode 100644 index 2bf192ec..00000000 --- a/transport/receiver.go +++ /dev/null @@ -1,50 +0,0 @@ -package transport - -import ( - "io" - "sync" -) - -// ResultWriter buffers result output that will be transported to the server. -type ResultWriter interface { - io.Writer - WriteResult(*Result) -} - -// A PayloadWriter sends payloads from the server to a consumer. -type PayloadWriter interface { - WritePayload([]byte) - // WriteError(error) -} - -// A PayloadHandler is responsible for consuming and acting upon payloads received from the server. -type PayloadHandler interface { - HandlePayload(ResultWriter, Payload, error) -} - -// PayloadHandlerFn is a function that implements PayloadHandler. -type PayloadHandlerFn func(ResultWriter, Payload, error) - -// HandlePayload wraps fn to provide an implementation of PayloadHandler. -func (fn PayloadHandlerFn) HandlePayload(w ResultWriter, data Payload, err error) { - fn(w, data, err) -} - -// A Receiver consumes payloads from the server. -type Receiver struct { - mu sync.Mutex - - ResultWriter - Decoder - Handler func(ResultWriter, Payload, error) -} - -// WritePayload implements PayloadWriter, passing data to the handler for consumption. It is safe -// for concurrent use by multiple transports. -func (recv *Receiver) WritePayload(data []byte) { - recv.mu.Lock() - defer recv.mu.Unlock() - - payload, err := recv.Decode(data) - recv.Handler(recv.ResultWriter, payload, err) -} diff --git a/transport/receiver_test.go b/transport/receiver_test.go deleted file mode 100644 index 3ff6570c..00000000 --- a/transport/receiver_test.go +++ /dev/null @@ -1,35 +0,0 @@ -package transport_test - -import ( - "testing" - - "github.com/golang/mock/gomock" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/mocks" - "github.com/stretchr/testify/require" -) - -func TestReceiver(t *testing.T) { - expectedData := []byte("here is a whole bunch of payload data") - expectedPayload := transport.Payload{} - - // Prepare mock controller - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // Prepare mocks - decoder := mocks.NewMockDecoder(ctrl) - decoder.EXPECT().Decode(expectedData).Return(expectedPayload, nil) - - recv := transport.Receiver{ - Decoder: transport.DecoderFn(func(data []byte) (transport.Payload, error) { - return decoder.Decode(data) - }), - Handler: transport.PayloadHandlerFn(func(w transport.ResultWriter, payload transport.Payload, err error) { - require.Equal(t, expectedPayload, payload) - require.NoError(t, err) - }), - } - - recv.WritePayload(expectedData) -} diff --git a/transport/registry.go b/transport/registry.go deleted file mode 100644 index 29fe3517..00000000 --- a/transport/registry.go +++ /dev/null @@ -1,113 +0,0 @@ -package transport - -import ( - "errors" - "io" - "sort" - "sync" -) - -// A Registry tracks transports and associated metadata. -type Registry struct { - SortBy func(t1, t2 Transport) bool - - mu sync.RWMutex - transports map[string]Transport - cache []Transport -} - -func (reg *Registry) set(transport Transport) { - reg.mu.Lock() - if reg.transports == nil { - reg.transports = map[string]Transport{} - } - reg.transports[transport.Name] = transport - reg.cache = nil - reg.mu.Unlock() -} - -// Add a transport to the registry. If a transport with the same name is already registered, this -// method is a no-op. Use Update() to modify existing transports. -func (reg *Registry) Add(transport Transport) { - if _, ok := reg.transports[transport.Name]; ok { - return - } - - reg.set(transport) -} - -// Get returns a registered transport with the given name. -func (reg *Registry) Get(name string) (Transport, error) { - reg.mu.RLock() - transport, ok := reg.transports[name] - reg.mu.RUnlock() - - if !ok { - return Transport{}, errors.New("transport not registered") - } - - return transport, nil -} - -// Update the transport with the provided name by applying options to it. -func (reg *Registry) Update(name string, options ...Option) error { - transport, err := reg.Get(name) - if err != nil { - return err - } - - for _, opt := range options { - opt(&transport) - } - - reg.set(transport) - - return nil -} - -// List transports, sorted by the provided SortBy method or by priority if no method was provided. -func (reg *Registry) List() []Transport { - if reg.cache != nil { - return reg.cache - } - - if reg.SortBy == nil { - reg.SortBy = func(t1, t2 Transport) bool { - return t1.Priority < t2.Priority - } - } - - transports := make([]Transport, 0, len(reg.transports)) - for _, transport := range reg.transports { - transports = append(transports, transport) - } - - sort.Slice(transports, func(i, j int) bool { - return reg.SortBy(transports[i], transports[j]) - }) - - reg.cache = transports - - return reg.cache -} - -// Close a transport by name. Close is a no-op if the transport is not an io.Closer. -func (reg *Registry) Close(name string) error { - transport, err := reg.Get(name) - if err != nil { - return err - } - - if closer, ok := transport.Writer.(io.Closer); ok { - return closer.Close() - } - - return nil -} - -// CloseAll attempts to close all transports. -func (reg *Registry) CloseAll() { - for name := range reg.transports { - reg.Close(name) - } -} diff --git a/transport/registry_test.go b/transport/registry_test.go deleted file mode 100644 index c56771a4..00000000 --- a/transport/registry_test.go +++ /dev/null @@ -1,52 +0,0 @@ -package transport_test - -import ( - "testing" - "time" - - "github.com/golang/mock/gomock" - "github.com/kcarretto/paragon/transport" - "github.com/kcarretto/paragon/transport/mocks" - "github.com/stretchr/testify/require" -) - -func TestRegistry(t *testing.T) { - // Prepare mock controller - ctrl := gomock.NewController(t) - defer ctrl.Finish() - - // Prepare mocks - writer1 := mocks.NewMockWriteCloser(ctrl) - writer2 := mocks.NewMockWriteCloser(ctrl) - gomock.InOrder( - writer1.EXPECT().Close(), - ) - - // Add transports to the registry - reg := transport.Registry{} - reg.Add(transport.New("RegTest1", writer1, transport.SetInterval(time.Second*100))) - reg.Add(transport.New("RegTest2", writer2)) - - // Get a transport - tp1, err := reg.Get("RegTest1") - require.NoError(t, err) - require.Equal(t, time.Second*100, tp1.Interval) - - // Update the transport - err = reg.Update("RegTest2", transport.SetInterval(time.Second*23), transport.SetPriority(-15)) - require.NoError(t, err) - tp2, err := reg.Get("RegTest2") - require.NoError(t, err) - require.Equal(t, time.Second*23, tp2.Interval) - require.Equal(t, -15, tp2.Priority) - - // List transports - lst := reg.List() - require.Equal(t, 2, len(lst)) - require.Equal(t, "RegTest2", lst[0].Name, "Transport list in unexpected order") - require.Equal(t, "RegTest1", lst[1].Name, "Transport list in unexpected order") - - // Close the first transport - err = reg.Close(tp1.Name) - require.NoError(t, err) -} diff --git a/transport/sender.go b/transport/sender.go deleted file mode 100644 index a0b3de03..00000000 --- a/transport/sender.go +++ /dev/null @@ -1,33 +0,0 @@ -package transport - -import ( - "io" - "time" -) - -// MultiSender is used to send a response buffer to the server using transports from the registry. -type MultiSender struct { - Transports *Registry - OnError func(Transport, error) -} - -// Send the response buffer to the server using available transports. If a transport fails to send -// the respond, OnError will be invoked if it is set. Transports will be attempted in the order -// defined by the registry sorting mechanism. If all transports are exhausted, ErrNoTransportAvailable -// will be returned. -func (sender MultiSender) Send(buffer TimestampWriterTo) (int64, error) { - for _, transport := range sender.Transports.List() { - delay := transport.Interval - time.Since(buffer.Timestamp()) - time.Sleep(delay) - - n, err := buffer.WriteTo(transport) - if err != nil && err != io.EOF && sender.OnError != nil { - sender.OnError(transport, err) - continue - } - - return n, nil - } - - return 0, ErrNoTransportAvailable -} diff --git a/transport/transport.go b/transport/transport.go deleted file mode 100644 index d1997d75..00000000 --- a/transport/transport.go +++ /dev/null @@ -1,54 +0,0 @@ -package transport - -import ( - "io" - "time" -) - -// An Option enables additional configuration of a transport. -type Option func(*Transport) - -// Transport wraps an underlying io.Writer with metadata. -type Transport struct { - io.Writer - - Name string - Priority int - Interval time.Duration - Jitter time.Duration -} - -// New creates and initializes a new Transport. -func New(name string, writer io.Writer, options ...Option) Transport { - transport := Transport{ - Writer: writer, - Name: name, - } - - for _, opt := range options { - opt(&transport) - } - - return transport -} - -// SetPriority metadata for the transport. -func SetPriority(priority int) Option { - return func(transport *Transport) { - transport.Priority = priority - } -} - -// SetInterval metadata for the transport. -func SetInterval(interval time.Duration) Option { - return func(transport *Transport) { - transport.Interval = interval - } -} - -// SetJitter metadata for the transport. -func SetJitter(jitter time.Duration) Option { - return func(transport *Transport) { - transport.Jitter = jitter - } -} diff --git a/transport/transport_test.go b/transport/transport_test.go deleted file mode 100644 index 2ecd0296..00000000 --- a/transport/transport_test.go +++ /dev/null @@ -1,30 +0,0 @@ -package transport_test - -import ( - "io/ioutil" - "testing" - "time" - - "github.com/kcarretto/paragon/transport" - "github.com/stretchr/testify/require" -) - -func TestNew(t *testing.T) { - name := "MyTransport" - interval := time.Hour * 5 - jitter := time.Nanosecond * 10 - priority := 1337 - - transport := transport.New( - name, - ioutil.Discard, - transport.SetInterval(interval), - transport.SetJitter(jitter), - transport.SetPriority(priority), - ) - - require.Equal(t, name, transport.Name) - require.Equal(t, interval, transport.Interval) - require.Equal(t, jitter, transport.Jitter) - require.Equal(t, priority, transport.Priority) -} From 18f53627947aadb8acaf89acf934378a5953e7cc Mon Sep 17 00:00:00 2001 From: Kyle Carretto Date: Wed, 16 Oct 2019 18:03:23 -0700 Subject: [PATCH 19/19] Comment out c2 code lol --- cmd/c2/main.go | 88 ++++++++++++++++++++++---------------------- cmd/c2/pubsub_gcp.go | 72 ++++++++++++++++++------------------ cmd/c2/pubsub_mem.go | 28 +++++++------- 3 files changed, 94 insertions(+), 94 deletions(-) diff --git a/cmd/c2/main.go b/cmd/c2/main.go index f9888c6f..3689aee1 100644 --- a/cmd/c2/main.go +++ b/cmd/c2/main.go @@ -1,52 +1,52 @@ package main -import ( - "context" - "net/http" - "os" +// import ( +// "context" +// "net/http" +// "os" - "github.com/kcarretto/paragon/c2" - "go.uber.org/zap" -) +// "github.com/kcarretto/paragon/c2" +// "go.uber.org/zap" +// ) func main() { - ctx := context.Background() - - logger, err := zap.NewDevelopment() - if err != nil { - panic(err) - } - - httpAddr := os.Getenv("HTTP_ADDR") - if httpAddr == "" { - httpAddr = "127.0.0.1:8080" - } - - resultTopic, err := openTopic(ctx, "tasks.result_received") - if err != nil { - logger.Panic("Failed to open pubsub topic", zap.Error(err)) - } - defer resultTopic.Shutdown(ctx) - - queueTopic, err := openSubscription(ctx, "tasks.queued") - if err != nil { - logger.Panic("Failed to subscribe to pubsub topic", zap.Error(err)) - } - defer queueTopic.Shutdown(ctx) - - srv := &c2.Server{ - Log: logger, - TaskResults: resultTopic, - } - - go func() { - - }() - - logger.Info("Started C2 server", zap.String("http_addr", httpAddr)) - if err := http.ListenAndServe(httpAddr, srv); err != nil { - logger.Panic("Failed to serve HTTP", zap.Error(err)) - } + // ctx := context.Background() + + // logger, err := zap.NewDevelopment() + // if err != nil { + // panic(err) + // } + + // httpAddr := os.Getenv("HTTP_ADDR") + // if httpAddr == "" { + // httpAddr = "127.0.0.1:8080" + // } + + // resultTopic, err := openTopic(ctx, "tasks.result_received") + // if err != nil { + // logger.Panic("Failed to open pubsub topic", zap.Error(err)) + // } + // defer resultTopic.Shutdown(ctx) + + // queueTopic, err := openSubscription(ctx, "tasks.queued") + // if err != nil { + // logger.Panic("Failed to subscribe to pubsub topic", zap.Error(err)) + // } + // defer queueTopic.Shutdown(ctx) + + // srv := &c2.Server{ + // Log: logger, + // TaskResults: resultTopic, + // } + + // go func() { + + // }() + + // logger.Info("Started C2 server", zap.String("http_addr", httpAddr)) + // if err := http.ListenAndServe(httpAddr, srv); err != nil { + // logger.Panic("Failed to serve HTTP", zap.Error(err)) + // } // router := http.NewServeMux() // router.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) { diff --git a/cmd/c2/pubsub_gcp.go b/cmd/c2/pubsub_gcp.go index ffab2371..22be27af 100644 --- a/cmd/c2/pubsub_gcp.go +++ b/cmd/c2/pubsub_gcp.go @@ -2,39 +2,39 @@ package main -import ( - "context" - "fmt" - "os" - - "gocloud.dev/pubsub" - _ "gocloud.dev/pubsub/gcppubsub" -) - -func getURI(topic string) (string, error) { - project := os.Getenv("GCP_PROJECT") - if project == "" { - return "", fmt.Errorf("must set GCP_PROJECT environment variable to use GCP pubsub") - } - - uri := fmt.Sprintf("gcppubsub://projects/%s/topics/%s", project, topic) - return uri, nil -} - -func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { - uri, err := getURI(topic) - if err != nil { - return nil, err - } - - return pubsub.OpenTopic(ctx, uri) -} - -func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { - uri, err := getURI(topic) - if err != nil { - return nil, err - } - - return pubsub.OpenSubscription(ctx, uri) -} +// import ( +// "context" +// "fmt" +// "os" + +// "gocloud.dev/pubsub" +// _ "gocloud.dev/pubsub/gcppubsub" +// ) + +// func getURI(topic string) (string, error) { +// project := os.Getenv("GCP_PROJECT") +// if project == "" { +// return "", fmt.Errorf("must set GCP_PROJECT environment variable to use GCP pubsub") +// } + +// uri := fmt.Sprintf("gcppubsub://projects/%s/topics/%s", project, topic) +// return uri, nil +// } + +// func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { +// uri, err := getURI(topic) +// if err != nil { +// return nil, err +// } + +// return pubsub.OpenTopic(ctx, uri) +// } + +// func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { +// uri, err := getURI(topic) +// if err != nil { +// return nil, err +// } + +// return pubsub.OpenSubscription(ctx, uri) +// } diff --git a/cmd/c2/pubsub_mem.go b/cmd/c2/pubsub_mem.go index 652c964a..73806845 100644 --- a/cmd/c2/pubsub_mem.go +++ b/cmd/c2/pubsub_mem.go @@ -2,20 +2,20 @@ package main -import ( - "context" - "fmt" +// import ( +// "context" +// "fmt" - "gocloud.dev/pubsub" - _ "gocloud.dev/pubsub/mempubsub" -) +// "gocloud.dev/pubsub" +// _ "gocloud.dev/pubsub/mempubsub" +// ) -func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { - uri := fmt.Sprintf("mem://%s", topic) - return pubsub.OpenTopic(ctx, uri) -} +// func openTopic(ctx context.Context, topic string) (*pubsub.Topic, error) { +// uri := fmt.Sprintf("mem://%s", topic) +// return pubsub.OpenTopic(ctx, uri) +// } -func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { - uri := fmt.Sprintf("mem://%s", topic) - return pubsub.OpenSubscription(ctx, uri) -} +// func openSubscription(ctx context.Context, topic string) (*pubsub.Subscription, error) { +// uri := fmt.Sprintf("mem://%s", topic) +// return pubsub.OpenSubscription(ctx, uri) +// }