Rename proto.Request.Params to .Args. Rename proto.CreateRequestParam…

…s to proto.CreateRequest. Remove util/: XID to RM.Create, NewTLSConfig to config/. Use time.Now().UTC() consistently.

config/config.go

@@ -1,14 +1,39 @@
-// Copyright 2017, Square, Inc.
+// Copyright 2017-2019, Square, Inc.
 
 package config
 
 import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
 	"io/ioutil"
 	"os"
 
 	"gopkg.in/yaml.v2"
 )
 
+// NewTLSConfig takes a cert, key, and ca file and creates a *tls.Config.
+func NewTLSConfig(caFile, certFile, keyFile string) (*tls.Config, error) {
+	cert, err := tls.LoadX509KeyPair(certFile, keyFile)
+	if err != nil {
+		return nil, fmt.Errorf("tls.LoadX509KeyPair: %s", err)
+	}
+
+	caCert, err := ioutil.ReadFile(caFile)
+	if err != nil {
+		return nil, err
+	}
+	caCertPool := x509.NewCertPool()
+	caCertPool.AppendCertsFromPEM(caCert)
+	tlsConfig := &tls.Config{
+		Certificates: []tls.Certificate{cert},
+		RootCAs:      caCertPool,
+	}
+	tlsConfig.BuildNameToCertificate()
+
+	return tlsConfig, nil
+}
+
 ///////////////////////////////////////////////////////////////////////////////
 // High-Level Config Structs
 ///////////////////////////////////////////////////////////////////////////////

job-runner/app/app.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2018, Square, Inc.
+// Copyright 2017-2019, Square, Inc.
 
 package app
 
@@ -12,7 +12,6 @@ import (
 	"github.com/square/spincycle/config"
 	"github.com/square/spincycle/job-runner/chain"
 	"github.com/square/spincycle/request-manager"
-	"github.com/square/spincycle/util"
 )
 
 type Context struct {
@@ -115,7 +114,7 @@ func MakeRequestManagerClient(appCtx Context) (rm.Client, error) {
 	cfg := appCtx.Config
 	httpClient := &http.Client{}
 	if cfg.RMClient.TLS.CertFile != "" && cfg.RMClient.TLS.KeyFile != "" && cfg.RMClient.TLS.CAFile != "" {
-		tlsConfig, err := util.NewTLSConfig(
+		tlsConfig, err := config.NewTLSConfig(
 			cfg.RMClient.TLS.CAFile,
 			cfg.RMClient.TLS.CertFile,
 			cfg.RMClient.TLS.KeyFile,

job-runner/chain/reaper.go

@@ -1,4 +1,4 @@
-// Copyright 2018, Square, Inc.
+// Copyright 2018-2019, Square, Inc.
 
 package chain
 
@@ -268,7 +268,7 @@ func (r *RunningChainReaper) Reap(job proto.Job) {
 // Finalize determines the final state of the chain and sends it to the Request
 // Manager.
 func (r *RunningChainReaper) Finalize(complete bool) {
-	finishedAt := time.Now()
+	finishedAt := time.Now().UTC()
 	if complete {
 		r.logger.Infof("chain is done, all jobs finished successfully")
 		r.chain.SetState(proto.STATE_COMPLETE)
@@ -393,7 +393,7 @@ func (r *SuspendedChainReaper) Reap(job proto.Job) {
 // either sends the Request Manager the chain's final state or a SuspendedJobChain
 // that can be used to resume running the chain.
 func (r *SuspendedChainReaper) Finalize() {
-	finishedAt := time.Now()
+	finishedAt := time.Now().UTC()
 
 	// Mark any jobs that didn't respond to Stop in time as Failed
 	for _, jobStatus := range r.chain.Running() {
@@ -521,7 +521,7 @@ func (r *StoppedChainReaper) Reap(job proto.Job) {
 // Finalize determines the final state of the chain and sends it to the Request
 // Manager.
 func (r *StoppedChainReaper) Finalize() {
-	finishedAt := time.Now()
+	finishedAt := time.Now().UTC()
 
 	// Mark any jobs that didn't respond to Stop in time as Failed
 	for _, jobStatus := range r.chain.Running() {

job-runner/runner/runner.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2018, Square, Inc.
+// Copyright 2017-2019, Square, Inc.
 
 // Package runner implements running a job.
 package runner
@@ -95,7 +95,7 @@ func (r *runner) Run(jobData map[string]interface{}) Return {
 	// the run fails.
 	var finalState byte = proto.STATE_PENDING
 
-	r.startTime = time.Now()
+	r.startTime = time.Now().UTC()
 
 	tryNo := uint(1)
 TRY_LOOP:

proto/proto.go

@@ -79,11 +79,11 @@ type JobChain struct {
 
 // Request represents something that a user asks Spin Cycle to do.
 type Request struct {
-	Id     string                 `json:"id"`         // unique identifier for the request
-	Type   string                 `json:"type"`       // the type of request
-	State  byte                   `json:"state"`      // STATE_* const
-	User   string                 `json:"user"`       // the user who made the request
-	Params map[string]interface{} `json:",omitempty"` // the jobArgs
+	Id    string                 `json:"id"`         // unique identifier for the request
+	Type  string                 `json:"type"`       // the type of request
+	State byte                   `json:"state"`      // STATE_* const
+	User  string                 `json:"user"`       // the user who made the request
+	Args  map[string]interface{} `json:",omitempty"` // the jobArgs
 
 	CreatedAt  time.Time  `json:"createdAt"`  // when the request was created
 	StartedAt  *time.Time `json:"startedAt"`  // when the request was sent to the job runner
@@ -182,9 +182,8 @@ type RunningStatus struct {
 	Requests map[string]Request `json:"requests,omitempty"` // keyed on RequestId
 }
 
-// CreateRequestParams represents the payload that is required to create a new
-// request in the RM.
-type CreateRequestParams struct {
+// CreateRequest represents the payload to create and start a new request.
+type CreateRequest struct {
 	Type string                 // the type of request being made
 	Args map[string]interface{} // the arguments for the request
 	User string                 // the user making the request

request-manager/api/api.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2018, Square, Inc.
+// Copyright 2017-2019, Square, Inc.
 
 // Package api provides controllers for each api endpoint. Controllers are
 // "dumb wiring"; there is little to no application logic in this package.
@@ -160,8 +160,8 @@ func (api *API) createRequestHandler(c echo.Context) error {
 	// ----------------------------------------------------------------------
 	// Make and validate request
 
-	// Convert the payload into a proto.CreateRequestParams.
-	var reqParams proto.CreateRequestParams
+	// Convert the payload into a proto.CreateRequest.
+	var reqParams proto.CreateRequest
 	if err := c.Bind(&reqParams); err != nil {
 		return err
 	}

request-manager/api/api_test.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2018, Square, Inc.
+// Copyright 2017-2019, Square, Inc.
 
 package api_test
 
@@ -82,9 +82,9 @@ func TestNewRequestHandlerRMError(t *testing.T) {
 	payload := `{"type":"something","args":{"first":"arg1"},"user":"mike"}`
 	// Create a mock request manager that will return an error and record the
 	// request params it receives.
-	var rmReqParams proto.CreateRequestParams
+	var rmReqParams proto.CreateRequest
 	rm := &mock.RequestManager{
-		CreateFunc: func(reqParams proto.CreateRequestParams) (proto.Request, error) {
+		CreateFunc: func(reqParams proto.CreateRequest) (proto.Request, error) {
 			rmReqParams = reqParams
 			return proto.Request{}, mock.ErrRequestManager
 		},
@@ -104,7 +104,7 @@ func TestNewRequestHandlerRMError(t *testing.T) {
 	}
 
 	// Check the request params sent to the request manager.
-	expectedReqParams := proto.CreateRequestParams{
+	expectedReqParams := proto.CreateRequest{
 		Type: "something",
 		Args: map[string]interface{}{
 			"first": "arg1",
@@ -125,9 +125,9 @@ func TestNewRequestHandlerSuccess(t *testing.T) {
 	}
 	// Create a mock request manager that will return a request and record the
 	// request params it receives.
-	var rmReqParams proto.CreateRequestParams
+	var rmReqParams proto.CreateRequest
 	rm := &mock.RequestManager{
-		CreateFunc: func(reqParams proto.CreateRequestParams) (proto.Request, error) {
+		CreateFunc: func(reqParams proto.CreateRequest) (proto.Request, error) {
 			rmReqParams = reqParams
 			return req, nil
 		},
@@ -164,7 +164,7 @@ func TestNewRequestHandlerSuccess(t *testing.T) {
 	}
 
 	// Check the request params sent to the request manager.
-	expectedReqParams := proto.CreateRequestParams{
+	expectedReqParams := proto.CreateRequest{
 		Type: "something",
 		Args: map[string]interface{}{
 			"first":  "arg1",
@@ -685,7 +685,7 @@ func TestAuth(t *testing.T) {
 		State: proto.STATE_PENDING,
 	}
 	ctx.RM = &mock.RequestManager{
-		CreateFunc: func(proto.CreateRequestParams) (proto.Request, error) {
+		CreateFunc: func(proto.CreateRequest) (proto.Request, error) {
 			createCalled = true
 			return req, nil
 		},

request-manager/app/app.go

@@ -23,7 +23,6 @@ import (
 	"github.com/square/spincycle/request-manager/joblog"
 	"github.com/square/spincycle/request-manager/request"
 	"github.com/square/spincycle/request-manager/status"
-	"github.com/square/spincycle/util"
 )
 
 // Context represents the config, core service singletons, and 3rd-party extensions.
@@ -183,7 +182,7 @@ func MakeJobRunnerClient(ctx Context) (jr.Client, error) {
 	httpClient := &http.Client{}
 	jrcfg := ctx.Config.JRClient
 	if jrcfg.TLS.CertFile != "" && jrcfg.TLS.KeyFile != "" && jrcfg.TLS.CAFile != "" {
-		tlsConfig, err := util.NewTLSConfig(jrcfg.TLS.CAFile, jrcfg.TLS.CertFile, jrcfg.TLS.KeyFile)
+		tlsConfig, err := config.NewTLSConfig(jrcfg.TLS.CAFile, jrcfg.TLS.CertFile, jrcfg.TLS.KeyFile)
 		if err != nil {
 			return nil, fmt.Errorf("error loading JR client TLS config: %s", err)
 		}
@@ -200,7 +199,7 @@ func MakeDbConnPool(ctx Context) (*sql.DB, error) {
 	dbcfg := ctx.Config.Db
 	dsn := dbcfg.DSN + "?parseTime=true" // always needs to be set
 	if dbcfg.TLS.CAFile != "" && dbcfg.TLS.CertFile != "" && dbcfg.TLS.KeyFile != "" {
-		tlsConfig, err := util.NewTLSConfig(dbcfg.TLS.CAFile, dbcfg.TLS.CertFile, dbcfg.TLS.KeyFile)
+		tlsConfig, err := config.NewTLSConfig(dbcfg.TLS.CAFile, dbcfg.TLS.CertFile, dbcfg.TLS.KeyFile)
 		if err != nil {
 			log.Fatalf("error loading database TLS config: %s", err)
 		}

request-manager/client.go

@@ -1,4 +1,4 @@
-// Copyright 2017-2018, Square, Inc.
+// Copyright 2017-2019, Square, Inc.
 
 // Package rm provides an HTTP client for interacting with the Request Manager (RM) API.
 package rm
@@ -79,7 +79,7 @@ func (c *client) CreateRequest(reqType string, args map[string]interface{}) (str
 	url := c.baseUrl + "/api/v1/requests"
 
 	// Create the payload struct.
-	reqParams := &proto.CreateRequestParams{
+	reqParams := &proto.CreateRequest{
 		Type: reqType,
 		Args: args,
 	}

request-manager/client_test.go

@@ -1,3 +1,5 @@
+// Copyright 2017-2019, Square, Inc.
+
 package rm_test
 
 import (
@@ -79,7 +81,7 @@ func TestCreateRequestSuccess(t *testing.T) {
 	reqType := "something"
 	args := map[string]interface{}{"arg1": "val1"}
 	reqId := "abcd1234"
-	var payload proto.CreateRequestParams
+	var payload proto.CreateRequest
 
 	setup(t, &payload, http.StatusCreated, "{\"id\":\""+reqId+"\"}")
 	defer cleanup()
@@ -90,7 +92,7 @@ func TestCreateRequestSuccess(t *testing.T) {
 		t.Errorf("err = %s, expected nil", err)
 	}
 
-	expectedPayload := proto.CreateRequestParams{
+	expectedPayload := proto.CreateRequest{
 		Type: reqType,
 		Args: args,
 	}

request-manager/request/manager.go

@@ -14,18 +14,18 @@ import (
 	"time"
 
 	"github.com/go-sql-driver/mysql"
+	"github.com/rs/xid"
 
 	jr "github.com/square/spincycle/job-runner"
 	"github.com/square/spincycle/proto"
 	"github.com/square/spincycle/request-manager/db"
 	"github.com/square/spincycle/request-manager/grapher"
-	"github.com/square/spincycle/util"
 )
 
 // A Manager is used to create and manage requests.
 type Manager interface {
 	// Create creates a proto.Request and saves it to the db.
-	Create(proto.CreateRequestParams) (proto.Request, error)
+	Create(proto.CreateRequest) (proto.Request, error)
 
 	// Get retrieves the request corresponding to the provided id,
 	// without its job chain or parameters set.
@@ -90,18 +90,18 @@ func NewManager(config ManagerConfig) Manager {
 	}
 }
 
-func (m *manager) Create(reqParams proto.CreateRequestParams) (proto.Request, error) {
+func (m *manager) Create(reqParams proto.CreateRequest) (proto.Request, error) {
 	var req proto.Request
 	if reqParams.Type == "" {
 		return req, ErrInvalidParams
 	}
 
-	reqIdBytes := util.XID()
+	reqIdBytes := xid.New()
 	reqId := reqIdBytes.String()
 	req = proto.Request{
 		Id:        reqId,
 		Type:      reqParams.Type,
-		CreatedAt: time.Now(),
+		CreatedAt: time.Now().UTC(),
 		State:     proto.STATE_PENDING,
 		User:      reqParams.User,
 	}
@@ -123,7 +123,7 @@ func (m *manager) Create(reqParams proto.CreateRequestParams) (proto.Request, er
 	// the auth pluign to let Authorize() do fine-grain auth for the request based
 	// on the args.
 	// @todo: should this be reqParams.Args? i.e. initial args or final post-processing args?
-	req.Params = args
+	req.Args = args
 
 	jc := &proto.JobChain{
 		Jobs:          map[string]proto.Job{},
@@ -252,8 +252,6 @@ func (m *manager) Start(requestId string) error {
 		return err
 	}
 
-	now := time.Now()
-
 	// Only start the request if it's currently Pending.
 	if req.State != proto.STATE_PENDING {
 		return NewErrInvalidState(proto.StateName[proto.STATE_PENDING], proto.StateName[req.State])
@@ -266,6 +264,7 @@ func (m *manager) Start(requestId string) error {
 		return err
 	}
 
+	now := time.Now().UTC()
 	req.StartedAt = &now
 	req.State = proto.STATE_RUNNING
 
@@ -536,7 +535,7 @@ func (m *manager) GetWithJC(requestId string) (proto.Request, error) {
 	ctx := context.TODO()
 
 	var jc proto.JobChain
-	var params proto.CreateRequestParams
+	var params proto.CreateRequest
 	var rawJc []byte     // raw job chains are stored as blobs in the db.
 	var rawParams []byte // raw params are stored as blobs in the db.
 	q := "SELECT job_chain, request FROM raw_requests WHERE request_id = ?"
@@ -557,7 +556,7 @@ func (m *manager) GetWithJC(requestId string) (proto.Request, error) {
 	}
 
 	req.JobChain = &jc
-	req.Params = params.Args
+	req.Args = params.Args
 	return req, nil
 }
 

request-manager/request/manager_test.go

@@ -113,7 +113,7 @@ func TestCreateMissingType(t *testing.T) {
 	m := request.NewManager(cfg)
 	defer close(shutdownChan)
 
-	_, err := m.Create(proto.CreateRequestParams{})
+	_, err := m.Create(proto.CreateRequest{})
 	if err != request.ErrInvalidParams {
 		t.Errorf("err = %s, expected %s", err, request.ErrInvalidParams)
 	}
@@ -133,7 +133,7 @@ func TestCreate(t *testing.T) {
 	m := request.NewManager(cfg)
 
 	// gr uses spec a-b-c.yaml which has reqest "three-nodes"
-	reqParams := proto.CreateRequestParams{
+	reqParams := proto.CreateRequest{
 		Type: "three-nodes",
 		User: "john",
 		Args: map[string]interface{}{
@@ -223,7 +223,7 @@ func TestCreate(t *testing.T) {
 		User:      reqParams.User,
 		JobChain:  nil,
 		TotalJobs: 5,
-		Params: map[string]interface{}{
+		Args: map[string]interface{}{
 			"foo":  "foo-value",
 			"bar":  "175",
 			"aArg": "aValue",