From 694b36a1d59db832272a13ccb205a715a53cad6b Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Wed, 19 Dec 2018 16:47:11 -0500 Subject: [PATCH 01/21] atc: add user artifact api endpoints * the handler uses a worker client to create a volume * it then streams the payload of the request into the volume * currently there is no way to delete a user artifact, but if the worker_artifact record goes away the volume gets gc'ed * we can now use these endpoints for fly execute concourse/concourse#2826 Signed-off-by: Josh Winters --- atc/api/api_suite_test.go | 6 +- atc/api/artifacts_test.go | 351 ++++++++++++++++ atc/api/artifactserver/create.go | 73 ++++ atc/api/artifactserver/get.go | 66 +++ atc/api/artifactserver/server.go | 21 + atc/api/containers_test.go | 8 +- atc/api/containerserver/hijack.go | 2 +- atc/api/containerserver/server.go | 6 +- atc/api/handler.go | 12 +- atc/atccmd/command.go | 7 +- atc/db/dbfakes/fake_created_volume.go | 144 +++++++ atc/db/dbfakes/fake_team.go | 166 ++++++++ atc/db/dbfakes/fake_volume_repository.go | 82 ++++ atc/db/dbfakes/fake_worker_artifact.go | 377 ++++++++++++++++++ ...545249372_create_worker_artifacts.down.sql | 9 + .../1545249372_create_worker_artifacts.up.sql | 15 + atc/db/team.go | 34 ++ atc/db/team_test.go | 42 +- atc/db/volume.go | 54 ++- atc/db/volume_repository.go | 46 ++- atc/db/volume_repository_test.go | 27 +- atc/db/volume_test.go | 36 ++ atc/db/worker_artifact.go | 109 +++++ atc/exec/get_step.go | 2 +- atc/exec/get_step_test.go | 8 +- atc/exec/put_step.go | 2 +- atc/exec/put_step_test.go | 10 +- atc/exec/task_step.go | 2 +- atc/exec/task_step_test.go | 20 +- atc/radar/resource_scanner.go | 2 +- atc/radar/resource_scanner_test.go | 12 +- atc/radar/resource_type_scanner.go | 2 +- atc/radar/resource_type_scanner_test.go | 16 +- atc/routes.go | 6 + atc/worker/client.go | 69 ++++ atc/worker/client_test.go | 197 +++++++++ atc/worker/db_worker_provider.go | 24 ++ atc/worker/db_worker_provider_test.go | 90 +++++ atc/worker/pool.go | 34 +- atc/worker/pool_test.go | 60 +-- atc/worker/volume.go | 6 +- atc/worker/volume_client.go | 26 ++ atc/worker/volume_client_test.go | 107 ++++- atc/worker/worker.go | 6 + atc/worker/worker_test.go | 27 +- atc/worker/workerfakes/fake_client.go | 297 ++++++++++++++ atc/worker/workerfakes/fake_pool.go | 191 +++++---- atc/worker/workerfakes/fake_volume.go | 80 ++++ atc/worker/workerfakes/fake_volume_client.go | 86 ++++ atc/worker/workerfakes/fake_worker.go | 84 ++++ .../workerfakes/fake_worker_provider.go | 87 ++++ atc/worker_artifact.go | 8 + atc/wrappa/api_auth_wrappa.go | 4 +- atc/wrappa/api_auth_wrappa_test.go | 5 +- go.mod | 2 +- 55 files changed, 3015 insertions(+), 250 deletions(-) create mode 100644 atc/api/artifacts_test.go create mode 100644 atc/api/artifactserver/create.go create mode 100644 atc/api/artifactserver/get.go create mode 100644 atc/api/artifactserver/server.go create mode 100644 atc/db/dbfakes/fake_worker_artifact.go create mode 100755 atc/db/migration/migrations/1545249372_create_worker_artifacts.down.sql create mode 100755 atc/db/migration/migrations/1545249372_create_worker_artifacts.up.sql create mode 100644 atc/db/worker_artifact.go create mode 100644 atc/worker/client.go create mode 100644 atc/worker/client_test.go create mode 100644 atc/worker/workerfakes/fake_client.go create mode 100644 atc/worker_artifact.go diff --git a/atc/api/api_suite_test.go b/atc/api/api_suite_test.go index 605cd9ca04d..2e132c449f8 100644 --- a/atc/api/api_suite_test.go +++ b/atc/api/api_suite_test.go @@ -37,7 +37,7 @@ var ( externalURL = "https://example.com" fakeEngine *enginefakes.FakeEngine - fakeWorkerPool *workerfakes.FakePool + fakeWorkerClient *workerfakes.FakeClient fakeWorkerProvider *workerfakes.FakeWorkerProvider fakeVolumeRepository *dbfakes.FakeVolumeRepository fakeContainerRepository *dbfakes.FakeContainerRepository @@ -122,7 +122,7 @@ var _ = BeforeEach(func() { drain = make(chan struct{}) fakeEngine = new(enginefakes.FakeEngine) - fakeWorkerPool = new(workerfakes.FakePool) + fakeWorkerClient = new(workerfakes.FakeClient) fakeWorkerProvider = new(workerfakes.FakeWorkerProvider) fakeSchedulerFactory = new(jobserverfakes.FakeSchedulerFactory) @@ -187,7 +187,7 @@ var _ = BeforeEach(func() { drain, fakeEngine, - fakeWorkerPool, + fakeWorkerClient, fakeWorkerProvider, fakeSchedulerFactory, diff --git a/atc/api/artifacts_test.go b/atc/api/artifacts_test.go new file mode 100644 index 00000000000..548854cc06a --- /dev/null +++ b/atc/api/artifacts_test.go @@ -0,0 +1,351 @@ +package api_test + +import ( + "bytes" + "errors" + "io" + "io/ioutil" + "net/http" + "time" + + "github.com/concourse/baggageclaim" + "github.com/concourse/concourse/atc/api/accessor/accessorfakes" + "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/db/dbfakes" + "github.com/concourse/concourse/atc/worker/workerfakes" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Artifacts API", func() { + var fakeaccess *accessorfakes.FakeAccess + + BeforeEach(func() { + fakeaccess = new(accessorfakes.FakeAccess) + }) + + JustBeforeEach(func() { + fakeAccessor.CreateReturns(fakeaccess) + }) + + Describe("POST /api/v1/teams/:team_name/artifacts", func() { + var request *http.Request + var response *http.Response + + BeforeEach(func() { + fakeaccess = new(accessorfakes.FakeAccess) + fakeaccess.IsAuthenticatedReturns(true) + + fakeAccessor.CreateReturns(fakeaccess) + }) + + JustBeforeEach(func() { + var err error + request, err = http.NewRequest("POST", server.URL+"/api/v1/teams/some-team/artifacts", bytes.NewBuffer([]byte("some-data"))) + Expect(err).NotTo(HaveOccurred()) + + request.Header.Set("Content-Type", "application/json") + + response, err = client.Do(request) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("when not authenticated", func() { + BeforeEach(func() { + fakeaccess.IsAuthenticatedReturns(false) + }) + + It("returns 401 Unauthorized", func() { + Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + }) + + Context("when not authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(false) + }) + + It("returns 403 Forbidden", func() { + Expect(response.StatusCode).To(Equal(http.StatusForbidden)) + }) + }) + + Context("when authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(true) + }) + + Context("when creating a volume fails", func() { + BeforeEach(func() { + fakeWorkerClient.CreateVolumeReturns(nil, errors.New("nope")) + }) + + It("returns 500 InternalServerError", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when creating a volume succeeds", func() { + var fakeVolume *workerfakes.FakeVolume + + BeforeEach(func() { + fakeVolume = new(workerfakes.FakeVolume) + fakeVolume.InitializeArtifactReturns(nil, errors.New("nope")) + + fakeWorkerClient.CreateVolumeReturns(fakeVolume, nil) + }) + + It("creates the volume using the worker client", func() { + Expect(fakeWorkerClient.CreateVolumeCallCount()).To(Equal(1)) + + _, volumeSpec, teamID, volumeType := fakeWorkerClient.CreateVolumeArgsForCall(0) + Expect(volumeSpec.Strategy).To(Equal(baggageclaim.EmptyStrategy{})) + Expect(teamID).To(Equal(734)) + Expect(volumeType).To(Equal(db.VolumeTypeArtifact)) + }) + + Context("when associating a volume with an artifact fails", func() { + BeforeEach(func() { + fakeVolume.InitializeArtifactReturns(nil, errors.New("nope")) + }) + + It("returns 500 InternalServerError", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when associating a volume with an artifact succeeds", func() { + var fakeWorkerArtifact *dbfakes.FakeWorkerArtifact + + BeforeEach(func() { + fakeWorkerArtifact = new(dbfakes.FakeWorkerArtifact) + fakeWorkerArtifact.IDReturns(0) + fakeWorkerArtifact.CreatedAtReturns(time.Now()) + + fakeVolume.InitializeArtifactReturns(fakeWorkerArtifact, nil) + }) + + It("invokes the initialization of an artifact on a volume", func() { + Expect(fakeVolume.InitializeArtifactCallCount()).To(Equal(1)) + + path, checksum := fakeVolume.InitializeArtifactArgsForCall(0) + Expect(path).To(Equal("/")) + Expect(checksum).To(Equal("")) + }) + + Context("when streaming in data to a volume fails", func() { + BeforeEach(func() { + fakeVolume.StreamInReturns(errors.New("nope")) + }) + + It("returns 500 InternalServerError", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when streaming in data to a volume succeeds", func() { + + BeforeEach(func() { + fakeVolume.StreamInReturns(nil) + + fakeVolume.StreamInStub = func(path string, body io.Reader) error { + Expect(path).To(Equal("/")) + + contents, err := ioutil.ReadAll(body) + Expect(err).ToNot(HaveOccurred()) + + Expect(contents).To(Equal([]byte("some-data"))) + return nil + } + }) + + It("streams in the user contents to the new volume", func() { + Expect(fakeVolume.StreamInCallCount()).To(Equal(1)) + }) + + Context("when the request succeeds", func() { + + It("returns 201 Created", func() { + Expect(response.StatusCode).To(Equal(http.StatusCreated)) + }) + + It("returns Content-Type 'application/json'", func() { + Expect(response.Header.Get("Content-Type")).To(Equal("application/json")) + }) + + It("returns the artifact record", func() { + body, err := ioutil.ReadAll(response.Body) + Expect(err).NotTo(HaveOccurred()) + + Expect(body).To(MatchJSON(`{ + "id": 0, + "path": "/", + "created_at": 42, + "checksum": "" + }`)) + }) + }) + }) + }) + }) + }) + }) + + Describe("GET /api/v1/teams/:team_name/artifacts/:artifact_id", func() { + var response *http.Response + + BeforeEach(func() { + fakeaccess = new(accessorfakes.FakeAccess) + fakeaccess.IsAuthenticatedReturns(true) + + fakeAccessor.CreateReturns(fakeaccess) + }) + + JustBeforeEach(func() { + var err error + response, err = http.Get(server.URL + "/api/v1/teams/some-team/artifacts/18") + Expect(err).NotTo(HaveOccurred()) + }) + + Context("when not authenticated", func() { + BeforeEach(func() { + fakeaccess.IsAuthenticatedReturns(false) + }) + + It("returns 401 Unauthorized", func() { + Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + }) + + Context("when not authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(false) + }) + + It("returns 403 Forbidden", func() { + Expect(response.StatusCode).To(Equal(http.StatusForbidden)) + }) + }) + + Context("when authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(true) + }) + + It("uses the artifactID to fetch the db volume record", func() { + Expect(dbTeam.FindVolumeForWorkerArtifactCallCount()).To(Equal(1)) + + artifactID := dbTeam.FindVolumeForWorkerArtifactArgsForCall(0) + Expect(artifactID).To(Equal(18)) + }) + + Context("when retrieving db artifact volume fails", func() { + BeforeEach(func() { + dbTeam.FindVolumeForWorkerArtifactReturns(nil, false, errors.New("nope")) + }) + + It("errors", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when the db artifact volume is not found", func() { + BeforeEach(func() { + dbTeam.FindVolumeForWorkerArtifactReturns(nil, false, nil) + }) + + It("returns 404", func() { + Expect(response.StatusCode).To(Equal(http.StatusNotFound)) + }) + }) + + Context("when the db artifact volume is found", func() { + var fakeVolume *dbfakes.FakeCreatedVolume + + BeforeEach(func() { + fakeVolume = new(dbfakes.FakeCreatedVolume) + fakeVolume.HandleReturns("some-handle") + + dbTeam.FindVolumeForWorkerArtifactReturns(fakeVolume, true, nil) + }) + + It("uses the volume handle to lookup the worker volume", func() { + Expect(fakeWorkerClient.FindVolumeCallCount()).To(Equal(1)) + + _, teamID, handle := fakeWorkerClient.FindVolumeArgsForCall(0) + Expect(handle).To(Equal("some-handle")) + Expect(teamID).To(Equal(734)) + }) + + Context("when the worker client errors", func() { + BeforeEach(func() { + fakeWorkerClient.FindVolumeReturns(nil, false, errors.New("nope")) + }) + + It("returns 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when the worker client can't find the volume", func() { + BeforeEach(func() { + fakeWorkerClient.FindVolumeReturns(nil, false, nil) + }) + + It("returns 404", func() { + Expect(response.StatusCode).To(Equal(http.StatusNotFound)) + }) + }) + + Context("when the worker client finds the volume", func() { + var fakeWorkerVolume *workerfakes.FakeVolume + + BeforeEach(func() { + reader := ioutil.NopCloser(bytes.NewReader([]byte(""))) + + fakeWorkerVolume = new(workerfakes.FakeVolume) + fakeWorkerVolume.StreamOutReturns(reader, nil) + + fakeWorkerClient.FindVolumeReturns(fakeWorkerVolume, true, nil) + }) + + It("streams out the contents of the volume from the root path", func() { + Expect(fakeWorkerVolume.StreamOutCallCount()).To(Equal(1)) + + path := fakeWorkerVolume.StreamOutArgsForCall(0) + Expect(path).To(Equal("/")) + }) + + Context("when streaming volume contents fails", func() { + BeforeEach(func() { + fakeWorkerVolume.StreamOutReturns(nil, errors.New("nope")) + }) + + It("returns 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when streaming volume contents succeeds", func() { + BeforeEach(func() { + reader := ioutil.NopCloser(bytes.NewReader([]byte("some-content"))) + fakeWorkerVolume.StreamOutReturns(reader, nil) + }) + + It("returns 200", func() { + Expect(response.StatusCode).To(Equal(http.StatusOK)) + }) + + It("returns Content-Type 'application/octet-stream'", func() { + Expect(response.Header.Get("Content-Type")).To(Equal("application/octet-stream")) + }) + + It("returns the contents of the volume", func() { + Expect(ioutil.ReadAll(response.Body)).To(Equal([]byte("some-content"))) + }) + }) + }) + }) + }) + }) +}) diff --git a/atc/api/artifactserver/create.go b/atc/api/artifactserver/create.go new file mode 100644 index 00000000000..5cce1b3c7fb --- /dev/null +++ b/atc/api/artifactserver/create.go @@ -0,0 +1,73 @@ +package artifactserver + +import ( + "encoding/json" + "net/http" + + "github.com/concourse/baggageclaim" + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/worker" +) + +func (s *Server) CreateArtifact(team db.Team) http.Handler { + hLog := s.logger.Session("create-artifact") + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") + + // TODO: can probably check if fly sent us an etag header + // which we can lookup in the checksum field + // that way we don't have to create another volume. + + spec := worker.VolumeSpec{ + Strategy: baggageclaim.EmptyStrategy{}, + } + + volume, err := s.workerClient.CreateVolume(hLog, spec, team.ID(), db.VolumeTypeArtifact) + if err != nil { + hLog.Error("failed-to-create-volume", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + // NOTE: there's a race condition here between when the + // volume gets created and when the artifact gets initialized + + // Within this timeframe there's a chance that the volume could + // get garbage collector out from under us. + + // This happens because CreateVolume returns a 'created' instead + // of 'creating' volume. + + // In the long run CreateVolume should probably return a 'creating' + // volume, but there are other changes needed in FindOrCreateContainer + // with the way we create volumes for a container + + // I think leaving the race condition is fine for now. Worst case + // is a fly execute will fail and the user will need to rerun it. + + artifact, err := volume.InitializeArtifact("/", "") + if err != nil { + hLog.Error("failed-to-initialize-artifact", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + err = volume.StreamIn("/", r.Body) + if err != nil { + hLog.Error("failed-to-stream-volume-contents", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + w.WriteHeader(http.StatusCreated) + + json.NewEncoder(w).Encode(atc.WorkerArtifact{ + ID: artifact.ID(), + Checksum: "", + Path: "/", + CreatedAt: artifact.CreatedAt().Unix(), + }) + }) +} diff --git a/atc/api/artifactserver/get.go b/atc/api/artifactserver/get.go new file mode 100644 index 00000000000..44d5d36ea78 --- /dev/null +++ b/atc/api/artifactserver/get.go @@ -0,0 +1,66 @@ +package artifactserver + +import ( + "io" + "net/http" + "strconv" + + "github.com/concourse/concourse/atc/db" +) + +func (s *Server) GetArtifact(team db.Team) http.Handler { + logger := s.logger.Session("get-artifact") + + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + + w.Header().Set("Content-Type", "application/octet-stream") + + artifactID, err := strconv.Atoi(r.FormValue(":artifact_id")) + if err != nil { + logger.Error("failed-to-get-artifact-id", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + artifactVolume, found, err := team.FindVolumeForWorkerArtifact(artifactID) + if err != nil { + logger.Error("failed-to-get-artifact-volume", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if !found { + logger.Error("failed-to-find-artifact-volume", err) + w.WriteHeader(http.StatusNotFound) + return + } + + workerVolume, found, err := s.workerClient.FindVolume(logger, team.ID(), artifactVolume.Handle()) + if err != nil { + logger.Error("failed-to-get-worker-volume", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + if !found { + logger.Error("failed-to-find-worker-volume", err) + w.WriteHeader(http.StatusNotFound) + return + } + + reader, err := workerVolume.StreamOut("/") + if err != nil { + logger.Error("failed-to-stream-volume-contents", err) + w.WriteHeader(http.StatusInternalServerError) + return + } + + defer reader.Close() + + _, err = io.Copy(w, reader) + if err != nil { + logger.Error("failed-to-encode-artifact", err) + w.WriteHeader(http.StatusInternalServerError) + } + }) +} diff --git a/atc/api/artifactserver/server.go b/atc/api/artifactserver/server.go new file mode 100644 index 00000000000..7a0acd08888 --- /dev/null +++ b/atc/api/artifactserver/server.go @@ -0,0 +1,21 @@ +package artifactserver + +import ( + "code.cloudfoundry.org/lager" + "github.com/concourse/concourse/atc/worker" +) + +type Server struct { + logger lager.Logger + workerClient worker.Client +} + +func NewServer( + logger lager.Logger, + workerClient worker.Client, +) *Server { + return &Server{ + logger: logger, + workerClient: workerClient, + } +} diff --git a/atc/api/containers_test.go b/atc/api/containers_test.go index 0584a2ae91c..358a2dd1510 100644 --- a/atc/api/containers_test.go +++ b/atc/api/containers_test.go @@ -572,7 +572,7 @@ var _ = Describe("Containers API", func() { fakeDBContainer.HandleReturns("some-handle") fakeContainer = new(workerfakes.FakeContainer) - fakeWorkerPool.FindContainerByHandleReturns(fakeContainer, true, nil) + fakeWorkerClient.FindContainerReturns(fakeContainer, true, nil) }) Context("when the container is a check container", func() { @@ -666,7 +666,7 @@ var _ = Describe("Containers API", func() { BeforeEach(func() { expectBadHandshake = true - fakeWorkerPool.FindContainerByHandleReturns(nil, false, errors.New("nope")) + fakeWorkerClient.FindContainerReturns(nil, false, errors.New("nope")) }) It("returns 500 internal error", func() { @@ -678,7 +678,7 @@ var _ = Describe("Containers API", func() { BeforeEach(func() { expectBadHandshake = true - fakeWorkerPool.FindContainerByHandleReturns(nil, false, nil) + fakeWorkerClient.FindContainerReturns(nil, false, nil) }) It("returns 404 Not Found", func() { @@ -749,7 +749,7 @@ var _ = Describe("Containers API", func() { It("hijacks the build", func() { Eventually(fakeContainer.RunCallCount).Should(Equal(1)) - _, lookedUpTeamID, lookedUpHandle := fakeWorkerPool.FindContainerByHandleArgsForCall(0) + _, lookedUpTeamID, lookedUpHandle := fakeWorkerClient.FindContainerArgsForCall(0) Expect(lookedUpTeamID).To(Equal(734)) Expect(lookedUpHandle).To(Equal(handle)) diff --git a/atc/api/containerserver/hijack.go b/atc/api/containerserver/hijack.go index 603eff8f6e2..8a23f444624 100644 --- a/atc/api/containerserver/hijack.go +++ b/atc/api/containerserver/hijack.go @@ -88,7 +88,7 @@ func (s *Server) HijackContainer(team db.Team) http.Handler { "handle": handle, }) - container, found, err := s.workerPool.FindContainerByHandle(hLog, team.ID(), handle) + container, found, err := s.workerClient.FindContainer(hLog, team.ID(), handle) if err != nil { hLog.Error("failed-to-find-container", err) w.WriteHeader(http.StatusInternalServerError) diff --git a/atc/api/containerserver/server.go b/atc/api/containerserver/server.go index 3c39299fe5c..9531ecf5f63 100644 --- a/atc/api/containerserver/server.go +++ b/atc/api/containerserver/server.go @@ -11,7 +11,7 @@ import ( type Server struct { logger lager.Logger - workerPool worker.Pool + workerClient worker.Client variablesFactory creds.VariablesFactory interceptTimeoutFactory InterceptTimeoutFactory containerRepository db.ContainerRepository @@ -20,7 +20,7 @@ type Server struct { func NewServer( logger lager.Logger, - workerPool worker.Pool, + workerClient worker.Client, variablesFactory creds.VariablesFactory, interceptTimeoutFactory InterceptTimeoutFactory, containerRepository db.ContainerRepository, @@ -28,7 +28,7 @@ func NewServer( ) *Server { return &Server{ logger: logger, - workerPool: workerPool, + workerClient: workerClient, variablesFactory: variablesFactory, interceptTimeoutFactory: interceptTimeoutFactory, containerRepository: containerRepository, diff --git a/atc/api/handler.go b/atc/api/handler.go index 3a5cb1db225..234a5baadf3 100644 --- a/atc/api/handler.go +++ b/atc/api/handler.go @@ -5,9 +5,8 @@ import ( "path/filepath" "code.cloudfoundry.org/lager" - "github.com/tedsuo/rata" - "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/api/artifactserver" "github.com/concourse/concourse/atc/api/buildserver" "github.com/concourse/concourse/atc/api/ccserver" "github.com/concourse/concourse/atc/api/cliserver" @@ -29,6 +28,7 @@ import ( "github.com/concourse/concourse/atc/mainredirect" "github.com/concourse/concourse/atc/worker" "github.com/concourse/concourse/atc/wrappa" + "github.com/tedsuo/rata" ) func NewHandler( @@ -54,7 +54,7 @@ func NewHandler( drain <-chan struct{}, engine engine.Engine, - workerPool worker.Pool, + workerClient worker.Client, workerProvider worker.WorkerProvider, schedulerFactory jobserver.SchedulerFactory, @@ -91,10 +91,11 @@ func NewHandler( workerServer := workerserver.NewServer(logger, dbTeamFactory, dbWorkerFactory, workerProvider) logLevelServer := loglevelserver.NewServer(logger, sink) cliServer := cliserver.NewServer(logger, absCLIDownloadsDir) - containerServer := containerserver.NewServer(logger, workerPool, variablesFactory, interceptTimeoutFactory, containerRepository, destroyer) + containerServer := containerserver.NewServer(logger, workerClient, variablesFactory, interceptTimeoutFactory, containerRepository, destroyer) volumesServer := volumeserver.NewServer(logger, volumeRepository, destroyer) teamServer := teamserver.NewServer(logger, dbTeamFactory, externalURL) infoServer := infoserver.NewServer(logger, version, workerVersion, credsManagers) + artifactServer := artifactserver.NewServer(logger, workerClient) handlers := map[string]http.Handler{ atc.GetConfig: http.HandlerFunc(configServer.GetConfig), @@ -194,6 +195,9 @@ func NewHandler( atc.RenameTeam: http.HandlerFunc(teamServer.RenameTeam), atc.DestroyTeam: http.HandlerFunc(teamServer.DestroyTeam), atc.ListTeamBuilds: http.HandlerFunc(teamServer.ListTeamBuilds), + + atc.CreateArtifact: teamHandlerFactory.HandlerFor(artifactServer.CreateArtifact), + atc.GetArtifact: teamHandlerFactory.HandlerFor(artifactServer.GetArtifact), } return rata.NewRouter(atc.Routes, wrapper.Wrap(handlers)) diff --git a/atc/atccmd/command.go b/atc/atccmd/command.go index 42946811dbb..e2e8ae8fb76 100644 --- a/atc/atccmd/command.go +++ b/atc/atccmd/command.go @@ -537,6 +537,7 @@ func (cmd *RunCommand) constructAPIMembers( ) pool := worker.NewPool(workerProvider) + workerClient := worker.NewClient(pool, workerProvider) defaultLimits, err := cmd.parseDefaultLimits() if err != nil { @@ -598,7 +599,7 @@ func (cmd *RunCommand) constructAPIMembers( dbBuildFactory, dbResourceConfigFactory, engine, - pool, + workerClient, workerProvider, drain, radarSchedulerFactory, @@ -1282,7 +1283,7 @@ func (cmd *RunCommand) constructAPIHandler( dbBuildFactory db.BuildFactory, resourceConfigFactory db.ResourceConfigFactory, engine engine.Engine, - workerPool worker.Pool, + workerClient worker.Client, workerProvider worker.WorkerProvider, drain <-chan struct{}, radarSchedulerFactory pipelines.RadarSchedulerFactory, @@ -1330,7 +1331,7 @@ func (cmd *RunCommand) constructAPIHandler( drain, engine, - workerPool, + workerClient, workerProvider, radarSchedulerFactory, radarScannerFactory, diff --git a/atc/db/dbfakes/fake_created_volume.go b/atc/db/dbfakes/fake_created_volume.go index afc3214e229..4aa97ee87f8 100644 --- a/atc/db/dbfakes/fake_created_volume.go +++ b/atc/db/dbfakes/fake_created_volume.go @@ -66,6 +66,20 @@ type FakeCreatedVolume struct { handleReturnsOnCall map[int]struct { result1 string } + InitializeArtifactStub func(string, string) (db.WorkerArtifact, error) + initializeArtifactMutex sync.RWMutex + initializeArtifactArgsForCall []struct { + arg1 string + arg2 string + } + initializeArtifactReturns struct { + result1 db.WorkerArtifact + result2 error + } + initializeArtifactReturnsOnCall map[int]struct { + result1 db.WorkerArtifact + result2 error + } InitializeResourceCacheStub func(db.UsedResourceCache) error initializeResourceCacheMutex sync.RWMutex initializeResourceCacheArgsForCall []struct { @@ -158,6 +172,16 @@ type FakeCreatedVolume struct { typeReturnsOnCall map[int]struct { result1 db.VolumeType } + WorkerArtifactIDStub func() int + workerArtifactIDMutex sync.RWMutex + workerArtifactIDArgsForCall []struct { + } + workerArtifactIDReturns struct { + result1 int + } + workerArtifactIDReturnsOnCall map[int]struct { + result1 int + } WorkerNameStub func() string workerNameMutex sync.RWMutex workerNameArgsForCall []struct { @@ -450,6 +474,70 @@ func (fake *FakeCreatedVolume) HandleReturnsOnCall(i int, result1 string) { }{result1} } +func (fake *FakeCreatedVolume) InitializeArtifact(arg1 string, arg2 string) (db.WorkerArtifact, error) { + fake.initializeArtifactMutex.Lock() + ret, specificReturn := fake.initializeArtifactReturnsOnCall[len(fake.initializeArtifactArgsForCall)] + fake.initializeArtifactArgsForCall = append(fake.initializeArtifactArgsForCall, struct { + arg1 string + arg2 string + }{arg1, arg2}) + fake.recordInvocation("InitializeArtifact", []interface{}{arg1, arg2}) + fake.initializeArtifactMutex.Unlock() + if fake.InitializeArtifactStub != nil { + return fake.InitializeArtifactStub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.initializeArtifactReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeCreatedVolume) InitializeArtifactCallCount() int { + fake.initializeArtifactMutex.RLock() + defer fake.initializeArtifactMutex.RUnlock() + return len(fake.initializeArtifactArgsForCall) +} + +func (fake *FakeCreatedVolume) InitializeArtifactCalls(stub func(string, string) (db.WorkerArtifact, error)) { + fake.initializeArtifactMutex.Lock() + defer fake.initializeArtifactMutex.Unlock() + fake.InitializeArtifactStub = stub +} + +func (fake *FakeCreatedVolume) InitializeArtifactArgsForCall(i int) (string, string) { + fake.initializeArtifactMutex.RLock() + defer fake.initializeArtifactMutex.RUnlock() + argsForCall := fake.initializeArtifactArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeCreatedVolume) InitializeArtifactReturns(result1 db.WorkerArtifact, result2 error) { + fake.initializeArtifactMutex.Lock() + defer fake.initializeArtifactMutex.Unlock() + fake.InitializeArtifactStub = nil + fake.initializeArtifactReturns = struct { + result1 db.WorkerArtifact + result2 error + }{result1, result2} +} + +func (fake *FakeCreatedVolume) InitializeArtifactReturnsOnCall(i int, result1 db.WorkerArtifact, result2 error) { + fake.initializeArtifactMutex.Lock() + defer fake.initializeArtifactMutex.Unlock() + fake.InitializeArtifactStub = nil + if fake.initializeArtifactReturnsOnCall == nil { + fake.initializeArtifactReturnsOnCall = make(map[int]struct { + result1 db.WorkerArtifact + result2 error + }) + } + fake.initializeArtifactReturnsOnCall[i] = struct { + result1 db.WorkerArtifact + result2 error + }{result1, result2} +} + func (fake *FakeCreatedVolume) InitializeResourceCache(arg1 db.UsedResourceCache) error { fake.initializeResourceCacheMutex.Lock() ret, specificReturn := fake.initializeResourceCacheReturnsOnCall[len(fake.initializeResourceCacheArgsForCall)] @@ -896,6 +984,58 @@ func (fake *FakeCreatedVolume) TypeReturnsOnCall(i int, result1 db.VolumeType) { }{result1} } +func (fake *FakeCreatedVolume) WorkerArtifactID() int { + fake.workerArtifactIDMutex.Lock() + ret, specificReturn := fake.workerArtifactIDReturnsOnCall[len(fake.workerArtifactIDArgsForCall)] + fake.workerArtifactIDArgsForCall = append(fake.workerArtifactIDArgsForCall, struct { + }{}) + fake.recordInvocation("WorkerArtifactID", []interface{}{}) + fake.workerArtifactIDMutex.Unlock() + if fake.WorkerArtifactIDStub != nil { + return fake.WorkerArtifactIDStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.workerArtifactIDReturns + return fakeReturns.result1 +} + +func (fake *FakeCreatedVolume) WorkerArtifactIDCallCount() int { + fake.workerArtifactIDMutex.RLock() + defer fake.workerArtifactIDMutex.RUnlock() + return len(fake.workerArtifactIDArgsForCall) +} + +func (fake *FakeCreatedVolume) WorkerArtifactIDCalls(stub func() int) { + fake.workerArtifactIDMutex.Lock() + defer fake.workerArtifactIDMutex.Unlock() + fake.WorkerArtifactIDStub = stub +} + +func (fake *FakeCreatedVolume) WorkerArtifactIDReturns(result1 int) { + fake.workerArtifactIDMutex.Lock() + defer fake.workerArtifactIDMutex.Unlock() + fake.WorkerArtifactIDStub = nil + fake.workerArtifactIDReturns = struct { + result1 int + }{result1} +} + +func (fake *FakeCreatedVolume) WorkerArtifactIDReturnsOnCall(i int, result1 int) { + fake.workerArtifactIDMutex.Lock() + defer fake.workerArtifactIDMutex.Unlock() + fake.WorkerArtifactIDStub = nil + if fake.workerArtifactIDReturnsOnCall == nil { + fake.workerArtifactIDReturnsOnCall = make(map[int]struct { + result1 int + }) + } + fake.workerArtifactIDReturnsOnCall[i] = struct { + result1 int + }{result1} +} + func (fake *FakeCreatedVolume) WorkerName() string { fake.workerNameMutex.Lock() ret, specificReturn := fake.workerNameReturnsOnCall[len(fake.workerNameArgsForCall)] @@ -961,6 +1101,8 @@ func (fake *FakeCreatedVolume) Invocations() map[string][][]interface{} { defer fake.destroyingMutex.RUnlock() fake.handleMutex.RLock() defer fake.handleMutex.RUnlock() + fake.initializeArtifactMutex.RLock() + defer fake.initializeArtifactMutex.RUnlock() fake.initializeResourceCacheMutex.RLock() defer fake.initializeResourceCacheMutex.RUnlock() fake.initializeTaskCacheMutex.RLock() @@ -977,6 +1119,8 @@ func (fake *FakeCreatedVolume) Invocations() map[string][][]interface{} { defer fake.teamIDMutex.RUnlock() fake.typeMutex.RLock() defer fake.typeMutex.RUnlock() + fake.workerArtifactIDMutex.RLock() + defer fake.workerArtifactIDMutex.RUnlock() fake.workerNameMutex.RLock() defer fake.workerNameMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/atc/db/dbfakes/fake_team.go b/atc/db/dbfakes/fake_team.go index 22707cfdbde..6f1d96f593e 100644 --- a/atc/db/dbfakes/fake_team.go +++ b/atc/db/dbfakes/fake_team.go @@ -158,6 +158,21 @@ type FakeTeam struct { result2 bool result3 error } + FindVolumeForWorkerArtifactStub func(int) (db.CreatedVolume, bool, error) + findVolumeForWorkerArtifactMutex sync.RWMutex + findVolumeForWorkerArtifactArgsForCall []struct { + arg1 int + } + findVolumeForWorkerArtifactReturns struct { + result1 db.CreatedVolume + result2 bool + result3 error + } + findVolumeForWorkerArtifactReturnsOnCall map[int]struct { + result1 db.CreatedVolume + result2 bool + result3 error + } FindWorkerForContainerStub func(string) (db.Worker, bool, error) findWorkerForContainerMutex sync.RWMutex findWorkerForContainerArgsForCall []struct { @@ -173,6 +188,21 @@ type FakeTeam struct { result2 bool result3 error } + FindWorkerForVolumeStub func(string) (db.Worker, bool, error) + findWorkerForVolumeMutex sync.RWMutex + findWorkerForVolumeArgsForCall []struct { + arg1 string + } + findWorkerForVolumeReturns struct { + result1 db.Worker + result2 bool + result3 error + } + findWorkerForVolumeReturnsOnCall map[int]struct { + result1 db.Worker + result2 bool + result3 error + } IDStub func() int iDMutex sync.RWMutex iDArgsForCall []struct { @@ -1037,6 +1067,72 @@ func (fake *FakeTeam) FindCreatedContainerByHandleReturnsOnCall(i int, result1 d }{result1, result2, result3} } +func (fake *FakeTeam) FindVolumeForWorkerArtifact(arg1 int) (db.CreatedVolume, bool, error) { + fake.findVolumeForWorkerArtifactMutex.Lock() + ret, specificReturn := fake.findVolumeForWorkerArtifactReturnsOnCall[len(fake.findVolumeForWorkerArtifactArgsForCall)] + fake.findVolumeForWorkerArtifactArgsForCall = append(fake.findVolumeForWorkerArtifactArgsForCall, struct { + arg1 int + }{arg1}) + fake.recordInvocation("FindVolumeForWorkerArtifact", []interface{}{arg1}) + fake.findVolumeForWorkerArtifactMutex.Unlock() + if fake.FindVolumeForWorkerArtifactStub != nil { + return fake.FindVolumeForWorkerArtifactStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.findVolumeForWorkerArtifactReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *FakeTeam) FindVolumeForWorkerArtifactCallCount() int { + fake.findVolumeForWorkerArtifactMutex.RLock() + defer fake.findVolumeForWorkerArtifactMutex.RUnlock() + return len(fake.findVolumeForWorkerArtifactArgsForCall) +} + +func (fake *FakeTeam) FindVolumeForWorkerArtifactCalls(stub func(int) (db.CreatedVolume, bool, error)) { + fake.findVolumeForWorkerArtifactMutex.Lock() + defer fake.findVolumeForWorkerArtifactMutex.Unlock() + fake.FindVolumeForWorkerArtifactStub = stub +} + +func (fake *FakeTeam) FindVolumeForWorkerArtifactArgsForCall(i int) int { + fake.findVolumeForWorkerArtifactMutex.RLock() + defer fake.findVolumeForWorkerArtifactMutex.RUnlock() + argsForCall := fake.findVolumeForWorkerArtifactArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeTeam) FindVolumeForWorkerArtifactReturns(result1 db.CreatedVolume, result2 bool, result3 error) { + fake.findVolumeForWorkerArtifactMutex.Lock() + defer fake.findVolumeForWorkerArtifactMutex.Unlock() + fake.FindVolumeForWorkerArtifactStub = nil + fake.findVolumeForWorkerArtifactReturns = struct { + result1 db.CreatedVolume + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeTeam) FindVolumeForWorkerArtifactReturnsOnCall(i int, result1 db.CreatedVolume, result2 bool, result3 error) { + fake.findVolumeForWorkerArtifactMutex.Lock() + defer fake.findVolumeForWorkerArtifactMutex.Unlock() + fake.FindVolumeForWorkerArtifactStub = nil + if fake.findVolumeForWorkerArtifactReturnsOnCall == nil { + fake.findVolumeForWorkerArtifactReturnsOnCall = make(map[int]struct { + result1 db.CreatedVolume + result2 bool + result3 error + }) + } + fake.findVolumeForWorkerArtifactReturnsOnCall[i] = struct { + result1 db.CreatedVolume + result2 bool + result3 error + }{result1, result2, result3} +} + func (fake *FakeTeam) FindWorkerForContainer(arg1 string) (db.Worker, bool, error) { fake.findWorkerForContainerMutex.Lock() ret, specificReturn := fake.findWorkerForContainerReturnsOnCall[len(fake.findWorkerForContainerArgsForCall)] @@ -1103,6 +1199,72 @@ func (fake *FakeTeam) FindWorkerForContainerReturnsOnCall(i int, result1 db.Work }{result1, result2, result3} } +func (fake *FakeTeam) FindWorkerForVolume(arg1 string) (db.Worker, bool, error) { + fake.findWorkerForVolumeMutex.Lock() + ret, specificReturn := fake.findWorkerForVolumeReturnsOnCall[len(fake.findWorkerForVolumeArgsForCall)] + fake.findWorkerForVolumeArgsForCall = append(fake.findWorkerForVolumeArgsForCall, struct { + arg1 string + }{arg1}) + fake.recordInvocation("FindWorkerForVolume", []interface{}{arg1}) + fake.findWorkerForVolumeMutex.Unlock() + if fake.FindWorkerForVolumeStub != nil { + return fake.FindWorkerForVolumeStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.findWorkerForVolumeReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *FakeTeam) FindWorkerForVolumeCallCount() int { + fake.findWorkerForVolumeMutex.RLock() + defer fake.findWorkerForVolumeMutex.RUnlock() + return len(fake.findWorkerForVolumeArgsForCall) +} + +func (fake *FakeTeam) FindWorkerForVolumeCalls(stub func(string) (db.Worker, bool, error)) { + fake.findWorkerForVolumeMutex.Lock() + defer fake.findWorkerForVolumeMutex.Unlock() + fake.FindWorkerForVolumeStub = stub +} + +func (fake *FakeTeam) FindWorkerForVolumeArgsForCall(i int) string { + fake.findWorkerForVolumeMutex.RLock() + defer fake.findWorkerForVolumeMutex.RUnlock() + argsForCall := fake.findWorkerForVolumeArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeTeam) FindWorkerForVolumeReturns(result1 db.Worker, result2 bool, result3 error) { + fake.findWorkerForVolumeMutex.Lock() + defer fake.findWorkerForVolumeMutex.Unlock() + fake.FindWorkerForVolumeStub = nil + fake.findWorkerForVolumeReturns = struct { + result1 db.Worker + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeTeam) FindWorkerForVolumeReturnsOnCall(i int, result1 db.Worker, result2 bool, result3 error) { + fake.findWorkerForVolumeMutex.Lock() + defer fake.findWorkerForVolumeMutex.Unlock() + fake.FindWorkerForVolumeStub = nil + if fake.findWorkerForVolumeReturnsOnCall == nil { + fake.findWorkerForVolumeReturnsOnCall = make(map[int]struct { + result1 db.Worker + result2 bool + result3 error + }) + } + fake.findWorkerForVolumeReturnsOnCall[i] = struct { + result1 db.Worker + result2 bool + result3 error + }{result1, result2, result3} +} + func (fake *FakeTeam) ID() int { fake.iDMutex.Lock() ret, specificReturn := fake.iDReturnsOnCall[len(fake.iDArgsForCall)] @@ -2029,8 +2191,12 @@ func (fake *FakeTeam) Invocations() map[string][][]interface{} { defer fake.findContainersByMetadataMutex.RUnlock() fake.findCreatedContainerByHandleMutex.RLock() defer fake.findCreatedContainerByHandleMutex.RUnlock() + fake.findVolumeForWorkerArtifactMutex.RLock() + defer fake.findVolumeForWorkerArtifactMutex.RUnlock() fake.findWorkerForContainerMutex.RLock() defer fake.findWorkerForContainerMutex.RUnlock() + fake.findWorkerForVolumeMutex.RLock() + defer fake.findWorkerForVolumeMutex.RUnlock() fake.iDMutex.RLock() defer fake.iDMutex.RUnlock() fake.isCheckContainerMutex.RLock() diff --git a/atc/db/dbfakes/fake_volume_repository.go b/atc/db/dbfakes/fake_volume_repository.go index 5cbfc3b3f4c..043746cefe7 100644 --- a/atc/db/dbfakes/fake_volume_repository.go +++ b/atc/db/dbfakes/fake_volume_repository.go @@ -66,6 +66,21 @@ type FakeVolumeRepository struct { result1 db.CreatingVolume result2 error } + CreateVolumeStub func(int, string, db.VolumeType) (db.CreatingVolume, error) + createVolumeMutex sync.RWMutex + createVolumeArgsForCall []struct { + arg1 int + arg2 string + arg3 db.VolumeType + } + createVolumeReturns struct { + result1 db.CreatingVolume + result2 error + } + createVolumeReturnsOnCall map[int]struct { + result1 db.CreatingVolume + result2 error + } DestroyFailedVolumesStub func() (int, error) destroyFailedVolumesMutex sync.RWMutex destroyFailedVolumesArgsForCall []struct { @@ -525,6 +540,71 @@ func (fake *FakeVolumeRepository) CreateTaskCacheVolumeReturnsOnCall(i int, resu }{result1, result2} } +func (fake *FakeVolumeRepository) CreateVolume(arg1 int, arg2 string, arg3 db.VolumeType) (db.CreatingVolume, error) { + fake.createVolumeMutex.Lock() + ret, specificReturn := fake.createVolumeReturnsOnCall[len(fake.createVolumeArgsForCall)] + fake.createVolumeArgsForCall = append(fake.createVolumeArgsForCall, struct { + arg1 int + arg2 string + arg3 db.VolumeType + }{arg1, arg2, arg3}) + fake.recordInvocation("CreateVolume", []interface{}{arg1, arg2, arg3}) + fake.createVolumeMutex.Unlock() + if fake.CreateVolumeStub != nil { + return fake.CreateVolumeStub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.createVolumeReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeVolumeRepository) CreateVolumeCallCount() int { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + return len(fake.createVolumeArgsForCall) +} + +func (fake *FakeVolumeRepository) CreateVolumeCalls(stub func(int, string, db.VolumeType) (db.CreatingVolume, error)) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = stub +} + +func (fake *FakeVolumeRepository) CreateVolumeArgsForCall(i int) (int, string, db.VolumeType) { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + argsForCall := fake.createVolumeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeVolumeRepository) CreateVolumeReturns(result1 db.CreatingVolume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + fake.createVolumeReturns = struct { + result1 db.CreatingVolume + result2 error + }{result1, result2} +} + +func (fake *FakeVolumeRepository) CreateVolumeReturnsOnCall(i int, result1 db.CreatingVolume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + if fake.createVolumeReturnsOnCall == nil { + fake.createVolumeReturnsOnCall = make(map[int]struct { + result1 db.CreatingVolume + result2 error + }) + } + fake.createVolumeReturnsOnCall[i] = struct { + result1 db.CreatingVolume + result2 error + }{result1, result2} +} + func (fake *FakeVolumeRepository) DestroyFailedVolumes() (int, error) { fake.destroyFailedVolumesMutex.Lock() ret, specificReturn := fake.destroyFailedVolumesReturnsOnCall[len(fake.destroyFailedVolumesArgsForCall)] @@ -1435,6 +1515,8 @@ func (fake *FakeVolumeRepository) Invocations() map[string][][]interface{} { defer fake.createResourceCertsVolumeMutex.RUnlock() fake.createTaskCacheVolumeMutex.RLock() defer fake.createTaskCacheVolumeMutex.RUnlock() + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() fake.destroyFailedVolumesMutex.RLock() defer fake.destroyFailedVolumesMutex.RUnlock() fake.findBaseResourceTypeVolumeMutex.RLock() diff --git a/atc/db/dbfakes/fake_worker_artifact.go b/atc/db/dbfakes/fake_worker_artifact.go new file mode 100644 index 00000000000..251e5cceb4a --- /dev/null +++ b/atc/db/dbfakes/fake_worker_artifact.go @@ -0,0 +1,377 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package dbfakes + +import ( + sync "sync" + time "time" + + db "github.com/concourse/concourse/atc/db" +) + +type FakeWorkerArtifact struct { + ChecksumStub func() string + checksumMutex sync.RWMutex + checksumArgsForCall []struct { + } + checksumReturns struct { + result1 string + } + checksumReturnsOnCall map[int]struct { + result1 string + } + CreatedAtStub func() time.Time + createdAtMutex sync.RWMutex + createdAtArgsForCall []struct { + } + createdAtReturns struct { + result1 time.Time + } + createdAtReturnsOnCall map[int]struct { + result1 time.Time + } + IDStub func() int + iDMutex sync.RWMutex + iDArgsForCall []struct { + } + iDReturns struct { + result1 int + } + iDReturnsOnCall map[int]struct { + result1 int + } + PathStub func() string + pathMutex sync.RWMutex + pathArgsForCall []struct { + } + pathReturns struct { + result1 string + } + pathReturnsOnCall map[int]struct { + result1 string + } + VolumeStub func(int) (db.CreatedVolume, bool, error) + volumeMutex sync.RWMutex + volumeArgsForCall []struct { + arg1 int + } + volumeReturns struct { + result1 db.CreatedVolume + result2 bool + result3 error + } + volumeReturnsOnCall map[int]struct { + result1 db.CreatedVolume + result2 bool + result3 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeWorkerArtifact) Checksum() string { + fake.checksumMutex.Lock() + ret, specificReturn := fake.checksumReturnsOnCall[len(fake.checksumArgsForCall)] + fake.checksumArgsForCall = append(fake.checksumArgsForCall, struct { + }{}) + fake.recordInvocation("Checksum", []interface{}{}) + fake.checksumMutex.Unlock() + if fake.ChecksumStub != nil { + return fake.ChecksumStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.checksumReturns + return fakeReturns.result1 +} + +func (fake *FakeWorkerArtifact) ChecksumCallCount() int { + fake.checksumMutex.RLock() + defer fake.checksumMutex.RUnlock() + return len(fake.checksumArgsForCall) +} + +func (fake *FakeWorkerArtifact) ChecksumCalls(stub func() string) { + fake.checksumMutex.Lock() + defer fake.checksumMutex.Unlock() + fake.ChecksumStub = stub +} + +func (fake *FakeWorkerArtifact) ChecksumReturns(result1 string) { + fake.checksumMutex.Lock() + defer fake.checksumMutex.Unlock() + fake.ChecksumStub = nil + fake.checksumReturns = struct { + result1 string + }{result1} +} + +func (fake *FakeWorkerArtifact) ChecksumReturnsOnCall(i int, result1 string) { + fake.checksumMutex.Lock() + defer fake.checksumMutex.Unlock() + fake.ChecksumStub = nil + if fake.checksumReturnsOnCall == nil { + fake.checksumReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.checksumReturnsOnCall[i] = struct { + result1 string + }{result1} +} + +func (fake *FakeWorkerArtifact) CreatedAt() time.Time { + fake.createdAtMutex.Lock() + ret, specificReturn := fake.createdAtReturnsOnCall[len(fake.createdAtArgsForCall)] + fake.createdAtArgsForCall = append(fake.createdAtArgsForCall, struct { + }{}) + fake.recordInvocation("CreatedAt", []interface{}{}) + fake.createdAtMutex.Unlock() + if fake.CreatedAtStub != nil { + return fake.CreatedAtStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.createdAtReturns + return fakeReturns.result1 +} + +func (fake *FakeWorkerArtifact) CreatedAtCallCount() int { + fake.createdAtMutex.RLock() + defer fake.createdAtMutex.RUnlock() + return len(fake.createdAtArgsForCall) +} + +func (fake *FakeWorkerArtifact) CreatedAtCalls(stub func() time.Time) { + fake.createdAtMutex.Lock() + defer fake.createdAtMutex.Unlock() + fake.CreatedAtStub = stub +} + +func (fake *FakeWorkerArtifact) CreatedAtReturns(result1 time.Time) { + fake.createdAtMutex.Lock() + defer fake.createdAtMutex.Unlock() + fake.CreatedAtStub = nil + fake.createdAtReturns = struct { + result1 time.Time + }{result1} +} + +func (fake *FakeWorkerArtifact) CreatedAtReturnsOnCall(i int, result1 time.Time) { + fake.createdAtMutex.Lock() + defer fake.createdAtMutex.Unlock() + fake.CreatedAtStub = nil + if fake.createdAtReturnsOnCall == nil { + fake.createdAtReturnsOnCall = make(map[int]struct { + result1 time.Time + }) + } + fake.createdAtReturnsOnCall[i] = struct { + result1 time.Time + }{result1} +} + +func (fake *FakeWorkerArtifact) ID() int { + fake.iDMutex.Lock() + ret, specificReturn := fake.iDReturnsOnCall[len(fake.iDArgsForCall)] + fake.iDArgsForCall = append(fake.iDArgsForCall, struct { + }{}) + fake.recordInvocation("ID", []interface{}{}) + fake.iDMutex.Unlock() + if fake.IDStub != nil { + return fake.IDStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.iDReturns + return fakeReturns.result1 +} + +func (fake *FakeWorkerArtifact) IDCallCount() int { + fake.iDMutex.RLock() + defer fake.iDMutex.RUnlock() + return len(fake.iDArgsForCall) +} + +func (fake *FakeWorkerArtifact) IDCalls(stub func() int) { + fake.iDMutex.Lock() + defer fake.iDMutex.Unlock() + fake.IDStub = stub +} + +func (fake *FakeWorkerArtifact) IDReturns(result1 int) { + fake.iDMutex.Lock() + defer fake.iDMutex.Unlock() + fake.IDStub = nil + fake.iDReturns = struct { + result1 int + }{result1} +} + +func (fake *FakeWorkerArtifact) IDReturnsOnCall(i int, result1 int) { + fake.iDMutex.Lock() + defer fake.iDMutex.Unlock() + fake.IDStub = nil + if fake.iDReturnsOnCall == nil { + fake.iDReturnsOnCall = make(map[int]struct { + result1 int + }) + } + fake.iDReturnsOnCall[i] = struct { + result1 int + }{result1} +} + +func (fake *FakeWorkerArtifact) Path() string { + fake.pathMutex.Lock() + ret, specificReturn := fake.pathReturnsOnCall[len(fake.pathArgsForCall)] + fake.pathArgsForCall = append(fake.pathArgsForCall, struct { + }{}) + fake.recordInvocation("Path", []interface{}{}) + fake.pathMutex.Unlock() + if fake.PathStub != nil { + return fake.PathStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.pathReturns + return fakeReturns.result1 +} + +func (fake *FakeWorkerArtifact) PathCallCount() int { + fake.pathMutex.RLock() + defer fake.pathMutex.RUnlock() + return len(fake.pathArgsForCall) +} + +func (fake *FakeWorkerArtifact) PathCalls(stub func() string) { + fake.pathMutex.Lock() + defer fake.pathMutex.Unlock() + fake.PathStub = stub +} + +func (fake *FakeWorkerArtifact) PathReturns(result1 string) { + fake.pathMutex.Lock() + defer fake.pathMutex.Unlock() + fake.PathStub = nil + fake.pathReturns = struct { + result1 string + }{result1} +} + +func (fake *FakeWorkerArtifact) PathReturnsOnCall(i int, result1 string) { + fake.pathMutex.Lock() + defer fake.pathMutex.Unlock() + fake.PathStub = nil + if fake.pathReturnsOnCall == nil { + fake.pathReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.pathReturnsOnCall[i] = struct { + result1 string + }{result1} +} + +func (fake *FakeWorkerArtifact) Volume(arg1 int) (db.CreatedVolume, bool, error) { + fake.volumeMutex.Lock() + ret, specificReturn := fake.volumeReturnsOnCall[len(fake.volumeArgsForCall)] + fake.volumeArgsForCall = append(fake.volumeArgsForCall, struct { + arg1 int + }{arg1}) + fake.recordInvocation("Volume", []interface{}{arg1}) + fake.volumeMutex.Unlock() + if fake.VolumeStub != nil { + return fake.VolumeStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.volumeReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *FakeWorkerArtifact) VolumeCallCount() int { + fake.volumeMutex.RLock() + defer fake.volumeMutex.RUnlock() + return len(fake.volumeArgsForCall) +} + +func (fake *FakeWorkerArtifact) VolumeCalls(stub func(int) (db.CreatedVolume, bool, error)) { + fake.volumeMutex.Lock() + defer fake.volumeMutex.Unlock() + fake.VolumeStub = stub +} + +func (fake *FakeWorkerArtifact) VolumeArgsForCall(i int) int { + fake.volumeMutex.RLock() + defer fake.volumeMutex.RUnlock() + argsForCall := fake.volumeArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeWorkerArtifact) VolumeReturns(result1 db.CreatedVolume, result2 bool, result3 error) { + fake.volumeMutex.Lock() + defer fake.volumeMutex.Unlock() + fake.VolumeStub = nil + fake.volumeReturns = struct { + result1 db.CreatedVolume + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeWorkerArtifact) VolumeReturnsOnCall(i int, result1 db.CreatedVolume, result2 bool, result3 error) { + fake.volumeMutex.Lock() + defer fake.volumeMutex.Unlock() + fake.VolumeStub = nil + if fake.volumeReturnsOnCall == nil { + fake.volumeReturnsOnCall = make(map[int]struct { + result1 db.CreatedVolume + result2 bool + result3 error + }) + } + fake.volumeReturnsOnCall[i] = struct { + result1 db.CreatedVolume + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeWorkerArtifact) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.checksumMutex.RLock() + defer fake.checksumMutex.RUnlock() + fake.createdAtMutex.RLock() + defer fake.createdAtMutex.RUnlock() + fake.iDMutex.RLock() + defer fake.iDMutex.RUnlock() + fake.pathMutex.RLock() + defer fake.pathMutex.RUnlock() + fake.volumeMutex.RLock() + defer fake.volumeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeWorkerArtifact) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ db.WorkerArtifact = new(FakeWorkerArtifact) diff --git a/atc/db/migration/migrations/1545249372_create_worker_artifacts.down.sql b/atc/db/migration/migrations/1545249372_create_worker_artifacts.down.sql new file mode 100755 index 00000000000..43b160d41b9 --- /dev/null +++ b/atc/db/migration/migrations/1545249372_create_worker_artifacts.down.sql @@ -0,0 +1,9 @@ +BEGIN; + ALTER TABLE volumes + DROP CONSTRAINT volumes_worker_artifact_id_fkey; + + ALTER TABLE volumes + DROP COLUMN worker_artifact_id; + + DROP TABLE worker_artifacts; +COMMIT; diff --git a/atc/db/migration/migrations/1545249372_create_worker_artifacts.up.sql b/atc/db/migration/migrations/1545249372_create_worker_artifacts.up.sql new file mode 100755 index 00000000000..a991c19ec2e --- /dev/null +++ b/atc/db/migration/migrations/1545249372_create_worker_artifacts.up.sql @@ -0,0 +1,15 @@ +BEGIN; + CREATE TABLE worker_artifacts ( + id SERIAL PRIMARY KEY, + path TEXT NOT NULL, + checksum TEXT NOT NULL, + created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL + ); + + ALTER TABLE volumes + ADD COLUMN worker_artifact_id integer; + + ALTER TABLE volumes + ADD CONSTRAINT volumes_worker_artifact_id_fkey FOREIGN KEY (worker_artifact_id) REFERENCES worker_artifacts(id) ON DELETE SET NULL; + +COMMIT; diff --git a/atc/db/team.go b/atc/db/team.go index 68b7dd6ad1b..40788d4c9f2 100644 --- a/atc/db/team.go +++ b/atc/db/team.go @@ -49,15 +49,18 @@ type Team interface { SaveWorker(atcWorker atc.Worker, ttl time.Duration) (Worker, error) Workers() ([]Worker, error) + FindVolumeForWorkerArtifact(int) (CreatedVolume, bool, error) Containers(lager.Logger) ([]Container, error) IsCheckContainer(string) (bool, error) IsContainerWithinTeam(string, bool) (bool, error) + FindContainerByHandle(string) (Container, bool, error) FindCheckContainers(lager.Logger, string, string, creds.VariablesFactory) ([]Container, map[int]time.Time, error) FindContainersByMetadata(ContainerMetadata) ([]Container, error) FindCreatedContainerByHandle(string) (CreatedContainer, bool, error) FindWorkerForContainer(handle string) (Worker, bool, error) + FindWorkerForVolume(handle string) (Worker, bool, error) UpdateProviderAuth(auth atc.TeamAuth) error } @@ -109,12 +112,43 @@ func (t *team) Workers() ([]Worker, error) { })) } +func (t *team) FindVolumeForWorkerArtifact(artifactID int) (CreatedVolume, bool, error) { + tx, err := t.conn.Begin() + if err != nil { + return nil, false, err + } + + defer Rollback(tx) + + artifact, found, err := getWorkerArtifact(tx, t.conn, artifactID) + if err != nil { + return nil, false, err + } + + err = tx.Commit() + if err != nil { + return nil, false, err + } + + if !found { + return nil, false, nil + } + + return artifact.Volume(t.ID()) +} + func (t *team) FindWorkerForContainer(handle string) (Worker, bool, error) { return getWorker(t.conn, workersQuery.Join("containers c ON c.worker_name = w.name").Where(sq.And{ sq.Eq{"c.handle": handle}, })) } +func (t *team) FindWorkerForVolume(handle string) (Worker, bool, error) { + return getWorker(t.conn, workersQuery.Join("volumes v ON v.worker_name = w.name").Where(sq.And{ + sq.Eq{"v.handle": handle}, + })) +} + func (t *team) Containers( logger lager.Logger, ) ([]Container, error) { diff --git a/atc/db/team_test.go b/atc/db/team_test.go index 93e49b1d550..32328ab5394 100644 --- a/atc/db/team_test.go +++ b/atc/db/team_test.go @@ -11,7 +11,6 @@ import ( "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/creds/credsfakes" "github.com/concourse/concourse/atc/db" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -693,6 +692,47 @@ var _ = Describe("Team", func() { }) }) + Describe("FindVolumeForWorkerArtifact", func() { + + Context("when the artifact doesn't exist", func() { + It("returns not found", func() { + _, found, err := defaultTeam.FindVolumeForWorkerArtifact(12) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeFalse()) + }) + }) + + Context("when the artifact exists", func() { + BeforeEach(func() { + _, err := dbConn.Exec("INSERT INTO worker_artifacts (id, path, checksum) VALUES ($1, '/', '')", 18) + Expect(err).NotTo(HaveOccurred()) + }) + + Context("when the associated volume doesn't exist", func() { + It("returns not found", func() { + _, found, err := defaultTeam.FindVolumeForWorkerArtifact(18) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeFalse()) + }) + }) + + Context("when the associated volume exists", func() { + BeforeEach(func() { + _, err := dbConn.Exec("INSERT INTO volumes (handle, team_id, worker_name, worker_artifact_id, state) VALUES ('some-handle', $1, $2, $3, $4)", defaultTeam.ID(), defaultWorker.Name(), 18, db.VolumeStateCreated) + Expect(err).NotTo(HaveOccurred()) + }) + + It("returns the volume", func() { + volume, found, err := defaultTeam.FindVolumeForWorkerArtifact(18) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(volume.Handle()).To(Equal("some-handle")) + Expect(volume.WorkerArtifactID()).To(Equal(18)) + }) + }) + }) + }) + Describe("FindWorkerForContainer", func() { var containerMetadata db.ContainerMetadata var defaultBuild db.Build diff --git a/atc/db/volume.go b/atc/db/volume.go index 63fb4beb7e5..93db3b779c1 100644 --- a/atc/db/volume.go +++ b/atc/db/volume.go @@ -7,10 +7,9 @@ import ( "fmt" sq "github.com/Masterminds/squirrel" + "github.com/concourse/concourse/atc" "github.com/lib/pq" uuid "github.com/nu7hatch/gouuid" - - "github.com/concourse/concourse/atc" ) var ( @@ -53,6 +52,7 @@ const ( VolumeTypeResourceType VolumeType = "resource-type" VolumeTypeResourceCerts VolumeType = "resource-certs" VolumeTypeTaskCache VolumeType = "task-cache" + VolumeTypeArtifact VolumeType = "artifact" VolumeTypeUknown VolumeType = "unknown" // for migration to life ) @@ -78,6 +78,7 @@ type creatingVolume struct { workerBaseResourceTypeID int workerTaskCacheID int workerResourceCertsID int + workerArtifactID int conn Conn } @@ -145,10 +146,12 @@ type CreatedVolume interface { Path() string Type() VolumeType TeamID() int + WorkerArtifactID() int CreateChildForContainer(CreatingContainer, string) (CreatingVolume, error) Destroying() (DestroyingVolume, error) WorkerName() string InitializeResourceCache(UsedResourceCache) error + InitializeArtifact(string, string) (WorkerArtifact, error) InitializeTaskCache(int, string, string) error ContainerHandle() string ParentHandle() string @@ -170,6 +173,7 @@ type createdVolume struct { workerBaseResourceTypeID int workerTaskCacheID int workerResourceCertsID int + workerArtifactID int conn Conn } @@ -186,6 +190,7 @@ func (volume *createdVolume) Type() VolumeType { return volume.typ } func (volume *createdVolume) TeamID() int { return volume.teamID } func (volume *createdVolume) ContainerHandle() string { return volume.containerHandle } func (volume *createdVolume) ParentHandle() string { return volume.parentHandle } +func (volume *createdVolume) WorkerArtifactID() int { return volume.workerArtifactID } func (volume *createdVolume) ResourceType() (*VolumeResourceType, error) { if volume.resourceCacheID == 0 { @@ -385,6 +390,51 @@ func (volume *createdVolume) InitializeResourceCache(resourceCache UsedResourceC return nil } +func (volume *createdVolume) InitializeArtifact(path string, checksum string) (WorkerArtifact, error) { + + tx, err := volume.conn.Begin() + if err != nil { + return nil, err + } + + defer Rollback(tx) + + atcWorkerArtifact := atc.WorkerArtifact{ + Checksum: checksum, + Path: path, + } + + workerArtifact, err := saveWorkerArtifact(tx, volume.conn, atcWorkerArtifact) + if err != nil { + return nil, err + } + + rows, err := psql.Update("volumes"). + Set("worker_artifact_id", workerArtifact.ID()). + Where(sq.Eq{"id": volume.id}). + RunWith(tx). + Exec() + if err != nil { + return nil, err + } + + affected, err := rows.RowsAffected() + if err != nil { + return nil, err + } + + if affected == 0 { + return nil, ErrVolumeMissing + } + + err = tx.Commit() + if err != nil { + return nil, err + } + + return workerArtifact, nil +} + func (volume *createdVolume) InitializeTaskCache(jobID int, stepName string, path string) error { tx, err := volume.conn.Begin() if err != nil { diff --git a/atc/db/volume_repository.go b/atc/db/volume_repository.go index ec59727641b..7547214e50e 100644 --- a/atc/db/volume_repository.go +++ b/atc/db/volume_repository.go @@ -17,8 +17,8 @@ type VolumeRepository interface { CreateContainerVolume(teamID int, workerName string, container CreatingContainer, mountPath string) (CreatingVolume, error) FindContainerVolume(teamID int, workerName string, container CreatingContainer, mountPath string) (CreatingVolume, CreatedVolume, error) - FindBaseResourceTypeVolume(usedWorkerBaseResourceType *UsedWorkerBaseResourceType) (CreatingVolume, CreatedVolume, error) - CreateBaseResourceTypeVolume(usedWorkerBaseResourceType *UsedWorkerBaseResourceType) (CreatingVolume, error) + FindBaseResourceTypeVolume(*UsedWorkerBaseResourceType) (CreatingVolume, CreatedVolume, error) + CreateBaseResourceTypeVolume(*UsedWorkerBaseResourceType) (CreatingVolume, error) FindResourceCacheVolume(workerName string, resourceCache UsedResourceCache) (CreatedVolume, bool, error) @@ -35,6 +35,7 @@ type VolumeRepository interface { GetDestroyingVolumes(workerName string) ([]string, error) + CreateVolume(int, string, VolumeType) (CreatingVolume, error) FindCreatedVolume(handle string) (CreatedVolume, bool, error) RemoveDestroyingVolumes(workerName string, handles []string) (int, error) @@ -254,6 +255,22 @@ func (repository *volumeRepository) CreateBaseResourceTypeVolume(uwbrt *UsedWork return volume, nil } +func (repository *volumeRepository) CreateVolume(teamID int, workerName string, volumeType VolumeType) (CreatingVolume, error) { + volume, err := repository.createVolume( + 0, + workerName, + map[string]interface{}{ + "team_id": teamID, + }, + volumeType, + ) + if err != nil { + return nil, err + } + + return volume, nil +} + func (repository *volumeRepository) CreateContainerVolume(teamID int, workerName string, container CreatingContainer, mountPath string) (CreatingVolume, error) { volume, err := repository.createVolume( teamID, @@ -395,7 +412,7 @@ func (repository *volumeRepository) FindResourceCacheVolume(workerName string, r } func (repository *volumeRepository) FindCreatedVolume(handle string) (CreatedVolume, bool, error) { - _, createdVolume, err := repository.findVolume(0, "", map[string]interface{}{ + _, createdVolume, err := getVolume(repository.conn, map[string]interface{}{ "v.handle": handle, }) if err != nil { @@ -423,6 +440,7 @@ func (repository *volumeRepository) GetOrphanedVolumes() ([]CreatedVolume, error "v.container_id": nil, "v.worker_task_cache_id": nil, "v.worker_resource_certs_id": nil, + "v.worker_artifact_id": nil, }, ). Where(sq.Eq{"v.state": string(VolumeStateCreated)}). @@ -566,16 +584,21 @@ func (repository *volumeRepository) findVolume(teamID int, workerName string, co whereClause[name] = value } + return getVolume(repository.conn, whereClause) +} + +func getVolume(conn Conn, where map[string]interface{}) (CreatingVolume, CreatedVolume, error) { row := psql.Select(volumeColumns...). From("volumes v"). LeftJoin("workers w ON v.worker_name = w.name"). LeftJoin("containers c ON v.container_id = c.id"). LeftJoin("volumes pv ON v.parent_id = pv.id"). LeftJoin("worker_resource_caches wrc ON wrc.id = v.worker_resource_cache_id"). - Where(whereClause). - RunWith(repository.conn). + Where(where). + RunWith(conn). QueryRow() - creatingVolume, createdVolume, _, _, err := scanVolume(row, repository.conn) + + creatingVolume, createdVolume, _, _, err := scanVolume(row, conn) if err != nil { if err == sql.ErrNoRows { return nil, nil, nil @@ -599,12 +622,14 @@ var volumeColumns = []string{ "v.worker_base_resource_type_id", "v.worker_task_cache_id", "v.worker_resource_certs_id", + "v.worker_artifact_id", `case when v.worker_base_resource_type_id is not NULL then 'resource-type' when v.worker_resource_cache_id is not NULL then 'resource' when v.container_id is not NULL then 'container' when v.worker_task_cache_id is not NULL then 'task-cache' when v.worker_resource_certs_id is not NULL then 'resource-certs' + when v.worker_artifact_id is not NULL then 'artifact' else 'unknown' end`, } @@ -622,6 +647,7 @@ func scanVolume(row sq.RowScanner, conn Conn) (CreatingVolume, CreatedVolume, De var sqWorkerBaseResourceTypeID sql.NullInt64 var sqWorkerTaskCacheID sql.NullInt64 var sqWorkerResourceCertsID sql.NullInt64 + var sqWorkerArtifactID sql.NullInt64 var volumeType VolumeType err := row.Scan( @@ -637,6 +663,7 @@ func scanVolume(row sq.RowScanner, conn Conn) (CreatingVolume, CreatedVolume, De &sqWorkerBaseResourceTypeID, &sqWorkerTaskCacheID, &sqWorkerResourceCertsID, + &sqWorkerArtifactID, &volumeType, ) if err != nil { @@ -683,6 +710,11 @@ func scanVolume(row sq.RowScanner, conn Conn) (CreatingVolume, CreatedVolume, De workerResourceCertsID = int(sqWorkerResourceCertsID.Int64) } + var workerArtifactID int + if sqWorkerArtifactID.Valid { + workerArtifactID = int(sqWorkerArtifactID.Int64) + } + switch VolumeState(state) { case VolumeStateCreated: return nil, &createdVolume{ @@ -698,6 +730,7 @@ func scanVolume(row sq.RowScanner, conn Conn) (CreatingVolume, CreatedVolume, De workerBaseResourceTypeID: workerBaseResourceTypeID, workerTaskCacheID: workerTaskCacheID, workerResourceCertsID: workerResourceCertsID, + workerArtifactID: workerArtifactID, conn: conn, }, nil, nil, nil case VolumeStateCreating: @@ -714,6 +747,7 @@ func scanVolume(row sq.RowScanner, conn Conn) (CreatingVolume, CreatedVolume, De workerBaseResourceTypeID: workerBaseResourceTypeID, workerTaskCacheID: workerTaskCacheID, workerResourceCertsID: workerResourceCertsID, + workerArtifactID: workerArtifactID, conn: conn, }, nil, nil, nil, nil case VolumeStateDestroying: diff --git a/atc/db/volume_repository_test.go b/atc/db/volume_repository_test.go index dbc69dbd523..b05bf10df97 100644 --- a/atc/db/volume_repository_test.go +++ b/atc/db/volume_repository_test.go @@ -208,6 +208,16 @@ var _ = Describe("VolumeFactory", func() { resourceCacheVolumeCreated, err := resourceCacheVolume.Created() Expect(err).NotTo(HaveOccurred()) + err = resourceCacheVolumeCreated.InitializeResourceCache(usedResourceCache) + Expect(err).NotTo(HaveOccurred()) + + artifactVolume, err := volumeRepository.CreateVolume(defaultTeam.ID(), defaultWorker.Name(), db.VolumeTypeArtifact) + Expect(err).NotTo(HaveOccurred()) + expectedCreatedHandles = append(expectedCreatedHandles, artifactVolume.Handle()) + + _, err = artifactVolume.Created() + Expect(err).NotTo(HaveOccurred()) + usedWorkerBaseResourceType, found, err := workerBaseResourceTypeFactory.Find(defaultWorkerResourceType.Type, defaultWorker) Expect(err).ToNot(HaveOccurred()) Expect(found).To(BeTrue()) @@ -228,9 +238,6 @@ var _ = Describe("VolumeFactory", func() { defaultWorker, err = workerFactory.SaveWorker(newWorker, 0) Expect(err).ToNot(HaveOccurred()) - err = resourceCacheVolumeCreated.InitializeResourceCache(usedResourceCache) - Expect(err).NotTo(HaveOccurred()) - tx, err := dbConn.Begin() Expect(err).NotTo(HaveOccurred()) workerResourceCerts, err := db.WorkerResourceCerts{ @@ -408,6 +415,20 @@ var _ = Describe("VolumeFactory", func() { }) }) + Describe("CreateVolume", func() { + It("creates a CreatingVolume of the given type with a teamID", func() { + volume, err := volumeRepository.CreateVolume(defaultTeam.ID(), defaultWorker.Name(), db.VolumeTypeArtifact) + Expect(err).NotTo(HaveOccurred()) + var teamID int + var workerName string + err = psql.Select("team_id, worker_name").From("volumes"). + Where(sq.Eq{"handle": volume.Handle()}).RunWith(dbConn).QueryRow().Scan(&teamID, &workerName) + Expect(err).NotTo(HaveOccurred()) + Expect(teamID).To(Equal(defaultTeam.ID())) + Expect(workerName).To(Equal(defaultWorker.Name())) + }) + }) + Describe("FindBaseResourceTypeVolume", func() { var usedWorkerBaseResourceType *db.UsedWorkerBaseResourceType BeforeEach(func() { diff --git a/atc/db/volume_test.go b/atc/db/volume_test.go index 23e7886051c..c5d84a532f0 100644 --- a/atc/db/volume_test.go +++ b/atc/db/volume_test.go @@ -260,6 +260,42 @@ var _ = Describe("Volume", func() { }) }) + Describe("createdVolume.InitializeArtifact", func() { + var ( + workerArtifact db.WorkerArtifact + creatingVolume db.CreatingVolume + createdVolume db.CreatedVolume + err error + ) + + BeforeEach(func() { + creatingVolume, err = volumeRepository.CreateVolume(defaultTeam.ID(), defaultWorker.Name(), db.VolumeTypeArtifact) + Expect(err).ToNot(HaveOccurred()) + + createdVolume, err = creatingVolume.Created() + Expect(err).ToNot(HaveOccurred()) + }) + + JustBeforeEach(func() { + workerArtifact, err = createdVolume.InitializeArtifact("/some/path", "some-checksum") + Expect(err).ToNot(HaveOccurred()) + }) + + It("initialize worker artifact", func() { + Expect(workerArtifact.ID()).To(Equal(1)) + Expect(workerArtifact.Path()).To(Equal("/some/path")) + Expect(workerArtifact.Checksum()).To(Equal("some-checksum")) + Expect(workerArtifact.CreatedAt()).ToNot(BeNil()) + }) + + It("associates worker artifact with the volume", func() { + created, found, err := volumeRepository.FindCreatedVolume(createdVolume.Handle()) + Expect(err).ToNot(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(created.WorkerArtifactID()).To(Equal(workerArtifact.ID())) + }) + }) + Describe("createdVolume.InitializeTaskCache", func() { Context("when there is a volume that belongs to worker task cache", func() { var ( diff --git a/atc/db/worker_artifact.go b/atc/db/worker_artifact.go new file mode 100644 index 00000000000..95f2c76ed01 --- /dev/null +++ b/atc/db/worker_artifact.go @@ -0,0 +1,109 @@ +package db + +import ( + "database/sql" + "errors" + "time" + + sq "github.com/Masterminds/squirrel" + "github.com/concourse/concourse/atc" + "github.com/lib/pq" +) + +//go:generate counterfeiter . WorkerArtifact + +type WorkerArtifact interface { + ID() int + Path() string + Checksum() string + CreatedAt() time.Time + Volume(int) (CreatedVolume, bool, error) +} + +type artifact struct { + conn Conn + + id int + path string + checksum string + createdAt time.Time +} + +func (a *artifact) ID() int { return a.id } +func (a *artifact) Path() string { return a.path } +func (a *artifact) Checksum() string { return a.checksum } +func (a *artifact) CreatedAt() time.Time { return a.createdAt } + +func (a *artifact) Volume(teamID int) (CreatedVolume, bool, error) { + where := map[string]interface{}{ + "v.team_id": teamID, + "v.worker_artifact_id": a.id, + } + + _, created, err := getVolume(a.conn, where) + if err != nil { + return nil, false, err + } + + if created == nil { + return nil, false, nil + } + + return created, true, nil +} + +func saveWorkerArtifact(tx Tx, conn Conn, atcArtifact atc.WorkerArtifact) (WorkerArtifact, error) { + + var artifactID int + + err := psql.Insert("worker_artifacts"). + SetMap(map[string]interface{}{ + "path": atcArtifact.Path, + "checksum": atcArtifact.Checksum, + }). + Suffix("RETURNING id"). + RunWith(tx). + QueryRow(). + Scan(&artifactID) + + if err != nil { + return nil, err + } + + artifact, found, err := getWorkerArtifact(tx, conn, artifactID) + + if err != nil { + return nil, err + } + + if !found { + return nil, errors.New("Not found") + } + + return artifact, nil +} + +func getWorkerArtifact(tx Tx, conn Conn, id int) (WorkerArtifact, bool, error) { + var createdAtTime pq.NullTime + + artifact := &artifact{conn: conn} + + err := psql.Select("id", "created_at", "path", "checksum"). + From("worker_artifacts"). + Where(sq.Eq{ + "id": id, + }). + RunWith(tx). + QueryRow(). + Scan(&artifact.id, &createdAtTime, &artifact.path, &artifact.checksum) + if err != nil { + if err == sql.ErrNoRows { + return nil, false, nil + } + + return nil, false, err + } + + artifact.createdAt = createdAtTime.Time + return artifact, true, nil +} diff --git a/atc/exec/get_step.go b/atc/exec/get_step.go index 42938b7b7e5..c590ac20dde 100644 --- a/atc/exec/get_step.go +++ b/atc/exec/get_step.go @@ -205,7 +205,7 @@ func (step *GetStep) Run(ctx context.Context, state RunState) error { ResourceTypes: step.resourceTypes, } - chosenWorker, err := step.workerPool.FindOrChooseWorker(logger, resourceInstance.ContainerOwner(), containerSpec, workerSpec, step.strategy) + chosenWorker, err := step.workerPool.FindOrChooseWorkerForContainer(logger, resourceInstance.ContainerOwner(), containerSpec, workerSpec, step.strategy) if err != nil { return err } diff --git a/atc/exec/get_step_test.go b/atc/exec/get_step_test.go index a7cea5b3a35..58608033fa8 100644 --- a/atc/exec/get_step_test.go +++ b/atc/exec/get_step_test.go @@ -146,8 +146,8 @@ var _ = Describe("GetStep", func() { }) It("finds or chooses a worker", func() { - Expect(fakePool.FindOrChooseWorkerCallCount()).To(Equal(1)) - _, actualOwner, actualContainerSpec, actualWorkerSpec, strategy := fakePool.FindOrChooseWorkerArgsForCall(0) + Expect(fakePool.FindOrChooseWorkerForContainerCallCount()).To(Equal(1)) + _, actualOwner, actualContainerSpec, actualWorkerSpec, strategy := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(actualOwner).To(Equal(db.NewBuildStepContainerOwner(buildID, atc.PlanID(planID), teamID))) Expect(actualContainerSpec).To(Equal(worker.ContainerSpec{ ImageSpec: worker.ImageSpec{ @@ -168,7 +168,7 @@ var _ = Describe("GetStep", func() { Context("when find or choosing worker succeeds", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) }) It("initializes the resource with the correct type and session id, making sure that it is not ephemeral", func() { @@ -566,7 +566,7 @@ var _ = Describe("GetStep", func() { disaster := errors.New("oh no") BeforeEach(func() { - fakePool.FindOrChooseWorkerReturns(nil, disaster) + fakePool.FindOrChooseWorkerForContainerReturns(nil, disaster) }) It("does not finish the step via the delegate", func() { diff --git a/atc/exec/put_step.go b/atc/exec/put_step.go index 94dbe3f37a3..3d5ea92144d 100644 --- a/atc/exec/put_step.go +++ b/atc/exec/put_step.go @@ -128,7 +128,7 @@ func (step *PutStep) Run(ctx context.Context, state RunState) error { } owner := db.NewBuildStepContainerOwner(step.build.ID(), step.planID, step.build.TeamID()) - chosenWorker, err := step.pool.FindOrChooseWorker(logger, owner, containerSpec, workerSpec, step.strategy) + chosenWorker, err := step.pool.FindOrChooseWorkerForContainer(logger, owner, containerSpec, workerSpec, step.strategy) if err != nil { return err } diff --git a/atc/exec/put_step_test.go b/atc/exec/put_step_test.go index 1a5e22a6ef9..6af4f1bef56 100644 --- a/atc/exec/put_step_test.go +++ b/atc/exec/put_step_test.go @@ -170,7 +170,7 @@ var _ = Describe("PutStep", func() { fakeVersionedSource.MetadataReturns([]atc.MetadataField{{"some", "metadata"}}) fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeResource = new(resourcefakes.FakeResource) fakeResource.PutReturns(fakeVersionedSource, nil) @@ -178,8 +178,8 @@ var _ = Describe("PutStep", func() { }) It("finds/chooses a worker and creates a container with the correct type, session, and sources with no inputs specified (meaning it takes all artifacts)", func() { - Expect(fakePool.FindOrChooseWorkerCallCount()).To(Equal(1)) - _, actualOwner, actualContainerSpec, actualWorkerSpec, strategy := fakePool.FindOrChooseWorkerArgsForCall(0) + Expect(fakePool.FindOrChooseWorkerForContainerCallCount()).To(Equal(1)) + _, actualOwner, actualContainerSpec, actualWorkerSpec, strategy := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(actualOwner).To(Equal(db.NewBuildStepContainerOwner(42, atc.PlanID(planID), 123))) Expect(actualContainerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ResourceType: "some-resource-type", @@ -381,7 +381,7 @@ var _ = Describe("PutStep", func() { disaster := errors.New("nope") BeforeEach(func() { - fakePool.FindOrChooseWorkerReturns(nil, disaster) + fakePool.FindOrChooseWorkerForContainerReturns(nil, disaster) }) It("returns the failure", func() { @@ -393,7 +393,7 @@ var _ = Describe("PutStep", func() { disaster := errors.New("nope") BeforeEach(func() { - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeWorker.FindOrCreateContainerReturns(nil, disaster) }) diff --git a/atc/exec/task_step.go b/atc/exec/task_step.go index 7871c6d851b..6f411c52c5e 100644 --- a/atc/exec/task_step.go +++ b/atc/exec/task_step.go @@ -187,7 +187,7 @@ func (action *TaskStep) Run(ctx context.Context, state RunState) error { } owner := db.NewBuildStepContainerOwner(action.buildID, action.planID, action.teamID) - chosenWorker, err := action.workerPool.FindOrChooseWorker(logger, owner, containerSpec, workerSpec, action.strategy) + chosenWorker, err := action.workerPool.FindOrChooseWorkerForContainer(logger, owner, containerSpec, workerSpec, action.strategy) if err != nil { return err } diff --git a/atc/exec/task_step_test.go b/atc/exec/task_step_test.go index 69132f115fd..d04e93179e7 100644 --- a/atc/exec/task_step_test.go +++ b/atc/exec/task_step_test.go @@ -173,15 +173,15 @@ var _ = Describe("TaskStep", func() { Context("when the worker is either found or chosen", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeContainer := new(workerfakes.FakeContainer) fakeWorker.FindOrCreateContainerReturns(fakeContainer, nil) }) It("finds or chooses a worker", func() { - Expect(fakePool.FindOrChooseWorkerCallCount()).To(Equal(1)) - _, owner, containerSpec, workerSpec, strategy := fakePool.FindOrChooseWorkerArgsForCall(0) + Expect(fakePool.FindOrChooseWorkerForContainerCallCount()).To(Equal(1)) + _, owner, containerSpec, workerSpec, strategy := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(owner).To(Equal(db.NewBuildStepContainerOwner(buildID, planID, teamID))) cpu := uint64(1024) @@ -1193,7 +1193,7 @@ var _ = Describe("TaskStep", func() { }) It("chooses a worker and creates the container with the image artifact source", func() { - _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, ImageArtifactName: worker.ArtifactName(imageArtifactName), @@ -1230,7 +1230,7 @@ var _ = Describe("TaskStep", func() { }) It("still chooses a worker and creates the container with the volume and a metadata stream", func() { - _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, ImageArtifactName: worker.ArtifactName(imageArtifactName), @@ -1261,7 +1261,7 @@ var _ = Describe("TaskStep", func() { }) It("still chooses a worker and creates the container with the volume and a metadata stream", func() { - _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, ImageArtifactName: worker.ArtifactName(imageArtifactName), @@ -1293,7 +1293,7 @@ var _ = Describe("TaskStep", func() { }) It("still chooses a worker and creates the container with the volume and a metadata stream", func() { - _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, ImageArtifactName: worker.ArtifactName(imageArtifactName), @@ -1338,7 +1338,7 @@ var _ = Describe("TaskStep", func() { }) It("creates the specs with the image resource", func() { - _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec.ImageResource).To(Equal(&worker.ImageResource{ Type: "docker", Source: creds.NewSource(template.StaticVariables{}, atc.Source{"some": "super-secret-source"}), @@ -1372,7 +1372,7 @@ var _ = Describe("TaskStep", func() { }) It("creates the specs with the image resource", func() { - _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec.ImageURL).To(Equal("some-image")) Expect(workerSpec).To(Equal(worker.WorkerSpec{ @@ -1647,7 +1647,7 @@ var _ = Describe("TaskStep", func() { disaster := errors.New("nope") BeforeEach(func() { - fakePool.FindOrChooseWorkerReturns(nil, disaster) + fakePool.FindOrChooseWorkerForContainerReturns(nil, disaster) }) It("returns the error", func() { diff --git a/atc/radar/resource_scanner.go b/atc/radar/resource_scanner.go index c527320a2e2..a556261016a 100644 --- a/atc/radar/resource_scanner.go +++ b/atc/radar/resource_scanner.go @@ -321,7 +321,7 @@ func (scanner *resourceScanner) check( Type: db.ContainerTypeCheck, } - chosenWorker, err := scanner.pool.FindOrChooseWorker(logger, owner, containerSpec, workerSpec, scanner.strategy) + chosenWorker, err := scanner.pool.FindOrChooseWorkerForContainer(logger, owner, containerSpec, workerSpec, scanner.strategy) if err != nil { logger.Error("failed-to-choose-a-worker", err) chkErr := resourceConfigScope.SetCheckError(err) diff --git a/atc/radar/resource_scanner_test.go b/atc/radar/resource_scanner_test.go index da44c061b3a..00f3e64018d 100644 --- a/atc/radar/resource_scanner_test.go +++ b/atc/radar/resource_scanner_test.go @@ -143,7 +143,7 @@ var _ = Describe("ResourceScanner", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeContainer.HandleReturns("some-handle") fakeWorker.FindOrCreateContainerReturns(fakeContainer, nil) @@ -248,7 +248,7 @@ var _ = Describe("ResourceScanner", func() { err := fakeDBResource.SetCheckSetupErrorArgsForCall(0) Expect(err).To(BeNil()) - _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(owner).To(Equal(db.NewResourceConfigCheckSessionContainerOwner(fakeResourceConfig, radar.ContainerExpiries))) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ResourceType: "git", @@ -596,7 +596,7 @@ var _ = Describe("ResourceScanner", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeContainer.HandleReturns("some-handle") fakeWorker.FindOrCreateContainerReturns(fakeContainer, nil) @@ -680,7 +680,7 @@ var _ = Describe("ResourceScanner", func() { err := fakeDBResource.SetCheckSetupErrorArgsForCall(0) Expect(err).To(BeNil()) - _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(owner).To(Equal(db.NewResourceConfigCheckSessionContainerOwner(fakeResourceConfig, radar.ContainerExpiries))) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ResourceType: "git", @@ -763,7 +763,7 @@ var _ = Describe("ResourceScanner", func() { Context("when find or choosing the worker fails", func() { BeforeEach(func() { - fakePool.FindOrChooseWorkerReturns(nil, errors.New("catastrophe")) + fakePool.FindOrChooseWorkerForContainerReturns(nil, errors.New("catastrophe")) }) It("sets the check error and returns the error", func() { @@ -1043,7 +1043,7 @@ var _ = Describe("ResourceScanner", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeContainer.HandleReturns("some-handle") fakeWorker.FindOrCreateContainerReturns(fakeContainer, nil) diff --git a/atc/radar/resource_type_scanner.go b/atc/radar/resource_type_scanner.go index fbf8a376d0f..9fcf7cc77ab 100644 --- a/atc/radar/resource_type_scanner.go +++ b/atc/radar/resource_type_scanner.go @@ -253,7 +253,7 @@ func (scanner *resourceTypeScanner) check( owner := db.NewResourceConfigCheckSessionContainerOwner(resourceConfigScope.ResourceConfig(), ContainerExpiries) - chosenWorker, err := scanner.pool.FindOrChooseWorker(logger, owner, containerSpec, workerSpec, scanner.strategy) + chosenWorker, err := scanner.pool.FindOrChooseWorkerForContainer(logger, owner, containerSpec, workerSpec, scanner.strategy) if err != nil { chkErr := resourceConfigScope.SetCheckError(err) if chkErr != nil { diff --git a/atc/radar/resource_type_scanner_test.go b/atc/radar/resource_type_scanner_test.go index 63b3a110260..5bb9cd6cd6c 100644 --- a/atc/radar/resource_type_scanner_test.go +++ b/atc/radar/resource_type_scanner_test.go @@ -121,7 +121,7 @@ var _ = Describe("ResourceTypeScanner", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeContainer.HandleReturns("some-handle") fakeWorker.FindOrCreateContainerReturns(fakeContainer, nil) @@ -184,7 +184,7 @@ var _ = Describe("ResourceTypeScanner", func() { Expect(resourceSource).To(Equal(atc.Source{"custom": "some-secret-sauce"})) Expect(resourceTypes).To(Equal(creds.VersionedResourceTypes{})) - _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(owner).To(Equal(db.NewResourceConfigCheckSessionContainerOwner(fakeResourceConfig, ContainerExpiries))) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ResourceType: "registry-image", @@ -243,7 +243,7 @@ var _ = Describe("ResourceTypeScanner", func() { err := fakeResourceType.SetCheckSetupErrorArgsForCall(0) Expect(err).To(BeNil()) - _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(owner).To(Equal(db.NewResourceConfigCheckSessionContainerOwner(fakeResourceConfig, ContainerExpiries))) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ResourceType: "registry-image", @@ -444,7 +444,7 @@ var _ = Describe("ResourceTypeScanner", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeContainer.HandleReturns("some-handle") fakeWorker.FindOrCreateContainerReturns(fakeContainer, nil) @@ -477,7 +477,7 @@ var _ = Describe("ResourceTypeScanner", func() { err := fakeResourceType.SetCheckSetupErrorArgsForCall(0) Expect(err).To(BeNil()) - _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(owner).To(Equal(db.NewResourceConfigCheckSessionContainerOwner(fakeResourceConfig, ContainerExpiries))) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ResourceType: "registry-image", @@ -603,7 +603,7 @@ var _ = Describe("ResourceTypeScanner", func() { versionedResourceType, }))) - _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerArgsForCall(0) + _, owner, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(owner).To(Equal(db.NewResourceConfigCheckSessionContainerOwner(fakeResourceConfig, ContainerExpiries))) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ResourceType: "registry-image", @@ -675,7 +675,7 @@ var _ = Describe("ResourceTypeScanner", func() { Context("when finding or choosing the worker fails", func() { BeforeEach(func() { - fakePool.FindOrChooseWorkerReturns(nil, errors.New("catastrophe")) + fakePool.FindOrChooseWorkerForContainerReturns(nil, errors.New("catastrophe")) }) It("sets the check error and returns the error", func() { @@ -876,7 +876,7 @@ var _ = Describe("ResourceTypeScanner", func() { BeforeEach(func() { fakeWorker.NameReturns("some-worker") - fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + fakePool.FindOrChooseWorkerForContainerReturns(fakeWorker, nil) fakeContainer.HandleReturns("some-handle") fakeWorker.FindOrCreateContainerReturns(fakeContainer, nil) diff --git a/atc/routes.go b/atc/routes.go index d7e9fa4270a..bd00b534701 100644 --- a/atc/routes.go +++ b/atc/routes.go @@ -100,6 +100,9 @@ const ( SendInputToBuildPlan = "SendInputToBuildPlan" ReadOutputFromBuildPlan = "ReadOutputFromBuildPlan" + + CreateArtifact = "CreateArtifact" + GetArtifact = "GetArtifact" ) const ( @@ -203,4 +206,7 @@ var Routes = rata.Routes([]rata.Route{ {Path: "/api/v1/teams/:team_name/rename", Method: "PUT", Name: RenameTeam}, {Path: "/api/v1/teams/:team_name", Method: "DELETE", Name: DestroyTeam}, {Path: "/api/v1/teams/:team_name/builds", Method: "GET", Name: ListTeamBuilds}, + + {Path: "/api/v1/teams/:team_name/artifacts", Method: "POST", Name: CreateArtifact}, + {Path: "/api/v1/teams/:team_name/artifacts/:artifact_id", Method: "GET", Name: GetArtifact}, }) diff --git a/atc/worker/client.go b/atc/worker/client.go new file mode 100644 index 00000000000..84388d0fba7 --- /dev/null +++ b/atc/worker/client.go @@ -0,0 +1,69 @@ +package worker + +import ( + "code.cloudfoundry.org/lager" + "github.com/concourse/concourse/atc/db" +) + +//go:generate counterfeiter . Client + +type Client interface { + FindContainer(logger lager.Logger, teamID int, handle string) (Container, bool, error) + FindVolume(logger lager.Logger, teamID int, handle string) (Volume, bool, error) + CreateVolume(logger lager.Logger, spec VolumeSpec, teamID int, volumeType db.VolumeType) (Volume, error) +} + +func NewClient(pool Pool, provider WorkerProvider) *client { + return &client{ + pool: pool, + provider: provider, + } +} + +type client struct { + pool Pool + provider WorkerProvider +} + +func (client *client) FindContainer(logger lager.Logger, teamID int, handle string) (Container, bool, error) { + worker, found, err := client.provider.FindWorkerForContainer( + logger.Session("find-worker"), + teamID, + handle, + ) + if err != nil { + return nil, false, err + } + + if !found { + return nil, false, nil + } + + return worker.FindContainerByHandle(logger, teamID, handle) +} + +func (client *client) FindVolume(logger lager.Logger, teamID int, handle string) (Volume, bool, error) { + worker, found, err := client.provider.FindWorkerForVolume( + logger.Session("find-worker"), + teamID, + handle, + ) + if err != nil { + return nil, false, err + } + + if !found { + return nil, false, nil + } + + return worker.LookupVolume(logger, handle) +} + +func (client *client) CreateVolume(logger lager.Logger, spec VolumeSpec, teamID int, volumeType db.VolumeType) (Volume, error) { + worker, err := client.pool.FindOrChooseWorker(logger, WorkerSpec{TeamID: teamID}) + if err != nil { + return nil, err + } + + return worker.CreateVolume(logger, spec, teamID, volumeType) +} diff --git a/atc/worker/client_test.go b/atc/worker/client_test.go new file mode 100644 index 00000000000..be731e225ad --- /dev/null +++ b/atc/worker/client_test.go @@ -0,0 +1,197 @@ +package worker_test + +import ( + "errors" + + "code.cloudfoundry.org/lager/lagertest" + "github.com/concourse/baggageclaim" + "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/worker" + "github.com/concourse/concourse/atc/worker/workerfakes" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Client", func() { + var ( + logger *lagertest.TestLogger + fakePool *workerfakes.FakePool + fakeProvider *workerfakes.FakeWorkerProvider + client worker.Client + ) + + BeforeEach(func() { + logger = lagertest.NewTestLogger("test") + fakePool = new(workerfakes.FakePool) + fakeProvider = new(workerfakes.FakeWorkerProvider) + + client = worker.NewClient(fakePool, fakeProvider) + }) + + Describe("FindContainer", func() { + var ( + foundContainer worker.Container + found bool + findErr error + ) + + JustBeforeEach(func() { + foundContainer, found, findErr = client.FindContainer( + logger, + 4567, + "some-handle", + ) + }) + + Context("when looking up the worker errors", func() { + BeforeEach(func() { + fakeProvider.FindWorkerForContainerReturns(nil, false, errors.New("nope")) + }) + + It("errors", func() { + Expect(findErr).To(HaveOccurred()) + }) + }) + + Context("when worker is not found", func() { + BeforeEach(func() { + fakeProvider.FindWorkerForContainerReturns(nil, false, nil) + }) + + It("returns not found", func() { + Expect(findErr).NotTo(HaveOccurred()) + Expect(found).To(BeFalse()) + }) + }) + + Context("when a worker is found with the container", func() { + var fakeWorker *workerfakes.FakeWorker + var fakeContainer *workerfakes.FakeContainer + + BeforeEach(func() { + fakeWorker = new(workerfakes.FakeWorker) + fakeProvider.FindWorkerForContainerReturns(fakeWorker, true, nil) + + fakeContainer = new(workerfakes.FakeContainer) + fakeWorker.FindContainerByHandleReturns(fakeContainer, true, nil) + }) + + It("succeeds", func() { + Expect(found).To(BeTrue()) + Expect(findErr).NotTo(HaveOccurred()) + }) + + It("returns the created container", func() { + Expect(foundContainer).To(Equal(fakeContainer)) + }) + }) + }) + + Describe("FindVolume", func() { + var ( + foundVolume worker.Volume + found bool + findErr error + ) + + JustBeforeEach(func() { + foundVolume, found, findErr = client.FindVolume( + logger, + 4567, + "some-handle", + ) + }) + + Context("when looking up the worker errors", func() { + BeforeEach(func() { + fakeProvider.FindWorkerForVolumeReturns(nil, false, errors.New("nope")) + }) + + It("errors", func() { + Expect(findErr).To(HaveOccurred()) + }) + }) + + Context("when worker is not found", func() { + BeforeEach(func() { + fakeProvider.FindWorkerForVolumeReturns(nil, false, nil) + }) + + It("returns not found", func() { + Expect(findErr).NotTo(HaveOccurred()) + Expect(found).To(BeFalse()) + }) + }) + + Context("when a worker is found with the volume", func() { + var fakeWorker *workerfakes.FakeWorker + var fakeVolume *workerfakes.FakeVolume + + BeforeEach(func() { + fakeWorker = new(workerfakes.FakeWorker) + fakeProvider.FindWorkerForVolumeReturns(fakeWorker, true, nil) + + fakeVolume = new(workerfakes.FakeVolume) + fakeWorker.LookupVolumeReturns(fakeVolume, true, nil) + }) + + It("succeeds", func() { + Expect(found).To(BeTrue()) + Expect(findErr).NotTo(HaveOccurred()) + }) + + It("returns the volume", func() { + Expect(foundVolume).To(Equal(fakeVolume)) + }) + }) + }) + + Describe("CreateVolume", func() { + var ( + fakeWorker *workerfakes.FakeWorker + volumeSpec worker.VolumeSpec + volumeType db.VolumeType + err error + ) + + BeforeEach(func() { + volumeSpec = worker.VolumeSpec{ + Strategy: baggageclaim.EmptyStrategy{}, + } + + volumeType = db.VolumeTypeArtifact + }) + + JustBeforeEach(func() { + _, err = client.CreateVolume(logger, volumeSpec, 1, volumeType) + }) + + Context("when no workers can be found", func() { + BeforeEach(func() { + fakePool.FindOrChooseWorkerReturns(nil, errors.New("nope")) + }) + + It("returns an error", func() { + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when the worker can be found", func() { + BeforeEach(func() { + fakeWorker = new(workerfakes.FakeWorker) + fakePool.FindOrChooseWorkerReturns(fakeWorker, nil) + }) + + It("creates the volume on the worker", func() { + Expect(err).ToNot(HaveOccurred()) + Expect(fakeWorker.CreateVolumeCallCount()).To(Equal(1)) + l, spec, id, t := fakeWorker.CreateVolumeArgsForCall(0) + Expect(l).To(Equal(logger)) + Expect(spec).To(Equal(volumeSpec)) + Expect(id).To(Equal(1)) + Expect(t).To(Equal(volumeType)) + }) + }) + }) +}) diff --git a/atc/worker/db_worker_provider.go b/atc/worker/db_worker_provider.go index 2c6f413e248..d0f0a9f5a3b 100644 --- a/atc/worker/db_worker_provider.go +++ b/atc/worker/db_worker_provider.go @@ -142,6 +142,30 @@ func (provider *dbWorkerProvider) FindWorkerForContainer( return worker, true, err } +func (provider *dbWorkerProvider) FindWorkerForVolume( + logger lager.Logger, + teamID int, + handle string, +) (Worker, bool, error) { + logger = logger.Session("worker-for-volume") + team := provider.dbTeamFactory.GetByID(teamID) + + dbWorker, found, err := team.FindWorkerForVolume(handle) + if err != nil { + return nil, false, err + } + + if !found { + return nil, false, nil + } + + worker := provider.NewGardenWorker(logger, clock.NewClock(), dbWorker, 0) + if !worker.IsVersionCompatible(logger, provider.workerVersion) { + return nil, false, nil + } + return worker, true, err +} + func (provider *dbWorkerProvider) NewGardenWorker(logger lager.Logger, tikTok clock.Clock, savedWorker db.Worker, buildContainersCount int) Worker { gcf := NewGardenClientFactory( provider.dbWorkerFactory, diff --git a/atc/worker/db_worker_provider_test.go b/atc/worker/db_worker_provider_test.go index 9298fada5ea..03626e4c8c6 100644 --- a/atc/worker/db_worker_provider_test.go +++ b/atc/worker/db_worker_provider_test.go @@ -597,6 +597,96 @@ var _ = Describe("DBProvider", func() { }) }) + Describe("FindWorkerForVolume", func() { + var ( + foundWorker Worker + found bool + findErr error + ) + + JustBeforeEach(func() { + foundWorker, found, findErr = provider.FindWorkerForVolume( + logger, + 345278, + "some-handle", + ) + }) + + Context("when the worker is found", func() { + var fakeExistingWorker *dbfakes.FakeWorker + + BeforeEach(func() { + addr := "1.2.3.4:7777" + + fakeExistingWorker = new(dbfakes.FakeWorker) + fakeExistingWorker.NameReturns("some-worker") + fakeExistingWorker.GardenAddrReturns(&addr) + workerVersion := "1.1.0" + fakeExistingWorker.VersionReturns(&workerVersion) + + fakeDBTeam.FindWorkerForVolumeReturns(fakeExistingWorker, true, nil) + }) + + It("returns true", func() { + Expect(found).To(BeTrue()) + Expect(findErr).ToNot(HaveOccurred()) + }) + + It("returns the worker", func() { + Expect(foundWorker).ToNot(BeNil()) + Expect(foundWorker.Name()).To(Equal("some-worker")) + }) + + It("found the worker for the right handle", func() { + handle := fakeDBTeam.FindWorkerForVolumeArgsForCall(0) + Expect(handle).To(Equal("some-handle")) + }) + + It("found the right team", func() { + actualTeam := fakeDBTeamFactory.GetByIDArgsForCall(0) + Expect(actualTeam).To(Equal(345278)) + }) + + Context("when the worker version is outdated", func() { + BeforeEach(func() { + fakeExistingWorker.VersionReturns(nil) + }) + + It("returns an error", func() { + Expect(findErr).ToNot(HaveOccurred()) + Expect(foundWorker).To(BeNil()) + Expect(found).To(BeFalse()) + }) + }) + }) + + Context("when the worker is not found", func() { + BeforeEach(func() { + fakeDBTeam.FindWorkerForVolumeReturns(nil, false, nil) + }) + + It("returns false", func() { + Expect(findErr).ToNot(HaveOccurred()) + Expect(foundWorker).To(BeNil()) + Expect(found).To(BeFalse()) + }) + }) + + Context("when finding the worker fails", func() { + disaster := errors.New("nope") + + BeforeEach(func() { + fakeDBTeam.FindWorkerForVolumeReturns(nil, false, disaster) + }) + + It("returns the error", func() { + Expect(findErr).To(Equal(disaster)) + Expect(foundWorker).To(BeNil()) + Expect(found).To(BeFalse()) + }) + }) + }) + Describe("FindWorkersForContainerByOwner", func() { var ( fakeOwner *dbfakes.FakeContainerOwner diff --git a/atc/worker/pool.go b/atc/worker/pool.go index cb4589a3205..cc91287a89c 100644 --- a/atc/worker/pool.go +++ b/atc/worker/pool.go @@ -22,6 +22,12 @@ type WorkerProvider interface { handle string, ) (Worker, bool, error) + FindWorkerForVolume( + logger lager.Logger, + teamID int, + handle string, + ) (Worker, bool, error) + FindWorkersForContainerByOwner( logger lager.Logger, owner db.ContainerOwner, @@ -50,7 +56,7 @@ func (err NoCompatibleWorkersError) Error() string { //go:generate counterfeiter . Pool type Pool interface { - FindOrChooseWorker( + FindOrChooseWorkerForContainer( lager.Logger, db.ContainerOwner, ContainerSpec, @@ -58,7 +64,10 @@ type Pool interface { ContainerPlacementStrategy, ) (Worker, error) - FindContainerByHandle(lager.Logger, int, string) (Container, bool, error) + FindOrChooseWorker( + lager.Logger, + WorkerSpec, + ) (Worker, error) } type pool struct { @@ -110,7 +119,7 @@ func (pool *pool) allSatisfying(logger lager.Logger, spec WorkerSpec) ([]Worker, } } -func (pool *pool) FindOrChooseWorker( +func (pool *pool) FindOrChooseWorkerForContainer( logger lager.Logger, owner db.ContainerOwner, containerSpec ContainerSpec, @@ -151,19 +160,14 @@ dance: return worker, nil } -func (pool *pool) FindContainerByHandle(logger lager.Logger, teamID int, handle string) (Container, bool, error) { - worker, found, err := pool.provider.FindWorkerForContainer( - logger.Session("find-worker"), - teamID, - handle, - ) +func (pool *pool) FindOrChooseWorker( + logger lager.Logger, + workerSpec WorkerSpec, +) (Worker, error) { + workers, err := pool.allSatisfying(logger, workerSpec) if err != nil { - return nil, false, err - } - - if !found { - return nil, false, nil + return nil, err } - return worker.FindContainerByHandle(logger, teamID, handle) + return workers[rand.Intn(len(workers))], nil } diff --git a/atc/worker/pool_test.go b/atc/worker/pool_test.go index 1e60b7863a4..5c520cb7381 100644 --- a/atc/worker/pool_test.go +++ b/atc/worker/pool_test.go @@ -11,7 +11,6 @@ import ( "github.com/concourse/concourse/atc/db/dbfakes" . "github.com/concourse/concourse/atc/worker" "github.com/concourse/concourse/atc/worker/workerfakes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -30,62 +29,7 @@ var _ = Describe("Pool", func() { pool = NewPool(fakeProvider) }) - Describe("FindContainerByHandle", func() { - var ( - foundContainer Container - found bool - findErr error - ) - - JustBeforeEach(func() { - foundContainer, found, findErr = pool.FindContainerByHandle( - logger, - 4567, - "some-handle", - ) - }) - - Context("when a worker is found with the container", func() { - var fakeWorker *workerfakes.FakeWorker - var fakeContainer *workerfakes.FakeContainer - - BeforeEach(func() { - fakeWorker = new(workerfakes.FakeWorker) - fakeProvider.FindWorkerForContainerReturns(fakeWorker, true, nil) - - fakeContainer = new(workerfakes.FakeContainer) - fakeWorker.FindContainerByHandleReturns(fakeContainer, true, nil) - }) - - It("succeeds", func() { - Expect(found).To(BeTrue()) - Expect(findErr).NotTo(HaveOccurred()) - }) - - It("returns the created container", func() { - Expect(foundContainer).To(Equal(fakeContainer)) - }) - - It("finds on the particular worker", func() { - Expect(fakeWorker.FindContainerByHandleCallCount()).To(Equal(1)) - - _, actualTeamID, actualHandle := fakeProvider.FindWorkerForContainerArgsForCall(0) - Expect(actualTeamID).To(Equal(4567)) - Expect(actualHandle).To(Equal("some-handle")) - }) - }) - - Context("when no worker is found with the container", func() { - BeforeEach(func() { - fakeProvider.FindWorkerForContainerReturns(nil, false, nil) - }) - - It("returns no container, false, and no error", func() { - }) - }) - }) - - Describe("FindOrChooseWorker", func() { + Describe("FindOrChooseWorkerForContainer", func() { var ( spec ContainerSpec workerSpec WorkerSpec @@ -170,7 +114,7 @@ var _ = Describe("Pool", func() { }) JustBeforeEach(func() { - chosenWorker, chooseErr = pool.FindOrChooseWorker( + chosenWorker, chooseErr = pool.FindOrChooseWorkerForContainer( logger, fakeOwner, spec, diff --git a/atc/worker/volume.go b/atc/worker/volume.go index e96cf02d1ce..eead574b1c1 100644 --- a/atc/worker/volume.go +++ b/atc/worker/volume.go @@ -4,7 +4,6 @@ import ( "io" "code.cloudfoundry.org/lager" - "github.com/concourse/baggageclaim" "github.com/concourse/concourse/atc/db" ) @@ -27,6 +26,7 @@ type Volume interface { InitializeResourceCache(db.UsedResourceCache) error InitializeTaskCache(lager.Logger, int, string, string, bool) error + InitializeArtifact(string, string) (db.WorkerArtifact, error) CreateChildForContainer(db.CreatingContainer, string) (db.CreatingVolume, error) @@ -113,6 +113,10 @@ func (v *volume) InitializeResourceCache(urc db.UsedResourceCache) error { return v.dbVolume.InitializeResourceCache(urc) } +func (v *volume) InitializeArtifact(path string, checksum string) (db.WorkerArtifact, error) { + return v.dbVolume.InitializeArtifact(path, checksum) +} + func (v *volume) InitializeTaskCache( logger lager.Logger, jobID int, diff --git a/atc/worker/volume_client.go b/atc/worker/volume_client.go index 116d5786caa..a6efab6b5d7 100644 --- a/atc/worker/volume_client.go +++ b/atc/worker/volume_client.go @@ -39,6 +39,13 @@ type VolumeClient interface { int, string, ) (Volume, error) + CreateVolume( + lager.Logger, + VolumeSpec, + int, + string, + db.VolumeType, + ) (Volume, error) FindVolumeForResourceCache( lager.Logger, db.UsedResourceCache, @@ -163,6 +170,25 @@ func (c *volumeClient) FindOrCreateCOWVolumeForContainer( ) } +func (c *volumeClient) CreateVolume( + logger lager.Logger, + volumeSpec VolumeSpec, + teamID int, + workerName string, + volumeType db.VolumeType, +) (Volume, error) { + return c.findOrCreateVolume( + logger.Session("find-or-create-volume-for-artifact"), + volumeSpec, + func() (db.CreatingVolume, db.CreatedVolume, error) { + return nil, nil, nil + }, + func() (db.CreatingVolume, error) { + return c.dbVolumeRepository.CreateVolume(teamID, workerName, volumeType) + }, + ) +} + func (c *volumeClient) FindOrCreateVolumeForBaseResourceType( logger lager.Logger, volumeSpec VolumeSpec, diff --git a/atc/worker/volume_client_test.go b/atc/worker/volume_client_test.go index 342fc6c1007..40341d7ec7b 100644 --- a/atc/worker/volume_client_test.go +++ b/atc/worker/volume_client_test.go @@ -8,15 +8,13 @@ import ( "code.cloudfoundry.org/lager" "code.cloudfoundry.org/lager/lagertest" "github.com/concourse/baggageclaim" + "github.com/concourse/baggageclaim/baggageclaimfakes" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/db/lock" "github.com/concourse/concourse/atc/db/lock/lockfakes" "github.com/concourse/concourse/atc/worker" - - "github.com/concourse/baggageclaim/baggageclaimfakes" "github.com/concourse/concourse/atc/worker/workerfakes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -641,6 +639,109 @@ var _ = Describe("VolumeClient", func() { }) }) + Describe("CreateVolume", func() { + var err error + var workerVolume worker.Volume + + BeforeEach(func() { + fakeLockFactory.AcquireReturns(nil, false, errors.New("nope")) + }) + + JustBeforeEach(func() { + workerVolume, err = volumeClient.CreateVolume(testLogger, worker.VolumeSpec{}, 42, "some-mount", db.VolumeTypeArtifact) + }) + + Context("when trying to create a new volume", func() { + + Context("when creating the volume in the db fails", func() { + BeforeEach(func() { + fakeDBVolumeRepository.CreateVolumeReturns(nil, errors.New("nope")) + }) + + It("errors", func() { + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when creating the volume in the db succeeds", func() { + var fakeCreatingVolume *dbfakes.FakeCreatingVolume + + BeforeEach(func() { + fakeCreatingVolume = new(dbfakes.FakeCreatingVolume) + + fakeDBVolumeRepository.CreateVolumeReturns(fakeCreatingVolume, nil) + }) + + Context("when it fails to acquire a lock", func() { + BeforeEach(func() { + fakeLockFactory.AcquireReturns(nil, false, errors.New("nope")) + }) + + It("errors", func() { + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when it acquires the lock", func() { + BeforeEach(func() { + fakeLockFactory.AcquireReturns(fakeLock, true, nil) + }) + + It("creates volume in baggageclaim", func() { + Expect(fakeBaggageclaimClient.CreateVolumeCallCount()).To(Equal(1)) + }) + + Context("when creating the volume in baggage claim fails", func() { + BeforeEach(func() { + fakeBaggageclaimClient.CreateVolumeReturns(nil, errors.New("nope")) + }) + + It("errors", func() { + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when creating the volume in baggage claim succeeds", func() { + var fakeBGVolume *baggageclaimfakes.FakeVolume + + BeforeEach(func() { + fakeBGVolume = new(baggageclaimfakes.FakeVolume) + fakeBaggageclaimClient.CreateVolumeReturns(fakeBGVolume, nil) + }) + + It("transitions the volume to created in the db", func() { + Expect(fakeCreatingVolume.CreatedCallCount()).To(Equal(1)) + }) + + Context("when transitioning the build fails", func() { + BeforeEach(func() { + fakeCreatingVolume.CreatedReturns(nil, errors.New("nope")) + }) + + It("errors", func() { + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when transitioning the build succeeds", func() { + var fakeCreatedVolume *dbfakes.FakeCreatedVolume + + BeforeEach(func() { + fakeCreatedVolume = new(dbfakes.FakeCreatedVolume) + fakeCreatingVolume.CreatedReturns(fakeCreatedVolume, nil) + }) + + It("returns a new volume with the bg volume and created volume", func() { + Expect(err).NotTo(HaveOccurred()) + Expect(workerVolume).To(Equal(worker.NewVolume(fakeBGVolume, fakeCreatedVolume, volumeClient))) + }) + }) + }) + }) + }) + }) + }) + Describe("LookupVolume", func() { var handle string diff --git a/atc/worker/worker.go b/atc/worker/worker.go index 1651be21c4c..2541e2653cc 100644 --- a/atc/worker/worker.go +++ b/atc/worker/worker.go @@ -49,6 +49,8 @@ type Worker interface { CertsVolume(lager.Logger) (volume Volume, found bool, err error) LookupVolume(lager.Logger, string) (Volume, bool, error) + CreateVolume(logger lager.Logger, spec VolumeSpec, teamID int, volumeType db.VolumeType) (Volume, error) + GardenClient() garden.Client } @@ -143,6 +145,10 @@ func (worker *gardenWorker) CertsVolume(logger lager.Logger) (Volume, bool, erro return worker.volumeClient.FindOrCreateVolumeForResourceCerts(logger.Session("find-or-create")) } +func (worker *gardenWorker) CreateVolume(logger lager.Logger, spec VolumeSpec, teamID int, volumeType db.VolumeType) (Volume, error) { + return worker.volumeClient.CreateVolume(logger.Session("find-or-create"), spec, teamID, worker.dbWorker.Name(), volumeType) +} + func (worker *gardenWorker) LookupVolume(logger lager.Logger, handle string) (Volume, bool, error) { return worker.volumeClient.LookupVolume(logger, handle) } diff --git a/atc/worker/worker_test.go b/atc/worker/worker_test.go index de7b5363b00..14349211b7c 100644 --- a/atc/worker/worker_test.go +++ b/atc/worker/worker_test.go @@ -4,16 +4,15 @@ import ( "time" "code.cloudfoundry.org/garden/gardenfakes" - "code.cloudfoundry.org/lager/lagertest" "github.com/cloudfoundry/bosh-cli/director/template" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/creds" + "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" . "github.com/concourse/concourse/atc/worker" wfakes "github.com/concourse/concourse/atc/worker/workerfakes" "github.com/cppforlife/go-semi-semantic/version" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -185,6 +184,30 @@ var _ = Describe("Worker", func() { }) }) + Describe("CreateVolume", func() { + var ( + fakeVolume *wfakes.FakeVolume + volume Volume + err error + ) + + BeforeEach(func() { + fakeVolume = new(wfakes.FakeVolume) + fakeVolumeClient.CreateVolumeReturns(fakeVolume, nil) + }) + + JustBeforeEach(func() { + volume, err = gardenWorker.CreateVolume(logger, VolumeSpec{}, 42, db.VolumeTypeArtifact) + }) + + It("calls the volume client", func() { + Expect(fakeVolumeClient.CreateVolumeCallCount()).To(Equal(1)) + + Expect(err).ToNot(HaveOccurred()) + Expect(volume).To(Equal(fakeVolume)) + }) + }) + Describe("Satisfies", func() { var ( spec WorkerSpec diff --git a/atc/worker/workerfakes/fake_client.go b/atc/worker/workerfakes/fake_client.go new file mode 100644 index 00000000000..b62eab19d92 --- /dev/null +++ b/atc/worker/workerfakes/fake_client.go @@ -0,0 +1,297 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package workerfakes + +import ( + sync "sync" + + lager "code.cloudfoundry.org/lager" + db "github.com/concourse/concourse/atc/db" + worker "github.com/concourse/concourse/atc/worker" +) + +type FakeClient struct { + CreateVolumeStub func(lager.Logger, worker.VolumeSpec, int, db.VolumeType) (worker.Volume, error) + createVolumeMutex sync.RWMutex + createVolumeArgsForCall []struct { + arg1 lager.Logger + arg2 worker.VolumeSpec + arg3 int + arg4 db.VolumeType + } + createVolumeReturns struct { + result1 worker.Volume + result2 error + } + createVolumeReturnsOnCall map[int]struct { + result1 worker.Volume + result2 error + } + FindContainerStub func(lager.Logger, int, string) (worker.Container, bool, error) + findContainerMutex sync.RWMutex + findContainerArgsForCall []struct { + arg1 lager.Logger + arg2 int + arg3 string + } + findContainerReturns struct { + result1 worker.Container + result2 bool + result3 error + } + findContainerReturnsOnCall map[int]struct { + result1 worker.Container + result2 bool + result3 error + } + FindVolumeStub func(lager.Logger, int, string) (worker.Volume, bool, error) + findVolumeMutex sync.RWMutex + findVolumeArgsForCall []struct { + arg1 lager.Logger + arg2 int + arg3 string + } + findVolumeReturns struct { + result1 worker.Volume + result2 bool + result3 error + } + findVolumeReturnsOnCall map[int]struct { + result1 worker.Volume + result2 bool + result3 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeClient) CreateVolume(arg1 lager.Logger, arg2 worker.VolumeSpec, arg3 int, arg4 db.VolumeType) (worker.Volume, error) { + fake.createVolumeMutex.Lock() + ret, specificReturn := fake.createVolumeReturnsOnCall[len(fake.createVolumeArgsForCall)] + fake.createVolumeArgsForCall = append(fake.createVolumeArgsForCall, struct { + arg1 lager.Logger + arg2 worker.VolumeSpec + arg3 int + arg4 db.VolumeType + }{arg1, arg2, arg3, arg4}) + fake.recordInvocation("CreateVolume", []interface{}{arg1, arg2, arg3, arg4}) + fake.createVolumeMutex.Unlock() + if fake.CreateVolumeStub != nil { + return fake.CreateVolumeStub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.createVolumeReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeClient) CreateVolumeCallCount() int { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + return len(fake.createVolumeArgsForCall) +} + +func (fake *FakeClient) CreateVolumeCalls(stub func(lager.Logger, worker.VolumeSpec, int, db.VolumeType) (worker.Volume, error)) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = stub +} + +func (fake *FakeClient) CreateVolumeArgsForCall(i int) (lager.Logger, worker.VolumeSpec, int, db.VolumeType) { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + argsForCall := fake.createVolumeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeClient) CreateVolumeReturns(result1 worker.Volume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + fake.createVolumeReturns = struct { + result1 worker.Volume + result2 error + }{result1, result2} +} + +func (fake *FakeClient) CreateVolumeReturnsOnCall(i int, result1 worker.Volume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + if fake.createVolumeReturnsOnCall == nil { + fake.createVolumeReturnsOnCall = make(map[int]struct { + result1 worker.Volume + result2 error + }) + } + fake.createVolumeReturnsOnCall[i] = struct { + result1 worker.Volume + result2 error + }{result1, result2} +} + +func (fake *FakeClient) FindContainer(arg1 lager.Logger, arg2 int, arg3 string) (worker.Container, bool, error) { + fake.findContainerMutex.Lock() + ret, specificReturn := fake.findContainerReturnsOnCall[len(fake.findContainerArgsForCall)] + fake.findContainerArgsForCall = append(fake.findContainerArgsForCall, struct { + arg1 lager.Logger + arg2 int + arg3 string + }{arg1, arg2, arg3}) + fake.recordInvocation("FindContainer", []interface{}{arg1, arg2, arg3}) + fake.findContainerMutex.Unlock() + if fake.FindContainerStub != nil { + return fake.FindContainerStub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.findContainerReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *FakeClient) FindContainerCallCount() int { + fake.findContainerMutex.RLock() + defer fake.findContainerMutex.RUnlock() + return len(fake.findContainerArgsForCall) +} + +func (fake *FakeClient) FindContainerCalls(stub func(lager.Logger, int, string) (worker.Container, bool, error)) { + fake.findContainerMutex.Lock() + defer fake.findContainerMutex.Unlock() + fake.FindContainerStub = stub +} + +func (fake *FakeClient) FindContainerArgsForCall(i int) (lager.Logger, int, string) { + fake.findContainerMutex.RLock() + defer fake.findContainerMutex.RUnlock() + argsForCall := fake.findContainerArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeClient) FindContainerReturns(result1 worker.Container, result2 bool, result3 error) { + fake.findContainerMutex.Lock() + defer fake.findContainerMutex.Unlock() + fake.FindContainerStub = nil + fake.findContainerReturns = struct { + result1 worker.Container + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeClient) FindContainerReturnsOnCall(i int, result1 worker.Container, result2 bool, result3 error) { + fake.findContainerMutex.Lock() + defer fake.findContainerMutex.Unlock() + fake.FindContainerStub = nil + if fake.findContainerReturnsOnCall == nil { + fake.findContainerReturnsOnCall = make(map[int]struct { + result1 worker.Container + result2 bool + result3 error + }) + } + fake.findContainerReturnsOnCall[i] = struct { + result1 worker.Container + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeClient) FindVolume(arg1 lager.Logger, arg2 int, arg3 string) (worker.Volume, bool, error) { + fake.findVolumeMutex.Lock() + ret, specificReturn := fake.findVolumeReturnsOnCall[len(fake.findVolumeArgsForCall)] + fake.findVolumeArgsForCall = append(fake.findVolumeArgsForCall, struct { + arg1 lager.Logger + arg2 int + arg3 string + }{arg1, arg2, arg3}) + fake.recordInvocation("FindVolume", []interface{}{arg1, arg2, arg3}) + fake.findVolumeMutex.Unlock() + if fake.FindVolumeStub != nil { + return fake.FindVolumeStub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.findVolumeReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *FakeClient) FindVolumeCallCount() int { + fake.findVolumeMutex.RLock() + defer fake.findVolumeMutex.RUnlock() + return len(fake.findVolumeArgsForCall) +} + +func (fake *FakeClient) FindVolumeCalls(stub func(lager.Logger, int, string) (worker.Volume, bool, error)) { + fake.findVolumeMutex.Lock() + defer fake.findVolumeMutex.Unlock() + fake.FindVolumeStub = stub +} + +func (fake *FakeClient) FindVolumeArgsForCall(i int) (lager.Logger, int, string) { + fake.findVolumeMutex.RLock() + defer fake.findVolumeMutex.RUnlock() + argsForCall := fake.findVolumeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeClient) FindVolumeReturns(result1 worker.Volume, result2 bool, result3 error) { + fake.findVolumeMutex.Lock() + defer fake.findVolumeMutex.Unlock() + fake.FindVolumeStub = nil + fake.findVolumeReturns = struct { + result1 worker.Volume + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeClient) FindVolumeReturnsOnCall(i int, result1 worker.Volume, result2 bool, result3 error) { + fake.findVolumeMutex.Lock() + defer fake.findVolumeMutex.Unlock() + fake.FindVolumeStub = nil + if fake.findVolumeReturnsOnCall == nil { + fake.findVolumeReturnsOnCall = make(map[int]struct { + result1 worker.Volume + result2 bool + result3 error + }) + } + fake.findVolumeReturnsOnCall[i] = struct { + result1 worker.Volume + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeClient) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + fake.findContainerMutex.RLock() + defer fake.findContainerMutex.RUnlock() + fake.findVolumeMutex.RLock() + defer fake.findVolumeMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeClient) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ worker.Client = new(FakeClient) diff --git a/atc/worker/workerfakes/fake_pool.go b/atc/worker/workerfakes/fake_pool.go index 8adbc81a580..9f17108ccc4 100644 --- a/atc/worker/workerfakes/fake_pool.go +++ b/atc/worker/workerfakes/fake_pool.go @@ -10,37 +10,34 @@ import ( ) type FakePool struct { - FindContainerByHandleStub func(lager.Logger, int, string) (worker.Container, bool, error) - findContainerByHandleMutex sync.RWMutex - findContainerByHandleArgsForCall []struct { + FindOrChooseWorkerStub func(lager.Logger, worker.WorkerSpec) (worker.Worker, error) + findOrChooseWorkerMutex sync.RWMutex + findOrChooseWorkerArgsForCall []struct { arg1 lager.Logger - arg2 int - arg3 string + arg2 worker.WorkerSpec } - findContainerByHandleReturns struct { - result1 worker.Container - result2 bool - result3 error + findOrChooseWorkerReturns struct { + result1 worker.Worker + result2 error } - findContainerByHandleReturnsOnCall map[int]struct { - result1 worker.Container - result2 bool - result3 error + findOrChooseWorkerReturnsOnCall map[int]struct { + result1 worker.Worker + result2 error } - FindOrChooseWorkerStub func(lager.Logger, db.ContainerOwner, worker.ContainerSpec, worker.WorkerSpec, worker.ContainerPlacementStrategy) (worker.Worker, error) - findOrChooseWorkerMutex sync.RWMutex - findOrChooseWorkerArgsForCall []struct { + FindOrChooseWorkerForContainerStub func(lager.Logger, db.ContainerOwner, worker.ContainerSpec, worker.WorkerSpec, worker.ContainerPlacementStrategy) (worker.Worker, error) + findOrChooseWorkerForContainerMutex sync.RWMutex + findOrChooseWorkerForContainerArgsForCall []struct { arg1 lager.Logger arg2 db.ContainerOwner arg3 worker.ContainerSpec arg4 worker.WorkerSpec arg5 worker.ContainerPlacementStrategy } - findOrChooseWorkerReturns struct { + findOrChooseWorkerForContainerReturns struct { result1 worker.Worker result2 error } - findOrChooseWorkerReturnsOnCall map[int]struct { + findOrChooseWorkerForContainerReturnsOnCall map[int]struct { result1 worker.Worker result2 error } @@ -48,88 +45,17 @@ type FakePool struct { invocationsMutex sync.RWMutex } -func (fake *FakePool) FindContainerByHandle(arg1 lager.Logger, arg2 int, arg3 string) (worker.Container, bool, error) { - fake.findContainerByHandleMutex.Lock() - ret, specificReturn := fake.findContainerByHandleReturnsOnCall[len(fake.findContainerByHandleArgsForCall)] - fake.findContainerByHandleArgsForCall = append(fake.findContainerByHandleArgsForCall, struct { - arg1 lager.Logger - arg2 int - arg3 string - }{arg1, arg2, arg3}) - fake.recordInvocation("FindContainerByHandle", []interface{}{arg1, arg2, arg3}) - fake.findContainerByHandleMutex.Unlock() - if fake.FindContainerByHandleStub != nil { - return fake.FindContainerByHandleStub(arg1, arg2, arg3) - } - if specificReturn { - return ret.result1, ret.result2, ret.result3 - } - fakeReturns := fake.findContainerByHandleReturns - return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 -} - -func (fake *FakePool) FindContainerByHandleCallCount() int { - fake.findContainerByHandleMutex.RLock() - defer fake.findContainerByHandleMutex.RUnlock() - return len(fake.findContainerByHandleArgsForCall) -} - -func (fake *FakePool) FindContainerByHandleCalls(stub func(lager.Logger, int, string) (worker.Container, bool, error)) { - fake.findContainerByHandleMutex.Lock() - defer fake.findContainerByHandleMutex.Unlock() - fake.FindContainerByHandleStub = stub -} - -func (fake *FakePool) FindContainerByHandleArgsForCall(i int) (lager.Logger, int, string) { - fake.findContainerByHandleMutex.RLock() - defer fake.findContainerByHandleMutex.RUnlock() - argsForCall := fake.findContainerByHandleArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - -func (fake *FakePool) FindContainerByHandleReturns(result1 worker.Container, result2 bool, result3 error) { - fake.findContainerByHandleMutex.Lock() - defer fake.findContainerByHandleMutex.Unlock() - fake.FindContainerByHandleStub = nil - fake.findContainerByHandleReturns = struct { - result1 worker.Container - result2 bool - result3 error - }{result1, result2, result3} -} - -func (fake *FakePool) FindContainerByHandleReturnsOnCall(i int, result1 worker.Container, result2 bool, result3 error) { - fake.findContainerByHandleMutex.Lock() - defer fake.findContainerByHandleMutex.Unlock() - fake.FindContainerByHandleStub = nil - if fake.findContainerByHandleReturnsOnCall == nil { - fake.findContainerByHandleReturnsOnCall = make(map[int]struct { - result1 worker.Container - result2 bool - result3 error - }) - } - fake.findContainerByHandleReturnsOnCall[i] = struct { - result1 worker.Container - result2 bool - result3 error - }{result1, result2, result3} -} - -func (fake *FakePool) FindOrChooseWorker(arg1 lager.Logger, arg2 db.ContainerOwner, arg3 worker.ContainerSpec, arg4 worker.WorkerSpec, arg5 worker.ContainerPlacementStrategy) (worker.Worker, error) { +func (fake *FakePool) FindOrChooseWorker(arg1 lager.Logger, arg2 worker.WorkerSpec) (worker.Worker, error) { fake.findOrChooseWorkerMutex.Lock() ret, specificReturn := fake.findOrChooseWorkerReturnsOnCall[len(fake.findOrChooseWorkerArgsForCall)] fake.findOrChooseWorkerArgsForCall = append(fake.findOrChooseWorkerArgsForCall, struct { arg1 lager.Logger - arg2 db.ContainerOwner - arg3 worker.ContainerSpec - arg4 worker.WorkerSpec - arg5 worker.ContainerPlacementStrategy - }{arg1, arg2, arg3, arg4, arg5}) - fake.recordInvocation("FindOrChooseWorker", []interface{}{arg1, arg2, arg3, arg4, arg5}) + arg2 worker.WorkerSpec + }{arg1, arg2}) + fake.recordInvocation("FindOrChooseWorker", []interface{}{arg1, arg2}) fake.findOrChooseWorkerMutex.Unlock() if fake.FindOrChooseWorkerStub != nil { - return fake.FindOrChooseWorkerStub(arg1, arg2, arg3, arg4, arg5) + return fake.FindOrChooseWorkerStub(arg1, arg2) } if specificReturn { return ret.result1, ret.result2 @@ -144,17 +70,17 @@ func (fake *FakePool) FindOrChooseWorkerCallCount() int { return len(fake.findOrChooseWorkerArgsForCall) } -func (fake *FakePool) FindOrChooseWorkerCalls(stub func(lager.Logger, db.ContainerOwner, worker.ContainerSpec, worker.WorkerSpec, worker.ContainerPlacementStrategy) (worker.Worker, error)) { +func (fake *FakePool) FindOrChooseWorkerCalls(stub func(lager.Logger, worker.WorkerSpec) (worker.Worker, error)) { fake.findOrChooseWorkerMutex.Lock() defer fake.findOrChooseWorkerMutex.Unlock() fake.FindOrChooseWorkerStub = stub } -func (fake *FakePool) FindOrChooseWorkerArgsForCall(i int) (lager.Logger, db.ContainerOwner, worker.ContainerSpec, worker.WorkerSpec, worker.ContainerPlacementStrategy) { +func (fake *FakePool) FindOrChooseWorkerArgsForCall(i int) (lager.Logger, worker.WorkerSpec) { fake.findOrChooseWorkerMutex.RLock() defer fake.findOrChooseWorkerMutex.RUnlock() argsForCall := fake.findOrChooseWorkerArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakePool) FindOrChooseWorkerReturns(result1 worker.Worker, result2 error) { @@ -183,13 +109,80 @@ func (fake *FakePool) FindOrChooseWorkerReturnsOnCall(i int, result1 worker.Work }{result1, result2} } +func (fake *FakePool) FindOrChooseWorkerForContainer(arg1 lager.Logger, arg2 db.ContainerOwner, arg3 worker.ContainerSpec, arg4 worker.WorkerSpec, arg5 worker.ContainerPlacementStrategy) (worker.Worker, error) { + fake.findOrChooseWorkerForContainerMutex.Lock() + ret, specificReturn := fake.findOrChooseWorkerForContainerReturnsOnCall[len(fake.findOrChooseWorkerForContainerArgsForCall)] + fake.findOrChooseWorkerForContainerArgsForCall = append(fake.findOrChooseWorkerForContainerArgsForCall, struct { + arg1 lager.Logger + arg2 db.ContainerOwner + arg3 worker.ContainerSpec + arg4 worker.WorkerSpec + arg5 worker.ContainerPlacementStrategy + }{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("FindOrChooseWorkerForContainer", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.findOrChooseWorkerForContainerMutex.Unlock() + if fake.FindOrChooseWorkerForContainerStub != nil { + return fake.FindOrChooseWorkerForContainerStub(arg1, arg2, arg3, arg4, arg5) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.findOrChooseWorkerForContainerReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakePool) FindOrChooseWorkerForContainerCallCount() int { + fake.findOrChooseWorkerForContainerMutex.RLock() + defer fake.findOrChooseWorkerForContainerMutex.RUnlock() + return len(fake.findOrChooseWorkerForContainerArgsForCall) +} + +func (fake *FakePool) FindOrChooseWorkerForContainerCalls(stub func(lager.Logger, db.ContainerOwner, worker.ContainerSpec, worker.WorkerSpec, worker.ContainerPlacementStrategy) (worker.Worker, error)) { + fake.findOrChooseWorkerForContainerMutex.Lock() + defer fake.findOrChooseWorkerForContainerMutex.Unlock() + fake.FindOrChooseWorkerForContainerStub = stub +} + +func (fake *FakePool) FindOrChooseWorkerForContainerArgsForCall(i int) (lager.Logger, db.ContainerOwner, worker.ContainerSpec, worker.WorkerSpec, worker.ContainerPlacementStrategy) { + fake.findOrChooseWorkerForContainerMutex.RLock() + defer fake.findOrChooseWorkerForContainerMutex.RUnlock() + argsForCall := fake.findOrChooseWorkerForContainerArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 +} + +func (fake *FakePool) FindOrChooseWorkerForContainerReturns(result1 worker.Worker, result2 error) { + fake.findOrChooseWorkerForContainerMutex.Lock() + defer fake.findOrChooseWorkerForContainerMutex.Unlock() + fake.FindOrChooseWorkerForContainerStub = nil + fake.findOrChooseWorkerForContainerReturns = struct { + result1 worker.Worker + result2 error + }{result1, result2} +} + +func (fake *FakePool) FindOrChooseWorkerForContainerReturnsOnCall(i int, result1 worker.Worker, result2 error) { + fake.findOrChooseWorkerForContainerMutex.Lock() + defer fake.findOrChooseWorkerForContainerMutex.Unlock() + fake.FindOrChooseWorkerForContainerStub = nil + if fake.findOrChooseWorkerForContainerReturnsOnCall == nil { + fake.findOrChooseWorkerForContainerReturnsOnCall = make(map[int]struct { + result1 worker.Worker + result2 error + }) + } + fake.findOrChooseWorkerForContainerReturnsOnCall[i] = struct { + result1 worker.Worker + result2 error + }{result1, result2} +} + func (fake *FakePool) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.findContainerByHandleMutex.RLock() - defer fake.findContainerByHandleMutex.RUnlock() fake.findOrChooseWorkerMutex.RLock() defer fake.findOrChooseWorkerMutex.RUnlock() + fake.findOrChooseWorkerForContainerMutex.RLock() + defer fake.findOrChooseWorkerForContainerMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/atc/worker/workerfakes/fake_volume.go b/atc/worker/workerfakes/fake_volume.go index dc20b26bc0f..e86b3118112 100644 --- a/atc/worker/workerfakes/fake_volume.go +++ b/atc/worker/workerfakes/fake_volume.go @@ -56,6 +56,20 @@ type FakeVolume struct { handleReturnsOnCall map[int]struct { result1 string } + InitializeArtifactStub func(string, string) (db.WorkerArtifact, error) + initializeArtifactMutex sync.RWMutex + initializeArtifactArgsForCall []struct { + arg1 string + arg2 string + } + initializeArtifactReturns struct { + result1 db.WorkerArtifact + result2 error + } + initializeArtifactReturnsOnCall map[int]struct { + result1 db.WorkerArtifact + result2 error + } InitializeResourceCacheStub func(db.UsedResourceCache) error initializeResourceCacheMutex sync.RWMutex initializeResourceCacheArgsForCall []struct { @@ -386,6 +400,70 @@ func (fake *FakeVolume) HandleReturnsOnCall(i int, result1 string) { }{result1} } +func (fake *FakeVolume) InitializeArtifact(arg1 string, arg2 string) (db.WorkerArtifact, error) { + fake.initializeArtifactMutex.Lock() + ret, specificReturn := fake.initializeArtifactReturnsOnCall[len(fake.initializeArtifactArgsForCall)] + fake.initializeArtifactArgsForCall = append(fake.initializeArtifactArgsForCall, struct { + arg1 string + arg2 string + }{arg1, arg2}) + fake.recordInvocation("InitializeArtifact", []interface{}{arg1, arg2}) + fake.initializeArtifactMutex.Unlock() + if fake.InitializeArtifactStub != nil { + return fake.InitializeArtifactStub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.initializeArtifactReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeVolume) InitializeArtifactCallCount() int { + fake.initializeArtifactMutex.RLock() + defer fake.initializeArtifactMutex.RUnlock() + return len(fake.initializeArtifactArgsForCall) +} + +func (fake *FakeVolume) InitializeArtifactCalls(stub func(string, string) (db.WorkerArtifact, error)) { + fake.initializeArtifactMutex.Lock() + defer fake.initializeArtifactMutex.Unlock() + fake.InitializeArtifactStub = stub +} + +func (fake *FakeVolume) InitializeArtifactArgsForCall(i int) (string, string) { + fake.initializeArtifactMutex.RLock() + defer fake.initializeArtifactMutex.RUnlock() + argsForCall := fake.initializeArtifactArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeVolume) InitializeArtifactReturns(result1 db.WorkerArtifact, result2 error) { + fake.initializeArtifactMutex.Lock() + defer fake.initializeArtifactMutex.Unlock() + fake.InitializeArtifactStub = nil + fake.initializeArtifactReturns = struct { + result1 db.WorkerArtifact + result2 error + }{result1, result2} +} + +func (fake *FakeVolume) InitializeArtifactReturnsOnCall(i int, result1 db.WorkerArtifact, result2 error) { + fake.initializeArtifactMutex.Lock() + defer fake.initializeArtifactMutex.Unlock() + fake.InitializeArtifactStub = nil + if fake.initializeArtifactReturnsOnCall == nil { + fake.initializeArtifactReturnsOnCall = make(map[int]struct { + result1 db.WorkerArtifact + result2 error + }) + } + fake.initializeArtifactReturnsOnCall[i] = struct { + result1 db.WorkerArtifact + result2 error + }{result1, result2} +} + func (fake *FakeVolume) InitializeResourceCache(arg1 db.UsedResourceCache) error { fake.initializeResourceCacheMutex.Lock() ret, specificReturn := fake.initializeResourceCacheReturnsOnCall[len(fake.initializeResourceCacheArgsForCall)] @@ -925,6 +1003,8 @@ func (fake *FakeVolume) Invocations() map[string][][]interface{} { defer fake.destroyMutex.RUnlock() fake.handleMutex.RLock() defer fake.handleMutex.RUnlock() + fake.initializeArtifactMutex.RLock() + defer fake.initializeArtifactMutex.RUnlock() fake.initializeResourceCacheMutex.RLock() defer fake.initializeResourceCacheMutex.RUnlock() fake.initializeTaskCacheMutex.RLock() diff --git a/atc/worker/workerfakes/fake_volume_client.go b/atc/worker/workerfakes/fake_volume_client.go index 5f10331fe6d..f0af37539c1 100644 --- a/atc/worker/workerfakes/fake_volume_client.go +++ b/atc/worker/workerfakes/fake_volume_client.go @@ -10,6 +10,23 @@ import ( ) type FakeVolumeClient struct { + CreateVolumeStub func(lager.Logger, worker.VolumeSpec, int, string, db.VolumeType) (worker.Volume, error) + createVolumeMutex sync.RWMutex + createVolumeArgsForCall []struct { + arg1 lager.Logger + arg2 worker.VolumeSpec + arg3 int + arg4 string + arg5 db.VolumeType + } + createVolumeReturns struct { + result1 worker.Volume + result2 error + } + createVolumeReturnsOnCall map[int]struct { + result1 worker.Volume + result2 error + } CreateVolumeForTaskCacheStub func(lager.Logger, worker.VolumeSpec, int, int, string, string) (worker.Volume, error) createVolumeForTaskCacheMutex sync.RWMutex createVolumeForTaskCacheArgsForCall []struct { @@ -149,6 +166,73 @@ type FakeVolumeClient struct { invocationsMutex sync.RWMutex } +func (fake *FakeVolumeClient) CreateVolume(arg1 lager.Logger, arg2 worker.VolumeSpec, arg3 int, arg4 string, arg5 db.VolumeType) (worker.Volume, error) { + fake.createVolumeMutex.Lock() + ret, specificReturn := fake.createVolumeReturnsOnCall[len(fake.createVolumeArgsForCall)] + fake.createVolumeArgsForCall = append(fake.createVolumeArgsForCall, struct { + arg1 lager.Logger + arg2 worker.VolumeSpec + arg3 int + arg4 string + arg5 db.VolumeType + }{arg1, arg2, arg3, arg4, arg5}) + fake.recordInvocation("CreateVolume", []interface{}{arg1, arg2, arg3, arg4, arg5}) + fake.createVolumeMutex.Unlock() + if fake.CreateVolumeStub != nil { + return fake.CreateVolumeStub(arg1, arg2, arg3, arg4, arg5) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.createVolumeReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeVolumeClient) CreateVolumeCallCount() int { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + return len(fake.createVolumeArgsForCall) +} + +func (fake *FakeVolumeClient) CreateVolumeCalls(stub func(lager.Logger, worker.VolumeSpec, int, string, db.VolumeType) (worker.Volume, error)) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = stub +} + +func (fake *FakeVolumeClient) CreateVolumeArgsForCall(i int) (lager.Logger, worker.VolumeSpec, int, string, db.VolumeType) { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + argsForCall := fake.createVolumeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4, argsForCall.arg5 +} + +func (fake *FakeVolumeClient) CreateVolumeReturns(result1 worker.Volume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + fake.createVolumeReturns = struct { + result1 worker.Volume + result2 error + }{result1, result2} +} + +func (fake *FakeVolumeClient) CreateVolumeReturnsOnCall(i int, result1 worker.Volume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + if fake.createVolumeReturnsOnCall == nil { + fake.createVolumeReturnsOnCall = make(map[int]struct { + result1 worker.Volume + result2 error + }) + } + fake.createVolumeReturnsOnCall[i] = struct { + result1 worker.Volume + result2 error + }{result1, result2} +} + func (fake *FakeVolumeClient) CreateVolumeForTaskCache(arg1 lager.Logger, arg2 worker.VolumeSpec, arg3 int, arg4 int, arg5 string, arg6 string) (worker.Volume, error) { fake.createVolumeForTaskCacheMutex.Lock() ret, specificReturn := fake.createVolumeForTaskCacheReturnsOnCall[len(fake.createVolumeForTaskCacheArgsForCall)] @@ -691,6 +775,8 @@ func (fake *FakeVolumeClient) LookupVolumeReturnsOnCall(i int, result1 worker.Vo func (fake *FakeVolumeClient) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() fake.createVolumeForTaskCacheMutex.RLock() defer fake.createVolumeForTaskCacheMutex.RUnlock() fake.findOrCreateCOWVolumeForContainerMutex.RLock() diff --git a/atc/worker/workerfakes/fake_worker.go b/atc/worker/workerfakes/fake_worker.go index f6cbc500a94..1d53845a636 100644 --- a/atc/worker/workerfakes/fake_worker.go +++ b/atc/worker/workerfakes/fake_worker.go @@ -61,6 +61,22 @@ type FakeWorker struct { result2 bool result3 error } + CreateVolumeStub func(lager.Logger, worker.VolumeSpec, int, db.VolumeType) (worker.Volume, error) + createVolumeMutex sync.RWMutex + createVolumeArgsForCall []struct { + arg1 lager.Logger + arg2 worker.VolumeSpec + arg3 int + arg4 db.VolumeType + } + createVolumeReturns struct { + result1 worker.Volume + result2 error + } + createVolumeReturnsOnCall map[int]struct { + result1 worker.Volume + result2 error + } DescriptionStub func() string descriptionMutex sync.RWMutex descriptionArgsForCall []struct { @@ -478,6 +494,72 @@ func (fake *FakeWorker) CertsVolumeReturnsOnCall(i int, result1 worker.Volume, r }{result1, result2, result3} } +func (fake *FakeWorker) CreateVolume(arg1 lager.Logger, arg2 worker.VolumeSpec, arg3 int, arg4 db.VolumeType) (worker.Volume, error) { + fake.createVolumeMutex.Lock() + ret, specificReturn := fake.createVolumeReturnsOnCall[len(fake.createVolumeArgsForCall)] + fake.createVolumeArgsForCall = append(fake.createVolumeArgsForCall, struct { + arg1 lager.Logger + arg2 worker.VolumeSpec + arg3 int + arg4 db.VolumeType + }{arg1, arg2, arg3, arg4}) + fake.recordInvocation("CreateVolume", []interface{}{arg1, arg2, arg3, arg4}) + fake.createVolumeMutex.Unlock() + if fake.CreateVolumeStub != nil { + return fake.CreateVolumeStub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.createVolumeReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeWorker) CreateVolumeCallCount() int { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + return len(fake.createVolumeArgsForCall) +} + +func (fake *FakeWorker) CreateVolumeCalls(stub func(lager.Logger, worker.VolumeSpec, int, db.VolumeType) (worker.Volume, error)) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = stub +} + +func (fake *FakeWorker) CreateVolumeArgsForCall(i int) (lager.Logger, worker.VolumeSpec, int, db.VolumeType) { + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() + argsForCall := fake.createVolumeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeWorker) CreateVolumeReturns(result1 worker.Volume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + fake.createVolumeReturns = struct { + result1 worker.Volume + result2 error + }{result1, result2} +} + +func (fake *FakeWorker) CreateVolumeReturnsOnCall(i int, result1 worker.Volume, result2 error) { + fake.createVolumeMutex.Lock() + defer fake.createVolumeMutex.Unlock() + fake.CreateVolumeStub = nil + if fake.createVolumeReturnsOnCall == nil { + fake.createVolumeReturnsOnCall = make(map[int]struct { + result1 worker.Volume + result2 error + }) + } + fake.createVolumeReturnsOnCall[i] = struct { + result1 worker.Volume + result2 error + }{result1, result2} +} + func (fake *FakeWorker) Description() string { fake.descriptionMutex.Lock() ret, specificReturn := fake.descriptionReturnsOnCall[len(fake.descriptionArgsForCall)] @@ -1368,6 +1450,8 @@ func (fake *FakeWorker) Invocations() map[string][][]interface{} { defer fake.buildContainersMutex.RUnlock() fake.certsVolumeMutex.RLock() defer fake.certsVolumeMutex.RUnlock() + fake.createVolumeMutex.RLock() + defer fake.createVolumeMutex.RUnlock() fake.descriptionMutex.RLock() defer fake.descriptionMutex.RUnlock() fake.ephemeralMutex.RLock() diff --git a/atc/worker/workerfakes/fake_worker_provider.go b/atc/worker/workerfakes/fake_worker_provider.go index c0cebd65171..de1741c2658 100644 --- a/atc/worker/workerfakes/fake_worker_provider.go +++ b/atc/worker/workerfakes/fake_worker_provider.go @@ -28,6 +28,23 @@ type FakeWorkerProvider struct { result2 bool result3 error } + FindWorkerForVolumeStub func(lager.Logger, int, string) (worker.Worker, bool, error) + findWorkerForVolumeMutex sync.RWMutex + findWorkerForVolumeArgsForCall []struct { + arg1 lager.Logger + arg2 int + arg3 string + } + findWorkerForVolumeReturns struct { + result1 worker.Worker + result2 bool + result3 error + } + findWorkerForVolumeReturnsOnCall map[int]struct { + result1 worker.Worker + result2 bool + result3 error + } FindWorkersForContainerByOwnerStub func(lager.Logger, db.ContainerOwner) ([]worker.Worker, error) findWorkersForContainerByOwnerMutex sync.RWMutex findWorkersForContainerByOwnerArgsForCall []struct { @@ -141,6 +158,74 @@ func (fake *FakeWorkerProvider) FindWorkerForContainerReturnsOnCall(i int, resul }{result1, result2, result3} } +func (fake *FakeWorkerProvider) FindWorkerForVolume(arg1 lager.Logger, arg2 int, arg3 string) (worker.Worker, bool, error) { + fake.findWorkerForVolumeMutex.Lock() + ret, specificReturn := fake.findWorkerForVolumeReturnsOnCall[len(fake.findWorkerForVolumeArgsForCall)] + fake.findWorkerForVolumeArgsForCall = append(fake.findWorkerForVolumeArgsForCall, struct { + arg1 lager.Logger + arg2 int + arg3 string + }{arg1, arg2, arg3}) + fake.recordInvocation("FindWorkerForVolume", []interface{}{arg1, arg2, arg3}) + fake.findWorkerForVolumeMutex.Unlock() + if fake.FindWorkerForVolumeStub != nil { + return fake.FindWorkerForVolumeStub(arg1, arg2, arg3) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.findWorkerForVolumeReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *FakeWorkerProvider) FindWorkerForVolumeCallCount() int { + fake.findWorkerForVolumeMutex.RLock() + defer fake.findWorkerForVolumeMutex.RUnlock() + return len(fake.findWorkerForVolumeArgsForCall) +} + +func (fake *FakeWorkerProvider) FindWorkerForVolumeCalls(stub func(lager.Logger, int, string) (worker.Worker, bool, error)) { + fake.findWorkerForVolumeMutex.Lock() + defer fake.findWorkerForVolumeMutex.Unlock() + fake.FindWorkerForVolumeStub = stub +} + +func (fake *FakeWorkerProvider) FindWorkerForVolumeArgsForCall(i int) (lager.Logger, int, string) { + fake.findWorkerForVolumeMutex.RLock() + defer fake.findWorkerForVolumeMutex.RUnlock() + argsForCall := fake.findWorkerForVolumeArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 +} + +func (fake *FakeWorkerProvider) FindWorkerForVolumeReturns(result1 worker.Worker, result2 bool, result3 error) { + fake.findWorkerForVolumeMutex.Lock() + defer fake.findWorkerForVolumeMutex.Unlock() + fake.FindWorkerForVolumeStub = nil + fake.findWorkerForVolumeReturns = struct { + result1 worker.Worker + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeWorkerProvider) FindWorkerForVolumeReturnsOnCall(i int, result1 worker.Worker, result2 bool, result3 error) { + fake.findWorkerForVolumeMutex.Lock() + defer fake.findWorkerForVolumeMutex.Unlock() + fake.FindWorkerForVolumeStub = nil + if fake.findWorkerForVolumeReturnsOnCall == nil { + fake.findWorkerForVolumeReturnsOnCall = make(map[int]struct { + result1 worker.Worker + result2 bool + result3 error + }) + } + fake.findWorkerForVolumeReturnsOnCall[i] = struct { + result1 worker.Worker + result2 bool + result3 error + }{result1, result2, result3} +} + func (fake *FakeWorkerProvider) FindWorkersForContainerByOwner(arg1 lager.Logger, arg2 db.ContainerOwner) ([]worker.Worker, error) { fake.findWorkersForContainerByOwnerMutex.Lock() ret, specificReturn := fake.findWorkersForContainerByOwnerReturnsOnCall[len(fake.findWorkersForContainerByOwnerArgsForCall)] @@ -336,6 +421,8 @@ func (fake *FakeWorkerProvider) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.findWorkerForContainerMutex.RLock() defer fake.findWorkerForContainerMutex.RUnlock() + fake.findWorkerForVolumeMutex.RLock() + defer fake.findWorkerForVolumeMutex.RUnlock() fake.findWorkersForContainerByOwnerMutex.RLock() defer fake.findWorkersForContainerByOwnerMutex.RUnlock() fake.newGardenWorkerMutex.RLock() diff --git a/atc/worker_artifact.go b/atc/worker_artifact.go new file mode 100644 index 00000000000..9e462b3dece --- /dev/null +++ b/atc/worker_artifact.go @@ -0,0 +1,8 @@ +package atc + +type WorkerArtifact struct { + ID int `json:"id"` + Path string `json:"path"` + Checksum string `json:"checksum"` + CreatedAt int64 `json:"created_at"` +} diff --git a/atc/wrappa/api_auth_wrappa.go b/atc/wrappa/api_auth_wrappa.go index 6a0aa77b699..bd2ba9a1283 100644 --- a/atc/wrappa/api_auth_wrappa.go +++ b/atc/wrappa/api_auth_wrappa.go @@ -139,7 +139,9 @@ func (wrappa *APIAuthWrappa) Wrap(handlers rata.Handlers) rata.Handlers { atc.ExposePipeline, atc.HidePipeline, atc.SaveConfig, - atc.ClearTaskCache: + atc.ClearTaskCache, + atc.CreateArtifact, + atc.GetArtifact: newHandler = auth.CheckAuthorizationHandler(handler, rejector) // think about it! diff --git a/atc/wrappa/api_auth_wrappa_test.go b/atc/wrappa/api_auth_wrappa_test.go index ee3cf9fe3d0..50af8813382 100644 --- a/atc/wrappa/api_auth_wrappa_test.go +++ b/atc/wrappa/api_auth_wrappa_test.go @@ -7,10 +7,9 @@ import ( "github.com/concourse/concourse/atc/api/auth" "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/wrappa" - "github.com/tedsuo/rata" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + "github.com/tedsuo/rata" ) var _ = Describe("APIAuthWrappa", func() { @@ -238,6 +237,8 @@ var _ = Describe("APIAuthWrappa", func() { atc.HidePipeline: authorized(inputHandlers[atc.HidePipeline]), atc.CreatePipelineBuild: authorized(inputHandlers[atc.CreatePipelineBuild]), atc.ClearTaskCache: authorized(inputHandlers[atc.ClearTaskCache]), + atc.CreateArtifact: authorized(inputHandlers[atc.CreateArtifact]), + atc.GetArtifact: authorized(inputHandlers[atc.GetArtifact]), } }) diff --git a/go.mod b/go.mod index 44cdd1940e8..219e4826ae1 100644 --- a/go.mod +++ b/go.mod @@ -154,7 +154,7 @@ require ( golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect - golang.org/x/sys v0.0.0-20190124100055-b90733256f2e + golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect gopkg.in/cheggaaa/pb.v1 v1.0.27 From 2d8a9179cdd309d7981936351222f1ae04514df3 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 20 Dec 2018 13:29:43 -0500 Subject: [PATCH 02/21] atc: add artifacts to go-concourse client Signed-off-by: Josh Winters --- go-concourse/concourse/artifacts.go | 51 ++++++++++++++ go-concourse/concourse/artifacts_test.go | 89 ++++++++++++++++++++++++ go-concourse/concourse/team.go | 5 ++ 3 files changed, 145 insertions(+) create mode 100644 go-concourse/concourse/artifacts.go create mode 100644 go-concourse/concourse/artifacts_test.go diff --git a/go-concourse/concourse/artifacts.go b/go-concourse/concourse/artifacts.go new file mode 100644 index 00000000000..3490d43ea68 --- /dev/null +++ b/go-concourse/concourse/artifacts.go @@ -0,0 +1,51 @@ +package concourse + +import ( + "io" + "net/http" + "strconv" + + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/go-concourse/concourse/internal" + "github.com/tedsuo/rata" +) + +func (team *team) CreateArtifact(src io.Reader) (atc.WorkerArtifact, error) { + var artifact atc.WorkerArtifact + + params := rata.Params{ + "team_name": team.Name(), + } + + err := team.connection.Send(internal.Request{ + Header: http.Header{"Content-Type": {"application/octet-stream"}}, + RequestName: atc.CreateArtifact, + Params: params, + Body: src, + }, &internal.Response{ + Result: &artifact, + }) + + return artifact, err +} + +func (team *team) GetArtifact(artifactID int) (io.ReadCloser, error) { + + params := rata.Params{ + "team_name": team.Name(), + "artifact_id": strconv.Itoa(artifactID), + } + + response := internal.Response{} + err := team.connection.Send(internal.Request{ + RequestName: atc.GetArtifact, + Params: params, + ReturnResponseBody: true, + }, &response) + + if err != nil { + return nil, err + } + + return response.Result.(io.ReadCloser), nil +} diff --git a/go-concourse/concourse/artifacts_test.go b/go-concourse/concourse/artifacts_test.go new file mode 100644 index 00000000000..4910df5fbdf --- /dev/null +++ b/go-concourse/concourse/artifacts_test.go @@ -0,0 +1,89 @@ +package concourse_test + +import ( + "bytes" + "io/ioutil" + "net/http" + + "github.com/concourse/concourse/atc" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/onsi/gomega/ghttp" +) + +var _ = Describe("Artifacts", func() { + + Describe("CreateArtifact", func() { + Context("when creating the artifact fails", func() { + BeforeEach(func() { + atcServer.AppendHandlers( + ghttp.CombineHandlers( + ghttp.VerifyRequest("POST", "/api/v1/teams/some-team/artifacts"), + ghttp.VerifyHeader(http.Header{"Content-Type": {"application/octet-stream"}}), + ghttp.VerifyBody([]byte("some-contents")), + ghttp.RespondWith(http.StatusInternalServerError, nil), + ), + ) + }) + + It("errors", func() { + _, err := team.CreateArtifact(bytes.NewBufferString("some-contents")) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when creating the artifact succeeds", func() { + BeforeEach(func() { + atcServer.AppendHandlers( + ghttp.CombineHandlers( + ghttp.VerifyRequest("POST", "/api/v1/teams/some-team/artifacts"), + ghttp.VerifyHeader(http.Header{"Content-Type": {"application/octet-stream"}}), + ghttp.VerifyBody([]byte("some-contents")), + ghttp.RespondWithJSONEncoded(http.StatusCreated, atc.WorkerArtifact{ID: 17}), + ), + ) + }) + + It("returns json", func() { + artifact, err := team.CreateArtifact(bytes.NewBufferString("some-contents")) + Expect(err).NotTo(HaveOccurred()) + Expect(artifact.ID).To(Equal(17)) + }) + }) + }) + + Describe("GetArtifact", func() { + Context("when getting the artifact fails", func() { + BeforeEach(func() { + atcServer.AppendHandlers( + ghttp.CombineHandlers( + ghttp.VerifyRequest("GET", "/api/v1/teams/some-team/artifacts/17"), + ghttp.RespondWith(http.StatusInternalServerError, ""), + ), + ) + }) + + It("errors", func() { + _, err := team.GetArtifact(17) + Expect(err).To(HaveOccurred()) + }) + }) + + Context("when the artifact exsits", func() { + BeforeEach(func() { + atcServer.AppendHandlers( + ghttp.CombineHandlers( + ghttp.VerifyRequest("GET", "/api/v1/teams/some-team/artifacts/17"), + ghttp.RespondWith(http.StatusOK, "some-other-contents"), + ), + ) + }) + + It("returns the contents", func() { + contents, err := team.GetArtifact(17) + Expect(err).NotTo(HaveOccurred()) + Expect(ioutil.ReadAll(contents)).To(Equal([]byte("some-other-contents"))) + }) + }) + }) +}) diff --git a/go-concourse/concourse/team.go b/go-concourse/concourse/team.go index 10f5b11e3d6..4334ed611e7 100644 --- a/go-concourse/concourse/team.go +++ b/go-concourse/concourse/team.go @@ -1,6 +1,8 @@ package concourse import ( + "io" + "github.com/concourse/concourse/atc" "github.com/concourse/concourse/go-concourse/concourse/internal" ) @@ -58,6 +60,9 @@ type Team interface { CreateBuild(plan atc.Plan) (atc.Build, error) Builds(page Page) ([]atc.Build, Pagination, error) OrderingPipelines(pipelineNames []string) error + + CreateArtifact(io.Reader) (atc.WorkerArtifact, error) + GetArtifact(int) (io.ReadCloser, error) } type team struct { From cf7154aca777a37dbf7a7c49e50e51c62d0f7903 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 3 Jan 2019 17:05:59 -0500 Subject: [PATCH 03/21] atc: remove the send input/read output endpoints Co-authored-by: Topher Bullock Signed-off-by: Josh Winters --- atc/api/accessor/accessor.go | 2 - atc/api/accessor/accessor_test.go | 9 - atc/api/builds_test.go | 385 ------------------------- atc/api/buildserver/read_output.go | 73 ----- atc/api/buildserver/send_input.go | 113 -------- atc/api/handler.go | 11 - atc/engine/db_engine.go | 45 --- atc/engine/db_engine_test.go | 60 +--- atc/engine/engine.go | 7 - atc/engine/exec_engine.go | 9 - atc/engine/exec_v1_dummy_engine.go | 7 - atc/routes.go | 5 - atc/worker/artifact_repository.go | 82 ------ atc/worker/artifact_repository_test.go | 113 +------- atc/wrappa/api_auth_wrappa.go | 6 +- atc/wrappa/api_auth_wrappa_test.go | 4 +- 16 files changed, 6 insertions(+), 925 deletions(-) delete mode 100644 atc/api/buildserver/read_output.go delete mode 100644 atc/api/buildserver/send_input.go diff --git a/atc/api/accessor/accessor.go b/atc/api/accessor/accessor.go index f53e9a3f85f..e939d85cd15 100644 --- a/atc/api/accessor/accessor.go +++ b/atc/api/accessor/accessor.go @@ -197,6 +197,4 @@ var requiredRoles = map[string]string{ atc.RenameTeam: "owner", atc.DestroyTeam: "owner", atc.ListTeamBuilds: "viewer", - atc.SendInputToBuildPlan: "member", - atc.ReadOutputFromBuildPlan: "member", } diff --git a/atc/api/accessor/accessor_test.go b/atc/api/accessor/accessor_test.go index a42654d5981..a91e94d0b19 100644 --- a/atc/api/accessor/accessor_test.go +++ b/atc/api/accessor/accessor_test.go @@ -9,7 +9,6 @@ import ( "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/api/accessor" jwt "github.com/dgrijalva/jwt-go" - . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" @@ -650,13 +649,5 @@ var _ = Describe("Accessor", func() { Entry("owner :: "+atc.ListTeamBuilds, atc.ListTeamBuilds, "owner", true), Entry("member :: "+atc.ListTeamBuilds, atc.ListTeamBuilds, "member", true), Entry("viewer :: "+atc.ListTeamBuilds, atc.ListTeamBuilds, "viewer", true), - - Entry("owner :: "+atc.SendInputToBuildPlan, atc.SendInputToBuildPlan, "owner", true), - Entry("member :: "+atc.SendInputToBuildPlan, atc.SendInputToBuildPlan, "member", true), - Entry("viewer :: "+atc.SendInputToBuildPlan, atc.SendInputToBuildPlan, "viewer", false), - - Entry("owner :: "+atc.ReadOutputFromBuildPlan, atc.ReadOutputFromBuildPlan, "owner", true), - Entry("member :: "+atc.ReadOutputFromBuildPlan, atc.ReadOutputFromBuildPlan, "member", true), - Entry("viewer :: "+atc.ReadOutputFromBuildPlan, atc.ReadOutputFromBuildPlan, "viewer", false), ) }) diff --git a/atc/api/builds_test.go b/atc/api/builds_test.go index 1ea693b7258..133bff9aac7 100644 --- a/atc/api/builds_test.go +++ b/atc/api/builds_test.go @@ -5,7 +5,6 @@ import ( "encoding/json" "errors" "fmt" - "io" "io/ioutil" "net/http" "sync" @@ -19,7 +18,6 @@ import ( "github.com/concourse/concourse/atc/engine/enginefakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/onsi/gomega/ghttp" ) var _ = Describe("Builds API", func() { @@ -1454,387 +1452,4 @@ var _ = Describe("Builds API", func() { }) }) }) - - Describe("PUT /api/v1/builds/:build_id/plan/:plan_id/input", func() { - var ( - otherTracker *ghttp.Server - - response *http.Response - ) - - BeforeEach(func() { - otherTracker = ghttp.NewServer() - - otherTracker.AppendHandlers( - ghttp.CombineHandlers( - ghttp.VerifyRequest("PUT", "/api/v1/builds/128/plan/some-plan/input"), - ghttp.VerifyBody([]byte("some-payload")), - ghttp.RespondWith(http.StatusTeapot, "im a teapot"), - ), - ) - }) - - JustBeforeEach(func() { - var err error - - req, err := http.NewRequest("PUT", server.URL+"/api/v1/builds/128/plan/some-plan/input", bytes.NewBufferString("some-payload")) - Expect(err).NotTo(HaveOccurred()) - - response, err = client.Do(req) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - otherTracker.Close() - }) - - Context("when authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(true) - }) - - Context("when the build can be found", func() { - BeforeEach(func() { - build.TeamNameReturns("some-team") - dbBuildFactory.BuildReturns(build, true, nil) - }) - - Context("when accessing same teams build", func() { - BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(true) - }) - - Context("when the build is tracked by the current ATC", func() { - BeforeEach(func() { - build.TrackerReturns("http://127.0.0.1:1234") - }) - - Context("when the engine returns a build", func() { - var engineBuild *enginefakes.FakeBuild - var streamedBody string - - BeforeEach(func() { - engineBuild = new(enginefakes.FakeBuild) - fakeEngine.LookupBuildReturns(engineBuild, nil) - - engineBuild.ReceiveInputStub = func(logger lager.Logger, id atc.PlanID, stream io.ReadCloser) { - p, err := ioutil.ReadAll(stream) - Expect(err).ToNot(HaveOccurred()) - - streamedBody = string(p) - } - }) - - It("sends the request body to the plan", func() { - Expect(engineBuild.ReceiveInputCallCount()).To(Equal(1)) - - _, id, _ := engineBuild.ReceiveInputArgsForCall(0) - Expect(id).To(Equal(atc.PlanID("some-plan"))) - Expect(streamedBody).To(Equal("some-payload")) - }) - - It("returns No Content", func() { - Expect(response.StatusCode).To(Equal(http.StatusNoContent)) - }) - - Context("when the build is initially not tracked", func() { - BeforeEach(func() { - build.ReloadReturns(true, nil) - - build.TrackerReturnsOnCall(0, "") - build.TrackerReturnsOnCall(2, "http://127.0.0.1:1234") - }) - - It("waits until it is", func() { - Expect(response.StatusCode).To(Equal(http.StatusNoContent)) - - Expect(engineBuild.ReceiveInputCallCount()).To(Equal(1)) - - _, id, _ := engineBuild.ReceiveInputArgsForCall(0) - Expect(id).To(Equal(atc.PlanID("some-plan"))) - Expect(streamedBody).To(Equal("some-payload")) - }) - }) - }) - - Context("when the engine returns no build", func() { - BeforeEach(func() { - fakeEngine.LookupBuildReturns(nil, errors.New("oh no!")) - }) - - It("returns Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - - Context("when the build is tracked by another ATC", func() { - BeforeEach(func() { - build.TrackerReturns(otherTracker.URL()) - }) - - It("does not bother looking up the build in the engine", func() { - Expect(fakeEngine.LookupBuildCallCount()).To(Equal(0)) - }) - - It("forwards the request to the other ATC", func() { - Expect(otherTracker.ReceivedRequests()).To(HaveLen(1)) - Expect(response.StatusCode).To(Equal(http.StatusTeapot)) - Expect(ioutil.ReadAll(response.Body)).To(Equal([]byte("im a teapot"))) - }) - - Context("when the build is initially not tracked", func() { - BeforeEach(func() { - build.ReloadReturns(true, nil) - - build.TrackerReturnsOnCall(0, "") - build.TrackerReturnsOnCall(2, otherTracker.URL()) - }) - - It("waits until it is", func() { - Expect(fakeEngine.LookupBuildCallCount()).To(Equal(0)) - - Expect(otherTracker.ReceivedRequests()).To(HaveLen(1)) - Expect(response.StatusCode).To(Equal(http.StatusTeapot)) - Expect(ioutil.ReadAll(response.Body)).To(Equal([]byte("im a teapot"))) - }) - }) - }) - }) - - Context("when accessing other teams build", func() { - BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(false) - }) - - It("returns 403", func() { - Expect(response.StatusCode).To(Equal(http.StatusForbidden)) - }) - }) - }) - - Context("when the build can not be found", func() { - BeforeEach(func() { - dbBuildFactory.BuildReturns(nil, false, nil) - }) - - It("returns Not Found", func() { - Expect(response.StatusCode).To(Equal(http.StatusNotFound)) - }) - }) - - Context("when calling the database fails", func() { - BeforeEach(func() { - dbBuildFactory.BuildReturns(nil, false, errors.New("nope")) - }) - - It("returns Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - - Context("when not authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(false) - }) - - It("returns 401", func() { - Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - - It("does not abort the build", func() { - Expect(otherTracker.ReceivedRequests()).To(BeEmpty()) - }) - }) - }) - - Describe("GET /api/v1/builds/:build_id/plan/:plan_id/output", func() { - var ( - otherTracker *ghttp.Server - - response *http.Response - ) - - BeforeEach(func() { - otherTracker = ghttp.NewServer() - - otherTracker.AppendHandlers( - ghttp.CombineHandlers( - ghttp.VerifyRequest("GET", "/api/v1/builds/128/plan/some-plan/output"), - ghttp.RespondWith(http.StatusTeapot, "im a teapot"), - ), - ) - }) - - JustBeforeEach(func() { - var err error - - req, err := http.NewRequest("GET", server.URL+"/api/v1/builds/128/plan/some-plan/output", nil) - Expect(err).NotTo(HaveOccurred()) - - response, err = client.Do(req) - Expect(err).NotTo(HaveOccurred()) - }) - - AfterEach(func() { - otherTracker.Close() - }) - - Context("when authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(true) - }) - - Context("when the build can be found", func() { - BeforeEach(func() { - build.TeamNameReturns("some-team") - dbBuildFactory.BuildReturns(build, true, nil) - }) - - Context("when accessing same team's build", func() { - BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(true) - }) - - Context("when the build is tracked by the current ATC", func() { - BeforeEach(func() { - build.TrackerReturns("http://127.0.0.1:1234") - }) - - Context("when the engine returns a build", func() { - var engineBuild *enginefakes.FakeBuild - - BeforeEach(func() { - engineBuild = new(enginefakes.FakeBuild) - fakeEngine.LookupBuildReturns(engineBuild, nil) - - engineBuild.SendOutputStub = func(logger lager.Logger, id atc.PlanID, dest io.Writer) { - fmt.Fprintln(dest, "hello from build") - } - }) - - It("sends the plan's output to the response body", func() { - Expect(engineBuild.SendOutputCallCount()).To(Equal(1)) - - _, id, _ := engineBuild.SendOutputArgsForCall(0) - Expect(id).To(Equal(atc.PlanID("some-plan"))) - Expect(ioutil.ReadAll(response.Body)).To(Equal([]byte("hello from build\n"))) - }) - - It("returns OK", func() { - Expect(response.StatusCode).To(Equal(http.StatusOK)) - }) - - Context("when the build is initially not tracked", func() { - BeforeEach(func() { - build.ReloadReturns(true, nil) - - build.TrackerReturnsOnCall(0, "") - build.TrackerReturnsOnCall(2, "http://127.0.0.1:1234") - }) - - It("waits until it is", func() { - Expect(response.StatusCode).To(Equal(http.StatusOK)) - - Expect(engineBuild.SendOutputCallCount()).To(Equal(1)) - - _, id, _ := engineBuild.SendOutputArgsForCall(0) - Expect(id).To(Equal(atc.PlanID("some-plan"))) - Expect(ioutil.ReadAll(response.Body)).To(Equal([]byte("hello from build\n"))) - }) - }) - }) - - Context("when the engine returns no build", func() { - BeforeEach(func() { - fakeEngine.LookupBuildReturns(nil, errors.New("oh no!")) - }) - - It("returns Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - - Context("when the build is tracked by another ATC", func() { - BeforeEach(func() { - build.TrackerReturns(otherTracker.URL()) - }) - - It("does not bother looking up the build in the engine", func() { - Expect(fakeEngine.LookupBuildCallCount()).To(Equal(0)) - }) - - It("forwards the request to the other ATC", func() { - Expect(otherTracker.ReceivedRequests()).To(HaveLen(1)) - Expect(response.StatusCode).To(Equal(http.StatusTeapot)) - Expect(ioutil.ReadAll(response.Body)).To(Equal([]byte("im a teapot"))) - }) - - Context("when the build is initially not tracked", func() { - BeforeEach(func() { - build.ReloadReturns(true, nil) - - build.TrackerReturnsOnCall(0, "") - build.TrackerReturnsOnCall(2, otherTracker.URL()) - }) - - It("waits until it is", func() { - Expect(fakeEngine.LookupBuildCallCount()).To(Equal(0)) - - Expect(otherTracker.ReceivedRequests()).To(HaveLen(1)) - Expect(response.StatusCode).To(Equal(http.StatusTeapot)) - Expect(ioutil.ReadAll(response.Body)).To(Equal([]byte("im a teapot"))) - }) - }) - }) - }) - - Context("when accessing other team's build", func() { - BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(false) - }) - - It("returns 403", func() { - Expect(response.StatusCode).To(Equal(http.StatusForbidden)) - }) - }) - }) - - Context("when the build can not be found", func() { - BeforeEach(func() { - dbBuildFactory.BuildReturns(nil, false, nil) - }) - - It("returns Not Found", func() { - Expect(response.StatusCode).To(Equal(http.StatusNotFound)) - }) - }) - - Context("when calling the database fails", func() { - BeforeEach(func() { - dbBuildFactory.BuildReturns(nil, false, errors.New("nope")) - }) - - It("returns Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - - Context("when not authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(false) - }) - - It("returns 401", func() { - Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - - It("does not abort the build", func() { - Expect(otherTracker.ReceivedRequests()).To(BeEmpty()) - }) - }) - }) }) diff --git a/atc/api/buildserver/read_output.go b/atc/api/buildserver/read_output.go deleted file mode 100644 index bff1e3726c6..00000000000 --- a/atc/api/buildserver/read_output.go +++ /dev/null @@ -1,73 +0,0 @@ -package buildserver - -import ( - "net/http" - "time" - - "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/db" - - "code.cloudfoundry.org/lager" -) - -func (s *Server) ReadOutputFromBuildPlan(build db.Build) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - logger := s.logger.Session("read-output", lager.Data{ - "build": build.ID(), - }) - - planID := atc.PlanID(r.FormValue(":plan_id")) - if len(planID) == 0 { - logger.Info("no-plan-id-specified") - w.WriteHeader(http.StatusBadRequest) - return - } - - logger.Debug("reading-output", lager.Data{"plan": planID}) - - cn := w.(http.CloseNotifier).CloseNotify() - - for build.Tracker() == "" { - found, err := build.Reload() - if err != nil { - logger.Error("failed-to-reload-build", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - if !found { - logger.Info("build-disappeared") - w.WriteHeader(http.StatusNotFound) - return - } - - select { - case <-time.After(time.Second): - case <-cn: - return - } - } - - if build.Tracker() == s.peerURL { - engineBuild, err := s.engine.LookupBuild(logger, build) - if err != nil { - logger.Error("failed-to-lookup-build", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - w.WriteHeader(http.StatusOK) - - engineBuild.SendOutput(logger, planID, w) - } else { - logger.Debug("forwarding", lager.Data{"to": build.Tracker()}) - - err := s.forwardRequest(w, r, build.Tracker(), atc.ReadOutputFromBuildPlan) - if err != nil { - logger.Error("failed-to-forward-request", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - } - }) -} diff --git a/atc/api/buildserver/send_input.go b/atc/api/buildserver/send_input.go deleted file mode 100644 index 834669e4a7e..00000000000 --- a/atc/api/buildserver/send_input.go +++ /dev/null @@ -1,113 +0,0 @@ -package buildserver - -import ( - "io" - "net/http" - "time" - - "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/db" - "github.com/tedsuo/rata" - - "code.cloudfoundry.org/lager" -) - -func (s *Server) SendInputToBuildPlan(build db.Build) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - logger := s.logger.Session("send-input", lager.Data{ - "build": build.ID(), - }) - - planID := atc.PlanID(r.FormValue(":plan_id")) - if len(planID) == 0 { - logger.Info("no-plan-id-specified") - w.WriteHeader(http.StatusBadRequest) - return - } - - logger.Debug("sending-input", lager.Data{"plan": planID}) - - cn := w.(http.CloseNotifier).CloseNotify() - - for build.Tracker() == "" { - found, err := build.Reload() - if err != nil { - logger.Error("failed-to-reload-build", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - if !found { - logger.Info("build-disappeared") - w.WriteHeader(http.StatusNotFound) - return - } - - select { - case <-time.After(time.Second): - case <-cn: - return - } - } - - if build.Tracker() == s.peerURL { - engineBuild, err := s.engine.LookupBuild(logger, build) - if err != nil { - logger.Error("failed-to-lookup-build", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - engineBuild.ReceiveInput(logger, planID, r.Body) - - w.WriteHeader(http.StatusNoContent) - } else { - logger.Debug("forwarding", lager.Data{"to": build.Tracker()}) - - err := s.forwardRequest(w, r, build.Tracker(), atc.SendInputToBuildPlan) - if err != nil { - logger.Error("failed-to-forward-request", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - } - }) -} - -func (s *Server) forwardRequest(w http.ResponseWriter, r *http.Request, host string, route string) error { - generator := rata.NewRequestGenerator(host, atc.Routes) - - req, err := generator.CreateRequest( - route, - rata.Params{ - "build_id": r.FormValue(":build_id"), - "plan_id": r.FormValue(":plan_id"), - }, - r.Body, - ) - if err != nil { - return err - } - - req.Header = r.Header - - client := &http.Client{ - Transport: &http.Transport{ - DisableKeepAlives: true, - }, - } - - response, err := client.Do(req) - if err != nil { - return err - } - - w.WriteHeader(response.StatusCode) - - _, err = io.Copy(w, response.Body) - if err != nil { - return err - } - - return nil -} diff --git a/atc/api/handler.go b/atc/api/handler.go index 234a5baadf3..689e51b6347 100644 --- a/atc/api/handler.go +++ b/atc/api/handler.go @@ -103,17 +103,6 @@ func NewHandler( atc.GetCC: http.HandlerFunc(ccServer.GetCC), - atc.ListBuilds: http.HandlerFunc(buildServer.ListBuilds), - atc.CreateBuild: teamHandlerFactory.HandlerFor(buildServer.CreateBuild), - atc.GetBuild: buildHandlerFactory.HandlerFor(buildServer.GetBuild), - atc.BuildResources: buildHandlerFactory.HandlerFor(buildServer.BuildResources), - atc.AbortBuild: buildHandlerFactory.HandlerFor(buildServer.AbortBuild), - atc.GetBuildPlan: buildHandlerFactory.HandlerFor(buildServer.GetBuildPlan), - atc.GetBuildPreparation: buildHandlerFactory.HandlerFor(buildServer.GetBuildPreparation), - atc.BuildEvents: buildHandlerFactory.HandlerFor(buildServer.BuildEvents), - atc.SendInputToBuildPlan: buildHandlerFactory.HandlerFor(buildServer.SendInputToBuildPlan), - atc.ReadOutputFromBuildPlan: buildHandlerFactory.HandlerFor(buildServer.ReadOutputFromBuildPlan), - atc.ListAllJobs: http.HandlerFunc(jobServer.ListAllJobs), atc.ListJobs: pipelineHandlerFactory.HandlerFor(jobServer.ListJobs), atc.GetJob: pipelineHandlerFactory.HandlerFor(jobServer.GetJob), diff --git a/atc/engine/db_engine.go b/atc/engine/db_engine.go index f798d53a078..08b4c378fce 100644 --- a/atc/engine/db_engine.go +++ b/atc/engine/db_engine.go @@ -2,7 +2,6 @@ package engine import ( "fmt" - "io" "strconv" "sync" "time" @@ -305,50 +304,6 @@ func (build *dbBuild) Resume(logger lager.Logger) { } } -func (build *dbBuild) ReceiveInput(logger lager.Logger, id atc.PlanID, input io.ReadCloser) { - buildEngineName := build.build.Engine() - if buildEngineName == "" { - logger.Info("sending-input-to-build-with-no-engine") - return - } - - buildEngine, found := build.engines.Lookup(buildEngineName) - if !found { - logger.Error("unknown-engine", nil, lager.Data{"engine": buildEngineName}) - return - } - - engineBuild, err := buildEngine.LookupBuild(logger, build.build) - if err != nil { - logger.Error("failed-to-lookup-build-in-engine", err) - return - } - - engineBuild.ReceiveInput(logger, id, input) -} - -func (build *dbBuild) SendOutput(logger lager.Logger, id atc.PlanID, output io.Writer) { - buildEngineName := build.build.Engine() - if buildEngineName == "" { - logger.Info("sending-input-to-build-with-no-engine") - return - } - - buildEngine, found := build.engines.Lookup(buildEngineName) - if !found { - logger.Error("unknown-engine", nil, lager.Data{"engine": buildEngineName}) - return - } - - engineBuild, err := buildEngine.LookupBuild(logger, build.build) - if err != nil { - logger.Error("failed-to-lookup-build-in-engine", err) - return - } - - engineBuild.SendOutput(logger, id, output) -} - func (build *dbBuild) finishWithError(logger lager.Logger, finishErr error) { err := build.build.FinishWithError(finishErr) if err != nil { diff --git a/atc/engine/db_engine_test.go b/atc/engine/db_engine_test.go index 2640d5b61c9..816a72203b9 100644 --- a/atc/engine/db_engine_test.go +++ b/atc/engine/db_engine_test.go @@ -1,23 +1,19 @@ package engine_test import ( - "bytes" "errors" - "io" - "io/ioutil" "time" "code.cloudfoundry.org/lager" "code.cloudfoundry.org/lager/lagertest" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/db/lock/lockfakes" . "github.com/concourse/concourse/atc/engine" "github.com/concourse/concourse/atc/engine/enginefakes" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("DBEngine", func() { @@ -711,57 +707,5 @@ var _ = Describe("DBEngine", func() { }) }) }) - - Describe("ReceiveInput", func() { - var ( - input io.ReadCloser - - realBuild *enginefakes.FakeBuild - ) - - BeforeEach(func() { - input = ioutil.NopCloser(bytes.NewBufferString("some-payload")) - dbBuild.EngineReturns("fake-engine-b") - realBuild = new(enginefakes.FakeBuild) - fakeEngineB.LookupBuildReturns(realBuild, nil) - }) - - JustBeforeEach(func() { - build.ReceiveInput(lagertest.NewTestLogger("test"), "some-plan-id", input) - }) - - It("delegates to the real build", func() { - Expect(realBuild.ReceiveInputCallCount()).To(Equal(1)) - _, id, in := realBuild.ReceiveInputArgsForCall(0) - Expect(id).To(Equal(atc.PlanID("some-plan-id"))) - Expect(in).To(Equal(input)) - }) - }) - - Describe("SendOutput", func() { - var ( - output *bytes.Buffer - - realBuild *enginefakes.FakeBuild - ) - - BeforeEach(func() { - output = new(bytes.Buffer) - dbBuild.EngineReturns("fake-engine-b") - realBuild = new(enginefakes.FakeBuild) - fakeEngineB.LookupBuildReturns(realBuild, nil) - }) - - JustBeforeEach(func() { - build.SendOutput(lagertest.NewTestLogger("test"), "some-plan-id", output) - }) - - It("delegates to the real build", func() { - Expect(realBuild.SendOutputCallCount()).To(Equal(1)) - _, id, out := realBuild.SendOutputArgsForCall(0) - Expect(id).To(Equal(atc.PlanID("some-plan-id"))) - Expect(out).To(Equal(output)) - }) - }) }) }) diff --git a/atc/engine/engine.go b/atc/engine/engine.go index bb69104b2f2..a507ef28310 100644 --- a/atc/engine/engine.go +++ b/atc/engine/engine.go @@ -1,8 +1,6 @@ package engine import ( - "io" - "code.cloudfoundry.org/lager" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/db" @@ -12,7 +10,6 @@ import ( type Engine interface { Name() string - CreateBuild(lager.Logger, db.Build, atc.Plan) (Build, error) LookupBuild(lager.Logger, db.Build) (Build, error) ReleaseAll(lager.Logger) @@ -22,12 +19,8 @@ type Engine interface { type Build interface { Metadata() string - Abort(lager.Logger) error Resume(lager.Logger) - - ReceiveInput(lager.Logger, atc.PlanID, io.ReadCloser) - SendOutput(lager.Logger, atc.PlanID, io.Writer) } type Engines []Engine diff --git a/atc/engine/exec_engine.go b/atc/engine/exec_engine.go index 36dc2c65880..18a2baf716c 100644 --- a/atc/engine/exec_engine.go +++ b/atc/engine/exec_engine.go @@ -3,7 +3,6 @@ package engine import ( "context" "encoding/json" - "io" "strconv" "strings" "sync" @@ -170,14 +169,6 @@ func (build *execBuild) Resume(logger lager.Logger) { } } -func (build *execBuild) ReceiveInput(logger lager.Logger, plan atc.PlanID, stream io.ReadCloser) { - build.runState().SendUserInput(plan, stream) -} - -func (build *execBuild) SendOutput(logger lager.Logger, plan atc.PlanID, output io.Writer) { - build.runState().ReadPlanOutput(plan, output) -} - func (build *execBuild) runState() exec.RunState { existingState, _ := build.trackedStates.LoadOrStore(build.dbBuild.ID(), exec.NewRunState()) return existingState.(exec.RunState) diff --git a/atc/engine/exec_v1_dummy_engine.go b/atc/engine/exec_v1_dummy_engine.go index 66119306181..89d3e8df28f 100644 --- a/atc/engine/exec_v1_dummy_engine.go +++ b/atc/engine/exec_v1_dummy_engine.go @@ -2,7 +2,6 @@ package engine import ( "errors" - "io" "code.cloudfoundry.org/lager" "github.com/concourse/concourse/atc" @@ -52,9 +51,3 @@ func (execV1DummyBuild) Abort(lager.Logger) error { func (execV1DummyBuild) Resume(logger lager.Logger) { } - -func (execV1DummyBuild) ReceiveInput(logger lager.Logger, id atc.PlanID, input io.ReadCloser) { -} - -func (execV1DummyBuild) SendOutput(logger lager.Logger, id atc.PlanID, output io.Writer) { -} diff --git a/atc/routes.go b/atc/routes.go index bd00b534701..cd466728f5f 100644 --- a/atc/routes.go +++ b/atc/routes.go @@ -98,9 +98,6 @@ const ( DestroyTeam = "DestroyTeam" ListTeamBuilds = "ListTeamBuilds" - SendInputToBuildPlan = "SendInputToBuildPlan" - ReadOutputFromBuildPlan = "ReadOutputFromBuildPlan" - CreateArtifact = "CreateArtifact" GetArtifact = "GetArtifact" ) @@ -119,8 +116,6 @@ var Routes = rata.Routes([]rata.Route{ {Path: "/api/v1/builds", Method: "GET", Name: ListBuilds}, {Path: "/api/v1/builds/:build_id", Method: "GET", Name: GetBuild}, {Path: "/api/v1/builds/:build_id/plan", Method: "GET", Name: GetBuildPlan}, - {Path: "/api/v1/builds/:build_id/plan/:plan_id/input", Method: "PUT", Name: SendInputToBuildPlan}, - {Path: "/api/v1/builds/:build_id/plan/:plan_id/output", Method: "GET", Name: ReadOutputFromBuildPlan}, {Path: "/api/v1/builds/:build_id/events", Method: "GET", Name: BuildEvents}, {Path: "/api/v1/builds/:build_id/resources", Method: "GET", Name: BuildResources}, {Path: "/api/v1/builds/:build_id/abort", Method: "PUT", Name: AbortBuild}, diff --git a/atc/worker/artifact_repository.go b/atc/worker/artifact_repository.go index 986894f6120..e020c991f71 100644 --- a/atc/worker/artifact_repository.go +++ b/atc/worker/artifact_repository.go @@ -1,12 +1,7 @@ package worker import ( - "fmt" - "io" - "strings" "sync" - - "code.cloudfoundry.org/lager" ) // ArtifactName is just a string, with its own type to make interfaces using it @@ -54,62 +49,6 @@ func (repo *ArtifactRepository) SourceFor(name ArtifactName) (ArtifactSource, bo return source, found } -// StreamTo will stream all currently registered artifacts to the destination. -// This is used by the Put step, which currently does not have an explicit set -// of dependencies, and instead just pulls in everything. -// -// Each ArtifactSource will be streamed to a subdirectory matching its -// ArtifactName. -func (repo *ArtifactRepository) StreamTo(logger lager.Logger, dest ArtifactDestination) error { - sources := map[ArtifactName]ArtifactSource{} - - repo.repoL.RLock() - for k, v := range repo.repo { - sources[k] = v - } - repo.repoL.RUnlock() - - for name, src := range sources { - err := src.StreamTo(logger, subdirectoryDestination{dest, string(name)}) - if err != nil { - return err - } - } - - return nil -} - -// StreamFile streams a single file out of the repository, using the first path -// segment to determine the ArtifactSource to stream out of. For example, -// StreamFile("a/b.yml") will look up the "a" ArtifactSource and return the -// result of StreamFile("b.yml") on it. -// -// If the ArtifactSource determined by the path is not present, -// FileNotFoundError will be returned. -func (repo *ArtifactRepository) StreamFile(logger lager.Logger, path string) (io.ReadCloser, error) { - sources := map[ArtifactName]ArtifactSource{} - - repo.repoL.RLock() - for k, v := range repo.repo { - sources[k] = v - } - repo.repoL.RUnlock() - - for name, src := range sources { - if strings.HasPrefix(path, string(name)+"/") { - return src.StreamFile(logger, path[len(name)+1:]) - } - } - - return nil, FileNotFoundError{Path: path} -} - -// VolumeOn returns nothing, as it's impossible for there to be a single volume -// representing all ArtifactSources. -func (repo *ArtifactRepository) VolumeOn(logger lager.Logger, worker Worker) (Volume, bool, error) { - return nil, false, nil -} - // AsMap extracts the current contents of the ArtifactRepository into a new map // and returns it. Changes to the returned map or the ArtifactRepository will not // affect each other. @@ -124,24 +63,3 @@ func (repo *ArtifactRepository) AsMap() map[ArtifactName]ArtifactSource { return result } - -type subdirectoryDestination struct { - destination ArtifactDestination - subdirectory string -} - -func (dest subdirectoryDestination) StreamIn(dst string, src io.Reader) error { - return dest.destination.StreamIn(dest.subdirectory+"/"+dst, src) -} - -// FileNotFoundError is the error to return from StreamFile when the given path -// does not exist. -type FileNotFoundError struct { - Path string -} - -// Error prints a helpful message including the file path. The user will see -// this message if e.g. their task config path does not exist. -func (err FileNotFoundError) Error() string { - return fmt.Sprintf("file not found: %s", err.Path) -} diff --git a/atc/worker/artifact_repository_test.go b/atc/worker/artifact_repository_test.go index 586e805041c..c1ea33e5b6b 100644 --- a/atc/worker/artifact_repository_test.go +++ b/atc/worker/artifact_repository_test.go @@ -1,28 +1,19 @@ package worker_test import ( - "bytes" - "errors" - "io" - - "code.cloudfoundry.org/lager/lagertest" . "github.com/concourse/concourse/atc/worker" "github.com/concourse/concourse/atc/worker/workerfakes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - "github.com/onsi/gomega/gbytes" ) var _ = Describe("ArtifactRepository", func() { var ( - repo *ArtifactRepository - logger *lagertest.TestLogger + repo *ArtifactRepository ) BeforeEach(func() { repo = NewArtifactRepository() - logger = lagertest.NewTestLogger("artifact-repository-test") }) It("initially does not contain any sources", func() { @@ -81,108 +72,6 @@ var _ = Describe("ArtifactRepository", func() { }) }) - Describe("StreamTo", func() { - var fakeDestination *workerfakes.FakeArtifactDestination - var streamErr error - - BeforeEach(func() { - fakeDestination = new(workerfakes.FakeArtifactDestination) - }) - - JustBeforeEach(func() { - streamErr = repo.StreamTo(logger, fakeDestination) - }) - - It("succeeds", func() { - Expect(streamErr).NotTo(HaveOccurred()) - }) - - It("streams both sources to the destination under subdirectories", func() { - someStream := new(bytes.Buffer) - - Expect(firstSource.StreamToCallCount()).To(Equal(1)) - Expect(secondSource.StreamToCallCount()).To(Equal(1)) - - _, firstDestination := firstSource.StreamToArgsForCall(0) - _, secondDestination := secondSource.StreamToArgsForCall(0) - - Expect(firstDestination.StreamIn("foo", someStream)).To(Succeed()) - - Expect(fakeDestination.StreamInCallCount()).To(Equal(1)) - destDir, stream := fakeDestination.StreamInArgsForCall(0) - Expect(destDir).To(Equal("first-source/foo")) - Expect(stream).To(Equal(someStream)) - - Expect(secondDestination.StreamIn("foo", someStream)).To(Succeed()) - - Expect(fakeDestination.StreamInCallCount()).To(Equal(2)) - destDir, stream = fakeDestination.StreamInArgsForCall(1) - Expect(destDir).To(Equal("second-source/foo")) - Expect(stream).To(Equal(someStream)) - }) - - Context("when the any of the sources fails to stream", func() { - disaster := errors.New("nope") - - BeforeEach(func() { - secondSource.StreamToReturns(disaster) - }) - - It("returns the error", func() { - Expect(streamErr).To(Equal(disaster)) - }) - }) - }) - - Describe("StreamFile", func() { - var path string - - var stream io.Reader - var streamErr error - - JustBeforeEach(func() { - stream, streamErr = repo.StreamFile(logger, path) - }) - - Context("from a path not referring to any source", func() { - BeforeEach(func() { - path = "bogus" - }) - - It("returns ErrFileNotFound", func() { - Expect(streamErr).To(MatchError(FileNotFoundError{Path: "bogus"})) - }) - }) - - Context("from a path referring to a source", func() { - var outStream io.ReadCloser - - BeforeEach(func() { - path = "first-source/foo" - - outStream = gbytes.NewBuffer() - firstSource.StreamFileReturns(outStream, nil) - }) - - It("streams out from the source", func() { - Expect(stream).To(Equal(outStream)) - _, file := firstSource.StreamFileArgsForCall(0) - Expect(file).To(Equal("foo")) - }) - - Context("when streaming out from the source fails", func() { - disaster := errors.New("nope") - - BeforeEach(func() { - firstSource.StreamFileReturns(nil, disaster) - }) - - It("returns the error", func() { - Expect(streamErr).To(Equal(disaster)) - }) - }) - }) - }) }) }) }) diff --git a/atc/wrappa/api_auth_wrappa.go b/atc/wrappa/api_auth_wrappa.go index bd2ba9a1283..9be05b2e6d3 100644 --- a/atc/wrappa/api_auth_wrappa.go +++ b/atc/wrappa/api_auth_wrappa.go @@ -59,10 +59,8 @@ func (wrappa *APIAuthWrappa) Wrap(handlers rata.Handlers) rata.Handlers { atc.BuildEvents: newHandler = wrappa.checkBuildReadAccessHandlerFactory.CheckIfPrivateJobHandler(handler, rejector) - // resource belongs to authorized team - case atc.AbortBuild, - atc.SendInputToBuildPlan, - atc.ReadOutputFromBuildPlan: + // resource belongs to authorized team + case atc.AbortBuild: newHandler = wrappa.checkBuildWriteAccessHandlerFactory.HandlerFor(handler, rejector) // requester is system, admin team, or worker owning team diff --git a/atc/wrappa/api_auth_wrappa_test.go b/atc/wrappa/api_auth_wrappa_test.go index 50af8813382..a8e2014b9d4 100644 --- a/atc/wrappa/api_auth_wrappa_test.go +++ b/atc/wrappa/api_auth_wrappa_test.go @@ -161,9 +161,7 @@ var _ = Describe("APIAuthWrappa", func() { atc.GetBuildPreparation: checksIfPrivateJob(inputHandlers[atc.GetBuildPreparation]), // resource belongs to authorized team - atc.AbortBuild: checkWritePermissionForBuild(inputHandlers[atc.AbortBuild]), - atc.SendInputToBuildPlan: checkWritePermissionForBuild(inputHandlers[atc.SendInputToBuildPlan]), - atc.ReadOutputFromBuildPlan: checkWritePermissionForBuild(inputHandlers[atc.ReadOutputFromBuildPlan]), + atc.AbortBuild: checkWritePermissionForBuild(inputHandlers[atc.AbortBuild]), // resource belongs to authorized team atc.PruneWorker: checkTeamAccessForWorker(inputHandlers[atc.PruneWorker]), From c69f4ac344b0d6383c89c9621ab32a04df4e1994 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Wed, 16 Jan 2019 11:04:54 -0500 Subject: [PATCH 04/21] atc: add accessor entries for artifact things Co-authored-by: Topher Bullock Signed-off-by: Josh Winters --- atc/api/accessor/accessor.go | 3 +++ atc/api/accessor/accessor_test.go | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/atc/api/accessor/accessor.go b/atc/api/accessor/accessor.go index e939d85cd15..f6d43b793fb 100644 --- a/atc/api/accessor/accessor.go +++ b/atc/api/accessor/accessor.go @@ -197,4 +197,7 @@ var requiredRoles = map[string]string{ atc.RenameTeam: "owner", atc.DestroyTeam: "owner", atc.ListTeamBuilds: "viewer", + atc.CreateArtifact: "member", + atc.GetArtifact: "viewer", + atc.ListBuildArtifacts: "viewer", } diff --git a/atc/api/accessor/accessor_test.go b/atc/api/accessor/accessor_test.go index a91e94d0b19..2bddc4a9aab 100644 --- a/atc/api/accessor/accessor_test.go +++ b/atc/api/accessor/accessor_test.go @@ -649,5 +649,17 @@ var _ = Describe("Accessor", func() { Entry("owner :: "+atc.ListTeamBuilds, atc.ListTeamBuilds, "owner", true), Entry("member :: "+atc.ListTeamBuilds, atc.ListTeamBuilds, "member", true), Entry("viewer :: "+atc.ListTeamBuilds, atc.ListTeamBuilds, "viewer", true), + + Entry("owner :: "+atc.CreateArtifact, atc.CreateArtifact, "owner", true), + Entry("member :: "+atc.CreateArtifact, atc.CreateArtifact, "member", true), + Entry("viewer :: "+atc.CreateArtifact, atc.CreateArtifact, "viewer", false), + + Entry("owner :: "+atc.GetArtifact, atc.GetArtifact, "owner", true), + Entry("member :: "+atc.GetArtifact, atc.GetArtifact, "member", true), + Entry("viewer :: "+atc.GetArtifact, atc.GetArtifact, "viewer", true), + + Entry("owner :: "+atc.ListBuildArtifacts, atc.ListBuildArtifacts, "owner", true), + Entry("member :: "+atc.ListBuildArtifacts, atc.ListBuildArtifacts, "member", true), + Entry("viewer :: "+atc.ListBuildArtifacts, atc.ListBuildArtifacts, "viewer", true), ) }) From b078b4c1c50ccc64af3c53df1801b13d8786a329 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 3 Jan 2019 17:06:22 -0500 Subject: [PATCH 05/21] atc: create artifacts for build outputs Co-authored-by: Topher Bullock Signed-off-by: Josh Winters --- atc/api/artifacts_test.go | 14 +- atc/api/artifactserver/create.go | 13 +- atc/api/buildserver/artifacts.go | 30 ++ atc/api/handler.go | 10 + atc/api/present/worker_artifacts.go | 23 ++ atc/atccmd/command.go | 28 +- atc/db/build.go | 55 +++- atc/db/dbfakes/fake_build.go | 147 +++++++++ atc/db/dbfakes/fake_created_volume.go | 12 +- atc/db/dbfakes/fake_worker_artifact.go | 154 +++++----- ...50159394_create_worker_artifacts.down.sql} | 0 ...1550159394_create_worker_artifacts.up.sql} | 9 +- atc/db/team_test.go | 2 +- atc/db/volume.go | 11 +- atc/db/volume_test.go | 8 +- atc/db/worker_artifact.go | 38 ++- atc/engine/build_step.go | 11 +- atc/engine/enginefakes/fake_build.go | 86 ------ atc/engine/exec_engine.go | 4 - atc/exec/aggregate_test.go | 7 +- atc/exec/artifact/artifact_suite_test.go | 13 + .../artifactfakes/fake_registerable_source.go | 280 ++++++++++++++++++ .../artifact/repository.go} | 34 ++- .../artifact/repository_test.go} | 19 +- atc/exec/artifact_output_step.go | 62 ---- atc/exec/artifact_output_step_test.go | 90 ------ atc/exec/ensure_step_test.go | 11 +- atc/exec/execfakes/fake_factory.go | 79 +++++ atc/exec/execfakes/fake_run_state.go | 253 +--------------- atc/exec/execfakes/fake_task_config_source.go | 14 +- atc/exec/factory.go | 7 + atc/exec/garden_factory.go | 18 +- atc/exec/get_step.go | 35 ++- atc/exec/get_step_test.go | 9 +- atc/exec/log_error_step_test.go | 7 +- atc/exec/on_abort_test.go | 11 +- atc/exec/on_failure_test.go | 11 +- atc/exec/on_success_test.go | 11 +- atc/exec/put_inputs.go | 13 +- atc/exec/put_step_test.go | 5 +- atc/exec/retry_step_test.go | 7 +- atc/exec/run_state.go | 83 +----- atc/exec/run_state_test.go | 100 ------- atc/exec/step.go | 10 +- atc/exec/task_config_source.go | 34 +-- atc/exec/task_config_source_test.go | 12 +- atc/exec/task_step.go | 91 ++---- atc/exec/task_step_test.go | 27 +- atc/exec/timeout_step_test.go | 7 +- atc/exec/try_step_test.go | 7 +- atc/exec/user_artifact_step.go | 93 +++--- atc/exec/user_artifact_step_test.go | 129 ++++++-- atc/plan.go | 3 +- atc/public_plan_test.go | 9 +- atc/routes.go | 6 +- atc/worker/container_spec.go | 1 - atc/worker/image/image_test.go | 2 - atc/worker/volume.go | 6 +- atc/worker/workerfakes/fake_volume.go | 12 +- atc/worker_artifact.go | 4 +- atc/wrappa/api_auth_wrappa.go | 3 +- atc/wrappa/api_auth_wrappa_test.go | 1 + fly/commands/execute.go | 90 +++--- .../internal/executehelpers/downloads.go | 21 +- .../internal/executehelpers/inputs.go | 43 ++- .../internal/executehelpers/uploads.go | 40 +-- ...est.go => execute_multiple_inputs_test.go} | 72 +++-- ...t.go => execute_overriding_inputs_test.go} | 61 ++-- fly/integration/execute_test.go | 69 +++-- fly/integration/execute_with_outputs_test.go | 103 ++++--- go-concourse/concourse/build_plan.go | 49 --- go-concourse/concourse/build_plan_test.go | 85 +----- go-concourse/concourse/builds.go | 17 ++ go-concourse/concourse/client.go | 3 +- .../concourse/concoursefakes/fake_client.go | 245 +++++---------- .../concourse/concoursefakes/fake_team.go | 157 ++++++++++ go.mod | 69 ++++- go.sum | 194 +++++++++++- 78 files changed, 1973 insertions(+), 1636 deletions(-) create mode 100644 atc/api/buildserver/artifacts.go create mode 100644 atc/api/present/worker_artifacts.go rename atc/db/migration/migrations/{1545249372_create_worker_artifacts.down.sql => 1550159394_create_worker_artifacts.down.sql} (100%) rename atc/db/migration/migrations/{1545249372_create_worker_artifacts.up.sql => 1550159394_create_worker_artifacts.up.sql} (57%) create mode 100644 atc/exec/artifact/artifact_suite_test.go create mode 100644 atc/exec/artifact/artifactfakes/fake_registerable_source.go rename atc/{worker/artifact_repository.go => exec/artifact/repository.go} (63%) rename atc/{worker/artifact_repository_test.go => exec/artifact/repository_test.go} (79%) delete mode 100644 atc/exec/artifact_output_step.go delete mode 100644 atc/exec/artifact_output_step_test.go rename fly/integration/{multiple_inputs_test.go => execute_multiple_inputs_test.go} (92%) rename fly/integration/{overriding_inputs_test.go => execute_overriding_inputs_test.go} (94%) diff --git a/atc/api/artifacts_test.go b/atc/api/artifacts_test.go index 548854cc06a..887d652576e 100644 --- a/atc/api/artifacts_test.go +++ b/atc/api/artifacts_test.go @@ -120,7 +120,7 @@ var _ = Describe("Artifacts API", func() { BeforeEach(func() { fakeWorkerArtifact = new(dbfakes.FakeWorkerArtifact) fakeWorkerArtifact.IDReturns(0) - fakeWorkerArtifact.CreatedAtReturns(time.Now()) + fakeWorkerArtifact.CreatedAtReturns(time.Unix(42, 0)) fakeVolume.InitializeArtifactReturns(fakeWorkerArtifact, nil) }) @@ -128,9 +128,9 @@ var _ = Describe("Artifacts API", func() { It("invokes the initialization of an artifact on a volume", func() { Expect(fakeVolume.InitializeArtifactCallCount()).To(Equal(1)) - path, checksum := fakeVolume.InitializeArtifactArgsForCall(0) - Expect(path).To(Equal("/")) - Expect(checksum).To(Equal("")) + name, buildID := fakeVolume.InitializeArtifactArgsForCall(0) + Expect(name).To(Equal("")) + Expect(buildID).To(Equal(0)) }) Context("when streaming in data to a volume fails", func() { @@ -179,9 +179,9 @@ var _ = Describe("Artifacts API", func() { Expect(body).To(MatchJSON(`{ "id": 0, - "path": "/", - "created_at": 42, - "checksum": "" + "name": "", + "build_id": 0, + "created_at": 42 }`)) }) }) diff --git a/atc/api/artifactserver/create.go b/atc/api/artifactserver/create.go index 5cce1b3c7fb..c6e88108855 100644 --- a/atc/api/artifactserver/create.go +++ b/atc/api/artifactserver/create.go @@ -5,7 +5,7 @@ import ( "net/http" "github.com/concourse/baggageclaim" - "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/api/present" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/worker" ) @@ -35,7 +35,7 @@ func (s *Server) CreateArtifact(team db.Team) http.Handler { // volume gets created and when the artifact gets initialized // Within this timeframe there's a chance that the volume could - // get garbage collector out from under us. + // get garbage collected out from under us. // This happens because CreateVolume returns a 'created' instead // of 'creating' volume. @@ -47,7 +47,7 @@ func (s *Server) CreateArtifact(team db.Team) http.Handler { // I think leaving the race condition is fine for now. Worst case // is a fly execute will fail and the user will need to rerun it. - artifact, err := volume.InitializeArtifact("/", "") + artifact, err := volume.InitializeArtifact("", 0) if err != nil { hLog.Error("failed-to-initialize-artifact", err) w.WriteHeader(http.StatusInternalServerError) @@ -63,11 +63,6 @@ func (s *Server) CreateArtifact(team db.Team) http.Handler { w.WriteHeader(http.StatusCreated) - json.NewEncoder(w).Encode(atc.WorkerArtifact{ - ID: artifact.ID(), - Checksum: "", - Path: "/", - CreatedAt: artifact.CreatedAt().Unix(), - }) + json.NewEncoder(w).Encode(present.WorkerArtifact(artifact)) }) } diff --git a/atc/api/buildserver/artifacts.go b/atc/api/buildserver/artifacts.go new file mode 100644 index 00000000000..1988708a1d3 --- /dev/null +++ b/atc/api/buildserver/artifacts.go @@ -0,0 +1,30 @@ +package buildserver + +import ( + "encoding/json" + "net/http" + + "github.com/concourse/concourse/atc/api/present" + "github.com/concourse/concourse/atc/db" +) + +func (s *Server) GetBuildArtifacts(build db.Build) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + logger := s.logger.Session("get-build-artifacts") + + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + + artifacts, err := build.Artifacts() + if err != nil { + logger.Error("failed-to-fetch-build-artifacts", err) + w.WriteHeader(http.StatusInternalServerError) + } + + err = json.NewEncoder(w).Encode(present.WorkerArtifacts(artifacts)) + if err != nil { + logger.Error("failed-to-encode-build-artifacts", err) + w.WriteHeader(http.StatusInternalServerError) + } + }) +} diff --git a/atc/api/handler.go b/atc/api/handler.go index 689e51b6347..dc3b4e3ac9b 100644 --- a/atc/api/handler.go +++ b/atc/api/handler.go @@ -103,6 +103,16 @@ func NewHandler( atc.GetCC: http.HandlerFunc(ccServer.GetCC), + atc.ListBuilds: http.HandlerFunc(buildServer.ListBuilds), + atc.CreateBuild: teamHandlerFactory.HandlerFor(buildServer.CreateBuild), + atc.GetBuild: buildHandlerFactory.HandlerFor(buildServer.GetBuild), + atc.BuildResources: buildHandlerFactory.HandlerFor(buildServer.BuildResources), + atc.AbortBuild: buildHandlerFactory.HandlerFor(buildServer.AbortBuild), + atc.GetBuildPlan: buildHandlerFactory.HandlerFor(buildServer.GetBuildPlan), + atc.GetBuildPreparation: buildHandlerFactory.HandlerFor(buildServer.GetBuildPreparation), + atc.BuildEvents: buildHandlerFactory.HandlerFor(buildServer.BuildEvents), + atc.ListBuildArtifacts: buildHandlerFactory.HandlerFor(buildServer.GetBuildArtifacts), + atc.ListAllJobs: http.HandlerFunc(jobServer.ListAllJobs), atc.ListJobs: pipelineHandlerFactory.HandlerFor(jobServer.ListJobs), atc.GetJob: pipelineHandlerFactory.HandlerFor(jobServer.GetJob), diff --git a/atc/api/present/worker_artifacts.go b/atc/api/present/worker_artifacts.go new file mode 100644 index 00000000000..1befa3dd0d1 --- /dev/null +++ b/atc/api/present/worker_artifacts.go @@ -0,0 +1,23 @@ +package present + +import ( + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/db" +) + +func WorkerArtifacts(artifacts []db.WorkerArtifact) []atc.WorkerArtifact { + wa := []atc.WorkerArtifact{} + for _, a := range artifacts { + wa = append(wa, WorkerArtifact(a)) + } + return wa +} + +func WorkerArtifact(artifact db.WorkerArtifact) atc.WorkerArtifact { + return atc.WorkerArtifact{ + ID: artifact.ID(), + Name: artifact.Name(), + BuildID: artifact.BuildID(), + CreatedAt: artifact.CreatedAt().Unix(), + } +} diff --git a/atc/atccmd/command.go b/atc/atccmd/command.go index e2e8ae8fb76..c3862709495 100644 --- a/atc/atccmd/command.go +++ b/atc/atccmd/command.go @@ -552,7 +552,17 @@ func (cmd *RunCommand) constructAPIMembers( buildContainerStrategy := cmd.chooseBuildContainerStrategy() checkContainerStrategy := worker.NewRandomPlacementStrategy() - engine := cmd.constructEngine(pool, resourceFetcher, dbResourceCacheFactory, dbResourceConfigFactory, variablesFactory, defaultLimits, buildContainerStrategy, resourceFactory) + engine := cmd.constructEngine( + pool, + workerClient, + resourceFetcher, + dbResourceCacheFactory, + dbResourceConfigFactory, + variablesFactory, + defaultLimits, + buildContainerStrategy, + resourceFactory, + ) radarSchedulerFactory := pipelines.NewRadarSchedulerFactory( pool, @@ -742,6 +752,7 @@ func (cmd *RunCommand) constructBackendMembers( ) pool := worker.NewPool(workerProvider) + workerClient := worker.NewClient(pool, workerProvider) defaultLimits, err := cmd.parseDefaultLimits() if err != nil { @@ -755,7 +766,18 @@ func (cmd *RunCommand) constructBackendMembers( buildContainerStrategy := cmd.chooseBuildContainerStrategy() checkContainerStrategy := worker.NewRandomPlacementStrategy() - engine := cmd.constructEngine(pool, resourceFetcher, dbResourceCacheFactory, dbResourceConfigFactory, variablesFactory, defaultLimits, buildContainerStrategy, resourceFactory) + + engine := cmd.constructEngine( + pool, + workerClient, + resourceFetcher, + dbResourceCacheFactory, + dbResourceConfigFactory, + variablesFactory, + defaultLimits, + buildContainerStrategy, + resourceFactory, + ) radarSchedulerFactory := pipelines.NewRadarSchedulerFactory( pool, @@ -1208,6 +1230,7 @@ func (cmd *RunCommand) configureAuthForDefaultTeam(teamFactory db.TeamFactory) e func (cmd *RunCommand) constructEngine( workerPool worker.Pool, + workerClient worker.Client, resourceFetcher resource.Fetcher, resourceCacheFactory db.ResourceCacheFactory, resourceConfigFactory db.ResourceConfigFactory, @@ -1218,6 +1241,7 @@ func (cmd *RunCommand) constructEngine( ) engine.Engine { gardenFactory := exec.NewGardenFactory( workerPool, + workerClient, resourceFetcher, resourceCacheFactory, resourceConfigFactory, diff --git a/atc/db/build.go b/atc/db/build.go index e4cbf50c801..3dc182862be 100644 --- a/atc/db/build.go +++ b/atc/db/build.go @@ -9,7 +9,6 @@ import ( "time" "code.cloudfoundry.org/lager" - sq "github.com/Masterminds/squirrel" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/creds" @@ -92,6 +91,9 @@ type Build interface { Events(uint) (EventSource, error) SaveEvent(event atc.Event) error + Artifacts() ([]WorkerArtifact, error) + Artifact(artifactID int) (WorkerArtifact, error) + SaveOutput(lager.Logger, string, atc.Source, creds.VersionedResourceTypes, atc.Version, ResourceConfigMetadataFields, string, string) error UseInputs(inputs []BuildInput) error @@ -142,6 +144,7 @@ type build struct { var ErrBuildDisappeared = errors.New("build disappeared from db") var ErrBuildHasNoPipeline = errors.New("build has no pipeline") +var ErrBuildArtifactNotFound = errors.New("build artifact not found") type ResourceNotFoundInPipeline struct { Resource string @@ -784,6 +787,56 @@ func (b *build) SaveEvent(event atc.Event) error { return b.conn.Bus().Notify(buildEventsChannel(b.id)) } +func (b *build) Artifact(artifactID int) (WorkerArtifact, error) { + + artifact := artifact{ + conn: b.conn, + } + + err := psql.Select("id", "name", "created_at"). + From("worker_artifacts"). + Where(sq.Eq{ + "id": artifactID, + }). + RunWith(b.conn). + Scan(&artifact.id, &artifact.name, &artifact.createdAt) + + return &artifact, err +} + +func (b *build) Artifacts() ([]WorkerArtifact, error) { + artifacts := []WorkerArtifact{} + + rows, err := psql.Select("id", "name", "created_at"). + From("worker_artifacts"). + Where(sq.Eq{ + "build_id": b.id, + }). + RunWith(b.conn). + Query() + if err != nil { + return nil, err + } + + defer Close(rows) + + for rows.Next() { + wa := artifact{ + conn: b.conn, + buildID: b.id, + } + + err = rows.Scan(&wa.id, &wa.name, &wa.createdAt) + if err != nil { + return nil, err + } + + artifacts = append(artifacts, &wa) + } + + return artifacts, nil +} + func (b *build) SaveOutput( logger lager.Logger, resourceType string, diff --git a/atc/db/dbfakes/fake_build.go b/atc/db/dbfakes/fake_build.go index bf322cf5477..6bd1759a460 100644 --- a/atc/db/dbfakes/fake_build.go +++ b/atc/db/dbfakes/fake_build.go @@ -42,6 +42,31 @@ type FakeBuild struct { result2 bool result3 error } + ArtifactStub func(int) (db.WorkerArtifact, error) + artifactMutex sync.RWMutex + artifactArgsForCall []struct { + arg1 int + } + artifactReturns struct { + result1 db.WorkerArtifact + result2 error + } + artifactReturnsOnCall map[int]struct { + result1 db.WorkerArtifact + result2 error + } + ArtifactsStub func() ([]db.WorkerArtifact, error) + artifactsMutex sync.RWMutex + artifactsArgsForCall []struct { + } + artifactsReturns struct { + result1 []db.WorkerArtifact + result2 error + } + artifactsReturnsOnCall map[int]struct { + result1 []db.WorkerArtifact + result2 error + } DeleteStub func() (bool, error) deleteMutex sync.RWMutex deleteArgsForCall []struct { @@ -602,6 +627,124 @@ func (fake *FakeBuild) AcquireTrackingLockReturnsOnCall(i int, result1 lock.Lock }{result1, result2, result3} } +func (fake *FakeBuild) Artifact(arg1 int) (db.WorkerArtifact, error) { + fake.artifactMutex.Lock() + ret, specificReturn := fake.artifactReturnsOnCall[len(fake.artifactArgsForCall)] + fake.artifactArgsForCall = append(fake.artifactArgsForCall, struct { + arg1 int + }{arg1}) + fake.recordInvocation("Artifact", []interface{}{arg1}) + fake.artifactMutex.Unlock() + if fake.ArtifactStub != nil { + return fake.ArtifactStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.artifactReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeBuild) ArtifactCallCount() int { + fake.artifactMutex.RLock() + defer fake.artifactMutex.RUnlock() + return len(fake.artifactArgsForCall) +} + +func (fake *FakeBuild) ArtifactCalls(stub func(int) (db.WorkerArtifact, error)) { + fake.artifactMutex.Lock() + defer fake.artifactMutex.Unlock() + fake.ArtifactStub = stub +} + +func (fake *FakeBuild) ArtifactArgsForCall(i int) int { + fake.artifactMutex.RLock() + defer fake.artifactMutex.RUnlock() + argsForCall := fake.artifactArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeBuild) ArtifactReturns(result1 db.WorkerArtifact, result2 error) { + fake.artifactMutex.Lock() + defer fake.artifactMutex.Unlock() + fake.ArtifactStub = nil + fake.artifactReturns = struct { + result1 db.WorkerArtifact + result2 error + }{result1, result2} +} + +func (fake *FakeBuild) ArtifactReturnsOnCall(i int, result1 db.WorkerArtifact, result2 error) { + fake.artifactMutex.Lock() + defer fake.artifactMutex.Unlock() + fake.ArtifactStub = nil + if fake.artifactReturnsOnCall == nil { + fake.artifactReturnsOnCall = make(map[int]struct { + result1 db.WorkerArtifact + result2 error + }) + } + fake.artifactReturnsOnCall[i] = struct { + result1 db.WorkerArtifact + result2 error + }{result1, result2} +} + +func (fake *FakeBuild) Artifacts() ([]db.WorkerArtifact, error) { + fake.artifactsMutex.Lock() + ret, specificReturn := fake.artifactsReturnsOnCall[len(fake.artifactsArgsForCall)] + fake.artifactsArgsForCall = append(fake.artifactsArgsForCall, struct { + }{}) + fake.recordInvocation("Artifacts", []interface{}{}) + fake.artifactsMutex.Unlock() + if fake.ArtifactsStub != nil { + return fake.ArtifactsStub() + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.artifactsReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeBuild) ArtifactsCallCount() int { + fake.artifactsMutex.RLock() + defer fake.artifactsMutex.RUnlock() + return len(fake.artifactsArgsForCall) +} + +func (fake *FakeBuild) ArtifactsCalls(stub func() ([]db.WorkerArtifact, error)) { + fake.artifactsMutex.Lock() + defer fake.artifactsMutex.Unlock() + fake.ArtifactsStub = stub +} + +func (fake *FakeBuild) ArtifactsReturns(result1 []db.WorkerArtifact, result2 error) { + fake.artifactsMutex.Lock() + defer fake.artifactsMutex.Unlock() + fake.ArtifactsStub = nil + fake.artifactsReturns = struct { + result1 []db.WorkerArtifact + result2 error + }{result1, result2} +} + +func (fake *FakeBuild) ArtifactsReturnsOnCall(i int, result1 []db.WorkerArtifact, result2 error) { + fake.artifactsMutex.Lock() + defer fake.artifactsMutex.Unlock() + fake.ArtifactsStub = nil + if fake.artifactsReturnsOnCall == nil { + fake.artifactsReturnsOnCall = make(map[int]struct { + result1 []db.WorkerArtifact + result2 error + }) + } + fake.artifactsReturnsOnCall[i] = struct { + result1 []db.WorkerArtifact + result2 error + }{result1, result2} +} + func (fake *FakeBuild) Delete() (bool, error) { fake.deleteMutex.Lock() ret, specificReturn := fake.deleteReturnsOnCall[len(fake.deleteArgsForCall)] @@ -2775,6 +2918,10 @@ func (fake *FakeBuild) Invocations() map[string][][]interface{} { defer fake.abortNotifierMutex.RUnlock() fake.acquireTrackingLockMutex.RLock() defer fake.acquireTrackingLockMutex.RUnlock() + fake.artifactMutex.RLock() + defer fake.artifactMutex.RUnlock() + fake.artifactsMutex.RLock() + defer fake.artifactsMutex.RUnlock() fake.deleteMutex.RLock() defer fake.deleteMutex.RUnlock() fake.endTimeMutex.RLock() diff --git a/atc/db/dbfakes/fake_created_volume.go b/atc/db/dbfakes/fake_created_volume.go index 4aa97ee87f8..eede5b6427e 100644 --- a/atc/db/dbfakes/fake_created_volume.go +++ b/atc/db/dbfakes/fake_created_volume.go @@ -66,11 +66,11 @@ type FakeCreatedVolume struct { handleReturnsOnCall map[int]struct { result1 string } - InitializeArtifactStub func(string, string) (db.WorkerArtifact, error) + InitializeArtifactStub func(string, int) (db.WorkerArtifact, error) initializeArtifactMutex sync.RWMutex initializeArtifactArgsForCall []struct { arg1 string - arg2 string + arg2 int } initializeArtifactReturns struct { result1 db.WorkerArtifact @@ -474,12 +474,12 @@ func (fake *FakeCreatedVolume) HandleReturnsOnCall(i int, result1 string) { }{result1} } -func (fake *FakeCreatedVolume) InitializeArtifact(arg1 string, arg2 string) (db.WorkerArtifact, error) { +func (fake *FakeCreatedVolume) InitializeArtifact(arg1 string, arg2 int) (db.WorkerArtifact, error) { fake.initializeArtifactMutex.Lock() ret, specificReturn := fake.initializeArtifactReturnsOnCall[len(fake.initializeArtifactArgsForCall)] fake.initializeArtifactArgsForCall = append(fake.initializeArtifactArgsForCall, struct { arg1 string - arg2 string + arg2 int }{arg1, arg2}) fake.recordInvocation("InitializeArtifact", []interface{}{arg1, arg2}) fake.initializeArtifactMutex.Unlock() @@ -499,13 +499,13 @@ func (fake *FakeCreatedVolume) InitializeArtifactCallCount() int { return len(fake.initializeArtifactArgsForCall) } -func (fake *FakeCreatedVolume) InitializeArtifactCalls(stub func(string, string) (db.WorkerArtifact, error)) { +func (fake *FakeCreatedVolume) InitializeArtifactCalls(stub func(string, int) (db.WorkerArtifact, error)) { fake.initializeArtifactMutex.Lock() defer fake.initializeArtifactMutex.Unlock() fake.InitializeArtifactStub = stub } -func (fake *FakeCreatedVolume) InitializeArtifactArgsForCall(i int) (string, string) { +func (fake *FakeCreatedVolume) InitializeArtifactArgsForCall(i int) (string, int) { fake.initializeArtifactMutex.RLock() defer fake.initializeArtifactMutex.RUnlock() argsForCall := fake.initializeArtifactArgsForCall[i] diff --git a/atc/db/dbfakes/fake_worker_artifact.go b/atc/db/dbfakes/fake_worker_artifact.go index 251e5cceb4a..2c6e04e7c35 100644 --- a/atc/db/dbfakes/fake_worker_artifact.go +++ b/atc/db/dbfakes/fake_worker_artifact.go @@ -9,15 +9,15 @@ import ( ) type FakeWorkerArtifact struct { - ChecksumStub func() string - checksumMutex sync.RWMutex - checksumArgsForCall []struct { + BuildIDStub func() int + buildIDMutex sync.RWMutex + buildIDArgsForCall []struct { } - checksumReturns struct { - result1 string + buildIDReturns struct { + result1 int } - checksumReturnsOnCall map[int]struct { - result1 string + buildIDReturnsOnCall map[int]struct { + result1 int } CreatedAtStub func() time.Time createdAtMutex sync.RWMutex @@ -39,14 +39,14 @@ type FakeWorkerArtifact struct { iDReturnsOnCall map[int]struct { result1 int } - PathStub func() string - pathMutex sync.RWMutex - pathArgsForCall []struct { + NameStub func() string + nameMutex sync.RWMutex + nameArgsForCall []struct { } - pathReturns struct { + nameReturns struct { result1 string } - pathReturnsOnCall map[int]struct { + nameReturnsOnCall map[int]struct { result1 string } VolumeStub func(int) (db.CreatedVolume, bool, error) @@ -68,55 +68,55 @@ type FakeWorkerArtifact struct { invocationsMutex sync.RWMutex } -func (fake *FakeWorkerArtifact) Checksum() string { - fake.checksumMutex.Lock() - ret, specificReturn := fake.checksumReturnsOnCall[len(fake.checksumArgsForCall)] - fake.checksumArgsForCall = append(fake.checksumArgsForCall, struct { +func (fake *FakeWorkerArtifact) BuildID() int { + fake.buildIDMutex.Lock() + ret, specificReturn := fake.buildIDReturnsOnCall[len(fake.buildIDArgsForCall)] + fake.buildIDArgsForCall = append(fake.buildIDArgsForCall, struct { }{}) - fake.recordInvocation("Checksum", []interface{}{}) - fake.checksumMutex.Unlock() - if fake.ChecksumStub != nil { - return fake.ChecksumStub() + fake.recordInvocation("BuildID", []interface{}{}) + fake.buildIDMutex.Unlock() + if fake.BuildIDStub != nil { + return fake.BuildIDStub() } if specificReturn { return ret.result1 } - fakeReturns := fake.checksumReturns + fakeReturns := fake.buildIDReturns return fakeReturns.result1 } -func (fake *FakeWorkerArtifact) ChecksumCallCount() int { - fake.checksumMutex.RLock() - defer fake.checksumMutex.RUnlock() - return len(fake.checksumArgsForCall) +func (fake *FakeWorkerArtifact) BuildIDCallCount() int { + fake.buildIDMutex.RLock() + defer fake.buildIDMutex.RUnlock() + return len(fake.buildIDArgsForCall) } -func (fake *FakeWorkerArtifact) ChecksumCalls(stub func() string) { - fake.checksumMutex.Lock() - defer fake.checksumMutex.Unlock() - fake.ChecksumStub = stub +func (fake *FakeWorkerArtifact) BuildIDCalls(stub func() int) { + fake.buildIDMutex.Lock() + defer fake.buildIDMutex.Unlock() + fake.BuildIDStub = stub } -func (fake *FakeWorkerArtifact) ChecksumReturns(result1 string) { - fake.checksumMutex.Lock() - defer fake.checksumMutex.Unlock() - fake.ChecksumStub = nil - fake.checksumReturns = struct { - result1 string +func (fake *FakeWorkerArtifact) BuildIDReturns(result1 int) { + fake.buildIDMutex.Lock() + defer fake.buildIDMutex.Unlock() + fake.BuildIDStub = nil + fake.buildIDReturns = struct { + result1 int }{result1} } -func (fake *FakeWorkerArtifact) ChecksumReturnsOnCall(i int, result1 string) { - fake.checksumMutex.Lock() - defer fake.checksumMutex.Unlock() - fake.ChecksumStub = nil - if fake.checksumReturnsOnCall == nil { - fake.checksumReturnsOnCall = make(map[int]struct { - result1 string +func (fake *FakeWorkerArtifact) BuildIDReturnsOnCall(i int, result1 int) { + fake.buildIDMutex.Lock() + defer fake.buildIDMutex.Unlock() + fake.BuildIDStub = nil + if fake.buildIDReturnsOnCall == nil { + fake.buildIDReturnsOnCall = make(map[int]struct { + result1 int }) } - fake.checksumReturnsOnCall[i] = struct { - result1 string + fake.buildIDReturnsOnCall[i] = struct { + result1 int }{result1} } @@ -224,54 +224,54 @@ func (fake *FakeWorkerArtifact) IDReturnsOnCall(i int, result1 int) { }{result1} } -func (fake *FakeWorkerArtifact) Path() string { - fake.pathMutex.Lock() - ret, specificReturn := fake.pathReturnsOnCall[len(fake.pathArgsForCall)] - fake.pathArgsForCall = append(fake.pathArgsForCall, struct { +func (fake *FakeWorkerArtifact) Name() string { + fake.nameMutex.Lock() + ret, specificReturn := fake.nameReturnsOnCall[len(fake.nameArgsForCall)] + fake.nameArgsForCall = append(fake.nameArgsForCall, struct { }{}) - fake.recordInvocation("Path", []interface{}{}) - fake.pathMutex.Unlock() - if fake.PathStub != nil { - return fake.PathStub() + fake.recordInvocation("Name", []interface{}{}) + fake.nameMutex.Unlock() + if fake.NameStub != nil { + return fake.NameStub() } if specificReturn { return ret.result1 } - fakeReturns := fake.pathReturns + fakeReturns := fake.nameReturns return fakeReturns.result1 } -func (fake *FakeWorkerArtifact) PathCallCount() int { - fake.pathMutex.RLock() - defer fake.pathMutex.RUnlock() - return len(fake.pathArgsForCall) +func (fake *FakeWorkerArtifact) NameCallCount() int { + fake.nameMutex.RLock() + defer fake.nameMutex.RUnlock() + return len(fake.nameArgsForCall) } -func (fake *FakeWorkerArtifact) PathCalls(stub func() string) { - fake.pathMutex.Lock() - defer fake.pathMutex.Unlock() - fake.PathStub = stub +func (fake *FakeWorkerArtifact) NameCalls(stub func() string) { + fake.nameMutex.Lock() + defer fake.nameMutex.Unlock() + fake.NameStub = stub } -func (fake *FakeWorkerArtifact) PathReturns(result1 string) { - fake.pathMutex.Lock() - defer fake.pathMutex.Unlock() - fake.PathStub = nil - fake.pathReturns = struct { +func (fake *FakeWorkerArtifact) NameReturns(result1 string) { + fake.nameMutex.Lock() + defer fake.nameMutex.Unlock() + fake.NameStub = nil + fake.nameReturns = struct { result1 string }{result1} } -func (fake *FakeWorkerArtifact) PathReturnsOnCall(i int, result1 string) { - fake.pathMutex.Lock() - defer fake.pathMutex.Unlock() - fake.PathStub = nil - if fake.pathReturnsOnCall == nil { - fake.pathReturnsOnCall = make(map[int]struct { +func (fake *FakeWorkerArtifact) NameReturnsOnCall(i int, result1 string) { + fake.nameMutex.Lock() + defer fake.nameMutex.Unlock() + fake.NameStub = nil + if fake.nameReturnsOnCall == nil { + fake.nameReturnsOnCall = make(map[int]struct { result1 string }) } - fake.pathReturnsOnCall[i] = struct { + fake.nameReturnsOnCall[i] = struct { result1 string }{result1} } @@ -345,14 +345,14 @@ func (fake *FakeWorkerArtifact) VolumeReturnsOnCall(i int, result1 db.CreatedVol func (fake *FakeWorkerArtifact) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.checksumMutex.RLock() - defer fake.checksumMutex.RUnlock() + fake.buildIDMutex.RLock() + defer fake.buildIDMutex.RUnlock() fake.createdAtMutex.RLock() defer fake.createdAtMutex.RUnlock() fake.iDMutex.RLock() defer fake.iDMutex.RUnlock() - fake.pathMutex.RLock() - defer fake.pathMutex.RUnlock() + fake.nameMutex.RLock() + defer fake.nameMutex.RUnlock() fake.volumeMutex.RLock() defer fake.volumeMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/atc/db/migration/migrations/1545249372_create_worker_artifacts.down.sql b/atc/db/migration/migrations/1550159394_create_worker_artifacts.down.sql similarity index 100% rename from atc/db/migration/migrations/1545249372_create_worker_artifacts.down.sql rename to atc/db/migration/migrations/1550159394_create_worker_artifacts.down.sql diff --git a/atc/db/migration/migrations/1545249372_create_worker_artifacts.up.sql b/atc/db/migration/migrations/1550159394_create_worker_artifacts.up.sql similarity index 57% rename from atc/db/migration/migrations/1545249372_create_worker_artifacts.up.sql rename to atc/db/migration/migrations/1550159394_create_worker_artifacts.up.sql index a991c19ec2e..7331470fa5a 100755 --- a/atc/db/migration/migrations/1545249372_create_worker_artifacts.up.sql +++ b/atc/db/migration/migrations/1550159394_create_worker_artifacts.up.sql @@ -1,13 +1,16 @@ BEGIN; CREATE TABLE worker_artifacts ( id SERIAL PRIMARY KEY, - path TEXT NOT NULL, - checksum TEXT NOT NULL, + name TEXT NOT NULL, + build_id INTEGER NULL, created_at TIMESTAMP WITH TIME ZONE DEFAULT NOW() NOT NULL ); + ALTER TABLE worker_artifacts + ADD CONSTRAINT worker_artifacts_build_id_fkey FOREIGN KEY (build_id) REFERENCES builds(id) ON DELETE SET NULL; + ALTER TABLE volumes - ADD COLUMN worker_artifact_id integer; + ADD COLUMN worker_artifact_id INTEGER; ALTER TABLE volumes ADD CONSTRAINT volumes_worker_artifact_id_fkey FOREIGN KEY (worker_artifact_id) REFERENCES worker_artifacts(id) ON DELETE SET NULL; diff --git a/atc/db/team_test.go b/atc/db/team_test.go index 32328ab5394..ab207f43cfb 100644 --- a/atc/db/team_test.go +++ b/atc/db/team_test.go @@ -704,7 +704,7 @@ var _ = Describe("Team", func() { Context("when the artifact exists", func() { BeforeEach(func() { - _, err := dbConn.Exec("INSERT INTO worker_artifacts (id, path, checksum) VALUES ($1, '/', '')", 18) + _, err := dbConn.Exec("INSERT INTO worker_artifacts (id, name) VALUES ($1, '')", 18) Expect(err).NotTo(HaveOccurred()) }) diff --git a/atc/db/volume.go b/atc/db/volume.go index 93db3b779c1..1e494afbf79 100644 --- a/atc/db/volume.go +++ b/atc/db/volume.go @@ -150,9 +150,11 @@ type CreatedVolume interface { CreateChildForContainer(CreatingContainer, string) (CreatingVolume, error) Destroying() (DestroyingVolume, error) WorkerName() string + InitializeResourceCache(UsedResourceCache) error - InitializeArtifact(string, string) (WorkerArtifact, error) + InitializeArtifact(name string, buildID int) (WorkerArtifact, error) InitializeTaskCache(int, string, string) error + ContainerHandle() string ParentHandle() string ResourceType() (*VolumeResourceType, error) @@ -390,8 +392,7 @@ func (volume *createdVolume) InitializeResourceCache(resourceCache UsedResourceC return nil } -func (volume *createdVolume) InitializeArtifact(path string, checksum string) (WorkerArtifact, error) { - +func (volume *createdVolume) InitializeArtifact(name string, buildID int) (WorkerArtifact, error) { tx, err := volume.conn.Begin() if err != nil { return nil, err @@ -400,8 +401,8 @@ func (volume *createdVolume) InitializeArtifact(path string, checksum string) (W defer Rollback(tx) atcWorkerArtifact := atc.WorkerArtifact{ - Checksum: checksum, - Path: path, + Name: name, + BuildID: buildID, } workerArtifact, err := saveWorkerArtifact(tx, volume.conn, atcWorkerArtifact) diff --git a/atc/db/volume_test.go b/atc/db/volume_test.go index c5d84a532f0..37cf2a9f832 100644 --- a/atc/db/volume_test.go +++ b/atc/db/volume_test.go @@ -277,14 +277,14 @@ var _ = Describe("Volume", func() { }) JustBeforeEach(func() { - workerArtifact, err = createdVolume.InitializeArtifact("/some/path", "some-checksum") + workerArtifact, err = createdVolume.InitializeArtifact("some-name", 0) Expect(err).ToNot(HaveOccurred()) }) - It("initialize worker artifact", func() { + It("initializes the worker artifact", func() { Expect(workerArtifact.ID()).To(Equal(1)) - Expect(workerArtifact.Path()).To(Equal("/some/path")) - Expect(workerArtifact.Checksum()).To(Equal("some-checksum")) + Expect(workerArtifact.Name()).To(Equal("some-name")) + Expect(workerArtifact.BuildID()).To(Equal(0)) Expect(workerArtifact.CreatedAt()).ToNot(BeNil()) }) diff --git a/atc/db/worker_artifact.go b/atc/db/worker_artifact.go index 95f2c76ed01..bfb173ee75b 100644 --- a/atc/db/worker_artifact.go +++ b/atc/db/worker_artifact.go @@ -14,24 +14,24 @@ import ( type WorkerArtifact interface { ID() int - Path() string - Checksum() string + Name() string + BuildID() int CreatedAt() time.Time - Volume(int) (CreatedVolume, bool, error) + Volume(teamID int) (CreatedVolume, bool, error) } type artifact struct { conn Conn id int - path string - checksum string + name string + buildID int createdAt time.Time } func (a *artifact) ID() int { return a.id } -func (a *artifact) Path() string { return a.path } -func (a *artifact) Checksum() string { return a.checksum } +func (a *artifact) Name() string { return a.name } +func (a *artifact) BuildID() int { return a.buildID } func (a *artifact) CreatedAt() time.Time { return a.createdAt } func (a *artifact) Volume(teamID int) (CreatedVolume, bool, error) { @@ -56,11 +56,16 @@ func saveWorkerArtifact(tx Tx, conn Conn, atcArtifact atc.WorkerArtifact) (Worke var artifactID int + values := map[string]interface{}{ + "name": atcArtifact.Name, + } + + if atcArtifact.BuildID != 0 { + values["build_id"] = atcArtifact.BuildID + } + err := psql.Insert("worker_artifacts"). - SetMap(map[string]interface{}{ - "path": atcArtifact.Path, - "checksum": atcArtifact.Checksum, - }). + SetMap(values). Suffix("RETURNING id"). RunWith(tx). QueryRow(). @@ -84,18 +89,21 @@ func saveWorkerArtifact(tx Tx, conn Conn, atcArtifact atc.WorkerArtifact) (Worke } func getWorkerArtifact(tx Tx, conn Conn, id int) (WorkerArtifact, bool, error) { - var createdAtTime pq.NullTime + var ( + createdAtTime pq.NullTime + buildID sql.NullInt64 + ) artifact := &artifact{conn: conn} - err := psql.Select("id", "created_at", "path", "checksum"). + err := psql.Select("id", "created_at", "name", "build_id"). From("worker_artifacts"). Where(sq.Eq{ "id": id, }). RunWith(tx). QueryRow(). - Scan(&artifact.id, &createdAtTime, &artifact.path, &artifact.checksum) + Scan(&artifact.id, &createdAtTime, &artifact.name, &buildID) if err != nil { if err == sql.ErrNoRows { return nil, false, nil @@ -105,5 +113,7 @@ func getWorkerArtifact(tx Tx, conn Conn, id int) (WorkerArtifact, bool, error) { } artifact.createdAt = createdAtTime.Time + artifact.buildID = int(buildID.Int64) + return artifact, true, nil } diff --git a/atc/engine/build_step.go b/atc/engine/build_step.go index 150f0cce7cf..9bc7a0e5c29 100644 --- a/atc/engine/build_step.go +++ b/atc/engine/build_step.go @@ -5,7 +5,6 @@ import ( "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/exec" - "github.com/concourse/concourse/atc/worker" ) func (build *execBuild) buildAggregateStep(logger lager.Logger, plan atc.Plan) exec.Step { @@ -159,9 +158,11 @@ func (build *execBuild) buildRetryStep(logger lager.Logger, plan atc.Plan) exec. } func (build *execBuild) buildUserArtifactStep(logger lager.Logger, plan atc.Plan) exec.Step { - return exec.UserArtifact(plan.ID, worker.ArtifactName(plan.UserArtifact.Name), build.delegate.BuildStepDelegate(plan.ID)) -} -func (build *execBuild) buildArtifactOutputStep(logger lager.Logger, plan atc.Plan) exec.Step { - return exec.ArtifactOutput(plan.ID, worker.ArtifactName(plan.ArtifactOutput.Name), build.delegate.BuildStepDelegate(plan.ID)) + return build.factory.ArtifactStep( + logger, + plan, + build.dbBuild, + build.delegate.BuildStepDelegate(plan.ID), + ) } diff --git a/atc/engine/enginefakes/fake_build.go b/atc/engine/enginefakes/fake_build.go index 79f16e35500..b1c9df62e3c 100644 --- a/atc/engine/enginefakes/fake_build.go +++ b/atc/engine/enginefakes/fake_build.go @@ -2,11 +2,9 @@ package enginefakes import ( - io "io" sync "sync" lager "code.cloudfoundry.org/lager" - atc "github.com/concourse/concourse/atc" engine "github.com/concourse/concourse/atc/engine" ) @@ -32,25 +30,11 @@ type FakeBuild struct { metadataReturnsOnCall map[int]struct { result1 string } - ReceiveInputStub func(lager.Logger, atc.PlanID, io.ReadCloser) - receiveInputMutex sync.RWMutex - receiveInputArgsForCall []struct { - arg1 lager.Logger - arg2 atc.PlanID - arg3 io.ReadCloser - } ResumeStub func(lager.Logger) resumeMutex sync.RWMutex resumeArgsForCall []struct { arg1 lager.Logger } - SendOutputStub func(lager.Logger, atc.PlanID, io.Writer) - sendOutputMutex sync.RWMutex - sendOutputArgsForCall []struct { - arg1 lager.Logger - arg2 atc.PlanID - arg3 io.Writer - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -167,39 +151,6 @@ func (fake *FakeBuild) MetadataReturnsOnCall(i int, result1 string) { }{result1} } -func (fake *FakeBuild) ReceiveInput(arg1 lager.Logger, arg2 atc.PlanID, arg3 io.ReadCloser) { - fake.receiveInputMutex.Lock() - fake.receiveInputArgsForCall = append(fake.receiveInputArgsForCall, struct { - arg1 lager.Logger - arg2 atc.PlanID - arg3 io.ReadCloser - }{arg1, arg2, arg3}) - fake.recordInvocation("ReceiveInput", []interface{}{arg1, arg2, arg3}) - fake.receiveInputMutex.Unlock() - if fake.ReceiveInputStub != nil { - fake.ReceiveInputStub(arg1, arg2, arg3) - } -} - -func (fake *FakeBuild) ReceiveInputCallCount() int { - fake.receiveInputMutex.RLock() - defer fake.receiveInputMutex.RUnlock() - return len(fake.receiveInputArgsForCall) -} - -func (fake *FakeBuild) ReceiveInputCalls(stub func(lager.Logger, atc.PlanID, io.ReadCloser)) { - fake.receiveInputMutex.Lock() - defer fake.receiveInputMutex.Unlock() - fake.ReceiveInputStub = stub -} - -func (fake *FakeBuild) ReceiveInputArgsForCall(i int) (lager.Logger, atc.PlanID, io.ReadCloser) { - fake.receiveInputMutex.RLock() - defer fake.receiveInputMutex.RUnlock() - argsForCall := fake.receiveInputArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - func (fake *FakeBuild) Resume(arg1 lager.Logger) { fake.resumeMutex.Lock() fake.resumeArgsForCall = append(fake.resumeArgsForCall, struct { @@ -231,39 +182,6 @@ func (fake *FakeBuild) ResumeArgsForCall(i int) lager.Logger { return argsForCall.arg1 } -func (fake *FakeBuild) SendOutput(arg1 lager.Logger, arg2 atc.PlanID, arg3 io.Writer) { - fake.sendOutputMutex.Lock() - fake.sendOutputArgsForCall = append(fake.sendOutputArgsForCall, struct { - arg1 lager.Logger - arg2 atc.PlanID - arg3 io.Writer - }{arg1, arg2, arg3}) - fake.recordInvocation("SendOutput", []interface{}{arg1, arg2, arg3}) - fake.sendOutputMutex.Unlock() - if fake.SendOutputStub != nil { - fake.SendOutputStub(arg1, arg2, arg3) - } -} - -func (fake *FakeBuild) SendOutputCallCount() int { - fake.sendOutputMutex.RLock() - defer fake.sendOutputMutex.RUnlock() - return len(fake.sendOutputArgsForCall) -} - -func (fake *FakeBuild) SendOutputCalls(stub func(lager.Logger, atc.PlanID, io.Writer)) { - fake.sendOutputMutex.Lock() - defer fake.sendOutputMutex.Unlock() - fake.SendOutputStub = stub -} - -func (fake *FakeBuild) SendOutputArgsForCall(i int) (lager.Logger, atc.PlanID, io.Writer) { - fake.sendOutputMutex.RLock() - defer fake.sendOutputMutex.RUnlock() - argsForCall := fake.sendOutputArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - func (fake *FakeBuild) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -271,12 +189,8 @@ func (fake *FakeBuild) Invocations() map[string][][]interface{} { defer fake.abortMutex.RUnlock() fake.metadataMutex.RLock() defer fake.metadataMutex.RUnlock() - fake.receiveInputMutex.RLock() - defer fake.receiveInputMutex.RUnlock() fake.resumeMutex.RLock() defer fake.resumeMutex.RUnlock() - fake.sendOutputMutex.RLock() - defer fake.sendOutputMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/atc/engine/exec_engine.go b/atc/engine/exec_engine.go index 18a2baf716c..b3855327d7c 100644 --- a/atc/engine/exec_engine.go +++ b/atc/engine/exec_engine.go @@ -231,10 +231,6 @@ func (build *execBuild) buildStep(logger lager.Logger, plan atc.Plan) exec.Step return build.buildUserArtifactStep(logger, plan) } - if plan.ArtifactOutput != nil { - return build.buildArtifactOutputStep(logger, plan) - } - return exec.IdentityStep{} } diff --git a/atc/exec/aggregate_test.go b/atc/exec/aggregate_test.go index c72d3a5766a..d12cfedf8fa 100644 --- a/atc/exec/aggregate_test.go +++ b/atc/exec/aggregate_test.go @@ -6,8 +6,7 @@ import ( "sync" . "github.com/concourse/concourse/atc/exec" - "github.com/concourse/concourse/atc/worker" - + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -21,7 +20,7 @@ var _ = Describe("Aggregate", func() { fakeStepA *execfakes.FakeStep fakeStepB *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState step Step @@ -39,7 +38,7 @@ var _ = Describe("Aggregate", func() { fakeStepB, } - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) }) diff --git a/atc/exec/artifact/artifact_suite_test.go b/atc/exec/artifact/artifact_suite_test.go new file mode 100644 index 00000000000..1bc36ea7343 --- /dev/null +++ b/atc/exec/artifact/artifact_suite_test.go @@ -0,0 +1,13 @@ +package artifact_test + +import ( + "testing" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +func TestArtifact(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "Artifact Suite") +} diff --git a/atc/exec/artifact/artifactfakes/fake_registerable_source.go b/atc/exec/artifact/artifactfakes/fake_registerable_source.go new file mode 100644 index 00000000000..f4f8d1460c9 --- /dev/null +++ b/atc/exec/artifact/artifactfakes/fake_registerable_source.go @@ -0,0 +1,280 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package artifactfakes + +import ( + io "io" + sync "sync" + + lager "code.cloudfoundry.org/lager" + artifact "github.com/concourse/concourse/atc/exec/artifact" + worker "github.com/concourse/concourse/atc/worker" +) + +type FakeRegisterableSource struct { + StreamFileStub func(lager.Logger, string) (io.ReadCloser, error) + streamFileMutex sync.RWMutex + streamFileArgsForCall []struct { + arg1 lager.Logger + arg2 string + } + streamFileReturns struct { + result1 io.ReadCloser + result2 error + } + streamFileReturnsOnCall map[int]struct { + result1 io.ReadCloser + result2 error + } + StreamToStub func(lager.Logger, worker.ArtifactDestination) error + streamToMutex sync.RWMutex + streamToArgsForCall []struct { + arg1 lager.Logger + arg2 worker.ArtifactDestination + } + streamToReturns struct { + result1 error + } + streamToReturnsOnCall map[int]struct { + result1 error + } + VolumeOnStub func(lager.Logger, worker.Worker) (worker.Volume, bool, error) + volumeOnMutex sync.RWMutex + volumeOnArgsForCall []struct { + arg1 lager.Logger + arg2 worker.Worker + } + volumeOnReturns struct { + result1 worker.Volume + result2 bool + result3 error + } + volumeOnReturnsOnCall map[int]struct { + result1 worker.Volume + result2 bool + result3 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeRegisterableSource) StreamFile(arg1 lager.Logger, arg2 string) (io.ReadCloser, error) { + fake.streamFileMutex.Lock() + ret, specificReturn := fake.streamFileReturnsOnCall[len(fake.streamFileArgsForCall)] + fake.streamFileArgsForCall = append(fake.streamFileArgsForCall, struct { + arg1 lager.Logger + arg2 string + }{arg1, arg2}) + fake.recordInvocation("StreamFile", []interface{}{arg1, arg2}) + fake.streamFileMutex.Unlock() + if fake.StreamFileStub != nil { + return fake.StreamFileStub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.streamFileReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeRegisterableSource) StreamFileCallCount() int { + fake.streamFileMutex.RLock() + defer fake.streamFileMutex.RUnlock() + return len(fake.streamFileArgsForCall) +} + +func (fake *FakeRegisterableSource) StreamFileCalls(stub func(lager.Logger, string) (io.ReadCloser, error)) { + fake.streamFileMutex.Lock() + defer fake.streamFileMutex.Unlock() + fake.StreamFileStub = stub +} + +func (fake *FakeRegisterableSource) StreamFileArgsForCall(i int) (lager.Logger, string) { + fake.streamFileMutex.RLock() + defer fake.streamFileMutex.RUnlock() + argsForCall := fake.streamFileArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeRegisterableSource) StreamFileReturns(result1 io.ReadCloser, result2 error) { + fake.streamFileMutex.Lock() + defer fake.streamFileMutex.Unlock() + fake.StreamFileStub = nil + fake.streamFileReturns = struct { + result1 io.ReadCloser + result2 error + }{result1, result2} +} + +func (fake *FakeRegisterableSource) StreamFileReturnsOnCall(i int, result1 io.ReadCloser, result2 error) { + fake.streamFileMutex.Lock() + defer fake.streamFileMutex.Unlock() + fake.StreamFileStub = nil + if fake.streamFileReturnsOnCall == nil { + fake.streamFileReturnsOnCall = make(map[int]struct { + result1 io.ReadCloser + result2 error + }) + } + fake.streamFileReturnsOnCall[i] = struct { + result1 io.ReadCloser + result2 error + }{result1, result2} +} + +func (fake *FakeRegisterableSource) StreamTo(arg1 lager.Logger, arg2 worker.ArtifactDestination) error { + fake.streamToMutex.Lock() + ret, specificReturn := fake.streamToReturnsOnCall[len(fake.streamToArgsForCall)] + fake.streamToArgsForCall = append(fake.streamToArgsForCall, struct { + arg1 lager.Logger + arg2 worker.ArtifactDestination + }{arg1, arg2}) + fake.recordInvocation("StreamTo", []interface{}{arg1, arg2}) + fake.streamToMutex.Unlock() + if fake.StreamToStub != nil { + return fake.StreamToStub(arg1, arg2) + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.streamToReturns + return fakeReturns.result1 +} + +func (fake *FakeRegisterableSource) StreamToCallCount() int { + fake.streamToMutex.RLock() + defer fake.streamToMutex.RUnlock() + return len(fake.streamToArgsForCall) +} + +func (fake *FakeRegisterableSource) StreamToCalls(stub func(lager.Logger, worker.ArtifactDestination) error) { + fake.streamToMutex.Lock() + defer fake.streamToMutex.Unlock() + fake.StreamToStub = stub +} + +func (fake *FakeRegisterableSource) StreamToArgsForCall(i int) (lager.Logger, worker.ArtifactDestination) { + fake.streamToMutex.RLock() + defer fake.streamToMutex.RUnlock() + argsForCall := fake.streamToArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeRegisterableSource) StreamToReturns(result1 error) { + fake.streamToMutex.Lock() + defer fake.streamToMutex.Unlock() + fake.StreamToStub = nil + fake.streamToReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeRegisterableSource) StreamToReturnsOnCall(i int, result1 error) { + fake.streamToMutex.Lock() + defer fake.streamToMutex.Unlock() + fake.StreamToStub = nil + if fake.streamToReturnsOnCall == nil { + fake.streamToReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.streamToReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeRegisterableSource) VolumeOn(arg1 lager.Logger, arg2 worker.Worker) (worker.Volume, bool, error) { + fake.volumeOnMutex.Lock() + ret, specificReturn := fake.volumeOnReturnsOnCall[len(fake.volumeOnArgsForCall)] + fake.volumeOnArgsForCall = append(fake.volumeOnArgsForCall, struct { + arg1 lager.Logger + arg2 worker.Worker + }{arg1, arg2}) + fake.recordInvocation("VolumeOn", []interface{}{arg1, arg2}) + fake.volumeOnMutex.Unlock() + if fake.VolumeOnStub != nil { + return fake.VolumeOnStub(arg1, arg2) + } + if specificReturn { + return ret.result1, ret.result2, ret.result3 + } + fakeReturns := fake.volumeOnReturns + return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 +} + +func (fake *FakeRegisterableSource) VolumeOnCallCount() int { + fake.volumeOnMutex.RLock() + defer fake.volumeOnMutex.RUnlock() + return len(fake.volumeOnArgsForCall) +} + +func (fake *FakeRegisterableSource) VolumeOnCalls(stub func(lager.Logger, worker.Worker) (worker.Volume, bool, error)) { + fake.volumeOnMutex.Lock() + defer fake.volumeOnMutex.Unlock() + fake.VolumeOnStub = stub +} + +func (fake *FakeRegisterableSource) VolumeOnArgsForCall(i int) (lager.Logger, worker.Worker) { + fake.volumeOnMutex.RLock() + defer fake.volumeOnMutex.RUnlock() + argsForCall := fake.volumeOnArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2 +} + +func (fake *FakeRegisterableSource) VolumeOnReturns(result1 worker.Volume, result2 bool, result3 error) { + fake.volumeOnMutex.Lock() + defer fake.volumeOnMutex.Unlock() + fake.VolumeOnStub = nil + fake.volumeOnReturns = struct { + result1 worker.Volume + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeRegisterableSource) VolumeOnReturnsOnCall(i int, result1 worker.Volume, result2 bool, result3 error) { + fake.volumeOnMutex.Lock() + defer fake.volumeOnMutex.Unlock() + fake.VolumeOnStub = nil + if fake.volumeOnReturnsOnCall == nil { + fake.volumeOnReturnsOnCall = make(map[int]struct { + result1 worker.Volume + result2 bool + result3 error + }) + } + fake.volumeOnReturnsOnCall[i] = struct { + result1 worker.Volume + result2 bool + result3 error + }{result1, result2, result3} +} + +func (fake *FakeRegisterableSource) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.streamFileMutex.RLock() + defer fake.streamFileMutex.RUnlock() + fake.streamToMutex.RLock() + defer fake.streamToMutex.RUnlock() + fake.volumeOnMutex.RLock() + defer fake.volumeOnMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeRegisterableSource) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ artifact.RegisterableSource = new(FakeRegisterableSource) diff --git a/atc/worker/artifact_repository.go b/atc/exec/artifact/repository.go similarity index 63% rename from atc/worker/artifact_repository.go rename to atc/exec/artifact/repository.go index e020c991f71..6ba929f21c1 100644 --- a/atc/worker/artifact_repository.go +++ b/atc/exec/artifact/repository.go @@ -1,14 +1,16 @@ -package worker +package artifact import ( "sync" + + "github.com/concourse/concourse/atc/worker" ) -// ArtifactName is just a string, with its own type to make interfaces using it +// Name is just a string, with its own type to make interfaces using it // more self-documenting. -type ArtifactName string +type Name string -// ArtifactRepository is the mapping from a ArtifactName to an ArtifactSource. +// Repository is the mapping from a Name to an ArtifactSource. // Steps will both populate this map with new artifacts (e.g. the resource // fetched by a Get step), and look up required artifacts (e.g. the inputs // configured for a Task step). @@ -19,22 +21,28 @@ type ArtifactName string // ArtifactRepository is, itself, an ArtifactSource. As an ArtifactSource it acts // as the set of all ArtifactSources it contains, as if they were each in // subdirectories corresponding to their ArtifactName. -type ArtifactRepository struct { - repo map[ArtifactName]ArtifactSource +type Repository struct { + repo map[Name]worker.ArtifactSource repoL sync.RWMutex } // NewArtifactRepository constructs a new repository. -func NewArtifactRepository() *ArtifactRepository { - return &ArtifactRepository{ - repo: make(map[ArtifactName]ArtifactSource), +func NewRepository() *Repository { + return &Repository{ + repo: make(map[Name]worker.ArtifactSource), } } +//go:generate counterfeiter . RegisterableSource +// A RegisterableSource artifact is an ArtifactSource which can be added to the registry +type RegisterableSource interface { + worker.ArtifactSource +} + // RegisterSource inserts an ArtifactSource into the map under the given // ArtifactName. Producers of artifacts, e.g. the Get step and the Task step, // will call this after they've successfully produced their artifact(s). -func (repo *ArtifactRepository) RegisterSource(name ArtifactName, source ArtifactSource) { +func (repo *Repository) RegisterSource(name Name, source RegisterableSource) { repo.repoL.Lock() repo.repo[name] = source repo.repoL.Unlock() @@ -42,7 +50,7 @@ func (repo *ArtifactRepository) RegisterSource(name ArtifactName, source Artifac // SourceFor looks up a Source for the given ArtifactName. Consumers of // artifacts, e.g. the Task step, will call this to locate their dependencies. -func (repo *ArtifactRepository) SourceFor(name ArtifactName) (ArtifactSource, bool) { +func (repo *Repository) SourceFor(name Name) (worker.ArtifactSource, bool) { repo.repoL.RLock() source, found := repo.repo[name] repo.repoL.RUnlock() @@ -52,8 +60,8 @@ func (repo *ArtifactRepository) SourceFor(name ArtifactName) (ArtifactSource, bo // AsMap extracts the current contents of the ArtifactRepository into a new map // and returns it. Changes to the returned map or the ArtifactRepository will not // affect each other. -func (repo *ArtifactRepository) AsMap() map[ArtifactName]ArtifactSource { - result := make(map[ArtifactName]ArtifactSource) +func (repo *Repository) AsMap() map[Name]worker.ArtifactSource { + result := make(map[Name]worker.ArtifactSource) repo.repoL.RLock() for name, source := range repo.repo { diff --git a/atc/worker/artifact_repository_test.go b/atc/exec/artifact/repository_test.go similarity index 79% rename from atc/worker/artifact_repository_test.go rename to atc/exec/artifact/repository_test.go index c1ea33e5b6b..e5d3ae6d5d4 100644 --- a/atc/worker/artifact_repository_test.go +++ b/atc/exec/artifact/repository_test.go @@ -1,19 +1,19 @@ -package worker_test +package artifact_test import ( - . "github.com/concourse/concourse/atc/worker" - "github.com/concourse/concourse/atc/worker/workerfakes" + . "github.com/concourse/concourse/atc/exec/artifact" + "github.com/concourse/concourse/atc/exec/artifact/artifactfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) var _ = Describe("ArtifactRepository", func() { var ( - repo *ArtifactRepository + repo *Repository ) BeforeEach(func() { - repo = NewArtifactRepository() + repo = NewRepository() }) It("initially does not contain any sources", func() { @@ -23,10 +23,10 @@ var _ = Describe("ArtifactRepository", func() { }) Context("when a source is registered", func() { - var firstSource *workerfakes.FakeArtifactSource + var firstSource *artifactfakes.FakeRegisterableSource BeforeEach(func() { - firstSource = new(workerfakes.FakeArtifactSource) + firstSource = new(artifactfakes.FakeRegisterableSource) repo.RegisterSource("first-source", firstSource) }) @@ -45,10 +45,10 @@ var _ = Describe("ArtifactRepository", func() { }) Context("when a second source is registered", func() { - var secondSource *workerfakes.FakeArtifactSource + var secondSource *artifactfakes.FakeRegisterableSource BeforeEach(func() { - secondSource = new(workerfakes.FakeArtifactSource) + secondSource = new(artifactfakes.FakeRegisterableSource) repo.RegisterSource("second-source", secondSource) }) @@ -71,7 +71,6 @@ var _ = Describe("ArtifactRepository", func() { Expect(found).To(BeFalse()) }) }) - }) }) }) diff --git a/atc/exec/artifact_output_step.go b/atc/exec/artifact_output_step.go deleted file mode 100644 index b3f10edf360..00000000000 --- a/atc/exec/artifact_output_step.go +++ /dev/null @@ -1,62 +0,0 @@ -package exec - -import ( - "context" - "io" - - "code.cloudfoundry.org/lager" - "code.cloudfoundry.org/lager/lagerctx" - "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/worker" -) - -type ArtifactOutputStep struct { - id atc.PlanID - name worker.ArtifactName - delegate BuildStepDelegate -} - -func ArtifactOutput(id atc.PlanID, name worker.ArtifactName, delegate BuildStepDelegate) Step { - return &ArtifactOutputStep{ - id: id, - name: name, - delegate: delegate, - } -} - -func (step *ArtifactOutputStep) Run(ctx context.Context, state RunState) error { - logger := lagerctx.FromContext(ctx).WithData(lager.Data{ - "plan-id": step.id, - "source": step.name, - }) - - source, found := state.Artifacts().SourceFor(step.name) - if !found { - return UnknownArtifactSourceError{ - SourceName: step.name, - } - } - - pb := progress(string(step.name)+":", step.delegate.Stdout()) - - return state.SendPlanOutput(step.id, func(w io.Writer) error { - pb.Start() - defer pb.Finish() - - logger.Debug("sending-plan-output") - return source.StreamTo(logger, streamDestination{io.MultiWriter(w, pb)}) - }) -} - -func (step *ArtifactOutputStep) Succeeded() bool { - return true -} - -type streamDestination struct { - w io.Writer -} - -func (dest streamDestination) StreamIn(path string, src io.Reader) error { - _, err := io.Copy(dest.w, src) - return err -} diff --git a/atc/exec/artifact_output_step_test.go b/atc/exec/artifact_output_step_test.go deleted file mode 100644 index ed39499c95f..00000000000 --- a/atc/exec/artifact_output_step_test.go +++ /dev/null @@ -1,90 +0,0 @@ -package exec_test - -import ( - "bytes" - "context" - "io/ioutil" - - "github.com/concourse/concourse/atc/exec" - "github.com/concourse/concourse/atc/exec/execfakes" - "github.com/concourse/concourse/atc/worker/workerfakes" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("ArtifactOutputStep", func() { - var ( - ctx context.Context - cancel func() - - state exec.RunState - delegate *execfakes.FakeBuildStepDelegate - - step exec.Step - stepErr error - ) - - BeforeEach(func() { - ctx, cancel = context.WithCancel(context.Background()) - - state = exec.NewRunState() - - delegate = new(execfakes.FakeBuildStepDelegate) - delegate.StdoutReturns(ioutil.Discard) - }) - - AfterEach(func() { - cancel() - }) - - JustBeforeEach(func() { - step = exec.ArtifactOutput( - "some-plan-id", - "some-name", - delegate, - ) - - stepErr = step.Run(ctx, state) - }) - - It("is successful", func() { - Expect(step.Succeeded()).To(BeTrue()) - }) - - Context("when the artifact is present", func() { - var ( - output *bytes.Buffer - source *workerfakes.FakeArtifactSource - ) - - BeforeEach(func() { - output = new(bytes.Buffer) - source = new(workerfakes.FakeArtifactSource) - - state.Artifacts().RegisterSource("some-name", source) - - go state.ReadPlanOutput("some-plan-id", output) - }) - - It("waits for a user and sends the artifact to them", func() { - Expect(source.StreamToCallCount()).To(Equal(1)) - - _, dest := source.StreamToArgsForCall(0) - Expect(dest.StreamIn(".", bytes.NewBufferString("hello"))).To(Succeed()) - - Expect(output.String()).To(Equal("hello")) - }) - }) - - Context("when the artifact is not present", func() { - BeforeEach(func() { - // do nothing - }) - - It("returns UnknownArtifactSourceError", func() { - Expect(stepErr).To(Equal(exec.UnknownArtifactSourceError{ - SourceName: "some-name", - })) - }) - }) -}) diff --git a/atc/exec/ensure_step_test.go b/atc/exec/ensure_step_test.go index 634c377fbf5..8a7235b95bd 100644 --- a/atc/exec/ensure_step_test.go +++ b/atc/exec/ensure_step_test.go @@ -4,12 +4,11 @@ import ( "context" "errors" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" - "github.com/concourse/concourse/atc/worker" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("Ensure Step", func() { @@ -20,7 +19,7 @@ var _ = Describe("Ensure Step", func() { step *execfakes.FakeStep hook *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState ensure exec.Step @@ -45,7 +44,7 @@ var _ = Describe("Ensure Step", func() { return ctx.Err() } - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/execfakes/fake_factory.go b/atc/exec/execfakes/fake_factory.go index f8223b6de19..fdb0ddc9597 100644 --- a/atc/exec/execfakes/fake_factory.go +++ b/atc/exec/execfakes/fake_factory.go @@ -11,6 +11,20 @@ import ( ) type FakeFactory struct { + ArtifactStepStub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step + artifactStepMutex sync.RWMutex + artifactStepArgsForCall []struct { + arg1 lager.Logger + arg2 atc.Plan + arg3 db.Build + arg4 exec.BuildStepDelegate + } + artifactStepReturns struct { + result1 exec.Step + } + artifactStepReturnsOnCall map[int]struct { + result1 exec.Step + } GetStub func(lager.Logger, atc.Plan, db.Build, exec.StepMetadata, db.ContainerMetadata, exec.GetDelegate) exec.Step getMutex sync.RWMutex getArgsForCall []struct { @@ -62,6 +76,69 @@ type FakeFactory struct { invocationsMutex sync.RWMutex } +func (fake *FakeFactory) ArtifactStep(arg1 lager.Logger, arg2 atc.Plan, arg3 db.Build, arg4 exec.BuildStepDelegate) exec.Step { + fake.artifactStepMutex.Lock() + ret, specificReturn := fake.artifactStepReturnsOnCall[len(fake.artifactStepArgsForCall)] + fake.artifactStepArgsForCall = append(fake.artifactStepArgsForCall, struct { + arg1 lager.Logger + arg2 atc.Plan + arg3 db.Build + arg4 exec.BuildStepDelegate + }{arg1, arg2, arg3, arg4}) + fake.recordInvocation("ArtifactStep", []interface{}{arg1, arg2, arg3, arg4}) + fake.artifactStepMutex.Unlock() + if fake.ArtifactStepStub != nil { + return fake.ArtifactStepStub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.artifactStepReturns + return fakeReturns.result1 +} + +func (fake *FakeFactory) ArtifactStepCallCount() int { + fake.artifactStepMutex.RLock() + defer fake.artifactStepMutex.RUnlock() + return len(fake.artifactStepArgsForCall) +} + +func (fake *FakeFactory) ArtifactStepCalls(stub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step) { + fake.artifactStepMutex.Lock() + defer fake.artifactStepMutex.Unlock() + fake.ArtifactStepStub = stub +} + +func (fake *FakeFactory) ArtifactStepArgsForCall(i int) (lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) { + fake.artifactStepMutex.RLock() + defer fake.artifactStepMutex.RUnlock() + argsForCall := fake.artifactStepArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeFactory) ArtifactStepReturns(result1 exec.Step) { + fake.artifactStepMutex.Lock() + defer fake.artifactStepMutex.Unlock() + fake.ArtifactStepStub = nil + fake.artifactStepReturns = struct { + result1 exec.Step + }{result1} +} + +func (fake *FakeFactory) ArtifactStepReturnsOnCall(i int, result1 exec.Step) { + fake.artifactStepMutex.Lock() + defer fake.artifactStepMutex.Unlock() + fake.ArtifactStepStub = nil + if fake.artifactStepReturnsOnCall == nil { + fake.artifactStepReturnsOnCall = make(map[int]struct { + result1 exec.Step + }) + } + fake.artifactStepReturnsOnCall[i] = struct { + result1 exec.Step + }{result1} +} + func (fake *FakeFactory) Get(arg1 lager.Logger, arg2 atc.Plan, arg3 db.Build, arg4 exec.StepMetadata, arg5 db.ContainerMetadata, arg6 exec.GetDelegate) exec.Step { fake.getMutex.Lock() ret, specificReturn := fake.getReturnsOnCall[len(fake.getArgsForCall)] @@ -259,6 +336,8 @@ func (fake *FakeFactory) TaskReturnsOnCall(i int, result1 exec.Step) { func (fake *FakeFactory) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() + fake.artifactStepMutex.RLock() + defer fake.artifactStepMutex.RUnlock() fake.getMutex.RLock() defer fake.getMutex.RUnlock() fake.putMutex.RLock() diff --git a/atc/exec/execfakes/fake_run_state.go b/atc/exec/execfakes/fake_run_state.go index 7fd101686ab..09c5fb1a7cc 100644 --- a/atc/exec/execfakes/fake_run_state.go +++ b/atc/exec/execfakes/fake_run_state.go @@ -2,42 +2,23 @@ package execfakes import ( - io "io" sync "sync" atc "github.com/concourse/concourse/atc" exec "github.com/concourse/concourse/atc/exec" - worker "github.com/concourse/concourse/atc/worker" + artifact "github.com/concourse/concourse/atc/exec/artifact" ) type FakeRunState struct { - ArtifactsStub func() *worker.ArtifactRepository + ArtifactsStub func() *artifact.Repository artifactsMutex sync.RWMutex artifactsArgsForCall []struct { } artifactsReturns struct { - result1 *worker.ArtifactRepository + result1 *artifact.Repository } artifactsReturnsOnCall map[int]struct { - result1 *worker.ArtifactRepository - } - ReadPlanOutputStub func(atc.PlanID, io.Writer) - readPlanOutputMutex sync.RWMutex - readPlanOutputArgsForCall []struct { - arg1 atc.PlanID - arg2 io.Writer - } - ReadUserInputStub func(atc.PlanID, exec.InputHandler) error - readUserInputMutex sync.RWMutex - readUserInputArgsForCall []struct { - arg1 atc.PlanID - arg2 exec.InputHandler - } - readUserInputReturns struct { - result1 error - } - readUserInputReturnsOnCall map[int]struct { - result1 error + result1 *artifact.Repository } ResultStub func(atc.PlanID, interface{}) bool resultMutex sync.RWMutex @@ -51,24 +32,6 @@ type FakeRunState struct { resultReturnsOnCall map[int]struct { result1 bool } - SendPlanOutputStub func(atc.PlanID, exec.OutputHandler) error - sendPlanOutputMutex sync.RWMutex - sendPlanOutputArgsForCall []struct { - arg1 atc.PlanID - arg2 exec.OutputHandler - } - sendPlanOutputReturns struct { - result1 error - } - sendPlanOutputReturnsOnCall map[int]struct { - result1 error - } - SendUserInputStub func(atc.PlanID, io.ReadCloser) - sendUserInputMutex sync.RWMutex - sendUserInputArgsForCall []struct { - arg1 atc.PlanID - arg2 io.ReadCloser - } StoreResultStub func(atc.PlanID, interface{}) storeResultMutex sync.RWMutex storeResultArgsForCall []struct { @@ -79,7 +42,7 @@ type FakeRunState struct { invocationsMutex sync.RWMutex } -func (fake *FakeRunState) Artifacts() *worker.ArtifactRepository { +func (fake *FakeRunState) Artifacts() *artifact.Repository { fake.artifactsMutex.Lock() ret, specificReturn := fake.artifactsReturnsOnCall[len(fake.artifactsArgsForCall)] fake.artifactsArgsForCall = append(fake.artifactsArgsForCall, struct { @@ -102,125 +65,32 @@ func (fake *FakeRunState) ArtifactsCallCount() int { return len(fake.artifactsArgsForCall) } -func (fake *FakeRunState) ArtifactsCalls(stub func() *worker.ArtifactRepository) { +func (fake *FakeRunState) ArtifactsCalls(stub func() *artifact.Repository) { fake.artifactsMutex.Lock() defer fake.artifactsMutex.Unlock() fake.ArtifactsStub = stub } -func (fake *FakeRunState) ArtifactsReturns(result1 *worker.ArtifactRepository) { +func (fake *FakeRunState) ArtifactsReturns(result1 *artifact.Repository) { fake.artifactsMutex.Lock() defer fake.artifactsMutex.Unlock() fake.ArtifactsStub = nil fake.artifactsReturns = struct { - result1 *worker.ArtifactRepository + result1 *artifact.Repository }{result1} } -func (fake *FakeRunState) ArtifactsReturnsOnCall(i int, result1 *worker.ArtifactRepository) { +func (fake *FakeRunState) ArtifactsReturnsOnCall(i int, result1 *artifact.Repository) { fake.artifactsMutex.Lock() defer fake.artifactsMutex.Unlock() fake.ArtifactsStub = nil if fake.artifactsReturnsOnCall == nil { fake.artifactsReturnsOnCall = make(map[int]struct { - result1 *worker.ArtifactRepository + result1 *artifact.Repository }) } fake.artifactsReturnsOnCall[i] = struct { - result1 *worker.ArtifactRepository - }{result1} -} - -func (fake *FakeRunState) ReadPlanOutput(arg1 atc.PlanID, arg2 io.Writer) { - fake.readPlanOutputMutex.Lock() - fake.readPlanOutputArgsForCall = append(fake.readPlanOutputArgsForCall, struct { - arg1 atc.PlanID - arg2 io.Writer - }{arg1, arg2}) - fake.recordInvocation("ReadPlanOutput", []interface{}{arg1, arg2}) - fake.readPlanOutputMutex.Unlock() - if fake.ReadPlanOutputStub != nil { - fake.ReadPlanOutputStub(arg1, arg2) - } -} - -func (fake *FakeRunState) ReadPlanOutputCallCount() int { - fake.readPlanOutputMutex.RLock() - defer fake.readPlanOutputMutex.RUnlock() - return len(fake.readPlanOutputArgsForCall) -} - -func (fake *FakeRunState) ReadPlanOutputCalls(stub func(atc.PlanID, io.Writer)) { - fake.readPlanOutputMutex.Lock() - defer fake.readPlanOutputMutex.Unlock() - fake.ReadPlanOutputStub = stub -} - -func (fake *FakeRunState) ReadPlanOutputArgsForCall(i int) (atc.PlanID, io.Writer) { - fake.readPlanOutputMutex.RLock() - defer fake.readPlanOutputMutex.RUnlock() - argsForCall := fake.readPlanOutputArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRunState) ReadUserInput(arg1 atc.PlanID, arg2 exec.InputHandler) error { - fake.readUserInputMutex.Lock() - ret, specificReturn := fake.readUserInputReturnsOnCall[len(fake.readUserInputArgsForCall)] - fake.readUserInputArgsForCall = append(fake.readUserInputArgsForCall, struct { - arg1 atc.PlanID - arg2 exec.InputHandler - }{arg1, arg2}) - fake.recordInvocation("ReadUserInput", []interface{}{arg1, arg2}) - fake.readUserInputMutex.Unlock() - if fake.ReadUserInputStub != nil { - return fake.ReadUserInputStub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - fakeReturns := fake.readUserInputReturns - return fakeReturns.result1 -} - -func (fake *FakeRunState) ReadUserInputCallCount() int { - fake.readUserInputMutex.RLock() - defer fake.readUserInputMutex.RUnlock() - return len(fake.readUserInputArgsForCall) -} - -func (fake *FakeRunState) ReadUserInputCalls(stub func(atc.PlanID, exec.InputHandler) error) { - fake.readUserInputMutex.Lock() - defer fake.readUserInputMutex.Unlock() - fake.ReadUserInputStub = stub -} - -func (fake *FakeRunState) ReadUserInputArgsForCall(i int) (atc.PlanID, exec.InputHandler) { - fake.readUserInputMutex.RLock() - defer fake.readUserInputMutex.RUnlock() - argsForCall := fake.readUserInputArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRunState) ReadUserInputReturns(result1 error) { - fake.readUserInputMutex.Lock() - defer fake.readUserInputMutex.Unlock() - fake.ReadUserInputStub = nil - fake.readUserInputReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRunState) ReadUserInputReturnsOnCall(i int, result1 error) { - fake.readUserInputMutex.Lock() - defer fake.readUserInputMutex.Unlock() - fake.ReadUserInputStub = nil - if fake.readUserInputReturnsOnCall == nil { - fake.readUserInputReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.readUserInputReturnsOnCall[i] = struct { - result1 error + result1 *artifact.Repository }{result1} } @@ -285,99 +155,6 @@ func (fake *FakeRunState) ResultReturnsOnCall(i int, result1 bool) { }{result1} } -func (fake *FakeRunState) SendPlanOutput(arg1 atc.PlanID, arg2 exec.OutputHandler) error { - fake.sendPlanOutputMutex.Lock() - ret, specificReturn := fake.sendPlanOutputReturnsOnCall[len(fake.sendPlanOutputArgsForCall)] - fake.sendPlanOutputArgsForCall = append(fake.sendPlanOutputArgsForCall, struct { - arg1 atc.PlanID - arg2 exec.OutputHandler - }{arg1, arg2}) - fake.recordInvocation("SendPlanOutput", []interface{}{arg1, arg2}) - fake.sendPlanOutputMutex.Unlock() - if fake.SendPlanOutputStub != nil { - return fake.SendPlanOutputStub(arg1, arg2) - } - if specificReturn { - return ret.result1 - } - fakeReturns := fake.sendPlanOutputReturns - return fakeReturns.result1 -} - -func (fake *FakeRunState) SendPlanOutputCallCount() int { - fake.sendPlanOutputMutex.RLock() - defer fake.sendPlanOutputMutex.RUnlock() - return len(fake.sendPlanOutputArgsForCall) -} - -func (fake *FakeRunState) SendPlanOutputCalls(stub func(atc.PlanID, exec.OutputHandler) error) { - fake.sendPlanOutputMutex.Lock() - defer fake.sendPlanOutputMutex.Unlock() - fake.SendPlanOutputStub = stub -} - -func (fake *FakeRunState) SendPlanOutputArgsForCall(i int) (atc.PlanID, exec.OutputHandler) { - fake.sendPlanOutputMutex.RLock() - defer fake.sendPlanOutputMutex.RUnlock() - argsForCall := fake.sendPlanOutputArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeRunState) SendPlanOutputReturns(result1 error) { - fake.sendPlanOutputMutex.Lock() - defer fake.sendPlanOutputMutex.Unlock() - fake.SendPlanOutputStub = nil - fake.sendPlanOutputReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeRunState) SendPlanOutputReturnsOnCall(i int, result1 error) { - fake.sendPlanOutputMutex.Lock() - defer fake.sendPlanOutputMutex.Unlock() - fake.SendPlanOutputStub = nil - if fake.sendPlanOutputReturnsOnCall == nil { - fake.sendPlanOutputReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.sendPlanOutputReturnsOnCall[i] = struct { - result1 error - }{result1} -} - -func (fake *FakeRunState) SendUserInput(arg1 atc.PlanID, arg2 io.ReadCloser) { - fake.sendUserInputMutex.Lock() - fake.sendUserInputArgsForCall = append(fake.sendUserInputArgsForCall, struct { - arg1 atc.PlanID - arg2 io.ReadCloser - }{arg1, arg2}) - fake.recordInvocation("SendUserInput", []interface{}{arg1, arg2}) - fake.sendUserInputMutex.Unlock() - if fake.SendUserInputStub != nil { - fake.SendUserInputStub(arg1, arg2) - } -} - -func (fake *FakeRunState) SendUserInputCallCount() int { - fake.sendUserInputMutex.RLock() - defer fake.sendUserInputMutex.RUnlock() - return len(fake.sendUserInputArgsForCall) -} - -func (fake *FakeRunState) SendUserInputCalls(stub func(atc.PlanID, io.ReadCloser)) { - fake.sendUserInputMutex.Lock() - defer fake.sendUserInputMutex.Unlock() - fake.SendUserInputStub = stub -} - -func (fake *FakeRunState) SendUserInputArgsForCall(i int) (atc.PlanID, io.ReadCloser) { - fake.sendUserInputMutex.RLock() - defer fake.sendUserInputMutex.RUnlock() - argsForCall := fake.sendUserInputArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - func (fake *FakeRunState) StoreResult(arg1 atc.PlanID, arg2 interface{}) { fake.storeResultMutex.Lock() fake.storeResultArgsForCall = append(fake.storeResultArgsForCall, struct { @@ -415,16 +192,8 @@ func (fake *FakeRunState) Invocations() map[string][][]interface{} { defer fake.invocationsMutex.RUnlock() fake.artifactsMutex.RLock() defer fake.artifactsMutex.RUnlock() - fake.readPlanOutputMutex.RLock() - defer fake.readPlanOutputMutex.RUnlock() - fake.readUserInputMutex.RLock() - defer fake.readUserInputMutex.RUnlock() fake.resultMutex.RLock() defer fake.resultMutex.RUnlock() - fake.sendPlanOutputMutex.RLock() - defer fake.sendPlanOutputMutex.RUnlock() - fake.sendUserInputMutex.RLock() - defer fake.sendUserInputMutex.RUnlock() fake.storeResultMutex.RLock() defer fake.storeResultMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} diff --git a/atc/exec/execfakes/fake_task_config_source.go b/atc/exec/execfakes/fake_task_config_source.go index 225ab0344fb..a7394c1c48d 100644 --- a/atc/exec/execfakes/fake_task_config_source.go +++ b/atc/exec/execfakes/fake_task_config_source.go @@ -7,15 +7,15 @@ import ( lager "code.cloudfoundry.org/lager" atc "github.com/concourse/concourse/atc" exec "github.com/concourse/concourse/atc/exec" - worker "github.com/concourse/concourse/atc/worker" + artifact "github.com/concourse/concourse/atc/exec/artifact" ) type FakeTaskConfigSource struct { - FetchConfigStub func(lager.Logger, *worker.ArtifactRepository) (atc.TaskConfig, error) + FetchConfigStub func(lager.Logger, *artifact.Repository) (atc.TaskConfig, error) fetchConfigMutex sync.RWMutex fetchConfigArgsForCall []struct { arg1 lager.Logger - arg2 *worker.ArtifactRepository + arg2 *artifact.Repository } fetchConfigReturns struct { result1 atc.TaskConfig @@ -39,12 +39,12 @@ type FakeTaskConfigSource struct { invocationsMutex sync.RWMutex } -func (fake *FakeTaskConfigSource) FetchConfig(arg1 lager.Logger, arg2 *worker.ArtifactRepository) (atc.TaskConfig, error) { +func (fake *FakeTaskConfigSource) FetchConfig(arg1 lager.Logger, arg2 *artifact.Repository) (atc.TaskConfig, error) { fake.fetchConfigMutex.Lock() ret, specificReturn := fake.fetchConfigReturnsOnCall[len(fake.fetchConfigArgsForCall)] fake.fetchConfigArgsForCall = append(fake.fetchConfigArgsForCall, struct { arg1 lager.Logger - arg2 *worker.ArtifactRepository + arg2 *artifact.Repository }{arg1, arg2}) fake.recordInvocation("FetchConfig", []interface{}{arg1, arg2}) fake.fetchConfigMutex.Unlock() @@ -64,13 +64,13 @@ func (fake *FakeTaskConfigSource) FetchConfigCallCount() int { return len(fake.fetchConfigArgsForCall) } -func (fake *FakeTaskConfigSource) FetchConfigCalls(stub func(lager.Logger, *worker.ArtifactRepository) (atc.TaskConfig, error)) { +func (fake *FakeTaskConfigSource) FetchConfigCalls(stub func(lager.Logger, *artifact.Repository) (atc.TaskConfig, error)) { fake.fetchConfigMutex.Lock() defer fake.fetchConfigMutex.Unlock() fake.FetchConfigStub = stub } -func (fake *FakeTaskConfigSource) FetchConfigArgsForCall(i int) (lager.Logger, *worker.ArtifactRepository) { +func (fake *FakeTaskConfigSource) FetchConfigArgsForCall(i int) (lager.Logger, *artifact.Repository) { fake.fetchConfigMutex.RLock() defer fake.fetchConfigMutex.RUnlock() argsForCall := fake.fetchConfigArgsForCall[i] diff --git a/atc/exec/factory.go b/atc/exec/factory.go index 5aecfebeba5..951fe9703bd 100644 --- a/atc/exec/factory.go +++ b/atc/exec/factory.go @@ -40,6 +40,13 @@ type Factory interface { db.ContainerMetadata, TaskDelegate, ) Step + + ArtifactStep( + lager.Logger, + atc.Plan, + db.Build, + BuildStepDelegate, + ) Step } // StepMetadata is used to inject metadata to make available to the step when diff --git a/atc/exec/garden_factory.go b/atc/exec/garden_factory.go index c134944f7f4..1655187dd08 100644 --- a/atc/exec/garden_factory.go +++ b/atc/exec/garden_factory.go @@ -6,17 +6,18 @@ import ( "path/filepath" "code.cloudfoundry.org/lager" - boshtemplate "github.com/cloudfoundry/bosh-cli/director/template" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/resource" "github.com/concourse/concourse/atc/worker" ) type gardenFactory struct { pool worker.Pool + client worker.Client resourceFetcher resource.Fetcher resourceCacheFactory db.ResourceCacheFactory resourceConfigFactory db.ResourceConfigFactory @@ -28,6 +29,7 @@ type gardenFactory struct { func NewGardenFactory( pool worker.Pool, + client worker.Client, resourceFetcher resource.Fetcher, resourceCacheFactory db.ResourceCacheFactory, resourceConfigFactory db.ResourceConfigFactory, @@ -38,6 +40,7 @@ func NewGardenFactory( ) Factory { return &gardenFactory{ pool: pool, + client: client, resourceFetcher: resourceFetcher, resourceCacheFactory: resourceCacheFactory, resourceConfigFactory: resourceConfigFactory, @@ -148,7 +151,7 @@ func (factory *gardenFactory) Task( containerMetadata db.ContainerMetadata, delegate TaskDelegate, ) Step { - workingDirectory := factory.taskWorkingDirectory(worker.ArtifactName(plan.Task.Name)) + workingDirectory := factory.taskWorkingDirectory(artifact.Name(plan.Task.Name)) containerMetadata.WorkingDirectory = workingDirectory credMgrVariables := factory.variablesFactory.NewVariables(build.TeamName(), build.PipelineName()) @@ -206,7 +209,16 @@ func (factory *gardenFactory) Task( return LogError(taskStep, delegate) } -func (factory *gardenFactory) taskWorkingDirectory(sourceName worker.ArtifactName) string { +func (factory *gardenFactory) ArtifactStep( + logger lager.Logger, + plan atc.Plan, + build db.Build, + delegate BuildStepDelegate, +) Step { + return NewArtifactStep(plan, build, factory.client, delegate) +} + +func (factory *gardenFactory) taskWorkingDirectory(sourceName artifact.Name) string { sum := sha1.Sum([]byte(sourceName)) return filepath.Join("/tmp", "build", fmt.Sprintf("%x", sum[:4])) } diff --git a/atc/exec/get_step.go b/atc/exec/get_step.go index c590ac20dde..f59f5486724 100644 --- a/atc/exec/get_step.go +++ b/atc/exec/get_step.go @@ -12,6 +12,7 @@ import ( "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/resource" "github.com/concourse/concourse/atc/worker" ) @@ -233,7 +234,7 @@ func (step *GetStep) Run(ctx context.Context, state RunState) error { return err } - state.Artifacts().RegisterSource(worker.ArtifactName(step.name), &getArtifactSource{ + state.Artifacts().RegisterSource(artifact.Name(step.name), &getArtifactSource{ resourceInstance: resourceInstance, versionedSource: versionedSource, }) @@ -299,19 +300,41 @@ func (s *getArtifactSource) VolumeOn(logger lager.Logger, worker worker.Worker) // StreamTo streams the resource's data to the destination. func (s *getArtifactSource) StreamTo(logger lager.Logger, destination worker.ArtifactDestination) error { - out, err := s.versionedSource.StreamOut(".") + return streamToHelper(s.versionedSource, logger, destination) +} + +// StreamFile streams a single file out of the resource. +func (s *getArtifactSource) StreamFile(logger lager.Logger, path string) (io.ReadCloser, error) { + return streamFileHelper(s.versionedSource, logger, path) +} + +func streamToHelper(s interface { + StreamOut(string) (io.ReadCloser, error) +}, logger lager.Logger, destination worker.ArtifactDestination) error { + logger.Debug("start") + + defer logger.Debug("end") + + out, err := s.StreamOut(".") if err != nil { + logger.Error("failed", err) return err } defer out.Close() - return destination.StreamIn(".", out) + err = destination.StreamIn(".", out) + if err != nil { + logger.Error("failed", err) + return err + } + return nil } -// StreamFile streams a single file out of the resource. -func (s *getArtifactSource) StreamFile(logger lager.Logger, path string) (io.ReadCloser, error) { - out, err := s.versionedSource.StreamOut(path) +func streamFileHelper(s interface { + StreamOut(string) (io.ReadCloser, error) +}, logger lager.Logger, path string) (io.ReadCloser, error) { + out, err := s.StreamOut(path) if err != nil { return nil, err } diff --git a/atc/exec/get_step_test.go b/atc/exec/get_step_test.go index 58608033fa8..f1578026a28 100644 --- a/atc/exec/get_step_test.go +++ b/atc/exec/get_step_test.go @@ -18,6 +18,7 @@ import ( "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" "github.com/concourse/concourse/atc/resource" "github.com/concourse/concourse/atc/resource/resourcefakes" @@ -36,6 +37,7 @@ var _ = Describe("GetStep", func() { fakeWorker *workerfakes.FakeWorker fakePool *workerfakes.FakePool + fakeClient *workerfakes.FakeClient fakeStrategy *workerfakes.FakeContainerPlacementStrategy fakeResourceFactory *resourcefakes.FakeResourceFactory fakeResourceFetcher *resourcefakes.FakeFetcher @@ -50,7 +52,7 @@ var _ = Describe("GetStep", func() { fakeVersionedSource *resourcefakes.FakeVersionedSource resourceTypes atc.VersionedResourceTypes - artifactRepository *worker.ArtifactRepository + artifactRepository *artifact.Repository state *execfakes.FakeRunState factory exec.Factory @@ -76,6 +78,7 @@ var _ = Describe("GetStep", func() { fakeWorker = new(workerfakes.FakeWorker) fakeResourceFetcher = new(resourcefakes.FakeFetcher) fakePool = new(workerfakes.FakePool) + fakeClient = new(workerfakes.FakeClient) fakeStrategy = new(workerfakes.FakeContainerPlacementStrategy) fakeResourceFactory = new(resourcefakes.FakeResourceFactory) fakeResourceCacheFactory = new(dbfakes.FakeResourceCacheFactory) @@ -86,7 +89,7 @@ var _ = Describe("GetStep", func() { } fakeVariablesFactory.NewVariablesReturns(variables) - artifactRepository = worker.NewArtifactRepository() + artifactRepository = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(artifactRepository) @@ -119,7 +122,7 @@ var _ = Describe("GetStep", func() { VersionedResourceTypes: resourceTypes, } - factory = exec.NewGardenFactory(fakePool, fakeResourceFetcher, fakeResourceCacheFactory, fakeResourceConfigFactory, fakeVariablesFactory, atc.ContainerLimits{}, fakeStrategy, fakeResourceFactory) + factory = exec.NewGardenFactory(fakePool, fakeClient, fakeResourceFetcher, fakeResourceCacheFactory, fakeResourceConfigFactory, fakeVariablesFactory, atc.ContainerLimits{}, fakeStrategy, fakeResourceFactory) fakeDelegate = new(execfakes.FakeGetDelegate) }) diff --git a/atc/exec/log_error_step_test.go b/atc/exec/log_error_step_test.go index 9789e2a7111..cbcc62278e5 100644 --- a/atc/exec/log_error_step_test.go +++ b/atc/exec/log_error_step_test.go @@ -5,8 +5,7 @@ import ( "errors" . "github.com/concourse/concourse/atc/exec" - "github.com/concourse/concourse/atc/worker" - + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -20,7 +19,7 @@ var _ = Describe("LogErrorStep", func() { fakeStep *execfakes.FakeStep fakeDelegate *execfakes.FakeBuildStepDelegate - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState step Step @@ -32,7 +31,7 @@ var _ = Describe("LogErrorStep", func() { fakeStep = new(execfakes.FakeStep) fakeDelegate = new(execfakes.FakeBuildStepDelegate) - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/on_abort_test.go b/atc/exec/on_abort_test.go index a00bd2729ff..7deee5bccd3 100644 --- a/atc/exec/on_abort_test.go +++ b/atc/exec/on_abort_test.go @@ -4,12 +4,11 @@ import ( "context" "errors" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" - "github.com/concourse/concourse/atc/worker" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("On Abort Step", func() { @@ -20,7 +19,7 @@ var _ = Describe("On Abort Step", func() { step *execfakes.FakeStep hook *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState onAbortStep exec.Step @@ -34,7 +33,7 @@ var _ = Describe("On Abort Step", func() { step = &execfakes.FakeStep{} hook = &execfakes.FakeStep{} - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/on_failure_test.go b/atc/exec/on_failure_test.go index f633bbfa600..afd3ba0c152 100644 --- a/atc/exec/on_failure_test.go +++ b/atc/exec/on_failure_test.go @@ -4,12 +4,11 @@ import ( "context" "errors" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" - "github.com/concourse/concourse/atc/worker" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("On Failure Step", func() { @@ -20,7 +19,7 @@ var _ = Describe("On Failure Step", func() { step *execfakes.FakeStep hook *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState onFailureStep exec.Step @@ -34,7 +33,7 @@ var _ = Describe("On Failure Step", func() { step = &execfakes.FakeStep{} hook = &execfakes.FakeStep{} - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/on_success_test.go b/atc/exec/on_success_test.go index c5ace340574..06d045454f0 100644 --- a/atc/exec/on_success_test.go +++ b/atc/exec/on_success_test.go @@ -4,12 +4,11 @@ import ( "context" "errors" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" - "github.com/concourse/concourse/atc/worker" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("On Success Step", func() { @@ -20,7 +19,7 @@ var _ = Describe("On Success Step", func() { step *execfakes.FakeStep hook *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState onSuccessStep exec.Step @@ -34,7 +33,7 @@ var _ = Describe("On Success Step", func() { step = &execfakes.FakeStep{} hook = &execfakes.FakeStep{} - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/put_inputs.go b/atc/exec/put_inputs.go index c89c6f3ce56..ca679c3309e 100644 --- a/atc/exec/put_inputs.go +++ b/atc/exec/put_inputs.go @@ -4,6 +4,7 @@ import ( "fmt" "code.cloudfoundry.org/lager" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/resource" "github.com/concourse/concourse/atc/worker" ) @@ -17,7 +18,7 @@ func (e PutInputNotFoundError) Error() string { } type PutInputs interface { - FindAll(*worker.ArtifactRepository) ([]worker.InputSource, error) + FindAll(*artifact.Repository) ([]worker.InputSource, error) } type allInputs struct{} @@ -26,7 +27,7 @@ func NewAllInputs() PutInputs { return &allInputs{} } -func (i allInputs) FindAll(artifacts *worker.ArtifactRepository) ([]worker.InputSource, error) { +func (i allInputs) FindAll(artifacts *artifact.Repository) ([]worker.InputSource, error) { inputs := []worker.InputSource{} for name, source := range artifacts.AsMap() { @@ -49,18 +50,18 @@ func NewSpecificInputs(inputs []string) PutInputs { } } -func (i specificInputs) FindAll(artifacts *worker.ArtifactRepository) ([]worker.InputSource, error) { +func (i specificInputs) FindAll(artifacts *artifact.Repository) ([]worker.InputSource, error) { artifactsMap := artifacts.AsMap() inputs := []worker.InputSource{} for _, i := range i.inputs { - artifactSource, found := artifactsMap[worker.ArtifactName(i)] + artifactSource, found := artifactsMap[artifact.Name(i)] if !found { return nil, PutInputNotFoundError{Input: i} } inputs = append(inputs, &putInputSource{ - name: worker.ArtifactName(i), + name: artifact.Name(i), source: PutResourceSource{artifactSource}, }) } @@ -69,7 +70,7 @@ func (i specificInputs) FindAll(artifacts *worker.ArtifactRepository) ([]worker. } type putInputSource struct { - name worker.ArtifactName + name artifact.Name source worker.ArtifactSource } diff --git a/atc/exec/put_step_test.go b/atc/exec/put_step_test.go index 6af4f1bef56..0a42b64ee83 100644 --- a/atc/exec/put_step_test.go +++ b/atc/exec/put_step_test.go @@ -10,6 +10,7 @@ import ( "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" "github.com/concourse/concourse/atc/resource" "github.com/concourse/concourse/atc/resource/resourcefakes" @@ -48,7 +49,7 @@ var _ = Describe("PutStep", func() { resourceTypes creds.VersionedResourceTypes - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState stdoutBuf *gbytes.Buffer @@ -87,7 +88,7 @@ var _ = Describe("PutStep", func() { fakeDelegate.StdoutReturns(stdoutBuf) fakeDelegate.StderrReturns(stderrBuf) - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/retry_step_test.go b/atc/exec/retry_step_test.go index bead8828025..25a7e45e24a 100644 --- a/atc/exec/retry_step_test.go +++ b/atc/exec/retry_step_test.go @@ -5,8 +5,7 @@ import ( "errors" . "github.com/concourse/concourse/atc/exec" - "github.com/concourse/concourse/atc/worker" - + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -21,7 +20,7 @@ var _ = Describe("Retry Step", func() { attempt2 *execfakes.FakeStep attempt3 *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState step Step @@ -34,7 +33,7 @@ var _ = Describe("Retry Step", func() { attempt2 = new(execfakes.FakeStep) attempt3 = new(execfakes.FakeStep) - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/run_state.go b/atc/exec/run_state.go index 4c227c08cce..66fd768657c 100644 --- a/atc/exec/run_state.go +++ b/atc/exec/run_state.go @@ -1,31 +1,26 @@ package exec import ( - "io" "reflect" "sync" "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/worker" + "github.com/concourse/concourse/atc/exec/artifact" ) type runState struct { - artifacts *worker.ArtifactRepository + artifacts *artifact.Repository results *sync.Map - inputs *sync.Map - outputs *sync.Map } func NewRunState() RunState { return &runState{ - artifacts: worker.NewArtifactRepository(), + artifacts: artifact.NewRepository(), results: &sync.Map{}, - inputs: &sync.Map{}, - outputs: &sync.Map{}, } } -func (state *runState) Artifacts() *worker.ArtifactRepository { +func (state *runState) Artifacts() *artifact.Repository { return state.artifacts } @@ -46,73 +41,3 @@ func (state *runState) Result(id atc.PlanID, to interface{}) bool { func (state *runState) StoreResult(id atc.PlanID, val interface{}) { state.results.Store(id, val) } - -func (state *runState) SendUserInput(id atc.PlanID, input io.ReadCloser) { - i, loaded := state.inputs.LoadOrStore(id, make(chan io.ReadCloser)) - if loaded { - // clear out the channel - state.inputs.Delete(id) - } - - inputs := i.(chan io.ReadCloser) - - // send input (blocking) to a reader - inputs <- input - - // wait for reader to close channel signifying that they're done - <-inputs -} - -func (state *runState) ReadUserInput(id atc.PlanID, handler InputHandler) error { - i, loaded := state.inputs.LoadOrStore(id, make(chan io.ReadCloser)) - if loaded { - // clear out the channel - state.inputs.Delete(id) - } - - inputs := i.(chan io.ReadCloser) - - // signal to sender that we're done - defer close(inputs) - - // wait for stream from request to arrive - stream := <-inputs - - // synchronously stream in - return handler(stream) -} - -func (state *runState) ReadPlanOutput(id atc.PlanID, output io.Writer) { - i, loaded := state.outputs.LoadOrStore(id, make(chan io.Writer)) - if loaded { - // clear out the channel - state.outputs.Delete(id) - } - - outputs := i.(chan io.Writer) - - // send input (blocking) to a reader - outputs <- output - - // wait for reader to close channel signifying that they're done - <-outputs -} - -func (state *runState) SendPlanOutput(id atc.PlanID, handler OutputHandler) error { - i, loaded := state.outputs.LoadOrStore(id, make(chan io.Writer)) - if loaded { - // clear out the channel - state.outputs.Delete(id) - } - - outputs := i.(chan io.Writer) - - // signal to sender that we're done - defer close(outputs) - - // wait for stream from request to arrive - stream := <-outputs - - // synchronously stream in - return handler(stream) -} diff --git a/atc/exec/run_state_test.go b/atc/exec/run_state_test.go index 85a71c30747..2a57619573d 100644 --- a/atc/exec/run_state_test.go +++ b/atc/exec/run_state_test.go @@ -1,12 +1,6 @@ package exec_test import ( - "bytes" - "errors" - "io" - "io/ioutil" - "runtime" - "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/exec" . "github.com/onsi/ginkgo" @@ -99,98 +93,4 @@ var _ = Describe("RunState", func() { }) }) }) - - Describe("User Input", func() { - It("can be passed around asynchronously", func() { - buf := ioutil.NopCloser(bytes.NewBufferString("some-payload")) - - for i := 0; i < 1000; i++ { - go state.SendUserInput("some-plan-id", buf) - - state.ReadUserInput("some-plan-id", func(rc io.ReadCloser) error { - Expect(rc).To(Equal(buf)) - return nil - }) - } - }) - - It("blocks the sender until the reader is finished", func() { - buf := ioutil.NopCloser(bytes.NewBufferString("some-payload")) - - var done bool - go func() { - defer GinkgoRecover() - state.SendUserInput("some-plan-id", buf) - Expect(done).To(BeTrue()) - }() - - runtime.Gosched() - - err := state.ReadUserInput("some-plan-id", func(rc io.ReadCloser) error { - Expect(rc).To(Equal(buf)) - done = true - return nil - }) - Expect(err).ToNot(HaveOccurred()) - }) - - It("bubbles up the handler error", func() { - buf := ioutil.NopCloser(bytes.NewBufferString("some-payload")) - - go state.SendUserInput("some-plan-id", buf) - - disaster := errors.New("nope") - err := state.ReadUserInput("some-plan-id", func(rc io.ReadCloser) error { - return disaster - }) - Expect(err).To(Equal(disaster)) - }) - }) - - Describe("Plan Output", func() { - It("can be passed around asynchronously", func() { - out := new(bytes.Buffer) - - for i := 0; i < 1000; i++ { - go state.ReadPlanOutput("some-plan-id", out) - - state.SendPlanOutput("some-plan-id", func(w io.Writer) error { - Expect(w).To(Equal(out)) - return nil - }) - } - }) - - It("blocks the sender until the reader is finished", func() { - buf := new(bytes.Buffer) - - var done bool - go func() { - defer GinkgoRecover() - state.ReadPlanOutput("some-plan-id", buf) - Expect(done).To(BeTrue()) - }() - - runtime.Gosched() - - err := state.SendPlanOutput("some-plan-id", func(w io.Writer) error { - Expect(w).To(Equal(buf)) - done = true - return nil - }) - Expect(err).ToNot(HaveOccurred()) - }) - - It("bubbles up the handler error", func() { - buf := new(bytes.Buffer) - - go state.ReadPlanOutput("some-plan-id", buf) - - disaster := errors.New("nope") - err := state.SendPlanOutput("some-plan-id", func(w io.Writer) error { - return disaster - }) - Expect(err).To(Equal(disaster)) - }) - }) }) diff --git a/atc/exec/step.go b/atc/exec/step.go index 4f173d540e0..b789d658400 100644 --- a/atc/exec/step.go +++ b/atc/exec/step.go @@ -5,7 +5,7 @@ import ( "io" "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/worker" + "github.com/concourse/concourse/atc/exec/artifact" ) //go:generate counterfeiter . Step @@ -35,16 +35,10 @@ type InputHandler func(io.ReadCloser) error type OutputHandler func(io.Writer) error type RunState interface { - Artifacts() *worker.ArtifactRepository + Artifacts() *artifact.Repository Result(atc.PlanID, interface{}) bool StoreResult(atc.PlanID, interface{}) - - SendUserInput(atc.PlanID, io.ReadCloser) - ReadUserInput(atc.PlanID, InputHandler) error - - ReadPlanOutput(atc.PlanID, io.Writer) - SendPlanOutput(atc.PlanID, OutputHandler) error } // ExitStatus is the resulting exit code from the process that the step ran. diff --git a/atc/exec/task_config_source.go b/atc/exec/task_config_source.go index 5e2b1ffbaf7..5390b93495d 100644 --- a/atc/exec/task_config_source.go +++ b/atc/exec/task_config_source.go @@ -3,17 +3,17 @@ package exec import ( "encoding/json" "fmt" - boshtemplate "github.com/cloudfoundry/bosh-cli/director/template" - "github.com/concourse/concourse/atc/template" "io/ioutil" "math" "strconv" "strings" "code.cloudfoundry.org/lager" + boshtemplate "github.com/cloudfoundry/bosh-cli/director/template" "github.com/concourse/baggageclaim" "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/worker" + "github.com/concourse/concourse/atc/exec/artifact" + "github.com/concourse/concourse/atc/template" ) //go:generate counterfeiter . TaskConfigSource @@ -21,8 +21,8 @@ import ( // TaskConfigSource is used to determine a Task step's TaskConfig. type TaskConfigSource interface { // FetchConfig returns the TaskConfig, and may have to a task config file out - // of the worker.ArtifactRepository. - FetchConfig(lager.Logger, *worker.ArtifactRepository) (atc.TaskConfig, error) + // of the artifact.Repository. + FetchConfig(lager.Logger, *artifact.Repository) (atc.TaskConfig, error) Warnings() []string } @@ -32,7 +32,7 @@ type StaticConfigSource struct { } // FetchConfig returns the configuration. -func (configSource StaticConfigSource) FetchConfig(lager.Logger, *worker.ArtifactRepository) (atc.TaskConfig, error) { +func (configSource StaticConfigSource) FetchConfig(lager.Logger, *artifact.Repository) (atc.TaskConfig, error) { taskConfig := atc.TaskConfig{} if configSource.Config != nil { taskConfig = *configSource.Config @@ -45,16 +45,16 @@ func (configSource StaticConfigSource) Warnings() []string { } // FileConfigSource represents a dynamically configured TaskConfig, which will -// be fetched from a specified file in the worker.ArtifactRepository. +// be fetched from a specified file in the artifact.Repository. type FileConfigSource struct { ConfigPath string } -// FetchConfig reads the specified file from the worker.ArtifactRepository and loads the +// FetchConfig reads the specified file from the artifact.Repository and loads the // TaskConfig contained therein (expecting it to be YAML format). // // The path must be in the format SOURCE_NAME/FILE/PATH.yml. The SOURCE_NAME -// will be used to determine the ArtifactSource in the worker.ArtifactRepository to +// will be used to determine the ArtifactSource in the artifact.Repository to // stream the file out of. // // If the source name is missing (i.e. if the path is just "foo.yml"), @@ -65,13 +65,13 @@ type FileConfigSource struct { // // If the task config file is not found, or is invalid YAML, or is an invalid // task configuration, the respective errors will be bubbled up. -func (configSource FileConfigSource) FetchConfig(logger lager.Logger, repo *worker.ArtifactRepository) (atc.TaskConfig, error) { +func (configSource FileConfigSource) FetchConfig(logger lager.Logger, repo *artifact.Repository) (atc.TaskConfig, error) { segs := strings.SplitN(configSource.ConfigPath, "/", 2) if len(segs) != 2 { return atc.TaskConfig{}, UnspecifiedArtifactSourceError{configSource.ConfigPath} } - sourceName := worker.ArtifactName(segs[0]) + sourceName := artifact.Name(segs[0]) filePath := segs[1] source, found := repo.SourceFor(sourceName) @@ -115,7 +115,7 @@ type OverrideParamsConfigSource struct { // FetchConfig overrides parameters, allowing the user to set params required by a task loaded // from a file by providing them in static configuration. -func (configSource *OverrideParamsConfigSource) FetchConfig(logger lager.Logger, source *worker.ArtifactRepository) (atc.TaskConfig, error) { +func (configSource *OverrideParamsConfigSource) FetchConfig(logger lager.Logger, source *artifact.Repository) (atc.TaskConfig, error) { taskConfig, err := configSource.ConfigSource.FetchConfig(logger, source) if err != nil { return atc.TaskConfig{}, err @@ -162,7 +162,7 @@ type InterpolateTemplateConfigSource struct { } // FetchConfig returns the interpolated configuration -func (configSource InterpolateTemplateConfigSource) FetchConfig(logger lager.Logger, source *worker.ArtifactRepository) (atc.TaskConfig, error) { +func (configSource InterpolateTemplateConfigSource) FetchConfig(logger lager.Logger, source *artifact.Repository) (atc.TaskConfig, error) { taskConfig, err := configSource.ConfigSource.FetchConfig(logger, source) if err != nil { return atc.TaskConfig{}, err @@ -199,7 +199,7 @@ type ValidatingConfigSource struct { // FetchConfig fetches the config using the underlying ConfigSource, and checks // that it's valid. -func (configSource ValidatingConfigSource) FetchConfig(logger lager.Logger, source *worker.ArtifactRepository) (atc.TaskConfig, error) { +func (configSource ValidatingConfigSource) FetchConfig(logger lager.Logger, source *artifact.Repository) (atc.TaskConfig, error) { config, err := configSource.ConfigSource.FetchConfig(logger, source) if err != nil { return atc.TaskConfig{}, err @@ -216,10 +216,10 @@ func (configSource ValidatingConfigSource) Warnings() []string { return configSource.ConfigSource.Warnings() } -// UnknownArtifactSourceError is returned when the worker.ArtifactName specified by the -// path does not exist in the worker.ArtifactRepository. +// UnknownArtifactSourceError is returned when the artifact.Name specified by the +// path does not exist in the artifact.Repository. type UnknownArtifactSourceError struct { - SourceName worker.ArtifactName + SourceName artifact.Name ConfigPath string } diff --git a/atc/exec/task_config_source_test.go b/atc/exec/task_config_source_test.go index b913e62357d..0667c30fd22 100644 --- a/atc/exec/task_config_source_test.go +++ b/atc/exec/task_config_source_test.go @@ -4,32 +4,30 @@ import ( "errors" "code.cloudfoundry.org/lager/lagertest" + boshtemplate "github.com/cloudfoundry/bosh-cli/director/template" "github.com/concourse/baggageclaim" "github.com/concourse/concourse/atc" . "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" - "github.com/concourse/concourse/atc/worker" "github.com/concourse/concourse/atc/worker/workerfakes" - "gopkg.in/yaml.v2" - - boshtemplate "github.com/cloudfoundry/bosh-cli/director/template" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" + "gopkg.in/yaml.v2" ) var _ = Describe("TaskConfigSource", func() { var ( taskConfig atc.TaskConfig taskVars atc.Params - repo *worker.ArtifactRepository + repo *artifact.Repository logger *lagertest.TestLogger ) BeforeEach(func() { logger = lagertest.NewTestLogger("task-config-source-test") - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() taskConfig = atc.TaskConfig{ Platform: "some-platform", RootfsURI: "some-image", diff --git a/atc/exec/task_step.go b/atc/exec/task_step.go index 6f411c52c5e..6e34fec5209 100644 --- a/atc/exec/task_step.go +++ b/atc/exec/task_step.go @@ -1,8 +1,6 @@ package exec import ( - "archive/tar" - "compress/gzip" "context" "errors" "fmt" @@ -12,14 +10,14 @@ import ( "strconv" "strings" - boshtemplate "github.com/cloudfoundry/bosh-cli/director/template" - "code.cloudfoundry.org/garden" "code.cloudfoundry.org/lager" "code.cloudfoundry.org/lager/lagerctx" + boshtemplate "github.com/cloudfoundry/bosh-cli/director/template" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/worker" ) @@ -66,7 +64,7 @@ type TaskDelegate interface { } // TaskStep executes a TaskConfig, whose inputs will be fetched from the -// worker.ArtifactRepository and outputs will be added to the worker.ArtifactRepository. +// artifact.Repository and outputs will be added to the artifact.Repository. type TaskStep struct { privileged Privileged configSource TaskConfigSource @@ -138,19 +136,19 @@ func NewTaskStep( } } -// Run will first selects the worker based on the TaskConfig's platform, the -// TaskStep's tags, and prioritized by availability of volumes for the TaskConfig's +// Run will first select the worker based on the TaskConfig's platform and the +// TaskStep's tags, and prioritize it by availability of volumes for the TaskConfig's // inputs. Inputs that did not have volumes available on the worker will be streamed // in to the container. // -// If any inputs are not available in the worker.ArtifactRepository, MissingInputsError +// If any inputs are not available in the artifact.Repository, MissingInputsError // is returned. // // Once all the inputs are satisfied, the task's script will be executed. If // the task is canceled via the context, the script will be interrupted. // // If the script exits successfully, the outputs specified in the TaskConfig -// are registered with the worker.ArtifactRepository. If no outputs are specified, the +// are registered with the artifact.Repository. If no outputs are specified, the // task's entire working directory is registered as an ArtifactSource under the // name of the task. func (action *TaskStep) Run(ctx context.Context, state RunState) error { @@ -214,7 +212,7 @@ func (action *TaskStep) Run(ctx context.Context, state RunState) error { return err } - action.succeeded = status == 0 + action.succeeded = (status == 0) err = action.registerOutputs(logger, repository, config, container) if err != nil { @@ -313,7 +311,7 @@ func (action *TaskStep) Succeeded() bool { return action.succeeded } -func (action *TaskStep) imageSpec(logger lager.Logger, repository *worker.ArtifactRepository, config atc.TaskConfig) (worker.ImageSpec, error) { +func (action *TaskStep) imageSpec(logger lager.Logger, repository *artifact.Repository, config atc.TaskConfig) (worker.ImageSpec, error) { imageSpec := worker.ImageSpec{ Privileged: bool(action.privileged), } @@ -321,13 +319,12 @@ func (action *TaskStep) imageSpec(logger lager.Logger, repository *worker.Artifa // Determine the source of the container image // a reference to an artifact (get step, task output) ? if action.imageArtifactName != "" { - source, found := repository.SourceFor(worker.ArtifactName(action.imageArtifactName)) + source, found := repository.SourceFor(artifact.Name(action.imageArtifactName)) if !found { return worker.ImageSpec{}, MissingTaskImageSourceError{action.imageArtifactName} } imageSpec.ImageArtifactSource = source - imageSpec.ImageArtifactName = worker.ArtifactName(action.imageArtifactName) //an image_resource } else if config.ImageResource != nil { @@ -345,7 +342,7 @@ func (action *TaskStep) imageSpec(logger lager.Logger, repository *worker.Artifa return imageSpec, nil } -func (action *TaskStep) containerInputs(logger lager.Logger, repository *worker.ArtifactRepository, config atc.TaskConfig) ([]worker.InputSource, error) { +func (action *TaskStep) containerInputs(logger lager.Logger, repository *artifact.Repository, config atc.TaskConfig) ([]worker.InputSource, error) { inputs := []worker.InputSource{} var missingRequiredInputs []string @@ -355,7 +352,7 @@ func (action *TaskStep) containerInputs(logger lager.Logger, repository *worker. inputName = sourceName } - source, found := repository.SourceFor(worker.ArtifactName(inputName)) + source, found := repository.SourceFor(artifact.Name(inputName)) if !found { if !input.Optional { missingRequiredInputs = append(missingRequiredInputs, inputName) @@ -386,7 +383,7 @@ func (action *TaskStep) containerInputs(logger lager.Logger, repository *worker. return inputs, nil } -func (action *TaskStep) containerSpec(logger lager.Logger, repository *worker.ArtifactRepository, config atc.TaskConfig) (worker.ContainerSpec, error) { +func (action *TaskStep) containerSpec(logger lager.Logger, repository *artifact.Repository, config atc.TaskConfig) (worker.ContainerSpec, error) { imageSpec, err := action.imageSpec(logger, repository, config) if err != nil { return worker.ContainerSpec{}, err @@ -419,7 +416,7 @@ func (action *TaskStep) containerSpec(logger lager.Logger, repository *worker.Ar return containerSpec, nil } -func (action *TaskStep) workerSpec(logger lager.Logger, resourceTypes creds.VersionedResourceTypes, repository *worker.ArtifactRepository, config atc.TaskConfig) (worker.WorkerSpec, error) { +func (action *TaskStep) workerSpec(logger lager.Logger, resourceTypes creds.VersionedResourceTypes, repository *artifact.Repository, config atc.TaskConfig) (worker.WorkerSpec, error) { workerSpec := worker.WorkerSpec{ Platform: config.Platform, Tags: action.tags, @@ -439,7 +436,7 @@ func (action *TaskStep) workerSpec(logger lager.Logger, resourceTypes creds.Vers return workerSpec, nil } -func (action *TaskStep) registerOutputs(logger lager.Logger, repository *worker.ArtifactRepository, config atc.TaskConfig, container worker.Container) error { +func (action *TaskStep) registerOutputs(logger lager.Logger, repository *artifact.Repository, config atc.TaskConfig, container worker.Container) error { volumeMounts := container.VolumeMounts() logger.Debug("registering-outputs", lager.Data{"outputs": config.Outputs}) @@ -454,8 +451,13 @@ func (action *TaskStep) registerOutputs(logger lager.Logger, repository *worker. for _, mount := range volumeMounts { if filepath.Clean(mount.MountPath) == filepath.Clean(outputPath) { - source := newTaskArtifactSource(mount.Volume) - repository.RegisterSource(worker.ArtifactName(outputName), source) + source := NewTaskArtifactSource(mount.Volume) + repository.RegisterSource(artifact.Name(outputName), source) + + _, err := mount.Volume.InitializeArtifact(outputName, action.buildID) + if err != nil { + return err + } } } } @@ -497,12 +499,8 @@ type taskArtifactSource struct { worker.Volume } -func newTaskArtifactSource( - volume worker.Volume, -) *taskArtifactSource { - return &taskArtifactSource{ - volume, - } +func NewTaskArtifactSource(volume worker.Volume) *taskArtifactSource { + return &taskArtifactSource{volume} } func (src *taskArtifactSource) StreamTo(logger lager.Logger, destination worker.ArtifactDestination) error { @@ -511,49 +509,12 @@ func (src *taskArtifactSource) StreamTo(logger lager.Logger, destination worker. "src-worker": src.WorkerName(), }) - logger.Debug("start") - - defer logger.Debug("end") - - out, err := src.StreamOut(".") - if err != nil { - logger.Error("failed", err) - return err - } - - defer out.Close() - - err = destination.StreamIn(".", out) - if err != nil { - logger.Error("failed", err) - return err - } - return nil + return streamToHelper(src, logger, destination) } func (src *taskArtifactSource) StreamFile(logger lager.Logger, filename string) (io.ReadCloser, error) { logger.Debug("streaming-file-from-volume") - out, err := src.StreamOut(filename) - if err != nil { - return nil, err - } - - gzReader, err := gzip.NewReader(out) - if err != nil { - return nil, FileNotFoundError{Path: filename} - } - - tarReader := tar.NewReader(gzReader) - - _, err = tarReader.Next() - if err != nil { - return nil, FileNotFoundError{Path: filename} - } - - return fileReadCloser{ - Reader: tarReader, - Closer: out, - }, nil + return streamFileHelper(src, logger, filename) } func (src *taskArtifactSource) VolumeOn(logger lager.Logger, w worker.Worker) (worker.Volume, bool, error) { diff --git a/atc/exec/task_step_test.go b/atc/exec/task_step_test.go index d04e93179e7..b4498d87b1a 100644 --- a/atc/exec/task_step_test.go +++ b/atc/exec/task_step_test.go @@ -18,6 +18,7 @@ import ( "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" "github.com/concourse/concourse/atc/worker" "github.com/concourse/concourse/atc/worker/workerfakes" @@ -56,7 +57,7 @@ var _ = Describe("TaskStep", func() { inputMapping map[string]string outputMapping map[string]string - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState taskStep exec.Step @@ -87,7 +88,7 @@ var _ = Describe("TaskStep", func() { jobID = 12345 configSource = new(execfakes.FakeTaskConfigSource) - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) @@ -325,6 +326,7 @@ var _ = Describe("TaskStep", func() { Expect(actualResourceTypes).To(Equal(resourceTypes)) }) + }) Context("when an exit status is already saved off", func() { @@ -427,6 +429,23 @@ var _ = Describe("TaskStep", func() { sourceMap := repo.AsMap() Expect(sourceMap).To(ConsistOf(artifactSource1, artifactSource2, artifactSource3)) }) + + It("initializes the outputs as artifacts", func() { + Expect(fakeVolume1.InitializeArtifactCallCount()).To(Equal(1)) + outputName, id := fakeVolume1.InitializeArtifactArgsForCall(0) + Expect(outputName).To(Equal("some-output")) + Expect(id).To(Equal(buildID)) + + Expect(fakeVolume2.InitializeArtifactCallCount()).To(Equal(1)) + outputName, id = fakeVolume2.InitializeArtifactArgsForCall(0) + Expect(outputName).To(Equal("some-other-output")) + Expect(id).To(Equal(buildID)) + + Expect(fakeVolume3.InitializeArtifactCallCount()).To(Equal(1)) + outputName, id = fakeVolume3.InitializeArtifactArgsForCall(0) + Expect(outputName).To(Equal("some-trailing-slash-output")) + Expect(id).To(Equal(buildID)) + }) }) }) @@ -1196,7 +1215,6 @@ var _ = Describe("TaskStep", func() { _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, - ImageArtifactName: worker.ArtifactName(imageArtifactName), })) Expect(workerSpec.ResourceType).To(Equal("")) @@ -1233,7 +1251,6 @@ var _ = Describe("TaskStep", func() { _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, - ImageArtifactName: worker.ArtifactName(imageArtifactName), })) Expect(workerSpec.ResourceType).To(Equal("")) @@ -1264,7 +1281,6 @@ var _ = Describe("TaskStep", func() { _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, - ImageArtifactName: worker.ArtifactName(imageArtifactName), })) Expect(workerSpec.ResourceType).To(Equal("")) @@ -1296,7 +1312,6 @@ var _ = Describe("TaskStep", func() { _, _, containerSpec, workerSpec, _ := fakePool.FindOrChooseWorkerForContainerArgsForCall(0) Expect(containerSpec.ImageSpec).To(Equal(worker.ImageSpec{ ImageArtifactSource: imageArtifactSource, - ImageArtifactName: worker.ArtifactName(imageArtifactName), })) Expect(workerSpec.ResourceType).To(Equal("")) }) diff --git a/atc/exec/timeout_step_test.go b/atc/exec/timeout_step_test.go index c59cebff305..e6a2ed621ce 100644 --- a/atc/exec/timeout_step_test.go +++ b/atc/exec/timeout_step_test.go @@ -6,8 +6,7 @@ import ( "time" . "github.com/concourse/concourse/atc/exec" - "github.com/concourse/concourse/atc/worker" - + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -20,7 +19,7 @@ var _ = Describe("Timeout Step", func() { fakeStep *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState step Step @@ -35,7 +34,7 @@ var _ = Describe("Timeout Step", func() { fakeStep = new(execfakes.FakeStep) - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/try_step_test.go b/atc/exec/try_step_test.go index bd546e02636..26d192f9630 100644 --- a/atc/exec/try_step_test.go +++ b/atc/exec/try_step_test.go @@ -5,8 +5,7 @@ import ( "errors" . "github.com/concourse/concourse/atc/exec" - "github.com/concourse/concourse/atc/worker" - + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/exec/execfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -19,7 +18,7 @@ var _ = Describe("Try Step", func() { runStep *execfakes.FakeStep - repo *worker.ArtifactRepository + repo *artifact.Repository state *execfakes.FakeRunState step Step @@ -30,7 +29,7 @@ var _ = Describe("Try Step", func() { runStep = new(execfakes.FakeStep) - repo = worker.NewArtifactRepository() + repo = artifact.NewRepository() state = new(execfakes.FakeRunState) state.ArtifactsReturns(repo) diff --git a/atc/exec/user_artifact_step.go b/atc/exec/user_artifact_step.go index cc64fa03c9c..57659290951 100644 --- a/atc/exec/user_artifact_step.go +++ b/atc/exec/user_artifact_step.go @@ -2,70 +2,79 @@ package exec import ( "context" - "errors" - "io" + "fmt" "code.cloudfoundry.org/lager" "code.cloudfoundry.org/lager/lagerctx" "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/exec/artifact" "github.com/concourse/concourse/atc/worker" ) -type UserArtifactStep struct { - id atc.PlanID - name worker.ArtifactName - delegate BuildStepDelegate +type ArtifactVolumeNotFoundErr string + +func (e ArtifactVolumeNotFoundErr) Error() string { + return fmt.Sprintf("volume for worker artifact '%s' not found", e) +} + +type ArtifactStep struct { + plan atc.Plan + build db.Build + workerClient worker.Client + delegate BuildStepDelegate + succeeded bool } -func UserArtifact(id atc.PlanID, name worker.ArtifactName, delegate BuildStepDelegate) Step { - return &UserArtifactStep{ - id: id, - name: name, - delegate: delegate, +func NewArtifactStep(plan atc.Plan, build db.Build, workerClient worker.Client, delegate BuildStepDelegate) Step { + return &ArtifactStep{ + plan: plan, + build: build, + workerClient: workerClient, + delegate: delegate, } } -func (step *UserArtifactStep) Run(ctx context.Context, state RunState) error { +func (step *ArtifactStep) Run(ctx context.Context, state RunState) error { logger := lagerctx.FromContext(ctx).WithData(lager.Data{ - "plan-id": step.id, - "name": step.name, - }) - - state.Artifacts().RegisterSource(step.name, streamSource{ - logger, - step, - state, + "plan-id": step.plan.ID, }) - return nil -} + buildArtifact, err := step.build.Artifact(step.plan.UserArtifact.ArtifactID) + if err != nil { + return err + } -func (step *UserArtifactStep) Succeeded() bool { - return true -} + volume, found, err := buildArtifact.Volume(step.build.TeamID()) + if err != nil { + return err + } -type streamSource struct { - logger lager.Logger - step *UserArtifactStep - state RunState -} + if !found { + return ArtifactVolumeNotFoundErr(buildArtifact.Name()) + } -func (source streamSource) StreamTo(logger lager.Logger, dest worker.ArtifactDestination) error { - pb := progress(string(source.step.name)+":", source.step.delegate.Stdout()) + workerVolume, found, err := step.workerClient.FindVolume(logger, volume.TeamID(), volume.Handle()) + if err != nil { + return err + } - return source.state.ReadUserInput(source.step.id, func(rc io.ReadCloser) error { - pb.Start() - defer pb.Finish() + if !found { + return ArtifactVolumeNotFoundErr(buildArtifact.Name()) + } - source.logger.Debug("reading-user-input") - return dest.StreamIn(".", pb.NewProxyReader(rc)) + logger.Info("register-artifact-source", lager.Data{ + "handle": workerVolume.Handle(), }) -} -func (source streamSource) StreamFile(logger lager.Logger, path string) (io.ReadCloser, error) { - return nil, errors.New("cannot stream single file from user artifact") + source := NewTaskArtifactSource(workerVolume) + state.Artifacts().RegisterSource(artifact.Name(step.plan.UserArtifact.Name), source) + + step.succeeded = true + + return nil } -func (source streamSource) VolumeOn(lager.Logger, worker.Worker) (worker.Volume, bool, error) { - return nil, false, nil +func (step *ArtifactStep) Succeeded() bool { + return step.succeeded } diff --git a/atc/exec/user_artifact_step_test.go b/atc/exec/user_artifact_step_test.go index 682829c53ce..23ebf90944a 100644 --- a/atc/exec/user_artifact_step_test.go +++ b/atc/exec/user_artifact_step_test.go @@ -1,11 +1,12 @@ package exec_test import ( - "bytes" "context" + "errors" "io/ioutil" - "code.cloudfoundry.org/lager/lagertest" + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/exec" "github.com/concourse/concourse/atc/exec/execfakes" "github.com/concourse/concourse/atc/worker/workerfakes" @@ -13,27 +14,33 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("UserArtifactStep", func() { +var _ = Describe("ArtifactStep", func() { var ( ctx context.Context cancel func() - logger *lagertest.TestLogger + // logger *lagertest.TestLogger state exec.RunState delegate *execfakes.FakeBuildStepDelegate - step exec.Step - stepErr error + step exec.Step + stepErr error + plan atc.Plan + fakeBuild *dbfakes.FakeBuild + fakeWorkerClient *workerfakes.FakeClient ) BeforeEach(func() { ctx, cancel = context.WithCancel(context.Background()) - logger = lagertest.NewTestLogger("user-artifact-step-test") + // logger = lagertest.NewTestLogger("user-artifact-step-test") state = exec.NewRunState() delegate = new(execfakes.FakeBuildStepDelegate) delegate.StdoutReturns(ioutil.Discard) + + fakeBuild = new(dbfakes.FakeBuild) + fakeWorkerClient = new(workerfakes.FakeClient) }) AfterEach(func() { @@ -41,35 +48,95 @@ var _ = Describe("UserArtifactStep", func() { }) JustBeforeEach(func() { - step = exec.UserArtifact( - "some-plan-id", - "some-name", - delegate, - ) + plan = atc.Plan{UserArtifact: &atc.UserArtifactPlan{0, "some-name"}} + step = exec.NewArtifactStep(plan, fakeBuild, fakeWorkerClient, delegate) stepErr = step.Run(ctx, state) }) - It("is successful", func() { - Expect(stepErr).ToNot(HaveOccurred()) - Expect(step.Succeeded()).To(BeTrue()) + Context("when looking up the build artifact errors", func() { + BeforeEach(func() { + fakeBuild.ArtifactReturns(nil, errors.New("nope")) + }) + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) }) - It("registers an artifact which reads from user input", func() { - source, found := state.Artifacts().SourceFor("some-name") - Expect(found).To(BeTrue()) - - dest := new(workerfakes.FakeArtifactDestination) - - input := ioutil.NopCloser(bytes.NewBufferString("hello")) - go state.SendUserInput("some-plan-id", input) - - Expect(dest.StreamInCallCount()).To(Equal(0)) - Expect(source.StreamTo(logger, dest)).To(Succeed()) - Expect(dest.StreamInCallCount()).To(Equal(1)) - - path, stream := dest.StreamInArgsForCall(0) - Expect(path).To(Equal(".")) - Expect(ioutil.ReadAll(stream)).To(Equal([]byte("hello"))) + Context("when looking up the build artifact succeeds", func() { + var fakeWorkerArtifact *dbfakes.FakeWorkerArtifact + + BeforeEach(func() { + fakeWorkerArtifact = new(dbfakes.FakeWorkerArtifact) + fakeBuild.ArtifactReturns(fakeWorkerArtifact, nil) + }) + + Context("when looking up the db volume fails", func() { + BeforeEach(func() { + fakeWorkerArtifact.VolumeReturns(nil, false, errors.New("nope")) + }) + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) + }) + + Context("when the db volume does not exist", func() { + BeforeEach(func() { + fakeWorkerArtifact.VolumeReturns(nil, false, nil) + }) + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) + }) + + Context("when the db volume does exist", func() { + var fakeVolume *dbfakes.FakeCreatedVolume + + BeforeEach(func() { + fakeVolume = new(dbfakes.FakeCreatedVolume) + fakeWorkerArtifact.VolumeReturns(fakeVolume, true, nil) + }) + + Context("when looking up the worker volume fails", func() { + BeforeEach(func() { + fakeWorkerClient.FindVolumeReturns(nil, false, errors.New("nope")) + }) + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) + }) + + Context("when the worker volume does not exist", func() { + BeforeEach(func() { + fakeWorkerClient.FindVolumeReturns(nil, false, nil) + }) + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) + }) + + Context("when the worker volume does exist", func() { + var fakeWorkerVolume *workerfakes.FakeVolume + + BeforeEach(func() { + fakeWorkerVolume = new(workerfakes.FakeVolume) + fakeWorkerVolume.HandleReturns("handle") + + fakeWorkerClient.FindVolumeReturns(fakeWorkerVolume, true, nil) + }) + + It("registers the worker volume as an artifact source", func() { + source, found := state.Artifacts().SourceFor("some-name") + + Expect(stepErr).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(source).To(Equal(exec.NewTaskArtifactSource(fakeWorkerVolume))) + }) + + It("succeeds", func() { + Expect(step.Succeeded()).To(BeTrue()) + }) + }) + }) }) }) diff --git a/atc/plan.go b/atc/plan.go index 2cc0cd09afe..64a4e2c5272 100644 --- a/atc/plan.go +++ b/atc/plan.go @@ -28,7 +28,8 @@ type Plan struct { type PlanID string type UserArtifactPlan struct { - Name string `json:"name"` + ArtifactID int `json:"artifact_id"` + Name string `json:"name"` } type ArtifactOutputPlan struct { diff --git a/atc/public_plan_test.go b/atc/public_plan_test.go index 1174f57cf7d..c1c73bd8f49 100644 --- a/atc/public_plan_test.go +++ b/atc/public_plan_test.go @@ -1,10 +1,9 @@ package atc_test import ( + "github.com/concourse/concourse/atc" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" - - "github.com/concourse/concourse/atc" ) var _ = Describe("Plan", func() { @@ -285,7 +284,8 @@ var _ = Describe("Plan", func() { atc.Plan{ ID: "31", UserArtifact: &atc.UserArtifactPlan{ - Name: "some-name", + ArtifactID: 17, + Name: "some-name", }, }, @@ -502,7 +502,8 @@ var _ = Describe("Plan", func() { { "id": "31", "user_artifact": { - "name": "some-name" + "artifact_id": 17, + "name" : "some-name" } }, { diff --git a/atc/routes.go b/atc/routes.go index cd466728f5f..4d76cde253c 100644 --- a/atc/routes.go +++ b/atc/routes.go @@ -98,8 +98,9 @@ const ( DestroyTeam = "DestroyTeam" ListTeamBuilds = "ListTeamBuilds" - CreateArtifact = "CreateArtifact" - GetArtifact = "GetArtifact" + CreateArtifact = "CreateArtifact" + GetArtifact = "GetArtifact" + ListBuildArtifacts = "ListBuildArtifacts" ) const ( @@ -120,6 +121,7 @@ var Routes = rata.Routes([]rata.Route{ {Path: "/api/v1/builds/:build_id/resources", Method: "GET", Name: BuildResources}, {Path: "/api/v1/builds/:build_id/abort", Method: "PUT", Name: AbortBuild}, {Path: "/api/v1/builds/:build_id/preparation", Method: "GET", Name: GetBuildPreparation}, + {Path: "/api/v1/builds/:build_id/artifacts", Method: "GET", Name: ListBuildArtifacts}, {Path: "/api/v1/jobs", Method: "GET", Name: ListAllJobs}, {Path: "/api/v1/teams/:team_name/pipelines/:pipeline_name/jobs", Method: "GET", Name: ListJobs}, diff --git a/atc/worker/container_spec.go b/atc/worker/container_spec.go index 0db9b8daf68..c3770d9ad9d 100644 --- a/atc/worker/container_spec.go +++ b/atc/worker/container_spec.go @@ -66,7 +66,6 @@ type ImageSpec struct { ImageURL string ImageResource *ImageResource ImageArtifactSource ArtifactSource - ImageArtifactName ArtifactName Privileged bool } diff --git a/atc/worker/image/image_test.go b/atc/worker/image/image_test.go index 5ecb9a352ce..212ece3d072 100644 --- a/atc/worker/image/image_test.go +++ b/atc/worker/image/image_test.go @@ -16,7 +16,6 @@ import ( "github.com/concourse/concourse/atc/worker/image" "github.com/concourse/concourse/atc/worker/image/imagefakes" "github.com/concourse/concourse/atc/worker/workerfakes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -151,7 +150,6 @@ var _ = Describe("Image", func() { fakeVolumeClient, worker.ImageSpec{ ImageArtifactSource: fakeImageArtifactSource, - ImageArtifactName: "some-image-artifact-name", Privileged: true, }, 42, diff --git a/atc/worker/volume.go b/atc/worker/volume.go index eead574b1c1..e5429651fe7 100644 --- a/atc/worker/volume.go +++ b/atc/worker/volume.go @@ -26,7 +26,7 @@ type Volume interface { InitializeResourceCache(db.UsedResourceCache) error InitializeTaskCache(lager.Logger, int, string, string, bool) error - InitializeArtifact(string, string) (db.WorkerArtifact, error) + InitializeArtifact(name string, buildID int) (db.WorkerArtifact, error) CreateChildForContainer(db.CreatingContainer, string) (db.CreatingVolume, error) @@ -113,8 +113,8 @@ func (v *volume) InitializeResourceCache(urc db.UsedResourceCache) error { return v.dbVolume.InitializeResourceCache(urc) } -func (v *volume) InitializeArtifact(path string, checksum string) (db.WorkerArtifact, error) { - return v.dbVolume.InitializeArtifact(path, checksum) +func (v *volume) InitializeArtifact(name string, buildID int) (db.WorkerArtifact, error) { + return v.dbVolume.InitializeArtifact(name, buildID) } func (v *volume) InitializeTaskCache( diff --git a/atc/worker/workerfakes/fake_volume.go b/atc/worker/workerfakes/fake_volume.go index e86b3118112..6d9ed0b3dd2 100644 --- a/atc/worker/workerfakes/fake_volume.go +++ b/atc/worker/workerfakes/fake_volume.go @@ -56,11 +56,11 @@ type FakeVolume struct { handleReturnsOnCall map[int]struct { result1 string } - InitializeArtifactStub func(string, string) (db.WorkerArtifact, error) + InitializeArtifactStub func(string, int) (db.WorkerArtifact, error) initializeArtifactMutex sync.RWMutex initializeArtifactArgsForCall []struct { arg1 string - arg2 string + arg2 int } initializeArtifactReturns struct { result1 db.WorkerArtifact @@ -400,12 +400,12 @@ func (fake *FakeVolume) HandleReturnsOnCall(i int, result1 string) { }{result1} } -func (fake *FakeVolume) InitializeArtifact(arg1 string, arg2 string) (db.WorkerArtifact, error) { +func (fake *FakeVolume) InitializeArtifact(arg1 string, arg2 int) (db.WorkerArtifact, error) { fake.initializeArtifactMutex.Lock() ret, specificReturn := fake.initializeArtifactReturnsOnCall[len(fake.initializeArtifactArgsForCall)] fake.initializeArtifactArgsForCall = append(fake.initializeArtifactArgsForCall, struct { arg1 string - arg2 string + arg2 int }{arg1, arg2}) fake.recordInvocation("InitializeArtifact", []interface{}{arg1, arg2}) fake.initializeArtifactMutex.Unlock() @@ -425,13 +425,13 @@ func (fake *FakeVolume) InitializeArtifactCallCount() int { return len(fake.initializeArtifactArgsForCall) } -func (fake *FakeVolume) InitializeArtifactCalls(stub func(string, string) (db.WorkerArtifact, error)) { +func (fake *FakeVolume) InitializeArtifactCalls(stub func(string, int) (db.WorkerArtifact, error)) { fake.initializeArtifactMutex.Lock() defer fake.initializeArtifactMutex.Unlock() fake.InitializeArtifactStub = stub } -func (fake *FakeVolume) InitializeArtifactArgsForCall(i int) (string, string) { +func (fake *FakeVolume) InitializeArtifactArgsForCall(i int) (string, int) { fake.initializeArtifactMutex.RLock() defer fake.initializeArtifactMutex.RUnlock() argsForCall := fake.initializeArtifactArgsForCall[i] diff --git a/atc/worker_artifact.go b/atc/worker_artifact.go index 9e462b3dece..30458f951c4 100644 --- a/atc/worker_artifact.go +++ b/atc/worker_artifact.go @@ -2,7 +2,7 @@ package atc type WorkerArtifact struct { ID int `json:"id"` - Path string `json:"path"` - Checksum string `json:"checksum"` + Name string `json:"name"` + BuildID int `json:"build_id"` CreatedAt int64 `json:"created_at"` } diff --git a/atc/wrappa/api_auth_wrappa.go b/atc/wrappa/api_auth_wrappa.go index 9be05b2e6d3..be984eea57a 100644 --- a/atc/wrappa/api_auth_wrappa.go +++ b/atc/wrappa/api_auth_wrappa.go @@ -56,7 +56,8 @@ func (wrappa *APIAuthWrappa) Wrap(handlers rata.Handlers) rata.Handlers { // pipeline and job are public or authorized case atc.GetBuildPreparation, - atc.BuildEvents: + atc.BuildEvents, + atc.ListBuildArtifacts: newHandler = wrappa.checkBuildReadAccessHandlerFactory.CheckIfPrivateJobHandler(handler, rejector) // resource belongs to authorized team diff --git a/atc/wrappa/api_auth_wrappa_test.go b/atc/wrappa/api_auth_wrappa_test.go index a8e2014b9d4..4d085fe4422 100644 --- a/atc/wrappa/api_auth_wrappa_test.go +++ b/atc/wrappa/api_auth_wrappa_test.go @@ -158,6 +158,7 @@ var _ = Describe("APIAuthWrappa", func() { // authorized or public pipeline and public job atc.BuildEvents: checksIfPrivateJob(inputHandlers[atc.BuildEvents]), + atc.ListBuildArtifacts: checksIfPrivateJob(inputHandlers[atc.ListBuildArtifacts]), atc.GetBuildPreparation: checksIfPrivateJob(inputHandlers[atc.GetBuildPreparation]), // resource belongs to authorized team diff --git a/fly/commands/execute.go b/fly/commands/execute.go index a498056c058..2de7867d2c7 100644 --- a/fly/commands/execute.go +++ b/fly/commands/execute.go @@ -46,32 +46,22 @@ func (command *ExecuteCommand) Execute(args []string) error { return err } - includeIgnored := command.IncludeIgnored - - taskTemplate := templatehelpers.NewYamlTemplateWithParams(command.TaskConfig, command.VarsFrom, command.Var, command.YAMLVar) - taskTemplateEvaluated, err := taskTemplate.Evaluate(false, false) - if err != nil { - return err - } - - taskConfig, err := config.OverrideTaskParams(taskTemplateEvaluated, args) + taskConfig, err := command.CreateTaskConfig(args) if err != nil { return err } - client := target.Client() - - fact := atc.NewPlanFactory(time.Now().Unix()) + planFactory := atc.NewPlanFactory(time.Now().Unix()) - inputMappings := executehelpers.DetermineInputMappings(command.InputMappings) - inputs, imageResource, err := executehelpers.DetermineInputs( - fact, + inputs, inputMappings, imageResource, err := executehelpers.DetermineInputs( + planFactory, target.Team(), taskConfig.Inputs, command.Inputs, - inputMappings, + command.InputMappings, command.Image, command.InputsFrom, + command.IncludeIgnored, ) if err != nil { return err @@ -82,7 +72,7 @@ func (command *ExecuteCommand) Execute(args []string) error { } outputs, err := executehelpers.DetermineOutputs( - fact, + planFactory, taskConfig.Outputs, command.Outputs, ) @@ -91,7 +81,7 @@ func (command *ExecuteCommand) Execute(args []string) error { } plan, err := executehelpers.CreateBuildPlan( - fact, + planFactory, target, command.Privileged, inputs, @@ -105,6 +95,7 @@ func (command *ExecuteCommand) Execute(args []string) error { return err } + client := target.Client() clientURL, err := url.Parse(client.URL()) if err != nil { return err @@ -144,31 +135,7 @@ func (command *ExecuteCommand) Execute(args []string) error { signal.Notify(terminate, syscall.SIGINT, syscall.SIGTERM) - inputChan := make(chan interface{}) - go func() { - for _, i := range inputs { - if i.Path != "" { - executehelpers.Upload(client, build.ID, i, includeIgnored) - } - } - close(inputChan) - }() - - var outputChans []chan (interface{}) - if len(outputs) > 0 { - for i, output := range outputs { - outputChans = append(outputChans, make(chan interface{}, 1)) - go func(o executehelpers.Output, outputChan chan<- interface{}) { - if o.Path != "" { - executehelpers.Download(client, build.ID, o) - } - - close(outputChan) - }(output, outputChans[i]) - } - } - - eventSource, err := client.BuildEvents(fmt.Sprintf("%d", build.ID)) + eventSource, err := client.BuildEvents(strconv.Itoa(build.ID)) if err != nil { return err } @@ -178,11 +145,23 @@ func (command *ExecuteCommand) Execute(args []string) error { exitCode := eventstream.Render(os.Stdout, eventSource, renderOptions) eventSource.Close() - <-inputChan + artifactList, err := client.ListBuildArtifacts(strconv.Itoa(build.ID)) + if err != nil { + return err + } + + artifacts := map[string]atc.WorkerArtifact{} + + for _, artifact := range artifactList { + artifacts[artifact.Name] = artifact + } - if len(outputs) > 0 { - for _, outputChan := range outputChans { - <-outputChan + for _, output := range outputs { + if artifact, ok := artifacts[output.Name]; ok { + err = executehelpers.Download(target.Team(), artifact.ID, output.Path) + if err != nil { + return err + } } } @@ -191,6 +170,23 @@ func (command *ExecuteCommand) Execute(args []string) error { return nil } +func (command *ExecuteCommand) CreateTaskConfig(args []string) (atc.TaskConfig, error) { + + taskTemplate := templatehelpers.NewYamlTemplateWithParams( + command.TaskConfig, + command.VarsFrom, + command.Var, + command.YAMLVar, + ) + + taskTemplateEvaluated, err := taskTemplate.Evaluate(false, false) + if err != nil { + return atc.TaskConfig{}, err + } + + return config.OverrideTaskParams(taskTemplateEvaluated, args) +} + func abortOnSignal( client concourse.Client, terminate <-chan os.Signal, diff --git a/fly/commands/internal/executehelpers/downloads.go b/fly/commands/internal/executehelpers/downloads.go index 82ca32ddd36..f41d1ad9428 100644 --- a/fly/commands/internal/executehelpers/downloads.go +++ b/fly/commands/internal/executehelpers/downloads.go @@ -1,30 +1,17 @@ package executehelpers import ( - "fmt" - - "github.com/concourse/concourse/fly/ui" "github.com/concourse/concourse/go-concourse/concourse" "github.com/concourse/go-archive/tgzfs" ) -func Download(client concourse.Client, buildID int, output Output) { - out, found, err := client.ReadOutputFromBuildPlan(buildID, output.Plan.ID) +func Download(team concourse.Team, artifactID int, path string) error { + out, err := team.GetArtifact(artifactID) if err != nil { - fmt.Fprintf(ui.Stderr, "failed to download output '%s': %s", output.Name, err) - return - } - - if !found { - fmt.Fprintf(ui.Stderr, "build disappeared while downloading '%s'", output.Name) - return + return err } defer out.Close() - err = tgzfs.Extract(out, output.Path) - if err != nil { - fmt.Fprintf(ui.Stderr, "failed to extract output '%s': %s", output.Name, err) - return - } + return tgzfs.Extract(out, path) } diff --git a/fly/commands/internal/executehelpers/inputs.go b/fly/commands/internal/executehelpers/inputs.go index 9a557398f41..285df009441 100644 --- a/fly/commands/internal/executehelpers/inputs.go +++ b/fly/commands/internal/executehelpers/inputs.go @@ -23,24 +23,28 @@ func DetermineInputs( team concourse.Team, taskInputs []atc.TaskInputConfig, localInputMappings []flaghelpers.InputPairFlag, - jobInputMappings map[string]string, + userInputMappings []flaghelpers.VariablePairFlag, jobInputImage string, inputsFrom flaghelpers.JobFlag, -) ([]Input, *atc.ImageResource, error) { + includeIgnored bool, +) ([]Input, map[string]string, *atc.ImageResource, error) { + + inputMappings := ConvertInputMappings(userInputMappings) + err := CheckForUnknownInputMappings(localInputMappings, taskInputs) if err != nil { - return nil, nil, err + return nil, nil, nil, err } err = CheckForInputType(localInputMappings) if err != nil { - return nil, nil, err + return nil, nil, nil, err } if len(localInputMappings) == 0 && inputsFrom.PipelineName == "" && inputsFrom.JobName == "" { wd, err := os.Getwd() if err != nil { - return nil, nil, err + return nil, nil, nil, err } localInputMappings = append(localInputMappings, flaghelpers.InputPairFlag{ @@ -49,14 +53,14 @@ func DetermineInputs( }) } - inputsFromLocal, err := GenerateLocalInputs(fact, localInputMappings) + inputsFromLocal, err := GenerateLocalInputs(fact, team, localInputMappings, includeIgnored) if err != nil { - return nil, nil, err + return nil, nil, nil, err } inputsFromJob, imageResourceFromJob, err := FetchInputsFromJob(fact, team, inputsFrom, jobInputImage) if err != nil { - return nil, nil, err + return nil, nil, nil, err } inputs := []Input{} @@ -65,7 +69,7 @@ func DetermineInputs( if !found { jobInputName := taskInput.Name - if name, ok := jobInputMappings[taskInput.Name]; ok { + if name, ok := inputMappings[taskInput.Name]; ok { jobInputName = name } @@ -74,7 +78,7 @@ func DetermineInputs( if taskInput.Optional { continue } else { - return nil, nil, fmt.Errorf("missing required input `%s`", taskInput.Name) + return nil, nil, nil, fmt.Errorf("missing required input `%s`", taskInput.Name) } } } @@ -82,10 +86,10 @@ func DetermineInputs( inputs = append(inputs, input) } - return inputs, imageResourceFromJob, nil + return inputs, inputMappings, imageResourceFromJob, nil } -func DetermineInputMappings(variables []flaghelpers.VariablePairFlag) map[string]string { +func ConvertInputMappings(variables []flaghelpers.VariablePairFlag) map[string]string { inputMappings := map[string]string{} for _, flag := range variables { inputMappings[flag.Name] = flag.Value @@ -127,10 +131,20 @@ func TaskInputsContainsName(inputs []atc.TaskInputConfig, name string) bool { return false } -func GenerateLocalInputs(fact atc.PlanFactory, inputMappings []flaghelpers.InputPairFlag) (map[string]Input, error) { +func GenerateLocalInputs( + fact atc.PlanFactory, + team concourse.Team, + inputMappings []flaghelpers.InputPairFlag, + includeIgnored bool) (map[string]Input, error) { + kvMap := map[string]Input{} for _, i := range inputMappings { + artifact, err := Upload(team, i.Path, includeIgnored) + if err != nil { + return nil, err + } + inputName := i.Name absPath := i.Path @@ -138,7 +152,8 @@ func GenerateLocalInputs(fact atc.PlanFactory, inputMappings []flaghelpers.Input Name: inputName, Path: absPath, Plan: fact.NewPlan(atc.UserArtifactPlan{ - Name: inputName, + ArtifactID: artifact.ID, + Name: inputName, }), } } diff --git a/fly/commands/internal/executehelpers/uploads.go b/fly/commands/internal/executehelpers/uploads.go index cd57c90bc92..d82c73f63ae 100644 --- a/fly/commands/internal/executehelpers/uploads.go +++ b/fly/commands/internal/executehelpers/uploads.go @@ -3,45 +3,45 @@ package executehelpers import ( "bufio" "bytes" - "fmt" + "errors" "io" "os/exec" - "github.com/concourse/concourse/fly/ui" + "github.com/concourse/concourse/atc" "github.com/concourse/concourse/go-concourse/concourse" "github.com/concourse/go-archive/tgzfs" ) -func Upload(client concourse.Client, buildID int, input Input, includeIgnored bool) { - path := input.Path +func Upload(team concourse.Team, path string, includeIgnored bool) (atc.WorkerArtifact, error) { + if path == "" { + return atc.WorkerArtifact{}, errors.New("Invalid path") + } + + files := getFiles(path, includeIgnored) + + archiveStream, archiveWriter := io.Pipe() + + go func() { + archiveWriter.CloseWithError(tgzfs.Compress(archiveWriter, path, files...)) + }() + + return team.CreateArtifact(archiveStream) +} +func getFiles(dir string, includeIgnored bool) []string { var files []string var err error if includeIgnored { files = []string{"."} } else { - files, err = getGitFiles(path) + files, err = getGitFiles(dir) if err != nil { files = []string{"."} } } - archiveStream, archiveWriter := io.Pipe() - - go func() { - archiveWriter.CloseWithError(tgzfs.Compress(archiveWriter, path, files...)) - }() - - found, err := client.SendInputToBuildPlan(buildID, input.Plan.ID, archiveStream) - if err != nil { - fmt.Fprintf(ui.Stderr, "failed to upload input '%s': %s", input.Name, err) - return - } - - if !found { - fmt.Fprintf(ui.Stderr, "build disappeared while uploading '%s'", input.Name) - } + return files } func getGitFiles(dir string) ([]string, error) { diff --git a/fly/integration/multiple_inputs_test.go b/fly/integration/execute_multiple_inputs_test.go similarity index 92% rename from fly/integration/multiple_inputs_test.go rename to fly/integration/execute_multiple_inputs_test.go index c16ea51946c..1a91fb891a2 100644 --- a/fly/integration/multiple_inputs_test.go +++ b/fly/integration/execute_multiple_inputs_test.go @@ -9,19 +9,17 @@ import ( "net/http" "os/exec" "path/filepath" - "regexp" "strings" "time" + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/event" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" "github.com/onsi/gomega/ghttp" "github.com/vito/go-sse/sse" - - "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/event" ) var _ = Describe("Fly CLI", func() { @@ -34,6 +32,10 @@ var _ = Describe("Fly CLI", func() { var uploadingTwo chan struct{} var expectedPlan atc.Plan + var workerArtifact = atc.WorkerArtifact{ + ID: 125, + Name: "some-dir", + } BeforeEach(func() { var err error @@ -86,10 +88,12 @@ run: expectedPlan = planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ planFactory.NewPlan(atc.UserArtifactPlan{ - Name: "some-input", + ArtifactID: 125, + Name: "some-input", }), planFactory.NewPlan(atc.UserArtifactPlan{ - Name: "some-other-input", + ArtifactID: 125, + Name: "some-other-input", }), }), planFactory.NewPlan(atc.TaskPlan{ @@ -124,6 +128,33 @@ run: uploading = make(chan struct{}) uploadingTwo = make(chan struct{}) + atcServer.RouteToHandler("POST", "/api/v1/teams/main/artifacts", + ghttp.CombineHandlers( + func(w http.ResponseWriter, req *http.Request) { + gr, err := gzip.NewReader(req.Body) + Expect(err).NotTo(HaveOccurred()) + + tr := tar.NewReader(gr) + + hdr, err := tr.Next() + Expect(err).NotTo(HaveOccurred()) + + Expect(hdr.Name).To(Equal("./")) + + hdr, err = tr.Next() + Expect(err).NotTo(HaveOccurred()) + + if strings.HasSuffix(hdr.Name, "task.yml") { + close(uploading) + } else if strings.HasSuffix(hdr.Name, "s3-asset-file") { + close(uploadingTwo) + } + + Expect(hdr.Name).To(MatchRegexp("(./)?(task.yml|s3-asset-file)$")) + }, + ghttp.RespondWith(201, `{"id":125}`), + ), + ) atcServer.RouteToHandler("POST", "/api/v1/teams/main/builds", ghttp.CombineHandlers( ghttp.VerifyRequest("POST", "/api/v1/teams/main/builds"), @@ -182,33 +213,10 @@ run: }, ), ) - atcServer.RouteToHandler("PUT", regexp.MustCompile(`/api/v1/builds/128/plan/.*/input`), - ghttp.CombineHandlers( - func(w http.ResponseWriter, req *http.Request) { - gr, err := gzip.NewReader(req.Body) - Expect(err).NotTo(HaveOccurred()) - - tr := tar.NewReader(gr) - - hdr, err := tr.Next() - Expect(err).NotTo(HaveOccurred()) - - Expect(hdr.Name).To(Equal("./")) - - hdr, err = tr.Next() - Expect(err).NotTo(HaveOccurred()) - - if strings.HasSuffix(hdr.Name, "task.yml") { - close(uploading) - } else if strings.HasSuffix(hdr.Name, "s3-asset-file") { - close(uploadingTwo) - } - - Expect(hdr.Name).To(MatchRegexp("(./)?(task.yml|s3-asset-file)$")) - }, - ghttp.RespondWith(200, ""), - ), + atcServer.RouteToHandler("GET", "/api/v1/builds/128/artifacts", + ghttp.RespondWithJSONEncoded(200, []atc.WorkerArtifact{workerArtifact}), ) + }) It("flies with multiple passengers", func() { diff --git a/fly/integration/overriding_inputs_test.go b/fly/integration/execute_overriding_inputs_test.go similarity index 94% rename from fly/integration/overriding_inputs_test.go rename to fly/integration/execute_overriding_inputs_test.go index d6b32a1a97b..91205112505 100644 --- a/fly/integration/overriding_inputs_test.go +++ b/fly/integration/execute_overriding_inputs_test.go @@ -9,18 +9,16 @@ import ( "net/http" "os/exec" "path/filepath" - "regexp" "time" + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/event" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" "github.com/onsi/gomega/ghttp" "github.com/vito/go-sse/sse" - - "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/event" ) var _ = Describe("Fly CLI", func() { @@ -32,6 +30,10 @@ var _ = Describe("Fly CLI", func() { var uploading chan struct{} var expectedPlan atc.Plan + var workerArtifact = atc.WorkerArtifact{ + ID: 125, + Name: "some-dir", + } BeforeEach(func() { var err error @@ -84,7 +86,8 @@ run: expectedPlan = planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ planFactory.NewPlan(atc.UserArtifactPlan{ - Name: "some-input", + ArtifactID: 125, + Name: "some-input", }), planFactory.NewPlan(atc.GetPlan{ Name: "some-other-input", @@ -156,6 +159,29 @@ run: ghttp.RespondWithJSONEncoded(http.StatusOK, nil), ), ) + atcServer.RouteToHandler("POST", "/api/v1/teams/main/artifacts", + ghttp.CombineHandlers( + func(w http.ResponseWriter, req *http.Request) { + close(uploading) + + gr, err := gzip.NewReader(req.Body) + Expect(err).NotTo(HaveOccurred()) + + tr := tar.NewReader(gr) + + hdr, err := tr.Next() + Expect(err).NotTo(HaveOccurred()) + + Expect(hdr.Name).To(Equal("./")) + + hdr, err = tr.Next() + Expect(err).NotTo(HaveOccurred()) + + Expect(hdr.Name).To(MatchRegexp("(./)?task.yml$")) + }, + ghttp.RespondWithJSONEncoded(201, workerArtifact), + ), + ) atcServer.RouteToHandler("POST", "/api/v1/teams/main/pipelines/some-pipeline/builds", ghttp.CombineHandlers( ghttp.VerifyRequest("POST", "/api/v1/teams/main/pipelines/some-pipeline/builds"), @@ -214,29 +240,10 @@ run: }, ), ) - atcServer.RouteToHandler("PUT", regexp.MustCompile(`/api/v1/builds/128/plan/.*/input`), - ghttp.CombineHandlers( - func(w http.ResponseWriter, req *http.Request) { - close(uploading) - - gr, err := gzip.NewReader(req.Body) - Expect(err).NotTo(HaveOccurred()) - - tr := tar.NewReader(gr) - - hdr, err := tr.Next() - Expect(err).NotTo(HaveOccurred()) - - Expect(hdr.Name).To(Equal("./")) - - hdr, err = tr.Next() - Expect(err).NotTo(HaveOccurred()) - - Expect(hdr.Name).To(MatchRegexp("(./)?task.yml$")) - }, - ghttp.RespondWith(200, ""), - ), + atcServer.RouteToHandler("GET", "/api/v1/builds/128/artifacts", + ghttp.RespondWithJSONEncoded(200, []atc.WorkerArtifact{workerArtifact}), ) + }) It("can base inputs on a job in the pipeline", func() { diff --git a/fly/integration/execute_test.go b/fly/integration/execute_test.go index 8916810c797..30fab35f1b7 100644 --- a/fly/integration/execute_test.go +++ b/fly/integration/execute_test.go @@ -12,21 +12,19 @@ import ( "os/exec" "path" "path/filepath" - "regexp" "runtime" "strings" "syscall" "time" + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/event" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" "github.com/onsi/gomega/ghttp" "github.com/vito/go-sse/sse" - - "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/event" ) var _ = Describe("Fly CLI", func() { @@ -39,6 +37,10 @@ var _ = Describe("Fly CLI", func() { var uploadingBits <-chan struct{} var expectedPlan atc.Plan + var workerArtifact = atc.WorkerArtifact{ + ID: 125, + Name: "some-dir", + } BeforeEach(func() { var err error @@ -86,7 +88,8 @@ run: expectedPlan = planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ planFactory.NewPlan(atc.UserArtifactPlan{ - Name: filepath.Base(buildDir), + ArtifactID: 125, + Name: filepath.Base(buildDir), }), }), planFactory.NewPlan(atc.TaskPlan{ @@ -124,6 +127,29 @@ run: uploading := make(chan struct{}) uploadingBits = uploading + atcServer.RouteToHandler("POST", "/api/v1/teams/main/artifacts", + ghttp.CombineHandlers( + func(w http.ResponseWriter, req *http.Request) { + close(uploading) + + gr, err := gzip.NewReader(req.Body) + Expect(err).NotTo(HaveOccurred()) + + tr := tar.NewReader(gr) + + hdr, err := tr.Next() + Expect(err).NotTo(HaveOccurred()) + + Expect(hdr.Name).To(Equal("./")) + + hdr, err = tr.Next() + Expect(err).NotTo(HaveOccurred()) + + Expect(hdr.Name).To(MatchRegexp("(./)?task.yml$")) + }, + ghttp.RespondWith(201, `{"id":125}`), + ), + ) atcServer.RouteToHandler("POST", "/api/v1/teams/main/builds", ghttp.CombineHandlers( ghttp.VerifyRequest("POST", "/api/v1/teams/main/builds"), @@ -182,29 +208,10 @@ run: }, ), ) - atcServer.RouteToHandler("PUT", regexp.MustCompile(`/api/v1/builds/128/plan/.*/input`), - ghttp.CombineHandlers( - func(w http.ResponseWriter, req *http.Request) { - close(uploading) - - gr, err := gzip.NewReader(req.Body) - Expect(err).NotTo(HaveOccurred()) - - tr := tar.NewReader(gr) - - hdr, err := tr.Next() - Expect(err).NotTo(HaveOccurred()) - - Expect(hdr.Name).To(Equal("./")) - - hdr, err = tr.Next() - Expect(err).NotTo(HaveOccurred()) - - Expect(hdr.Name).To(MatchRegexp("(./)?task.yml$")) - }, - ghttp.RespondWith(200, ""), - ), + atcServer.RouteToHandler("GET", "/api/v1/builds/128/artifacts", + ghttp.RespondWithJSONEncoded(200, []atc.WorkerArtifact{workerArtifact}), ) + }) It("creates a build, streams output, uploads the bits, and polls until completion", func() { @@ -314,7 +321,7 @@ run: {} It("by default apply .gitignore", func() { uploading := make(chan struct{}) uploadingBits = uploading - atcServer.RouteToHandler("PUT", regexp.MustCompile(`/api/v1/builds/128/plan/.*/input`), + atcServer.RouteToHandler("POST", "/api/v1/teams/main/artifacts", ghttp.CombineHandlers( func(w http.ResponseWriter, req *http.Request) { close(uploading) @@ -338,7 +345,7 @@ run: {} Expect(matchFound).To(Equal(false)) }, - ghttp.RespondWith(200, ""), + ghttp.RespondWith(201, `{"id":125}`), ), ) @@ -364,7 +371,7 @@ run: {} It("uploading with everything", func() { uploading := make(chan struct{}) uploadingBits = uploading - atcServer.RouteToHandler("PUT", regexp.MustCompile(`/api/v1/builds/128/plan/.*/input`), + atcServer.RouteToHandler("POST", "/api/v1/teams/main/artifacts", ghttp.CombineHandlers( func(w http.ResponseWriter, req *http.Request) { close(uploading) @@ -388,7 +395,7 @@ run: {} Expect(matchFound).To(Equal(true)) }, - ghttp.RespondWith(200, ""), + ghttp.RespondWith(201, `{"id":125}`), ), ) flyCmd := exec.Command(flyPath, "-t", targetName, "e", "-c", taskConfigPath, "--include-ignored") diff --git a/fly/integration/execute_with_outputs_test.go b/fly/integration/execute_with_outputs_test.go index b5f81745475..905e80d27a4 100644 --- a/fly/integration/execute_with_outputs_test.go +++ b/fly/integration/execute_with_outputs_test.go @@ -10,18 +10,16 @@ import ( "os" "os/exec" "path/filepath" - "regexp" "time" + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/event" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gbytes" "github.com/onsi/gomega/gexec" "github.com/onsi/gomega/ghttp" "github.com/vito/go-sse/sse" - - "github.com/concourse/concourse/atc" - "github.com/concourse/concourse/atc/event" ) var _ = Describe("Fly CLI", func() { @@ -34,6 +32,10 @@ var _ = Describe("Fly CLI", func() { var outputDir string var expectedPlan atc.Plan + var workerArtifact = atc.WorkerArtifact{ + ID: 125, + Name: "some-dir", + } BeforeEach(func() { var err error @@ -91,7 +93,8 @@ run: Step: planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ planFactory.NewPlan(atc.UserArtifactPlan{ - Name: filepath.Base(buildDir), + Name: filepath.Base(buildDir), + ArtifactID: 125, }), }), planFactory.NewPlan(atc.TaskPlan{ @@ -139,6 +142,27 @@ run: }) JustBeforeEach(func() { + atcServer.RouteToHandler("POST", "/api/v1/teams/main/artifacts", + ghttp.CombineHandlers( + func(w http.ResponseWriter, req *http.Request) { + gr, err := gzip.NewReader(req.Body) + Expect(err).NotTo(HaveOccurred()) + + tr := tar.NewReader(gr) + + hdr, err := tr.Next() + Expect(err).NotTo(HaveOccurred()) + + Expect(hdr.Name).To(Equal("./")) + + hdr, err = tr.Next() + Expect(err).NotTo(HaveOccurred()) + + Expect(hdr.Name).To(MatchRegexp("(./)?task.yml$")) + }, + ghttp.RespondWithJSONEncoded(201, workerArtifact), + ), + ) atcServer.RouteToHandler("POST", "/api/v1/teams/main/builds", ghttp.CombineHandlers( ghttp.VerifyRequest("POST", "/api/v1/teams/main/builds"), @@ -197,50 +221,11 @@ run: }, ), ) - atcServer.RouteToHandler("PUT", regexp.MustCompile(`/api/v1/builds/128/plan/.*/input`), - func(w http.ResponseWriter, req *http.Request) { - gr, err := gzip.NewReader(req.Body) - Expect(err).NotTo(HaveOccurred()) - - tr := tar.NewReader(gr) - - hdr, err := tr.Next() - Expect(err).NotTo(HaveOccurred()) - - Expect(hdr.Name).To(Equal("./")) - - hdr, err = tr.Next() - Expect(err).NotTo(HaveOccurred()) - - Expect(hdr.Name).To(MatchRegexp("(./)?task.yml$")) - - w.WriteHeader(http.StatusNoContent) - }, + atcServer.RouteToHandler("GET", "/api/v1/builds/128/artifacts", + ghttp.RespondWithJSONEncoded(200, []atc.WorkerArtifact{workerArtifact}), ) - atcServer.RouteToHandler("GET", regexp.MustCompile(`/api/v1/builds/128/plan/.*/output`), - func(w http.ResponseWriter, req *http.Request) { - gw := gzip.NewWriter(w) - tw := tar.NewWriter(gw) - - tarContents := []byte("tar-contents") - - err := tw.WriteHeader(&tar.Header{ - Name: "some-file", - Mode: 0644, - Size: int64(len(tarContents)), - }) - Expect(err).NotTo(HaveOccurred()) - - _, err = tw.Write(tarContents) - Expect(err).NotTo(HaveOccurred()) - - err = tw.Close() - Expect(err).NotTo(HaveOccurred()) - err = gw.Close() - Expect(err).NotTo(HaveOccurred()) - }, - ) + atcServer.RouteToHandler("GET", "/api/v1/teams/main/artifacts/125", tarHandler) }) Context("when running with --output", func() { @@ -288,3 +273,27 @@ run: }) }) }) + +func tarHandler(w http.ResponseWriter, req *http.Request) { + gw := gzip.NewWriter(w) + tw := tar.NewWriter(gw) + + tarContents := []byte("tar-contents") + + err := tw.WriteHeader(&tar.Header{ + Name: "some-file", + Mode: 0644, + Size: int64(len(tarContents)), + }) + Expect(err).NotTo(HaveOccurred()) + + _, err = tw.Write(tarContents) + Expect(err).NotTo(HaveOccurred()) + + err = tw.Close() + Expect(err).NotTo(HaveOccurred()) + + err = gw.Close() + Expect(err).NotTo(HaveOccurred()) + +} diff --git a/go-concourse/concourse/build_plan.go b/go-concourse/concourse/build_plan.go index 92fb2ff5336..0450b48f960 100644 --- a/go-concourse/concourse/build_plan.go +++ b/go-concourse/concourse/build_plan.go @@ -1,8 +1,6 @@ package concourse import ( - "io" - "net/http" "strconv" "github.com/concourse/concourse/atc" @@ -32,50 +30,3 @@ func (client *client) BuildPlan(buildID int) (atc.PublicBuildPlan, bool, error) return buildPlan, false, err } } - -func (client *client) SendInputToBuildPlan(buildID int, planID atc.PlanID, src io.Reader) (bool, error) { - params := rata.Params{ - "build_id": strconv.Itoa(buildID), - "plan_id": string(planID), - } - - response := internal.Response{} - err := client.connection.Send(internal.Request{ - Header: http.Header{"Content-Type": {"application/octet-stream"}}, - RequestName: atc.SendInputToBuildPlan, - Params: params, - Body: src, - }, &response) - - switch err.(type) { - case nil: - return true, nil - case internal.ResourceNotFoundError: - return false, nil - default: - return false, err - } -} - -func (client *client) ReadOutputFromBuildPlan(buildID int, planID atc.PlanID) (io.ReadCloser, bool, error) { - params := rata.Params{ - "build_id": strconv.Itoa(buildID), - "plan_id": string(planID), - } - - response := internal.Response{} - err := client.connection.Send(internal.Request{ - RequestName: atc.ReadOutputFromBuildPlan, - Params: params, - ReturnResponseBody: true, - }, &response) - - switch err.(type) { - case nil: - return response.Result.(io.ReadCloser), true, nil - case internal.ResourceNotFoundError: - return nil, false, nil - default: - return nil, false, err - } -} diff --git a/go-concourse/concourse/build_plan_test.go b/go-concourse/concourse/build_plan_test.go index 37304975666..4d58d452b56 100644 --- a/go-concourse/concourse/build_plan_test.go +++ b/go-concourse/concourse/build_plan_test.go @@ -1,13 +1,10 @@ package concourse_test import ( - "bytes" "encoding/json" - "io/ioutil" "net/http" "github.com/concourse/concourse/atc" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/ghttp" @@ -17,10 +14,10 @@ var _ = Describe("ATC Handler Build Plans", func() { Describe("BuildPlan", func() { Context("when build exists and has a plan", func() { // var plan json.RawMessage = json.RawMessage(`{}`) - expectedPlanJson := json.RawMessage(`{"do":"stuff"}`) + expectedPlanJSON := json.RawMessage(`{"do":"stuff"}`) expectedBuildPlan := atc.PublicBuildPlan{ Schema: "exec.v2", - Plan: &expectedPlanJson, + Plan: &expectedPlanJSON, } expectedURL := "/api/v1/builds/1234/plan" @@ -60,82 +57,4 @@ var _ = Describe("ATC Handler Build Plans", func() { }) }) }) - - Describe("SendInputToBuildPlan", func() { - expectedURL := "/api/v1/builds/1234/plan/some-id/input" - - Context("when build exists and has a plan", func() { - BeforeEach(func() { - atcServer.AppendHandlers( - ghttp.CombineHandlers( - ghttp.VerifyRequest("PUT", expectedURL), - ghttp.VerifyBody([]byte("some-body-once-told-me")), - ghttp.RespondWith(http.StatusNoContent, nil), - ), - ) - }) - - It("returns true and no error", func() { - found, err := client.SendInputToBuildPlan(1234, "some-id", bytes.NewBufferString("some-body-once-told-me")) - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - }) - }) - - Context("when build does not exist or has no plan", func() { - BeforeEach(func() { - atcServer.AppendHandlers( - ghttp.CombineHandlers( - ghttp.VerifyRequest("PUT", expectedURL), - ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil), - ), - ) - }) - - It("returns false and no error", func() { - found, err := client.SendInputToBuildPlan(1234, "some-id", bytes.NewBufferString("some-body-once-told-me")) - Expect(err).ToNot(HaveOccurred()) - Expect(found).To(BeFalse()) - }) - }) - }) - - Describe("ReadOutputFromBuildPlan", func() { - expectedURL := "/api/v1/builds/1234/plan/some-id/output" - - Context("when build exists and has a plan", func() { - BeforeEach(func() { - atcServer.AppendHandlers( - ghttp.CombineHandlers( - ghttp.VerifyRequest("GET", expectedURL), - ghttp.RespondWith(http.StatusOK, "the-world-is-gonna-roll-me"), - ), - ) - }) - - It("returns true and no error", func() { - out, found, err := client.ReadOutputFromBuildPlan(1234, "some-id") - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - Expect(ioutil.ReadAll(out)).To(Equal([]byte("the-world-is-gonna-roll-me"))) - }) - }) - - Context("when build does not exist or has no plan", func() { - BeforeEach(func() { - atcServer.AppendHandlers( - ghttp.CombineHandlers( - ghttp.VerifyRequest("GET", expectedURL), - ghttp.RespondWithJSONEncoded(http.StatusNotFound, nil), - ), - ) - }) - - It("returns false and no error", func() { - _, found, err := client.ReadOutputFromBuildPlan(1234, "some-id") - Expect(err).ToNot(HaveOccurred()) - Expect(found).To(BeFalse()) - }) - }) - }) }) diff --git a/go-concourse/concourse/builds.go b/go-concourse/concourse/builds.go index cb01f5eca9c..d8207d67df5 100644 --- a/go-concourse/concourse/builds.go +++ b/go-concourse/concourse/builds.go @@ -172,3 +172,20 @@ func (team *team) Builds(page Page) ([]atc.Build, Pagination, error) { return nil, Pagination{}, err } } + +func (client *client) ListBuildArtifacts(buildID string) ([]atc.WorkerArtifact, error) { + params := rata.Params{ + "build_id": buildID, + } + + var artifacts []atc.WorkerArtifact + + err := client.connection.Send(internal.Request{ + RequestName: atc.ListBuildArtifacts, + Params: params, + }, &internal.Response{ + Result: &artifacts, + }) + + return artifacts, err +} diff --git a/go-concourse/concourse/client.go b/go-concourse/concourse/client.go index 6175c2162ff..f37cb70ad85 100644 --- a/go-concourse/concourse/client.go +++ b/go-concourse/concourse/client.go @@ -18,10 +18,9 @@ type Client interface { Build(buildID string) (atc.Build, bool, error) BuildEvents(buildID string) (Events, error) BuildResources(buildID int) (atc.BuildInputsOutputs, bool, error) + ListBuildArtifacts(buildID string) ([]atc.WorkerArtifact, error) AbortBuild(buildID string) error BuildPlan(buildID int) (atc.PublicBuildPlan, bool, error) - SendInputToBuildPlan(buildID int, planID atc.PlanID, src io.Reader) (bool, error) - ReadOutputFromBuildPlan(buildID int, planID atc.PlanID) (io.ReadCloser, bool, error) SaveWorker(atc.Worker, *time.Duration) (*atc.Worker, error) ListWorkers() ([]atc.Worker, error) PruneWorker(workerName string) error diff --git a/go-concourse/concourse/concoursefakes/fake_client.go b/go-concourse/concourse/concoursefakes/fake_client.go index 6496106635e..ad72843b361 100644 --- a/go-concourse/concourse/concoursefakes/fake_client.go +++ b/go-concourse/concourse/concoursefakes/fake_client.go @@ -145,6 +145,19 @@ type FakeClient struct { landWorkerReturnsOnCall map[int]struct { result1 error } + ListBuildArtifactsStub func(string) ([]atc.WorkerArtifact, error) + listBuildArtifactsMutex sync.RWMutex + listBuildArtifactsArgsForCall []struct { + arg1 string + } + listBuildArtifactsReturns struct { + result1 []atc.WorkerArtifact + result2 error + } + listBuildArtifactsReturnsOnCall map[int]struct { + result1 []atc.WorkerArtifact + result2 error + } ListPipelinesStub func() ([]atc.Pipeline, error) listPipelinesMutex sync.RWMutex listPipelinesArgsForCall []struct { @@ -192,22 +205,6 @@ type FakeClient struct { pruneWorkerReturnsOnCall map[int]struct { result1 error } - ReadOutputFromBuildPlanStub func(int, atc.PlanID) (io.ReadCloser, bool, error) - readOutputFromBuildPlanMutex sync.RWMutex - readOutputFromBuildPlanArgsForCall []struct { - arg1 int - arg2 atc.PlanID - } - readOutputFromBuildPlanReturns struct { - result1 io.ReadCloser - result2 bool - result3 error - } - readOutputFromBuildPlanReturnsOnCall map[int]struct { - result1 io.ReadCloser - result2 bool - result3 error - } SaveWorkerStub func(atc.Worker, *time.Duration) (*atc.Worker, error) saveWorkerMutex sync.RWMutex saveWorkerArgsForCall []struct { @@ -222,21 +219,6 @@ type FakeClient struct { result1 *atc.Worker result2 error } - SendInputToBuildPlanStub func(int, atc.PlanID, io.Reader) (bool, error) - sendInputToBuildPlanMutex sync.RWMutex - sendInputToBuildPlanArgsForCall []struct { - arg1 int - arg2 atc.PlanID - arg3 io.Reader - } - sendInputToBuildPlanReturns struct { - result1 bool - result2 error - } - sendInputToBuildPlanReturnsOnCall map[int]struct { - result1 bool - result2 error - } TeamStub func(string) concourse.Team teamMutex sync.RWMutex teamArgsForCall []struct { @@ -895,6 +877,69 @@ func (fake *FakeClient) LandWorkerReturnsOnCall(i int, result1 error) { }{result1} } +func (fake *FakeClient) ListBuildArtifacts(arg1 string) ([]atc.WorkerArtifact, error) { + fake.listBuildArtifactsMutex.Lock() + ret, specificReturn := fake.listBuildArtifactsReturnsOnCall[len(fake.listBuildArtifactsArgsForCall)] + fake.listBuildArtifactsArgsForCall = append(fake.listBuildArtifactsArgsForCall, struct { + arg1 string + }{arg1}) + fake.recordInvocation("ListBuildArtifacts", []interface{}{arg1}) + fake.listBuildArtifactsMutex.Unlock() + if fake.ListBuildArtifactsStub != nil { + return fake.ListBuildArtifactsStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.listBuildArtifactsReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeClient) ListBuildArtifactsCallCount() int { + fake.listBuildArtifactsMutex.RLock() + defer fake.listBuildArtifactsMutex.RUnlock() + return len(fake.listBuildArtifactsArgsForCall) +} + +func (fake *FakeClient) ListBuildArtifactsCalls(stub func(string) ([]atc.WorkerArtifact, error)) { + fake.listBuildArtifactsMutex.Lock() + defer fake.listBuildArtifactsMutex.Unlock() + fake.ListBuildArtifactsStub = stub +} + +func (fake *FakeClient) ListBuildArtifactsArgsForCall(i int) string { + fake.listBuildArtifactsMutex.RLock() + defer fake.listBuildArtifactsMutex.RUnlock() + argsForCall := fake.listBuildArtifactsArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeClient) ListBuildArtifactsReturns(result1 []atc.WorkerArtifact, result2 error) { + fake.listBuildArtifactsMutex.Lock() + defer fake.listBuildArtifactsMutex.Unlock() + fake.ListBuildArtifactsStub = nil + fake.listBuildArtifactsReturns = struct { + result1 []atc.WorkerArtifact + result2 error + }{result1, result2} +} + +func (fake *FakeClient) ListBuildArtifactsReturnsOnCall(i int, result1 []atc.WorkerArtifact, result2 error) { + fake.listBuildArtifactsMutex.Lock() + defer fake.listBuildArtifactsMutex.Unlock() + fake.ListBuildArtifactsStub = nil + if fake.listBuildArtifactsReturnsOnCall == nil { + fake.listBuildArtifactsReturnsOnCall = make(map[int]struct { + result1 []atc.WorkerArtifact + result2 error + }) + } + fake.listBuildArtifactsReturnsOnCall[i] = struct { + result1 []atc.WorkerArtifact + result2 error + }{result1, result2} +} + func (fake *FakeClient) ListPipelines() ([]atc.Pipeline, error) { fake.listPipelinesMutex.Lock() ret, specificReturn := fake.listPipelinesReturnsOnCall[len(fake.listPipelinesArgsForCall)] @@ -1120,73 +1165,6 @@ func (fake *FakeClient) PruneWorkerReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeClient) ReadOutputFromBuildPlan(arg1 int, arg2 atc.PlanID) (io.ReadCloser, bool, error) { - fake.readOutputFromBuildPlanMutex.Lock() - ret, specificReturn := fake.readOutputFromBuildPlanReturnsOnCall[len(fake.readOutputFromBuildPlanArgsForCall)] - fake.readOutputFromBuildPlanArgsForCall = append(fake.readOutputFromBuildPlanArgsForCall, struct { - arg1 int - arg2 atc.PlanID - }{arg1, arg2}) - fake.recordInvocation("ReadOutputFromBuildPlan", []interface{}{arg1, arg2}) - fake.readOutputFromBuildPlanMutex.Unlock() - if fake.ReadOutputFromBuildPlanStub != nil { - return fake.ReadOutputFromBuildPlanStub(arg1, arg2) - } - if specificReturn { - return ret.result1, ret.result2, ret.result3 - } - fakeReturns := fake.readOutputFromBuildPlanReturns - return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 -} - -func (fake *FakeClient) ReadOutputFromBuildPlanCallCount() int { - fake.readOutputFromBuildPlanMutex.RLock() - defer fake.readOutputFromBuildPlanMutex.RUnlock() - return len(fake.readOutputFromBuildPlanArgsForCall) -} - -func (fake *FakeClient) ReadOutputFromBuildPlanCalls(stub func(int, atc.PlanID) (io.ReadCloser, bool, error)) { - fake.readOutputFromBuildPlanMutex.Lock() - defer fake.readOutputFromBuildPlanMutex.Unlock() - fake.ReadOutputFromBuildPlanStub = stub -} - -func (fake *FakeClient) ReadOutputFromBuildPlanArgsForCall(i int) (int, atc.PlanID) { - fake.readOutputFromBuildPlanMutex.RLock() - defer fake.readOutputFromBuildPlanMutex.RUnlock() - argsForCall := fake.readOutputFromBuildPlanArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2 -} - -func (fake *FakeClient) ReadOutputFromBuildPlanReturns(result1 io.ReadCloser, result2 bool, result3 error) { - fake.readOutputFromBuildPlanMutex.Lock() - defer fake.readOutputFromBuildPlanMutex.Unlock() - fake.ReadOutputFromBuildPlanStub = nil - fake.readOutputFromBuildPlanReturns = struct { - result1 io.ReadCloser - result2 bool - result3 error - }{result1, result2, result3} -} - -func (fake *FakeClient) ReadOutputFromBuildPlanReturnsOnCall(i int, result1 io.ReadCloser, result2 bool, result3 error) { - fake.readOutputFromBuildPlanMutex.Lock() - defer fake.readOutputFromBuildPlanMutex.Unlock() - fake.ReadOutputFromBuildPlanStub = nil - if fake.readOutputFromBuildPlanReturnsOnCall == nil { - fake.readOutputFromBuildPlanReturnsOnCall = make(map[int]struct { - result1 io.ReadCloser - result2 bool - result3 error - }) - } - fake.readOutputFromBuildPlanReturnsOnCall[i] = struct { - result1 io.ReadCloser - result2 bool - result3 error - }{result1, result2, result3} -} - func (fake *FakeClient) SaveWorker(arg1 atc.Worker, arg2 *time.Duration) (*atc.Worker, error) { fake.saveWorkerMutex.Lock() ret, specificReturn := fake.saveWorkerReturnsOnCall[len(fake.saveWorkerArgsForCall)] @@ -1251,71 +1229,6 @@ func (fake *FakeClient) SaveWorkerReturnsOnCall(i int, result1 *atc.Worker, resu }{result1, result2} } -func (fake *FakeClient) SendInputToBuildPlan(arg1 int, arg2 atc.PlanID, arg3 io.Reader) (bool, error) { - fake.sendInputToBuildPlanMutex.Lock() - ret, specificReturn := fake.sendInputToBuildPlanReturnsOnCall[len(fake.sendInputToBuildPlanArgsForCall)] - fake.sendInputToBuildPlanArgsForCall = append(fake.sendInputToBuildPlanArgsForCall, struct { - arg1 int - arg2 atc.PlanID - arg3 io.Reader - }{arg1, arg2, arg3}) - fake.recordInvocation("SendInputToBuildPlan", []interface{}{arg1, arg2, arg3}) - fake.sendInputToBuildPlanMutex.Unlock() - if fake.SendInputToBuildPlanStub != nil { - return fake.SendInputToBuildPlanStub(arg1, arg2, arg3) - } - if specificReturn { - return ret.result1, ret.result2 - } - fakeReturns := fake.sendInputToBuildPlanReturns - return fakeReturns.result1, fakeReturns.result2 -} - -func (fake *FakeClient) SendInputToBuildPlanCallCount() int { - fake.sendInputToBuildPlanMutex.RLock() - defer fake.sendInputToBuildPlanMutex.RUnlock() - return len(fake.sendInputToBuildPlanArgsForCall) -} - -func (fake *FakeClient) SendInputToBuildPlanCalls(stub func(int, atc.PlanID, io.Reader) (bool, error)) { - fake.sendInputToBuildPlanMutex.Lock() - defer fake.sendInputToBuildPlanMutex.Unlock() - fake.SendInputToBuildPlanStub = stub -} - -func (fake *FakeClient) SendInputToBuildPlanArgsForCall(i int) (int, atc.PlanID, io.Reader) { - fake.sendInputToBuildPlanMutex.RLock() - defer fake.sendInputToBuildPlanMutex.RUnlock() - argsForCall := fake.sendInputToBuildPlanArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - -func (fake *FakeClient) SendInputToBuildPlanReturns(result1 bool, result2 error) { - fake.sendInputToBuildPlanMutex.Lock() - defer fake.sendInputToBuildPlanMutex.Unlock() - fake.SendInputToBuildPlanStub = nil - fake.sendInputToBuildPlanReturns = struct { - result1 bool - result2 error - }{result1, result2} -} - -func (fake *FakeClient) SendInputToBuildPlanReturnsOnCall(i int, result1 bool, result2 error) { - fake.sendInputToBuildPlanMutex.Lock() - defer fake.sendInputToBuildPlanMutex.Unlock() - fake.SendInputToBuildPlanStub = nil - if fake.sendInputToBuildPlanReturnsOnCall == nil { - fake.sendInputToBuildPlanReturnsOnCall = make(map[int]struct { - result1 bool - result2 error - }) - } - fake.sendInputToBuildPlanReturnsOnCall[i] = struct { - result1 bool - result2 error - }{result1, result2} -} - func (fake *FakeClient) Team(arg1 string) concourse.Team { fake.teamMutex.Lock() ret, specificReturn := fake.teamReturnsOnCall[len(fake.teamArgsForCall)] @@ -1506,6 +1419,8 @@ func (fake *FakeClient) Invocations() map[string][][]interface{} { defer fake.hTTPClientMutex.RUnlock() fake.landWorkerMutex.RLock() defer fake.landWorkerMutex.RUnlock() + fake.listBuildArtifactsMutex.RLock() + defer fake.listBuildArtifactsMutex.RUnlock() fake.listPipelinesMutex.RLock() defer fake.listPipelinesMutex.RUnlock() fake.listTeamsMutex.RLock() @@ -1514,12 +1429,8 @@ func (fake *FakeClient) Invocations() map[string][][]interface{} { defer fake.listWorkersMutex.RUnlock() fake.pruneWorkerMutex.RLock() defer fake.pruneWorkerMutex.RUnlock() - fake.readOutputFromBuildPlanMutex.RLock() - defer fake.readOutputFromBuildPlanMutex.RUnlock() fake.saveWorkerMutex.RLock() defer fake.saveWorkerMutex.RUnlock() - fake.sendInputToBuildPlanMutex.RLock() - defer fake.sendInputToBuildPlanMutex.RUnlock() fake.teamMutex.RLock() defer fake.teamMutex.RUnlock() fake.uRLMutex.RLock() diff --git a/go-concourse/concourse/concoursefakes/fake_team.go b/go-concourse/concourse/concoursefakes/fake_team.go index df0653c8ad1..8df5f476863 100644 --- a/go-concourse/concourse/concoursefakes/fake_team.go +++ b/go-concourse/concourse/concoursefakes/fake_team.go @@ -2,6 +2,7 @@ package concoursefakes import ( + io "io" sync "sync" atc "github.com/concourse/concourse/atc" @@ -120,6 +121,19 @@ type FakeTeam struct { result1 int64 result2 error } + CreateArtifactStub func(io.Reader) (atc.WorkerArtifact, error) + createArtifactMutex sync.RWMutex + createArtifactArgsForCall []struct { + arg1 io.Reader + } + createArtifactReturns struct { + result1 atc.WorkerArtifact + result2 error + } + createArtifactReturnsOnCall map[int]struct { + result1 atc.WorkerArtifact + result2 error + } CreateBuildStub func(atc.Plan) (atc.Build, error) createBuildMutex sync.RWMutex createBuildArgsForCall []struct { @@ -265,6 +279,19 @@ type FakeTeam struct { result1 bool result2 error } + GetArtifactStub func(int) (io.ReadCloser, error) + getArtifactMutex sync.RWMutex + getArtifactArgsForCall []struct { + arg1 int + } + getArtifactReturns struct { + result1 io.ReadCloser + result2 error + } + getArtifactReturnsOnCall map[int]struct { + result1 io.ReadCloser + result2 error + } HidePipelineStub func(string) (bool, error) hidePipelineMutex sync.RWMutex hidePipelineArgsForCall []struct { @@ -1067,6 +1094,69 @@ func (fake *FakeTeam) ClearTaskCacheReturnsOnCall(i int, result1 int64, result2 }{result1, result2} } +func (fake *FakeTeam) CreateArtifact(arg1 io.Reader) (atc.WorkerArtifact, error) { + fake.createArtifactMutex.Lock() + ret, specificReturn := fake.createArtifactReturnsOnCall[len(fake.createArtifactArgsForCall)] + fake.createArtifactArgsForCall = append(fake.createArtifactArgsForCall, struct { + arg1 io.Reader + }{arg1}) + fake.recordInvocation("CreateArtifact", []interface{}{arg1}) + fake.createArtifactMutex.Unlock() + if fake.CreateArtifactStub != nil { + return fake.CreateArtifactStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.createArtifactReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeTeam) CreateArtifactCallCount() int { + fake.createArtifactMutex.RLock() + defer fake.createArtifactMutex.RUnlock() + return len(fake.createArtifactArgsForCall) +} + +func (fake *FakeTeam) CreateArtifactCalls(stub func(io.Reader) (atc.WorkerArtifact, error)) { + fake.createArtifactMutex.Lock() + defer fake.createArtifactMutex.Unlock() + fake.CreateArtifactStub = stub +} + +func (fake *FakeTeam) CreateArtifactArgsForCall(i int) io.Reader { + fake.createArtifactMutex.RLock() + defer fake.createArtifactMutex.RUnlock() + argsForCall := fake.createArtifactArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeTeam) CreateArtifactReturns(result1 atc.WorkerArtifact, result2 error) { + fake.createArtifactMutex.Lock() + defer fake.createArtifactMutex.Unlock() + fake.CreateArtifactStub = nil + fake.createArtifactReturns = struct { + result1 atc.WorkerArtifact + result2 error + }{result1, result2} +} + +func (fake *FakeTeam) CreateArtifactReturnsOnCall(i int, result1 atc.WorkerArtifact, result2 error) { + fake.createArtifactMutex.Lock() + defer fake.createArtifactMutex.Unlock() + fake.CreateArtifactStub = nil + if fake.createArtifactReturnsOnCall == nil { + fake.createArtifactReturnsOnCall = make(map[int]struct { + result1 atc.WorkerArtifact + result2 error + }) + } + fake.createArtifactReturnsOnCall[i] = struct { + result1 atc.WorkerArtifact + result2 error + }{result1, result2} +} + func (fake *FakeTeam) CreateBuild(arg1 atc.Plan) (atc.Build, error) { fake.createBuildMutex.Lock() ret, specificReturn := fake.createBuildReturnsOnCall[len(fake.createBuildArgsForCall)] @@ -1720,6 +1810,69 @@ func (fake *FakeTeam) ExposePipelineReturnsOnCall(i int, result1 bool, result2 e }{result1, result2} } +func (fake *FakeTeam) GetArtifact(arg1 int) (io.ReadCloser, error) { + fake.getArtifactMutex.Lock() + ret, specificReturn := fake.getArtifactReturnsOnCall[len(fake.getArtifactArgsForCall)] + fake.getArtifactArgsForCall = append(fake.getArtifactArgsForCall, struct { + arg1 int + }{arg1}) + fake.recordInvocation("GetArtifact", []interface{}{arg1}) + fake.getArtifactMutex.Unlock() + if fake.GetArtifactStub != nil { + return fake.GetArtifactStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.getArtifactReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeTeam) GetArtifactCallCount() int { + fake.getArtifactMutex.RLock() + defer fake.getArtifactMutex.RUnlock() + return len(fake.getArtifactArgsForCall) +} + +func (fake *FakeTeam) GetArtifactCalls(stub func(int) (io.ReadCloser, error)) { + fake.getArtifactMutex.Lock() + defer fake.getArtifactMutex.Unlock() + fake.GetArtifactStub = stub +} + +func (fake *FakeTeam) GetArtifactArgsForCall(i int) int { + fake.getArtifactMutex.RLock() + defer fake.getArtifactMutex.RUnlock() + argsForCall := fake.getArtifactArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeTeam) GetArtifactReturns(result1 io.ReadCloser, result2 error) { + fake.getArtifactMutex.Lock() + defer fake.getArtifactMutex.Unlock() + fake.GetArtifactStub = nil + fake.getArtifactReturns = struct { + result1 io.ReadCloser + result2 error + }{result1, result2} +} + +func (fake *FakeTeam) GetArtifactReturnsOnCall(i int, result1 io.ReadCloser, result2 error) { + fake.getArtifactMutex.Lock() + defer fake.getArtifactMutex.Unlock() + fake.GetArtifactStub = nil + if fake.getArtifactReturnsOnCall == nil { + fake.getArtifactReturnsOnCall = make(map[int]struct { + result1 io.ReadCloser + result2 error + }) + } + fake.getArtifactReturnsOnCall[i] = struct { + result1 io.ReadCloser + result2 error + }{result1, result2} +} + func (fake *FakeTeam) HidePipeline(arg1 string) (bool, error) { fake.hidePipelineMutex.Lock() ret, specificReturn := fake.hidePipelineReturnsOnCall[len(fake.hidePipelineArgsForCall)] @@ -3216,6 +3369,8 @@ func (fake *FakeTeam) Invocations() map[string][][]interface{} { defer fake.checkResourceTypeMutex.RUnlock() fake.clearTaskCacheMutex.RLock() defer fake.clearTaskCacheMutex.RUnlock() + fake.createArtifactMutex.RLock() + defer fake.createArtifactMutex.RUnlock() fake.createBuildMutex.RLock() defer fake.createBuildMutex.RUnlock() fake.createJobBuildMutex.RLock() @@ -3236,6 +3391,8 @@ func (fake *FakeTeam) Invocations() map[string][][]interface{} { defer fake.enableResourceVersionMutex.RUnlock() fake.exposePipelineMutex.RLock() defer fake.exposePipelineMutex.RUnlock() + fake.getArtifactMutex.RLock() + defer fake.getArtifactMutex.RUnlock() fake.hidePipelineMutex.RLock() defer fake.hidePipelineMutex.RUnlock() fake.jobMutex.RLock() diff --git a/go.mod b/go.mod index 219e4826ae1..8335b9a0de1 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,10 @@ require ( code.cloudfoundry.org/lager v2.0.0+incompatible code.cloudfoundry.org/localip v0.0.0-20170223024724-b88ad0dea95c code.cloudfoundry.org/urljoiner v0.0.0-20170223060717-5cabba6c0a50 + contrib.go.opencensus.io/exporter/ocagent v0.4.1 // indirect + github.com/Azure/azure-sdk-for-go v24.0.0+incompatible // indirect github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 // indirect + github.com/Azure/go-autorest v11.2.8+incompatible // indirect github.com/DataDog/datadog-go v0.0.0-20180702141236-ef3a9daf849d github.com/Jeffail/gabs v1.1.0 // indirect github.com/Masterminds/squirrel v0.0.0-20190107164353-fa735ea14f09 @@ -20,6 +23,8 @@ require ( github.com/SAP/go-hdb v0.13.1 // indirect github.com/SermoDigital/jose v0.9.1 // indirect github.com/The-Cloud-Source/goryman v0.0.0-20150410173800-c22b6e4a7ac1 + github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190107113132-5452bdb42a73 // indirect + github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect github.com/armon/go-radix v1.0.0 // indirect github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a @@ -28,14 +33,20 @@ require ( github.com/bitly/go-hostpool v0.0.0-20171023180738-a3a6125de932 // indirect github.com/bmatcuk/doublestar v1.1.1 // indirect github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect + github.com/boltdb/bolt v1.3.1 // indirect + github.com/boombuler/barcode v1.0.0 // indirect + github.com/briankassouf/jose v0.9.1 // indirect github.com/caarlos0/env v3.5.0+incompatible github.com/cenkalti/backoff v2.1.1+incompatible + github.com/centrify/cloud-golang-sdk v0.0.0-20180119173102-7c97cc6fde16 // indirect + github.com/chrismalek/oktasdk-go v0.0.0-20181212195951-3430665dfaa0 // indirect github.com/circonus-labs/circonus-gometrics v2.2.1+incompatible // indirect github.com/circonus-labs/circonusllhist v0.0.0-20180430145027-5eb751da55c6 // indirect github.com/cloudfoundry/bosh-cli v5.4.0+incompatible github.com/cloudfoundry/bosh-utils v0.0.0-20181224171034-c2cf699102bd // indirect github.com/cloudfoundry/go-socks5 v0.0.0-20180221174514-54f73bdb8a8e // indirect github.com/cloudfoundry/socks5-proxy v0.0.0-20180530211953-3659db090cb2 // indirect + github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 // indirect github.com/concourse/baggageclaim v1.3.5 github.com/concourse/dex v0.0.0-20181120155244-024cbea7e753 github.com/concourse/flag v1.0.0 @@ -43,10 +54,15 @@ require ( github.com/concourse/retryhttp v0.0.0-20181126170240-7ab5e29e634f github.com/containerd/continuity v0.0.0-20180919190352-508d86ade3c2 // indirect github.com/coreos/go-oidc v0.0.0-20170307191026-be73733bb8cc + github.com/coreos/go-semver v0.2.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76 // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect github.com/cppforlife/go-patch v0.0.0-20171006213518-250da0e0e68c // indirect github.com/cppforlife/go-semi-semantic v0.0.0-20160921010311-576b6af77ae4 + github.com/dancannon/gorethink v4.0.0+incompatible // indirect github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/dimchansky/utfbom v1.1.0 // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.3.3 // indirect github.com/duosecurity/duo_api_golang v0.0.0-20180315112207-d0530c80e49a // indirect @@ -55,30 +71,38 @@ require ( github.com/fatih/color v1.7.0 github.com/fatih/structs v1.0.0 // indirect github.com/felixge/httpsnoop v1.0.0 + github.com/fullsailor/pkcs7 v0.0.0-20180613152042-8306686428a5 // indirect + github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4 // indirect + github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92 // indirect + github.com/garyburd/redigo v1.6.0 // indirect + github.com/go-errors/errors v1.0.1 // indirect github.com/go-ldap/ldap v2.5.1+incompatible // indirect github.com/go-openapi/jsonpointer v0.0.0-20180825180259-52eb3d4b47c6 // indirect github.com/go-openapi/jsonreference v0.0.0-20180825180305-1c6a3fa339f2 // indirect github.com/go-openapi/spec v0.0.0-20180825180323-f1468acb3b29 // indirect github.com/go-openapi/swag v0.0.0-20180908172849-dd0dad036e67 // indirect github.com/go-sql-driver/mysql v0.0.0-20160802113842-0b58b37b664c // indirect + github.com/go-stomp/stomp v2.0.2+incompatible // indirect github.com/go-test/deep v1.0.1 // indirect github.com/gobuffalo/packr v1.13.7 github.com/gocql/gocql v0.0.0-20180920092337-799fb0373110 // indirect github.com/gogo/protobuf v1.1.1 // indirect - github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect + github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db // indirect github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c // indirect - github.com/google/go-cmp v0.2.0 // indirect github.com/google/go-github v17.0.0+incompatible // indirect github.com/google/go-querystring v1.0.0 // indirect github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf // indirect github.com/google/jsonapi v0.0.0-20180618021926-5d047c6bc66b + github.com/googleapis/gax-go v2.0.2+incompatible // indirect github.com/googleapis/gnostic v0.2.0 // indirect + github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 // indirect github.com/gorilla/websocket v1.4.0 github.com/gotestyourself/gotestyourself v2.1.0+incompatible // indirect github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 // indirect github.com/hashicorp/consul v1.2.3 // indirect github.com/hashicorp/go-cleanhttp v0.5.0 // indirect + github.com/hashicorp/go-gcp-common v0.0.0-20180425173946-763e39302965 // indirect github.com/hashicorp/go-hclog v0.0.0-20180910232447-e45cbeb79f04 // indirect github.com/hashicorp/go-immutable-radix v1.0.0 // indirect github.com/hashicorp/go-memdb v0.0.0-20180223233045-1289e7fffe71 // indirect @@ -91,32 +115,49 @@ require ( github.com/hashicorp/go-version v1.0.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hashicorp/memberlist v0.1.0 // indirect + github.com/hashicorp/nomad v0.8.6 // indirect + github.com/hashicorp/raft v1.0.0 // indirect github.com/hashicorp/serf v0.8.1 // indirect - github.com/hashicorp/vault v0.10.4 + github.com/hashicorp/vault v1.0.1 + github.com/hashicorp/vault-plugin-auth-alicloud v0.0.0-20181109180636-f278a59ca3e8 // indirect + github.com/hashicorp/vault-plugin-auth-azure v0.0.0-20181207232528-4c0b46069a22 // indirect + github.com/hashicorp/vault-plugin-auth-centrify v0.0.0-20180816201131-66b0a34a58bf // indirect + github.com/hashicorp/vault-plugin-auth-gcp v0.0.0-20181210200133-4d63bbfe6fcf // indirect + github.com/hashicorp/vault-plugin-auth-jwt v0.0.0-20181031195942-f428c7791733 // indirect + github.com/hashicorp/vault-plugin-auth-kubernetes v0.0.0-20181130162533-091d9e5d5fab // indirect + github.com/hashicorp/vault-plugin-secrets-ad v0.0.0-20181109182834-540c0b6f1f11 // indirect + github.com/hashicorp/vault-plugin-secrets-alicloud v0.0.0-20181109181453-2aee79cc5cbf // indirect + github.com/hashicorp/vault-plugin-secrets-azure v0.0.0-20181207232500-0087bdef705a // indirect + github.com/hashicorp/vault-plugin-secrets-gcp v0.0.0-20180921173200-d6445459e80c // indirect + github.com/hashicorp/vault-plugin-secrets-gcpkms v0.0.0-20181212182553-6cd991800a6d // indirect github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20180825215324-5a464a61f7de // indirect github.com/hashicorp/yamux v0.0.0-20180917205041-7221087c3d28 // indirect github.com/howeyc/gopass v0.0.0-20170109162249-bf9dde6d0d2c // indirect github.com/imdario/mergo v0.3.6 // indirect github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf github.com/influxdata/influxdb1-client v0.0.0-20190118215656-f8cdb5d5f175 + github.com/jeffchao/backoff v0.0.0-20140404060208-9d7fd7aa17f2 // indirect github.com/jefferai/jsonx v0.0.0-20160721235117-9cc31c3135ee // indirect github.com/jessevdk/go-flags v1.4.0 github.com/json-iterator/go v1.1.5 // indirect github.com/juju/ratelimit v1.0.1 // indirect github.com/keybase/go-crypto v0.0.0-20180920171116-0b2a91ace448 // indirect - github.com/kr/pretty v0.1.0 // indirect - github.com/kr/pty v1.1.3 + github.com/kr/pty v1.1.2 github.com/krishicks/yaml-patch v0.0.10 github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect + github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b // indirect github.com/mattn/go-colorable v0.1.1 github.com/mattn/go-isatty v0.0.6 github.com/mattn/go-runewidth v0.0.3 // indirect github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b + github.com/michaelklishin/rabbit-hole v1.4.0 // indirect + github.com/miekg/dns v1.1.4 github.com/miekg/dns v1.1.6 github.com/mitchellh/copystructure v1.0.0 // indirect github.com/mitchellh/go-homedir v1.0.0 // indirect github.com/mitchellh/go-testing-interface v1.0.0 // indirect + github.com/mitchellh/hashstructure v1.0.0 // indirect github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699 github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.1 // indirect @@ -127,38 +168,51 @@ require ( github.com/opencontainers/go-digest v1.0.0-rc1 // indirect github.com/opencontainers/image-spec v1.0.1 // indirect github.com/opencontainers/runc v0.1.1 // indirect + github.com/ory-am/common v0.4.0 // indirect github.com/ory/dockertest v3.3.2+incompatible // indirect github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c // indirect github.com/patrickmn/go-cache v2.1.0+incompatible // indirect + github.com/pborman/uuid v1.2.0 // indirect github.com/peterbourgon/diskv v2.0.1+incompatible // indirect github.com/peterhellberg/link v1.0.0 + github.com/pierrec/lz4 v2.0.5+incompatible // indirect github.com/pkg/errors v0.8.1 github.com/pkg/term v0.0.0-20190109203006-aa71e9d9e942 + github.com/pquerna/otp v1.1.0 // indirect github.com/prometheus/client_golang v0.9.2 github.com/racksec/srslog v0.0.0-20180709174129-a4725f04ec91 github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 // indirect + github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect github.com/satori/go.uuid v1.2.0 // indirect github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/sirupsen/logrus v1.3.0 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c + github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa // indirect github.com/spf13/pflag v1.0.3 // indirect github.com/square/certstrap v1.1.1 + github.com/streadway/amqp v0.0.0-20190225234609-30f8ed68076e // indirect github.com/tedsuo/ifrit v0.0.0-20180802180643-bea94bb476cc github.com/tedsuo/rata v1.0.1-0.20170830210128-07d200713958 github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 // indirect + github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 // indirect github.com/vito/go-interact v0.0.0-20171111012221-fa338ed9e9ec github.com/vito/go-sse v0.0.0-20160212001227-fd69d275caac github.com/vito/houdini v1.1.1 github.com/vito/twentythousandtonnesofcrudeoil v0.0.0-20180305154709-3b21ad808fcb + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 - golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be + golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 // indirect golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 // indirect - gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect + google.golang.org/api v0.1.0 // indirect + google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f // indirect gopkg.in/cheggaaa/pb.v1 v1.0.27 + gopkg.in/gorethink/gorethink.v4 v4.1.0 // indirect gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect + gopkg.in/ory-am/dockertest.v2 v2.2.3 // indirect + gopkg.in/square/go-jose.v2 v2.1.8 gopkg.in/square/go-jose.v2 v2.3.0 gopkg.in/yaml.v2 v2.2.2 gotest.tools v2.1.0+incompatible // indirect @@ -166,4 +220,5 @@ require ( k8s.io/apimachinery v0.0.0-20171027084411-18a564baac72 k8s.io/client-go v2.0.0-alpha.0.0.20171101191150-72e1c2a1ef30+incompatible k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c // indirect + layeh.com/radius v0.0.0-20190101232339-d3a4fc175dc9 // indirect ) diff --git a/go.sum b/go.sum index 30bd092d224..cc96da68cfb 100644 --- a/go.sum +++ b/go.sum @@ -1,3 +1,4 @@ +cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= cloud.google.com/go v0.28.0 h1:KZ/88LWSw8NxMkjdQyX7LQSGR9PkHr4PaVuNm8zgFq0= cloud.google.com/go v0.28.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= code.cloudfoundry.org/clock v0.0.0-20180518195852-02e53af36e6c h1:5eeuG0BHx1+DHeT3AP+ISKZ2ht1UjGhm581ljqYpVeQ= @@ -13,8 +14,15 @@ code.cloudfoundry.org/localip v0.0.0-20170223024724-b88ad0dea95c h1:dO6i+2uQgR1Z code.cloudfoundry.org/localip v0.0.0-20170223024724-b88ad0dea95c/go.mod h1:q9OZPRxTlJybLg3Qb0b3R+hYgJBk5/Gi05gKB8O/lSU= code.cloudfoundry.org/urljoiner v0.0.0-20170223060717-5cabba6c0a50 h1:y+DtLO/eX/9NZjGGHntWs1bNG6uxdql8SqrHzu6VH3Q= code.cloudfoundry.org/urljoiner v0.0.0-20170223060717-5cabba6c0a50/go.mod h1:GyubIUn2eHGSlpIqJhGKBKicAe6CUV/pQJosfNEHdo4= +contrib.go.opencensus.io/exporter/ocagent v0.4.1 h1:1lyr7duzSVn3G9skLcA4Ym15ufvQLOjNq+Mvg7eK70g= +contrib.go.opencensus.io/exporter/ocagent v0.4.1/go.mod h1:b6YwD5Q3Yvj4yk0CDK5vGXexygNzI09aXUdDEakQBgA= +git.apache.org/thrift.git v0.0.0-20180902110319-2566ecd5d999/go.mod h1:fPE2ZNJGynbRyZ4dJvy6G277gSllfV2HJqblrnkyeyg= +github.com/Azure/azure-sdk-for-go v24.0.0+incompatible h1:GdF0ozHojCPSZH1LPWA2+XHQ3G/mapn0G+PCIlMVZg4= +github.com/Azure/azure-sdk-for-go v24.0.0+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= +github.com/Azure/go-autorest v11.2.8+incompatible h1:Q2feRPMlcfVcqz3pF87PJzkm5lZrL+x6BDtzhODzNJM= +github.com/Azure/go-autorest v11.2.8+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24= github.com/DataDog/datadog-go v0.0.0-20180702141236-ef3a9daf849d h1:qGaiqpKg/VnndIRJu9Z0jdVVXebhTxzd6sDP//mh3/E= github.com/DataDog/datadog-go v0.0.0-20180702141236-ef3a9daf849d/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/Jeffail/gabs v1.1.0 h1:kw5zCcl9tlJNHTDme7qbi21fDHZmXrnjMoXos3Jw/NI= @@ -37,6 +45,10 @@ github.com/SermoDigital/jose v0.9.1 h1:atYaHPD3lPICcbK1owly3aPm0iaJGSGPi0WD4vLzn github.com/SermoDigital/jose v0.9.1/go.mod h1:ARgCUhI1MHQH+ONky/PAtmVHQrP5JlGY0F3poXOp/fA= github.com/The-Cloud-Source/goryman v0.0.0-20150410173800-c22b6e4a7ac1 h1:LirsQvQpiIRrGXR/7xEe/m589nEzHmqhyMeKatC9LBk= github.com/The-Cloud-Source/goryman v0.0.0-20150410173800-c22b6e4a7ac1/go.mod h1:eoPf+25GcZRRy4HU0o2kK55qlGgadQuhME3/GYcX/MM= +github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190107113132-5452bdb42a73 h1:yZaBtrpzD3RjYCSxZ/Q4EZSYaaX31sW7+GG+xcJqcIA= +github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190107113132-5452bdb42a73/go.mod h1:T9M45xf79ahXVelWoOBmH0y4aC1t5kXO5BxwyakgIGA= +github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 h1:Xz25cuW4REGC5W5UtpMU3QItMIImag615HiQcRbxqKQ= +github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61/go.mod h1:ikc1XA58M+Rx7SEbf0bLJCfBkwayZ8T5jBo5FXK8Uz8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= github.com/armon/go-radix v1.0.0 h1:F4z6KzEeeQIMeLFa97iZU6vupzoecKdU5TX24SNppXI= @@ -45,6 +57,8 @@ github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a h1:pv34s756C4pEXnjg github.com/aryann/difflib v0.0.0-20170710044230-e206f873d14a/go.mod h1:DAHtR1m6lCRdSC2Tm3DSWRPvIPr6xNKyeHdqDQSQT+A= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf h1:eg0MeVzsP1G42dRafH3vf+al2vQIJU0YHX+1Tw87oco= github.com/asaskevich/govalidator v0.0.0-20180720115003-f9ffefc3facf/go.mod h1:lB+ZfQJz7igIIfQNfa7Ml4HSf2uFQQRzpGGRXenZAgY= +github.com/aws/aws-sdk-go v1.17.4 h1:L2KFocQhg48kIzEAV98SnSz3nmIZ3UDFP+vU647KO3c= +github.com/aws/aws-sdk-go v1.17.4/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go v1.18.1 h1:Jldy1wK0NFIF91+AVieoAqDOgNCS5hpT18A1qdz2ulA= github.com/aws/aws-sdk-go v1.18.1/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beevik/etree v0.0.0-20161216042344-4cd0dd976db8 h1:83NNCRw/4bJwVOCZ5NKmRiqbffkDC/B2DFmKZ/EzU0c= @@ -60,16 +74,30 @@ github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4Yn github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40 h1:y4B3+GPxKlrigF1ha5FFErxK+sr6sWxQovRMzwMhejo= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= +github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= +github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= +github.com/boombuler/barcode v1.0.0 h1:s1TvRnXwL2xJRaccrdcBQMZxq6X7DvsMogtmJeHDdrc= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/briankassouf/jose v0.9.1 h1:HMeR+qpz2PBP7RgR+64nJPA9qcZI74tdkBv5NyD0Mzk= +github.com/briankassouf/jose v0.9.1/go.mod h1:HQhVmdUf7dBNwIIdBTivnCDxcf6IZY3/zrb+uKSJz6Y= github.com/caarlos0/env v3.5.0+incompatible h1:Yy0UN8o9Wtr/jGHZDpCBLpNrzcFLLM2yixi/rBrKyJs= github.com/caarlos0/env v3.5.0+incompatible/go.mod h1:tdCsowwCzMLdkqRYDlHpZCp2UooDD3MspDBjZ2AD02Y= +github.com/cenkalti/backoff v2.0.0+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff v2.1.1+incompatible h1:tKJnvO2kl0zmb/jA5UKAt4VoEVw1qxKWjE/Bpp46npY= github.com/cenkalti/backoff v2.1.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= +github.com/census-instrumentation/opencensus-proto v0.1.0 h1:VwZ9smxzX8u14/125wHIX7ARV+YhR+L4JADswwxWK0Y= +github.com/census-instrumentation/opencensus-proto v0.1.0/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/centrify/cloud-golang-sdk v0.0.0-20180119173102-7c97cc6fde16 h1:KXl7Kj2SRSZIvqafDrF3JoSX3olfAd+1V72deQaZGCs= +github.com/centrify/cloud-golang-sdk v0.0.0-20180119173102-7c97cc6fde16/go.mod h1:C0rtzmGXgN78pYR0tGJFhtHgkbAs0lIbHwkB81VxDQE= github.com/charlievieth/fs v0.0.0-20170613215519-7dc373669fa1 h1:vTlpHKxJqykyKdW9bkrDJNWeKNuSIAJ0TP/K4lRsz/Q= github.com/charlievieth/fs v0.0.0-20170613215519-7dc373669fa1/go.mod h1:sAoA1zHCH4FJPE2gne5iBiiVG66U7Nyp6JqlOo+FEyg= +github.com/chrismalek/oktasdk-go v0.0.0-20181212195951-3430665dfaa0 h1:CWU8piLyqoi9qXEUwzOh5KFKGgmSU5ZhktJyYcq6ryQ= +github.com/chrismalek/oktasdk-go v0.0.0-20181212195951-3430665dfaa0/go.mod h1:5d8DqS60xkj9k3aXfL3+mXBH0DPYO0FQjcKosxl+b/Q= github.com/circonus-labs/circonus-gometrics v2.2.1+incompatible h1:1L31gruXa06igfDg0HNf0lw40TEgGesySg9nZ3rI5v4= github.com/circonus-labs/circonus-gometrics v2.2.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= github.com/circonus-labs/circonusllhist v0.0.0-20180430145027-5eb751da55c6 h1:Rm//1hbNxCErrYx7+QSOvITlZtNpt3rW1Ov8lxifdsg= github.com/circonus-labs/circonusllhist v0.0.0-20180430145027-5eb751da55c6/go.mod h1:kMXHVDlOchFAehlya5ePtbp5jckzBHf4XRpQvBOLI+I= +github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudfoundry/bosh-cli v5.4.0+incompatible h1:KpT2PBB7nP1QnK8guXeZ/D2k7FZYAOxcveKgYTDEDBI= github.com/cloudfoundry/bosh-cli v5.4.0+incompatible/go.mod h1:rzIB+e1sn7wQL/TJ54bl/FemPKRhXby5BIMS3tLuWFM= github.com/cloudfoundry/bosh-utils v0.0.0-20181224171034-c2cf699102bd h1:MoLE+6GJml+nqHFXKXQSQsmRieKgyyttAYDUIK2YNfY= @@ -78,6 +106,8 @@ github.com/cloudfoundry/go-socks5 v0.0.0-20180221174514-54f73bdb8a8e h1:FQdRViao github.com/cloudfoundry/go-socks5 v0.0.0-20180221174514-54f73bdb8a8e/go.mod h1:PXmcacyJB/pJjSxEl15IU6rEIKXrhZQRzsr0UTkgNNs= github.com/cloudfoundry/socks5-proxy v0.0.0-20180530211953-3659db090cb2 h1:9j2KbUEQn5E7MEV3enSrkJTrBC0iDbosW5gXX+Z+dLE= github.com/cloudfoundry/socks5-proxy v0.0.0-20180530211953-3659db090cb2/go.mod h1:0a+Ghg38uB86Dx+de84dFSkILTnBHzCpFMRnjHgSzi4= +github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292 h1:dzj1/xcivGjNPwwifh/dWTczkwcuqsXXFHY1X/TZMtw= +github.com/cockroachdb/cmux v0.0.0-20170110192607-30d10be49292/go.mod h1:qRiX68mZX1lGBkTWyp3CLcenw9I94W2dLeRvMzcn9N4= github.com/concourse/baggageclaim v1.3.5 h1:cRHTq93MW0WyPgANjkt9YBnAl0wwquh//41fCla8Uic= github.com/concourse/baggageclaim v1.3.5/go.mod h1:9K1fnrJ8XyAm7OQtkdILDCj3bmCsnFkW/KCPBPyDy+s= github.com/concourse/dex v0.0.0-20181120155244-024cbea7e753 h1:1e4sAPr+xZn0LH8/yGQXdNEaCw4y7NGI3ikgAt0k7v0= @@ -94,13 +124,22 @@ github.com/concourse/retryhttp v0.0.0-20181126170240-7ab5e29e634f h1:YlKSrd1N2/W github.com/concourse/retryhttp v0.0.0-20181126170240-7ab5e29e634f/go.mod h1:4+V9YCkKuoV7rg+/No+ZM9FsO3BK4tIJNUiYMI7nki0= github.com/containerd/continuity v0.0.0-20180919190352-508d86ade3c2 h1:oiQ0OCfHdE7YXG94mc3HpKhbaZ8Nk17lw4E+ycI8Zgs= github.com/containerd/continuity v0.0.0-20180919190352-508d86ade3c2/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= +github.com/coreos/etcd v3.2.9+incompatible h1:3TbjfK5+aSRLTU/KgBC1xlgA2dn2ddYQngRqX6HFwlQ= github.com/coreos/etcd v3.2.9+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-oidc v0.0.0-20170307191026-be73733bb8cc h1:9yuvA19Q5WFkLwJcMDoYm8m89ilzqZ5zEHqdvU+Zbds= github.com/coreos/go-oidc v0.0.0-20170307191026-be73733bb8cc/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= +github.com/coreos/go-semver v0.2.0 h1:3Jm3tLmsgAYcjC+4Up7hJrFBPr+n7rAqYeSw/SZazuY= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76 h1:FE783w8WFh+Rvg+7bZ5g8p7gP4SeVS4AoNwkvazlsBg= +github.com/coreos/go-systemd v0.0.0-20190212144455-93d5ec2c7f76/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= +github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cppforlife/go-patch v0.0.0-20171006213518-250da0e0e68c h1:L6Qwcfk/qeD05lCaMxjhn8fCKNAVEWOBn1vqU7KJHtk= github.com/cppforlife/go-patch v0.0.0-20171006213518-250da0e0e68c/go.mod h1:67a7aIi94FHDZdoeGSJRRFDp66l9MhaAG1yGxpUoFD8= github.com/cppforlife/go-semi-semantic v0.0.0-20160921010311-576b6af77ae4 h1:J+ghqo7ZubTzelkjo9hntpTtP/9lUCWH9icEmAW+B+Q= github.com/cppforlife/go-semi-semantic v0.0.0-20160921010311-576b6af77ae4/go.mod h1:socxpf5+mELPbosI149vWpNlHK6mbfWFxSWOoSndXR8= +github.com/dancannon/gorethink v4.0.0+incompatible h1:KFV7Gha3AuqT+gr0B/eKvGhbjmUv0qGF43aKCIKVE9A= +github.com/dancannon/gorethink v4.0.0+incompatible/go.mod h1:BLvkat9KmZc1efyYwhz3WnybhRZtgF1K929FD8z1avU= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -108,6 +147,8 @@ github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6 h1:BZGp1dbKF github.com/denisenkom/go-mssqldb v0.0.0-20180901172138-1eb28afdf9b6/go.mod h1:xN/JuLBIz4bjkxNmByTiV1IbhfnYb6oo99phBn4Eqhc= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= +github.com/dimchansky/utfbom v1.1.0 h1:FcM3g+nofKgUteL8dm/UpdRXNC9KmADgTpLKsu0TRo4= +github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.3.3 h1:Xk8S3Xj5sLGlG5g67hJmYMmUgXv5N4PhkjJHHqrwnTk= @@ -126,8 +167,20 @@ github.com/felixge/httpsnoop v1.0.0 h1:gh8fMGz0rlOv/1WmRZm7OgncIOTsAj21iNJot48om github.com/felixge/httpsnoop v1.0.0/go.mod h1:3+D9sFq0ahK/JeJPhCBUV1xlf4/eIYrUQaxulT0VzX8= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/fullsailor/pkcs7 v0.0.0-20180613152042-8306686428a5 h1:v+vxrd9XS8uWIXG2RK0BHCnXc30qLVQXVqbK+IOmpXk= +github.com/fullsailor/pkcs7 v0.0.0-20180613152042-8306686428a5/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= +github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4 h1:R+19WKQClnfMXS60cP5BmMe1wjZ4u0evY2p2Ar0ZTXo= +github.com/gammazero/deque v0.0.0-20180920172122-f6adf94963e4/go.mod h1:GeIq9qoE43YdGnDXURnmKTnGg15pQz4mYkXSTChbneI= +github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92 h1:EipXK6U05IQ2wtuFRn4k3h0+2lXypzItoXGVyf4r9Io= +github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92/go.mod h1:w9RqFVO2BM3xwWEcAB8Fwp0OviTBBEiRmSBDfbXnd3w= +github.com/garyburd/redigo v1.6.0 h1:0VruCpn7yAIIu7pWVClQC8wxCJEcG3nyzpMSHKi1PQc= +github.com/garyburd/redigo v1.6.0/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/ghodss/yaml v0.0.0-20161020005002-bea76d6a4713 h1:ag3kFMoZZMrEBi4ySTdSWSPz8K7Slu++J9/QXzLBiLk= github.com/ghodss/yaml v0.0.0-20161020005002-bea76d6a4713/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= +github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/go-errors/errors v1.0.1 h1:LUHzmkK3GUKUrL/1gfBUxAHzcev3apQlezX/+O7ma6w= +github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-ldap/ldap v2.5.1+incompatible h1:Opaoft5zMW8IU/VRULB0eGMBQ9P5buRvCW6sFTRmMn8= github.com/go-ldap/ldap v2.5.1+incompatible/go.mod h1:qfd9rJvER9Q0/D/Sqn1DfHRoBp40uXYvFoEVrNEPqRc= github.com/go-openapi/jsonpointer v0.0.0-20180825180259-52eb3d4b47c6 h1:xYFQx+26lG+zQh0+P+SDqcKTMMwRnEwE2cF59TtnOQ8= @@ -140,6 +193,8 @@ github.com/go-openapi/swag v0.0.0-20180908172849-dd0dad036e67 h1:HSEYUsQFq79SfgU github.com/go-openapi/swag v0.0.0-20180908172849-dd0dad036e67/go.mod h1:DXUve3Dpr1UfpPtxFw+EFuQ41HhCWZfha5jSVRG7C7I= github.com/go-sql-driver/mysql v0.0.0-20160802113842-0b58b37b664c h1:jdWrt1yXmcXx/g3UFLn3XEqh7DulXBwbbkfUjKlMLWA= github.com/go-sql-driver/mysql v0.0.0-20160802113842-0b58b37b664c/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-stomp/stomp v2.0.2+incompatible h1:0yPknMJh32lE2xiCFGW5t/KgamhUC4OgCv10wIjx5aw= +github.com/go-stomp/stomp v2.0.2+incompatible/go.mod h1:VqCtqNZv1226A1/79yh+rMiFUcfY3R109np+7ke4n0c= github.com/go-test/deep v1.0.1 h1:UQhStjbkDClarlmv0am7OXXO4/GaPdCGiUiMTvi28sg= github.com/go-test/deep v1.0.1/go.mod h1:wGDj63lr65AM2AQyKZd/NYHGb0R+1RLqB8NKt3aSFNA= github.com/gobuffalo/packr v1.13.7 h1:2uZgLd6b/W4yRBZV/ScaORxZLNGMHO0VCvqQNkKukNA= @@ -150,6 +205,10 @@ github.com/gogo/protobuf v1.1.1 h1:72R+M5VuhED/KujmZVcIquuo8mBgX4oVda//DQb3PXo= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef h1:veQD95Isof8w9/WXiA+pa3tz3fJXkt5B7QaRBrM62gk= +github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:tluoj9z5200jBnyusfRPU2LqT6J+DAorxEvtC7LHB+E= +github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v0.0.0-20171113180720-1e59b77b52bf/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -168,8 +227,16 @@ github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf h1:+RRA9JqSOZFfKrOeq github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/jsonapi v0.0.0-20180618021926-5d047c6bc66b h1:ndHKV+Umsd7wiG2y6n8aTdFdzCFh1pJ6UjsOEUA3Kqw= github.com/google/jsonapi v0.0.0-20180618021926-5d047c6bc66b/go.mod h1:XSx4m2SziAqk9DXY9nz659easTq4q6TyrpYd9tHSm0g= +github.com/google/uuid v1.0.0 h1:b4Gk+7WdP/d3HZH8EJsZpvV7EtDOgaZLtnaNGIu1adA= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/gax-go v2.0.2+incompatible h1:silFMLAnr330+NRuag/VjIGF7TLp/LBrV2CJKFLWEww= +github.com/googleapis/gax-go v2.0.2+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gnostic v0.2.0 h1:l6N3VoaVzTncYYW+9yOz2LJJammFZGBO13sqgEhpy9g= github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY= +github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/gorilla/context v0.0.0-20160525203319-aed02d124ae4 h1:3nOfQt8sRPYbXORD5tJ8YyQ3HlL2Jt3LJ2U17CbNh6I= github.com/gorilla/context v0.0.0-20160525203319-aed02d124ae4/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v0.0.0-20161206055144-3a5767ca75ec h1:RDyBmwIQWsSnpRDzYBziOiD54eGHlAffUxF98z17bNY= @@ -182,7 +249,10 @@ github.com/gotestyourself/gotestyourself v2.1.0+incompatible h1:JdX/5sh/7yF7jRW5 github.com/gotestyourself/gotestyourself v2.1.0+incompatible/go.mod h1:zZKM6oeNM8k+FRljX1mnzVYeS8wiGgQyvST1/GafPbY= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7 h1:pdN6V1QBWetyv/0+wjACpqVH+eVULgEjkurDLq3goeM= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= +github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea h1:Bzd/0fcg24qAEJyr7pTtDOn806SRBtzyloCuLTEvSOo= github.com/grpc-ecosystem/go-grpc-prometheus v0.0.0-20170826090648-0dafe0d496ea/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= +github.com/grpc-ecosystem/grpc-gateway v1.5.0 h1:WcmKMm43DR7RdtlkEXQJyo5ws8iTp98CyhCCbOHMvNI= +github.com/grpc-ecosystem/grpc-gateway v1.5.0/go.mod h1:RSKVYQBd5MCa4OVpNdGskqpgL2+G+NZTnrVHpWWfpdw= github.com/gtank/cryptopasta v0.0.0-20160720052843-e7e23673cac3/go.mod h1:YLEMZOtU+AZ7dhN9T/IpGhXVGly2bvkJQ+zxj3WeVQo= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed h1:5upAirOpQc1Q53c0bnx2ufif5kANL7bfZWcc6VJWJd8= github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4= @@ -192,6 +262,8 @@ github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/U github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-cleanhttp v0.5.0 h1:wvCrVc9TjDls6+YGAF2hAifE1E5U1+b4tH6KdvN3Gig= github.com/hashicorp/go-cleanhttp v0.5.0/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80= +github.com/hashicorp/go-gcp-common v0.0.0-20180425173946-763e39302965 h1:CRWAJ7atEOq9OfAa5rBKvL4wFNbUrPil8hN1knLuEG0= +github.com/hashicorp/go-gcp-common v0.0.0-20180425173946-763e39302965/go.mod h1:LNbios2fdMAuLA1dsYUvUcoCYIfywcCEK8/ooaWjoOA= github.com/hashicorp/go-hclog v0.0.0-20180910232447-e45cbeb79f04 h1:o2dJ2/0Dttz3Snc/jfklGLGjKJlGG1+AM4D+Zn6Luh0= github.com/hashicorp/go-hclog v0.0.0-20180910232447-e45cbeb79f04/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0= @@ -220,10 +292,36 @@ github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/memberlist v0.1.0 h1:qSsCiC0WYD39lbSitKNt40e30uorm2Ss/d4JGU1hzH8= github.com/hashicorp/memberlist v0.1.0/go.mod h1:ncdBp14cuox2iFOq3kDiquKU6fqsTBc3W6JvZwjxxsE= +github.com/hashicorp/nomad v0.8.6 h1:z+gocir324zUa88k9bIXkf0RpSgjVa9Izut+iV8T2qg= +github.com/hashicorp/nomad v0.8.6/go.mod h1:WRaKjdO1G2iqi86TvTjIYtKTyxg4pl7NLr9InxtWaI0= +github.com/hashicorp/raft v1.0.0 h1:htBVktAOtGs4Le5Z7K8SF5H2+oWsQFYVmOgH5loro7Y= +github.com/hashicorp/raft v1.0.0/go.mod h1:DVSAWItjLjTOkVbSpWQ0j0kUADIvDaCtBxIcbNAQLkI= github.com/hashicorp/serf v0.8.1 h1:mYs6SMzu72+90OcPa5wr3nfznA4Dw9UyR791ZFNOIf4= github.com/hashicorp/serf v0.8.1/go.mod h1:h/Ru6tmZazX7WO/GDmwdpS975F019L4t5ng5IgwbNrE= -github.com/hashicorp/vault v0.10.4 h1:4x0lHxui/ZRp/B3E0Auv1QNBJpzETqHR2kQD3mHSBJU= -github.com/hashicorp/vault v0.10.4/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0= +github.com/hashicorp/vault v1.0.1 h1:x3hcjkJLd5L4ehPhZcraokFO7dq8MJ3oKvQtrkIiIU8= +github.com/hashicorp/vault v1.0.1/go.mod h1:KfSyffbKxoVyspOdlaGVjIuwLobi07qD1bAbosPMpP0= +github.com/hashicorp/vault-plugin-auth-alicloud v0.0.0-20181109180636-f278a59ca3e8 h1:r4d3rctTbU7dwo9cMpOo9KTxsdIIadBwrqUTs+Tvc6M= +github.com/hashicorp/vault-plugin-auth-alicloud v0.0.0-20181109180636-f278a59ca3e8/go.mod h1:o3i5QQWgV5+SYouIn++L9D0kbhLYB3FjxNRHNf6KS+Q= +github.com/hashicorp/vault-plugin-auth-azure v0.0.0-20181207232528-4c0b46069a22 h1:aAQcFVgrbfjVWsFFP0Iv0f7P4JPTxL4Ta9Z4uBkX2Jc= +github.com/hashicorp/vault-plugin-auth-azure v0.0.0-20181207232528-4c0b46069a22/go.mod h1:f+VmjSQIxxO+YTeO3FbPWRPCPbd3f3lwpP6jaO/YduQ= +github.com/hashicorp/vault-plugin-auth-centrify v0.0.0-20180816201131-66b0a34a58bf h1:AizyQbOM+h/ZjV0ao65aJ/kQpOxlOKgZORLP/XnJlTA= +github.com/hashicorp/vault-plugin-auth-centrify v0.0.0-20180816201131-66b0a34a58bf/go.mod h1:IIz+CMBKBEFyjeBeFUlpoUuMOyFb7mybOUNP6GX1xuk= +github.com/hashicorp/vault-plugin-auth-gcp v0.0.0-20181210200133-4d63bbfe6fcf h1:HDjFaQtt0WjqWqHS2DDIcyl3HF0/vRk4SvXO5A8kjEc= +github.com/hashicorp/vault-plugin-auth-gcp v0.0.0-20181210200133-4d63bbfe6fcf/go.mod h1:E/E+5CuQCjOn/YGCmZ/tA7GwLey/lN1PwwJOOa9Iqy0= +github.com/hashicorp/vault-plugin-auth-jwt v0.0.0-20181031195942-f428c7791733 h1:/BJ5C3Kk9c3DdWSeH7EGez6CsPqG29wu+gaDLNJZlvQ= +github.com/hashicorp/vault-plugin-auth-jwt v0.0.0-20181031195942-f428c7791733/go.mod h1:j6Xmkj3dzuC63mivquwVVTlxjwDndwNxi4cJUku40J8= +github.com/hashicorp/vault-plugin-auth-kubernetes v0.0.0-20181130162533-091d9e5d5fab h1:aH7uQ4qdJhMftBIWcfjXf53eCH8SwrX93fwXBDMjEug= +github.com/hashicorp/vault-plugin-auth-kubernetes v0.0.0-20181130162533-091d9e5d5fab/go.mod h1:PqRUU5TaQ6FwVTsHPLrJs1F+T5IjbSzlfTy9cTyGeHM= +github.com/hashicorp/vault-plugin-secrets-ad v0.0.0-20181109182834-540c0b6f1f11 h1:ziFCrdE41xXxinELLHZGhMJpW9Tlk/9MX/iGJL0YIE8= +github.com/hashicorp/vault-plugin-secrets-ad v0.0.0-20181109182834-540c0b6f1f11/go.mod h1:4vRQzvp3JI+g4oUqzcklIEj2UKyhQnpIo+BDbh2uzYM= +github.com/hashicorp/vault-plugin-secrets-alicloud v0.0.0-20181109181453-2aee79cc5cbf h1:Q3K2MOcbHpvcs+yacQYxQQfQGoiLGjpTJ/DGg0z5oj0= +github.com/hashicorp/vault-plugin-secrets-alicloud v0.0.0-20181109181453-2aee79cc5cbf/go.mod h1:rl8WzY7++fZMLXd6Z/k/o9wUmMbOqpTLhbtKs1loMU0= +github.com/hashicorp/vault-plugin-secrets-azure v0.0.0-20181207232500-0087bdef705a h1:1sAzFWTeuHSBT+IzQmmWCw2oN9ZBNyPGbG2y30TYjK8= +github.com/hashicorp/vault-plugin-secrets-azure v0.0.0-20181207232500-0087bdef705a/go.mod h1:/DhLpYuRP2o00gkj6S0Gy7NvKk5AaAtP6p3f+OmxDUI= +github.com/hashicorp/vault-plugin-secrets-gcp v0.0.0-20180921173200-d6445459e80c h1:zTXxl4Kof596fEzsedu3v9vb+U4K0tIep0QIW84HBss= +github.com/hashicorp/vault-plugin-secrets-gcp v0.0.0-20180921173200-d6445459e80c/go.mod h1:IV2OZZZ9FCtSYeKDLsnO5JipMdjwachVISz9pNuQjhs= +github.com/hashicorp/vault-plugin-secrets-gcpkms v0.0.0-20181212182553-6cd991800a6d h1:jicq98xyziRHfEcfQh7wAHrERpQika9mU+z/BvNUHFQ= +github.com/hashicorp/vault-plugin-secrets-gcpkms v0.0.0-20181212182553-6cd991800a6d/go.mod h1:2n62quNV4DvfMY5Lxx82NJmx+9pYtv4RltLIFKxEO4E= github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20180825215324-5a464a61f7de h1:JS4zw0gtKE0G2LDYchlFvQxnaGnmdN4PA6SPypBzhIU= github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20180825215324-5a464a61f7de/go.mod h1:VJHHT2SC1tAPrfENQeBhLlb5FbZoKZM+oC/ROmEftz0= github.com/hashicorp/yamux v0.0.0-20180917205041-7221087c3d28 h1:gACvD6C1jev87LpraglGaxsDUGA8qg7Uz+Xh6dWKtIQ= @@ -239,6 +337,8 @@ github.com/inconshreveable/go-update v0.0.0-20160112193335-8152e7eb6ccf/go.mod h github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/influxdata/influxdb1-client v0.0.0-20190118215656-f8cdb5d5f175 h1:NR6eyx+1eski3gN8uqdtjnfpmq0T3lO6HhrPheCPFFc= github.com/influxdata/influxdb1-client v0.0.0-20190118215656-f8cdb5d5f175/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= +github.com/jeffchao/backoff v0.0.0-20140404060208-9d7fd7aa17f2 h1:mex1izRBCD+7WjieGgRdy7e651vD/lvB1bD9vNE/3K4= +github.com/jeffchao/backoff v0.0.0-20140404060208-9d7fd7aa17f2/go.mod h1:xkfESuHriIekR+4RoV+fu91j/CfnYM29Zi2tMFw5iD4= github.com/jefferai/jsonx v0.0.0-20160721235117-9cc31c3135ee h1:AQ/QmCk6x8ECPpf2pkPtA4lyncEEBbs8VFnVXPYKhIs= github.com/jefferai/jsonx v0.0.0-20160721235117-9cc31c3135ee/go.mod h1:N0t2vlmpe8nyZB5ouIbJQPDSR+mH6oe7xHB9VZHSUzM= github.com/jessevdk/go-flags v1.4.0 h1:4IU2WS7AumrZ/40jfhf4QVDMsQwqA7VEHozFRrGARJA= @@ -249,17 +349,21 @@ github.com/jonboulle/clockwork v0.0.0-20160907122059-bcac9884e750 h1:ysDggIU+1Xi github.com/jonboulle/clockwork v0.0.0-20160907122059-bcac9884e750/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= github.com/json-iterator/go v1.1.5 h1:gL2yXlmiIo4+t+y32d4WGwOjKGYcGOuyrg46vadswDE= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= +github.com/jtolds/gls v4.2.1+incompatible h1:fSuqC+Gmlu6l/ZYAoZzx2pyucC8Xza35fpRVWLVmUEE= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ratelimit v1.0.1 h1:+7AIFJVQ0EQgq/K9+0Krm7m530Du7tIz0METWzN0RgY= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/keybase/go-crypto v0.0.0-20180920171116-0b2a91ace448 h1:V4HrZZ/KjBRQTxaMp1pHbXsYhPt32kewNCquzl7m2jc= github.com/keybase/go-crypto v0.0.0-20180920171116-0b2a91ace448/go.mod h1:ghbZscTyKdM07+Fw3KSi0hcJm+AlEUWj8QLlPtijN/M= +github.com/kisielk/gotool v1.0.0 h1:AV2c/EiW3KqPNT9ZKl07ehoAGi4C5/01Cfbblndcapg= +github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/pty v1.1.3 h1:/Um6a/ZmD5tF7peoOJ5oN5KMQ0DrGVQSXLNwyckutPk= -github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.2 h1:Q7kfkJVHag8Gix8Z5+eTo09NFHV8MXL9K66sv9qDaVI= +github.com/kr/pty v1.1.2/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/krishicks/yaml-patch v0.0.10 h1:H4FcHpnNwVmw8u0MjPRjWyIXtco6zM2F78t+57oNM3E= @@ -274,6 +378,8 @@ github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc h1:0pifi8wVV/YuUKBDmlH3koJg github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3bUBu+FXuk2pFbkN6tcwi/pjyaDic= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k= +github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= @@ -289,6 +395,8 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0j github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/michaelklishin/rabbit-hole v1.4.0 h1:g80Jk11TqfI09Yn7FRze547z0FqNtU0IQH1O1GpDQvk= +github.com/michaelklishin/rabbit-hole v1.4.0/go.mod h1:vvI1uOitYZi0O5HEGXhaWC1XT80Gy+HvFheJ+5Krlhk= github.com/miekg/dns v1.1.4 h1:rCMZsU2ScVSYcAsOXgmC6+AKOK+6pmQTOcw03nfwYV0= github.com/miekg/dns v1.1.4/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.6 h1:jVwb4GDwD65q/gtItR/lIZHjNH93QfeGxZUkzJcW9mc= @@ -299,6 +407,8 @@ github.com/mitchellh/go-homedir v1.0.0 h1:vKb8ShqSby24Yrqr/yDYkuFz8d0WUjys40rvnG github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0= github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI= +github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= +github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ= github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699 h1:KXZJFdun9knAVAR8tg/aHJEr5DgtcbqyvzacK+CDCaI= github.com/mitchellh/mapstructure v0.0.0-20180715050151-f15292f7a699/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/reflectwalk v1.0.0 h1:9D+8oIskB4VJBN5SFlmc27fSlIBZaov1Wpk/IfikLNY= @@ -324,16 +434,25 @@ github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVo github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg4X946/Y5Zwg= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/openzipkin/zipkin-go v0.1.1/go.mod h1:NtoC/o8u3JlF1lSlyPNswIbeQH9bJTmOf0Erfk+hxe8= +github.com/ory-am/common v0.4.0 h1:edGPoxYX4hno0IJHXh9TCMUPR6ZcJp+y6aClFYxeuUE= +github.com/ory-am/common v0.4.0/go.mod h1:oCYGuwwM8FyYMKqh9vrhBaeUoyz/edx0bgJN6uS6/+k= github.com/ory/dockertest v3.3.2+incompatible h1:uO+NcwH6GuFof/Uz8yzjNi1g0sGT5SLAJbdBvD8bUYc= github.com/ory/dockertest v3.3.2+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= +github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/peterbourgon/diskv v2.0.1+incompatible h1:UBdAOUP5p4RWqPBg048CAvpKN+vxiaj6gdUUzhl4XmI= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/peterhellberg/link v1.0.0 h1:mUWkiegowUXEcmlb+ybF75Q/8D2Y0BjZtR8cxoKhaQo= github.com/peterhellberg/link v1.0.0/go.mod h1:gtSlOT4jmkY8P47hbTc8PTgiDDWpdPbFYl75keYyBB8= +github.com/pierrec/lz4 v2.0.5+incompatible h1:2xWsjqPFWcplujydGg4WmhC/6fZqK42wMM8aXeqhl0I= +github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0 h1:WdK/asTD0HN+q6hsWO3/vpuAkAr+tw6aNJNDFFf0+qw= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= @@ -344,16 +463,28 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/cachecontrol v0.0.0-20160421231612-c97913dcbd76 h1:O60OlfVScwx/OixpMy8gIPeKNIN3bI9BrOuTIUexlbc= github.com/pquerna/cachecontrol v0.0.0-20160421231612-c97913dcbd76/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/pquerna/otp v1.1.0 h1:q2gMsMuMl3JzneUaAX1MRGxLvOG6bzXV51hivBaStf0= +github.com/pquerna/otp v1.1.0/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk= +github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= +github.com/prometheus/client_golang v0.9.0-pre1 h1:AWTOhsOI9qxeirTuA0A4By/1Es1+y9EcCGY6bBZ2fhM= github.com/prometheus/client_golang v0.9.0-pre1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.2 h1:awm861/B8OKDd2I/6o1dy3ra4BamzKhYOiGItCeZ740= github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_model v0.0.0-20150212101744-fa8ad6fec335/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910 h1:idejC8f05m9MGOsuEi1ATq9shN03HrxNkD/luQvxCv8= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= +github.com/prometheus/common v0.0.0-20170220103846-49fee292b27b h1:nure2StBXEgV+CtAOZSggLGJ7bfuSfvuitPnwEQSKWQ= github.com/prometheus/common v0.0.0-20170220103846-49fee292b27b/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20170220103846-49fee292b27b/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e h1:n/3MEhJQjQxrOUCzh1Y3Re6aJUUWRp2M9+Oc3eVn/54= +github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275 h1:PnBWHBf+6L0jOqq0gIVUe6Yk0/QMZ640k6NvkxcBf+8= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= +github.com/prometheus/procfs v0.0.0-20170216223256-a1dba9ce8bae h1:nbLP9B5vU3a/0hOXzolmZHxr2SQ2MEu6vhZappUZY9c= +github.com/prometheus/procfs v0.0.0-20170216223256-a1dba9ce8bae/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20170216223256-a1dba9ce8bae/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273 h1:agujYaXJSxSo18YNX3jzl+4G6Bstwt+kqv47GS12uL0= +github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a h1:9a8MnZMP0X2nLJdBg+pBmGgkJlSaKC2KaQmTCk1XDtE= github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/racksec/srslog v0.0.0-20180709174129-a4725f04ec91 h1:3hihQaxFTzBL1t5bTYaPhEwL4rxD3zjSgu4afGzgQqI= @@ -362,15 +493,22 @@ github.com/russellhaering/goxmldsig v0.0.0-20170324122954-eaac44c63fe0 h1:jhWWGM github.com/russellhaering/goxmldsig v0.0.0-20170324122954-eaac44c63fe0/go.mod h1:Oz4y6ImuOQZxynhbSXk7btjEfNBtGlj2dcaOvXl2FSM= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= +github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec h1:6ncX5ko6B9LntYM0YBRXkiSaZMmLYeZ/NWcmeB43mMY= +github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sirupsen/logrus v0.0.0-20170713114250-a3f95b5c4235/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.3.0 h1:hI/7Q+DtNZ2kINb6qt/lS+IyXnHQe9e90POfeewL/ME= github.com/sirupsen/logrus v1.3.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c h1:fyKiXKO1/I/B6Y2U8T7WdQGWzwehOuGIrljPtt7YTTI= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= +github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/spf13/cobra v0.0.0-20160615143614-bc81c21bd0d8/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.3 h1:ZlrZ4XsMRm04Fr5pSFxBgfND2EBVa1nLpiy1stUsX/8= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= @@ -380,7 +518,10 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/square/certstrap v1.1.1 h1:GAzfXUcuior+dIWBo0OYODROz71P3mf0B9AQsVX0FSI= github.com/square/certstrap v1.1.1/go.mod h1:1+xoDwJbjCv1e3erNygZ/sHwgq8dr8CgQB3M5mMI6ds= +github.com/streadway/amqp v0.0.0-20190225234609-30f8ed68076e h1:IsT9JYWmthEsrdMpyp2ISwNIokvp2QDZcvcyPvFf7Ng= +github.com/streadway/amqp v0.0.0-20190225234609-30f8ed68076e/go.mod h1:1WNBiOZtZQLpVAyu0iTduoJL9hEsMloAK5XWrtW0xdY= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= @@ -392,6 +533,8 @@ github.com/tedsuo/rata v1.0.1-0.20170830210128-07d200713958 h1:mueRRuRjR35dEOkHd github.com/tedsuo/rata v1.0.1-0.20170830210128-07d200713958/go.mod h1:X47ELzhOoLbfFIY0Cql9P6yo3Cdwf2CMX3FVZxRzJPc= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926 h1:G3dpKMzFDjgEh2q1Z7zUUtKa8ViPtH+ocF0bE0g00O8= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= +github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2 h1:EICbibRW4JNKMcY+LsWmuwob+CRS1BmdRdjphAm9mH4= +github.com/ugorji/go/codec v0.0.0-20181209151446-772ced7fd4c2/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/vito/go-interact v0.0.0-20171111012221-fa338ed9e9ec h1:Klu98tQ9Z1t23gvC7p7sCmvxkZxLhBHLNyrUPsWsYFg= github.com/vito/go-interact v0.0.0-20171111012221-fa338ed9e9ec/go.mod h1:wPlfmglZmRWMYv/qJy3P+fK/UnoQB5ISk4txfNd9tDo= github.com/vito/go-sse v0.0.0-20160212001227-fd69d275caac h1:W7dFvBGUW6cNTGkOzxnb/zIPTrJiM/biDuycvlo3/ek= @@ -400,28 +543,44 @@ github.com/vito/houdini v1.1.1 h1:6oYBTNNcJAtd/WxrxokyFr+WXxoBN43IL7r2TYuGrHg= github.com/vito/houdini v1.1.1/go.mod h1:9T7AKNV+G5ckCYdDeGEnonREED7IqPz/LTKIkIz3MM4= github.com/vito/twentythousandtonnesofcrudeoil v0.0.0-20180305154709-3b21ad808fcb h1:OlLJlAyoyYjUAQIbycSXFSyYSC7tBf0Fos2Fyu7GeDY= github.com/vito/twentythousandtonnesofcrudeoil v0.0.0-20180305154709-3b21ad808fcb/go.mod h1:NhbXFsoSa/atpWDPj5okj24Vnvd5bgEW+W5+hFAFodA= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= +github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= +go.opencensus.io v0.18.0 h1:Mk5rgZcggtbvtAun5aJzAtjKKN/t0R3jJPlWILlv938= +go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= +go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b h1:6ayHMBPtdP3jNuk+Sfhso+PTB7ZJQ5E1FBo403m2H8w= +go.opencensus.io v0.18.1-0.20181204023538-aab39bd6a98b/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= golang.org/x/crypto v0.0.0-20160711182412-2c99acdd1e9b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180820150726-614d502a4dac/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180910181607-0e37d006457b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b h1:Elez2XeF2p9uyVj0yEUDqQ56NFcDtcBNkYP7yv8YbUE= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20181023182221-1baf3a9d7d67/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/net v0.0.0-20170413175226-5602c733f70a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180911220305-26e67e76b6c3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181106065722-10aee1819953/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3 h1:ulvT7fqt0yHWzpJwI57MezWnYDVpCAYBVuYst/L+fAY= golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/oauth2 v0.0.0-20160718223228-08c8d727d239/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be h1:vEDujvNQGv4jgYKudGeI/+DAX4Jffq6hpD55MmoEvKs= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890 h1:uESlIz09WIHT2I+pasSXcpLYqYK8wHcdCetU3VuMBJE= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f h1:Bl/8QSvNqXvPGPGXa2z5xUTmV7VDcZyvRZ+QQXkXTZQ= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4 h1:YUO/7uOKsKeq9UokNS62b8FYywz3ker1l1vDZRCRefw= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20151211033651-833a04a10549/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180828065106-d99a578cf41b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180918153733-ee1b12c67af4/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -434,13 +593,31 @@ golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2 h1:+DCIGbF/swA92ohVg0//6X2IVY3KZs6p9mix0ziNYJM= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181024171208-a2dc47679d30 h1:iZIABIEHjQFp5zqGZgQiaXi5Ue5czJhXyylr2CTtdRY= golang.org/x/tools v0.0.0-20181024171208-a2dc47679d30/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +google.golang.org/api v0.0.0-20180910000450-7ca32eb868bf/go.mod h1:4mhQ8q/RsB7i+udVvVy5NUi08OU8ZlA0gRVgrF7VFY0= +google.golang.org/api v0.1.0 h1:K6z2u68e86TPdSdefXdzvXgR1zEMa+459vBSfWYAZkI= +google.golang.org/api v0.1.0/go.mod h1:UGEZY7KEX120AnNLIHFMKIo4obdJhkp2tPbaPlQx13Y= google.golang.org/appengine v0.0.0-20160621060416-267c27e74922 h1:gHCUPwst+IxLPSNSG+jClGv+Dz5Tly/Y8D73bvQwIXM= google.golang.org/appengine v0.0.0-20160621060416-267c27e74922/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0 h1:FBSsiFRMz3LBeXIomRnVzrQwSDj4ibvcRexLG0LZGQk= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20170404132009-411e09b969b1 h1:HEurpBgyZQ15ngKW89Tw1gNMezbyH69oBSyrKIBvOLw= google.golang.org/genproto v0.0.0-20170404132009-411e09b969b1/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= +google.golang.org/genproto v0.0.0-20181202183823-bd91e49a0898/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= +google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f h1:eT3B0O2ghdSPzjAOznr3oOLyN1HFeYUncYl7FRwg4VI= +google.golang.org/genproto v0.0.0-20181221175505-bd9b4fb69e2f/go.mod h1:7Ep/1NZk928CDR8SjdVbjWNpdIf6nzjE3BTgJDr2Atg= google.golang.org/grpc v0.0.0-20170413033559-0e8b58d22f34 h1:iKtEgI7FElNp1rHnHbtWcwcLRfHR7+PomIFwnVpYjHo= google.golang.org/grpc v0.0.0-20170413033559-0e8b58d22f34/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= +google.golang.org/grpc v1.15.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= +google.golang.org/grpc v1.17.0 h1:TRJYBgMclJvGYn2rIMjj+h9KtMt5r1Ij7ODVRIZkwhk= +google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175 h1:nn6Zav2sOQHCFJHEspya8KqxhFwKci30UxHy3HXPTyQ= gopkg.in/asn1-ber.v1 v1.0.0-20150924051756-4e86f4367175/go.mod h1:cuepJuh7vyXfUyUwEgHQXw849cJrilpS5NeIjOWESAw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -448,14 +625,20 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/cheggaaa/pb.v1 v1.0.27 h1:kJdccidYzt3CaHD1crCFTS1hxyhSi059NhOFUf03YFo= gopkg.in/cheggaaa/pb.v1 v1.0.27/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/fatih/pool.v2 v2.0.0 h1:xIFeWtxifuQJGk/IEPKsTduEKcKvPmhoiVDGpC40nKg= +gopkg.in/fatih/pool.v2 v2.0.0/go.mod h1:8xVGeu1/2jr2wm5V9SPuMht2H5AEmf5aFMGSQixtjTY= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/gorethink/gorethink.v4 v4.1.0 h1:xoE9qJ9Ae9KdKEsiQGCF44u2JdnjyohrMBRDtts3Gjw= +gopkg.in/gorethink/gorethink.v4 v4.1.0/go.mod h1:M7JgwrUAmshJ3iUbEK0Pt049MPyPK+CYDGGaEjdZb/c= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ldap.v2 v2.5.1 h1:wiu0okdNfjlBzg6UWvd1Hn8Y+Ux17/u/4nlk4CQr6tU= gopkg.in/ldap.v2 v2.5.1/go.mod h1:oI0cpe/D7HRtBQl8aTg+ZmzFUAvu4lsv3eLXMLGFxWk= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce h1:xcEWjVhvbDy+nHP67nPDDpbYrY+ILlfndk4bRioVHaU= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= +gopkg.in/ory-am/dockertest.v2 v2.2.3 h1:vSYvP7tvyfAm9merq0gHmcI4yk5nkPpfXmoBCnSP3/4= +gopkg.in/ory-am/dockertest.v2 v2.2.3/go.mod h1:kDHEsan1UcKFYH1c28sDmqnmeqIpB4Nj682gSNhYDYM= gopkg.in/square/go-jose.v2 v2.1.8 h1:yECBkTX7ypNaRFILw4trAAYXRLvcGxTeHCBKj/fc8gU= gopkg.in/square/go-jose.v2 v2.1.8/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.0 h1:nLzhkFyl5bkblqYBoiWJUt5JkWOzmiaBtCxdJAqJd3U= @@ -468,6 +651,7 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.1.0+incompatible h1:5USw7CrJBYKqjg9R7QlA6jzqZKEAtvW82aNmsxxGPxw= gotest.tools v2.1.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= +honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= k8s.io/api v0.0.0-20171027084545-218912509d74 h1:CYism0UbF96TF8s8sYrKagsJn3oqR456q/ER/cELIuA= k8s.io/api v0.0.0-20171027084545-218912509d74/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= k8s.io/apimachinery v0.0.0-20171027084411-18a564baac72 h1:pPbfmsjOvePqojmf5AhR0o7bxZTCxE0nkDakanV50CM= @@ -476,3 +660,5 @@ k8s.io/client-go v2.0.0-alpha.0.0.20171101191150-72e1c2a1ef30+incompatible h1:mF k8s.io/client-go v2.0.0-alpha.0.0.20171101191150-72e1c2a1ef30+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c h1:3KSCztE7gPitlZmWbNwue/2U0YruD65DqX3INopDAQM= k8s.io/kube-openapi v0.0.0-20180731170545-e3762e86a74c/go.mod h1:BXM9ceUBTj2QnfH2MK1odQs778ajze1RxcmP6S8RVVc= +layeh.com/radius v0.0.0-20190101232339-d3a4fc175dc9 h1:hcd14Os1u5SRB18e0TkmySNtvb8sCwZHkAD7w8Hr0B8= +layeh.com/radius v0.0.0-20190101232339-d3a4fc175dc9/go.mod h1:fywZKyu//X7iRzaxLgPWsvc0L26IUpVvE/aeIL2JtIQ= From 4f458e39d2442ad1995d0a4408b8a2fbc14336cb Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Mon, 14 Jan 2019 16:34:15 -0500 Subject: [PATCH 06/21] atc: add artifact collector Co-authored-by: Krishna Mannem Signed-off-by: Josh Winters --- atc/atccmd/command.go | 2 + .../dbfakes/fake_worker_artifact_lifecycle.go | 101 ++++++++++++++++++ atc/db/worker_artifact_lifecycle.go | 31 ++++++ atc/db/worker_artifact_lifecycle_test.go | 55 ++++++++++ atc/exec/user_artifact_step_test.go | 2 - atc/gc/artifacts_collector.go | 27 +++++ atc/gc/artifacts_collector_test.go | 30 ++++++ atc/gc/collector.go | 8 ++ atc/gc/collector_test.go | 68 +++++++++--- .../internal/executehelpers/inputs.go | 3 +- go.mod | 1 + go.sum | 2 + 12 files changed, 310 insertions(+), 20 deletions(-) create mode 100644 atc/db/dbfakes/fake_worker_artifact_lifecycle.go create mode 100644 atc/db/worker_artifact_lifecycle.go create mode 100644 atc/db/worker_artifact_lifecycle_test.go create mode 100644 atc/gc/artifacts_collector.go create mode 100644 atc/gc/artifacts_collector_test.go diff --git a/atc/atccmd/command.go b/atc/atccmd/command.go index c3862709495..1d9a4e28374 100644 --- a/atc/atccmd/command.go +++ b/atc/atccmd/command.go @@ -791,6 +791,7 @@ func (cmd *RunCommand) constructBackendMembers( dbWorkerLifecycle := db.NewWorkerLifecycle(dbConn) dbResourceCacheLifecycle := db.NewResourceCacheLifecycle(dbConn) dbContainerRepository := db.NewContainerRepository(dbConn) + dbArtifactLifecycle := db.NewArtifactLifecycle(dbConn) resourceConfigCheckSessionLifecycle := db.NewResourceConfigCheckSessionLifecycle(dbConn) dbBuildFactory := db.NewBuildFactory(dbConn, lockFactory, cmd.GC.OneOffBuildGracePeriod) bus := dbConn.Bus() @@ -836,6 +837,7 @@ func (cmd *RunCommand) constructBackendMembers( gc.NewResourceCacheUseCollector(dbResourceCacheLifecycle), gc.NewResourceConfigCollector(dbResourceConfigFactory), gc.NewResourceCacheCollector(dbResourceCacheLifecycle), + gc.NewArtifactCollector(dbArtifactLifecycle), gc.NewVolumeCollector( dbVolumeRepository, cmd.GC.MissingGracePeriod, diff --git a/atc/db/dbfakes/fake_worker_artifact_lifecycle.go b/atc/db/dbfakes/fake_worker_artifact_lifecycle.go new file mode 100644 index 00000000000..37b3b178951 --- /dev/null +++ b/atc/db/dbfakes/fake_worker_artifact_lifecycle.go @@ -0,0 +1,101 @@ +// Code generated by counterfeiter. DO NOT EDIT. +package dbfakes + +import ( + sync "sync" + + db "github.com/concourse/concourse/atc/db" +) + +type FakeWorkerArtifactLifecycle struct { + RemoveExpiredArtifactsStub func() error + removeExpiredArtifactsMutex sync.RWMutex + removeExpiredArtifactsArgsForCall []struct { + } + removeExpiredArtifactsReturns struct { + result1 error + } + removeExpiredArtifactsReturnsOnCall map[int]struct { + result1 error + } + invocations map[string][][]interface{} + invocationsMutex sync.RWMutex +} + +func (fake *FakeWorkerArtifactLifecycle) RemoveExpiredArtifacts() error { + fake.removeExpiredArtifactsMutex.Lock() + ret, specificReturn := fake.removeExpiredArtifactsReturnsOnCall[len(fake.removeExpiredArtifactsArgsForCall)] + fake.removeExpiredArtifactsArgsForCall = append(fake.removeExpiredArtifactsArgsForCall, struct { + }{}) + fake.recordInvocation("RemoveExpiredArtifacts", []interface{}{}) + fake.removeExpiredArtifactsMutex.Unlock() + if fake.RemoveExpiredArtifactsStub != nil { + return fake.RemoveExpiredArtifactsStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.removeExpiredArtifactsReturns + return fakeReturns.result1 +} + +func (fake *FakeWorkerArtifactLifecycle) RemoveExpiredArtifactsCallCount() int { + fake.removeExpiredArtifactsMutex.RLock() + defer fake.removeExpiredArtifactsMutex.RUnlock() + return len(fake.removeExpiredArtifactsArgsForCall) +} + +func (fake *FakeWorkerArtifactLifecycle) RemoveExpiredArtifactsCalls(stub func() error) { + fake.removeExpiredArtifactsMutex.Lock() + defer fake.removeExpiredArtifactsMutex.Unlock() + fake.RemoveExpiredArtifactsStub = stub +} + +func (fake *FakeWorkerArtifactLifecycle) RemoveExpiredArtifactsReturns(result1 error) { + fake.removeExpiredArtifactsMutex.Lock() + defer fake.removeExpiredArtifactsMutex.Unlock() + fake.RemoveExpiredArtifactsStub = nil + fake.removeExpiredArtifactsReturns = struct { + result1 error + }{result1} +} + +func (fake *FakeWorkerArtifactLifecycle) RemoveExpiredArtifactsReturnsOnCall(i int, result1 error) { + fake.removeExpiredArtifactsMutex.Lock() + defer fake.removeExpiredArtifactsMutex.Unlock() + fake.RemoveExpiredArtifactsStub = nil + if fake.removeExpiredArtifactsReturnsOnCall == nil { + fake.removeExpiredArtifactsReturnsOnCall = make(map[int]struct { + result1 error + }) + } + fake.removeExpiredArtifactsReturnsOnCall[i] = struct { + result1 error + }{result1} +} + +func (fake *FakeWorkerArtifactLifecycle) Invocations() map[string][][]interface{} { + fake.invocationsMutex.RLock() + defer fake.invocationsMutex.RUnlock() + fake.removeExpiredArtifactsMutex.RLock() + defer fake.removeExpiredArtifactsMutex.RUnlock() + copiedInvocations := map[string][][]interface{}{} + for key, value := range fake.invocations { + copiedInvocations[key] = value + } + return copiedInvocations +} + +func (fake *FakeWorkerArtifactLifecycle) recordInvocation(key string, args []interface{}) { + fake.invocationsMutex.Lock() + defer fake.invocationsMutex.Unlock() + if fake.invocations == nil { + fake.invocations = map[string][][]interface{}{} + } + if fake.invocations[key] == nil { + fake.invocations[key] = [][]interface{}{} + } + fake.invocations[key] = append(fake.invocations[key], args) +} + +var _ db.WorkerArtifactLifecycle = new(FakeWorkerArtifactLifecycle) diff --git a/atc/db/worker_artifact_lifecycle.go b/atc/db/worker_artifact_lifecycle.go new file mode 100644 index 00000000000..8dd46bf00a4 --- /dev/null +++ b/atc/db/worker_artifact_lifecycle.go @@ -0,0 +1,31 @@ +package db + +import ( + sq "github.com/Masterminds/squirrel" +) + +//go:generate counterfeiter . WorkerArtifactLifecycle + +type WorkerArtifactLifecycle interface { + RemoveExpiredArtifacts() error +} + +type artifactLifecycle struct { + conn Conn +} + +func NewArtifactLifecycle(conn Conn) *artifactLifecycle { + return &artifactLifecycle{ + conn: conn, + } +} + +func (lifecycle *artifactLifecycle) RemoveExpiredArtifacts() error { + + _, err := psql.Delete("worker_artifacts"). + Where(sq.Expr("created_at < NOW() - interval '12 hours'")). + RunWith(lifecycle.conn). + Exec() + + return err +} diff --git a/atc/db/worker_artifact_lifecycle_test.go b/atc/db/worker_artifact_lifecycle_test.go new file mode 100644 index 00000000000..99c70d19aeb --- /dev/null +++ b/atc/db/worker_artifact_lifecycle_test.go @@ -0,0 +1,55 @@ +package db_test + +import ( + "github.com/concourse/concourse/atc/db" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("WorkerArtifactLifecycle", func() { + var workerArtifactLifecycle db.WorkerArtifactLifecycle + + BeforeEach(func() { + workerArtifactLifecycle = db.NewArtifactLifecycle(dbConn) + }) + + Describe("RemoveExpiredArtifacts", func() { + JustBeforeEach(func() { + err := workerArtifactLifecycle.RemoveExpiredArtifacts() + Expect(err).ToNot(HaveOccurred()) + }) + + Context("removes artifacts created more than 12 hours ago", func() { + + BeforeEach(func() { + _, err := dbConn.Exec("INSERT INTO worker_artifacts(name, created_at) VALUES('some-name', NOW() - '13 hours'::interval)") + Expect(err).ToNot(HaveOccurred()) + }) + + It("removes the record", func() { + var count int + err := dbConn.QueryRow("SELECT count(*) from worker_artifacts").Scan(&count) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(0)) + }) + }) + + Context("keeps artifacts for 12 hours", func() { + + BeforeEach(func() { + _, err := dbConn.Exec("INSERT INTO worker_artifacts(name, created_at) VALUES('some-name', NOW() - '13 hours'::interval)") + Expect(err).ToNot(HaveOccurred()) + + _, err = dbConn.Exec("INSERT INTO worker_artifacts(name, created_at) VALUES('some-other-name', NOW())") + Expect(err).ToNot(HaveOccurred()) + }) + + It("does not remove the record", func() { + var count int + err := dbConn.QueryRow("SELECT count(*) from worker_artifacts").Scan(&count) + Expect(err).ToNot(HaveOccurred()) + Expect(count).To(Equal(1)) + }) + }) + }) +}) diff --git a/atc/exec/user_artifact_step_test.go b/atc/exec/user_artifact_step_test.go index 23ebf90944a..5e935d89b85 100644 --- a/atc/exec/user_artifact_step_test.go +++ b/atc/exec/user_artifact_step_test.go @@ -18,7 +18,6 @@ var _ = Describe("ArtifactStep", func() { var ( ctx context.Context cancel func() - // logger *lagertest.TestLogger state exec.RunState delegate *execfakes.FakeBuildStepDelegate @@ -32,7 +31,6 @@ var _ = Describe("ArtifactStep", func() { BeforeEach(func() { ctx, cancel = context.WithCancel(context.Background()) - // logger = lagertest.NewTestLogger("user-artifact-step-test") state = exec.NewRunState() diff --git a/atc/gc/artifacts_collector.go b/atc/gc/artifacts_collector.go new file mode 100644 index 00000000000..f9ba4ce34ea --- /dev/null +++ b/atc/gc/artifacts_collector.go @@ -0,0 +1,27 @@ +package gc + +import ( + "context" + + "code.cloudfoundry.org/lager/lagerctx" + "github.com/concourse/concourse/atc/db" +) + +type artifactCollector struct { + artifactLifecycle db.WorkerArtifactLifecycle +} + +func NewArtifactCollector(artifactLifecycle db.WorkerArtifactLifecycle) *artifactCollector { + return &artifactCollector{ + artifactLifecycle: artifactLifecycle, + } +} + +func (a *artifactCollector) Run(ctx context.Context) error { + logger := lagerctx.FromContext(ctx).Session("artifact-collector") + + logger.Debug("start") + defer logger.Debug("done") + + return a.artifactLifecycle.RemoveExpiredArtifacts() +} diff --git a/atc/gc/artifacts_collector_test.go b/atc/gc/artifacts_collector_test.go new file mode 100644 index 00000000000..61419cdfc1e --- /dev/null +++ b/atc/gc/artifacts_collector_test.go @@ -0,0 +1,30 @@ +package gc_test + +import ( + "context" + + "github.com/concourse/concourse/atc/db/dbfakes" + "github.com/concourse/concourse/atc/gc" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("ArtifactCollector", func() { + var collector gc.Collector + var fakeArtifactLifecycle *dbfakes.FakeWorkerArtifactLifecycle + + BeforeEach(func() { + fakeArtifactLifecycle = new(dbfakes.FakeWorkerArtifactLifecycle) + + collector = gc.NewArtifactCollector(fakeArtifactLifecycle) + }) + + Describe("Run", func() { + It("tells the artifact lifecycle to remove expired artifacts", func() { + err := collector.Run(context.TODO()) + Expect(err).NotTo(HaveOccurred()) + + Expect(fakeArtifactLifecycle.RemoveExpiredArtifactsCallCount()).To(Equal(1)) + }) + }) +}) diff --git a/atc/gc/collector.go b/atc/gc/collector.go index 3261f0f8139..c0c3d0ade37 100644 --- a/atc/gc/collector.go +++ b/atc/gc/collector.go @@ -21,6 +21,7 @@ type aggregateCollector struct { volumeCollector Collector containerCollector Collector resourceConfigCheckSessionCollector Collector + artifactCollector Collector } func NewCollector( @@ -29,6 +30,7 @@ func NewCollector( resourceCacheUses Collector, resourceConfigs Collector, resourceCaches Collector, + artifactCollector Collector, volumes Collector, containers Collector, resourceConfigCheckSessionCollector Collector, @@ -39,6 +41,7 @@ func NewCollector( resourceCacheUseCollector: resourceCacheUses, resourceConfigCollector: resourceConfigs, resourceCacheCollector: resourceCaches, + artifactCollector: artifactCollector, volumeCollector: volumes, containerCollector: containers, resourceConfigCheckSessionCollector: resourceConfigCheckSessionCollector, @@ -80,6 +83,11 @@ func (c *aggregateCollector) Run(ctx context.Context) error { logger.Error("resource-config-check-session-collector", err) } + err = c.artifactCollector.Run(ctx) + if err != nil { + logger.Error("artifact-collector", err) + } + err = c.containerCollector.Run(ctx) if err != nil { logger.Error("container-collector", err) diff --git a/atc/gc/collector_test.go b/atc/gc/collector_test.go index 9ef2ca352c7..92db0c0d201 100644 --- a/atc/gc/collector_test.go +++ b/atc/gc/collector_test.go @@ -6,7 +6,6 @@ import ( . "github.com/concourse/concourse/atc/gc" "github.com/concourse/concourse/atc/gc/gcfakes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -20,6 +19,7 @@ var _ = Describe("Aggregate Collector", func() { fakeResourceCacheUseCollector *gcfakes.FakeCollector fakeResourceConfigCollector *gcfakes.FakeCollector fakeResourceCacheCollector *gcfakes.FakeCollector + fakeArtifactCollector *gcfakes.FakeCollector fakeVolumeCollector *gcfakes.FakeCollector fakeContainerCollector *gcfakes.FakeCollector fakeResourceConfigCheckSessionCollector *gcfakes.FakeCollector @@ -34,6 +34,7 @@ var _ = Describe("Aggregate Collector", func() { fakeResourceCacheUseCollector = new(gcfakes.FakeCollector) fakeResourceConfigCollector = new(gcfakes.FakeCollector) fakeResourceCacheCollector = new(gcfakes.FakeCollector) + fakeArtifactCollector = new(gcfakes.FakeCollector) fakeVolumeCollector = new(gcfakes.FakeCollector) fakeContainerCollector = new(gcfakes.FakeCollector) fakeResourceConfigCheckSessionCollector = new(gcfakes.FakeCollector) @@ -44,6 +45,7 @@ var _ = Describe("Aggregate Collector", func() { fakeResourceCacheUseCollector, fakeResourceConfigCollector, fakeResourceCacheCollector, + fakeArtifactCollector, fakeVolumeCollector, fakeContainerCollector, fakeResourceConfigCheckSessionCollector, @@ -57,11 +59,11 @@ var _ = Describe("Aggregate Collector", func() { err = subject.Run(context.TODO()) }) - It("runs the build collector", func() { + It("runs the collector", func() { Expect(fakeBuildCollector.RunCallCount()).To(Equal(1)) }) - Context("when the build collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeBuildCollector.RunReturns(disaster) }) @@ -75,6 +77,7 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) Expect(fakeContainerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) @@ -82,11 +85,11 @@ var _ = Describe("Aggregate Collector", func() { }) Context("when the build collector succeeds", func() { - It("attempts to collect workers", func() { + It("attempts to collect", func() { Expect(fakeWorkerCollector.RunCallCount()).To(Equal(1)) }) - Context("when the worker collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeWorkerCollector.RunReturns(disaster) }) @@ -99,6 +102,7 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) Expect(fakeContainerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) @@ -106,11 +110,11 @@ var _ = Describe("Aggregate Collector", func() { }) Context("when the worker collector succeeds", func() { - It("attempts to collect cache uses", func() { + It("attempts to collect", func() { Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) }) - Context("when the cache use collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeResourceCacheUseCollector.RunReturns(disaster) }) @@ -123,6 +127,7 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeWorkerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) Expect(fakeContainerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) @@ -130,11 +135,11 @@ var _ = Describe("Aggregate Collector", func() { }) Context("when the cache use collector succeeds", func() { - It("attempts to collect configs", func() { + It("attempts to collect", func() { Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) }) - Context("when the config collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeResourceConfigCollector.RunReturns(disaster) }) @@ -147,6 +152,7 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeWorkerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) Expect(fakeContainerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) @@ -154,11 +160,11 @@ var _ = Describe("Aggregate Collector", func() { }) Context("when the config collector succeeds", func() { - It("attempts to collect caches", func() { + It("attempts to collect", func() { Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) }) - Context("when the cache collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeResourceCacheCollector.RunReturns(disaster) }) @@ -171,6 +177,7 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeWorkerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) Expect(fakeContainerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) @@ -178,11 +185,11 @@ var _ = Describe("Aggregate Collector", func() { }) Context("when the config use collector succeeds", func() { - It("attempts to collect volumes", func() { + It("attempts to collect", func() { Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) }) - Context("when the volume collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeVolumeCollector.RunReturns(disaster) }) @@ -196,17 +203,18 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeContainerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) }) }) Context("when the volume collector succeeds", func() { - It("attempts to collect containers", func() { + It("attempts to collect", func() { Expect(fakeContainerCollector.RunCallCount()).To(Equal(1)) }) - Context("when the container collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeContainerCollector.RunReturns(disaster) }) @@ -219,17 +227,18 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeWorkerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) }) }) Context("when the resource config check session collector succeeds", func() { - It("attempts to collect containers", func() { + It("attempts to collect", func() { Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) }) - Context("when the container collector errors", func() { + Context("when the collector errors", func() { BeforeEach(func() { fakeResourceConfigCheckSessionCollector.RunReturns(disaster) }) @@ -242,10 +251,35 @@ var _ = Describe("Aggregate Collector", func() { Expect(fakeWorkerCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) }) }) + Context("when the artifact collector succeeds", func() { + It("attempts to collect", func() { + Expect(fakeArtifactCollector.RunCallCount()).To(Equal(1)) + }) + + Context("when the collector errors", func() { + BeforeEach(func() { + fakeArtifactCollector.RunReturns(disaster) + }) + + It("does not return an error", func() { + Expect(err).NotTo(HaveOccurred()) + }) + + It("runs the rest of collectors", func() { + Expect(fakeWorkerCollector.RunCallCount()).To(Equal(1)) + Expect(fakeResourceCacheUseCollector.RunCallCount()).To(Equal(1)) + Expect(fakeResourceConfigCollector.RunCallCount()).To(Equal(1)) + Expect(fakeResourceCacheCollector.RunCallCount()).To(Equal(1)) + Expect(fakeResourceConfigCheckSessionCollector.RunCallCount()).To(Equal(1)) + Expect(fakeVolumeCollector.RunCallCount()).To(Equal(1)) + }) + }) + }) }) }) }) diff --git a/fly/commands/internal/executehelpers/inputs.go b/fly/commands/internal/executehelpers/inputs.go index 285df009441..b824dbd13d4 100644 --- a/fly/commands/internal/executehelpers/inputs.go +++ b/fly/commands/internal/executehelpers/inputs.go @@ -135,7 +135,8 @@ func GenerateLocalInputs( fact atc.PlanFactory, team concourse.Team, inputMappings []flaghelpers.InputPairFlag, - includeIgnored bool) (map[string]Input, error) { + includeIgnored bool, +) (map[string]Input, error) { kvMap := map[string]Input{} diff --git a/go.mod b/go.mod index 8335b9a0de1..87b730f60f4 100644 --- a/go.mod +++ b/go.mod @@ -187,6 +187,7 @@ require ( github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect github.com/sirupsen/logrus v1.3.0 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c + github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 // indirect github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa // indirect github.com/spf13/pflag v1.0.3 // indirect github.com/square/certstrap v1.1.1 diff --git a/go.sum b/go.sum index cc96da68cfb..247c00fed8c 100644 --- a/go.sum +++ b/go.sum @@ -507,6 +507,8 @@ github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c h1:fyKiXKO1 github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304 h1:Jpy1PXuP99tXNrhbq2BaPz9B+jNAvH1JPQQpG/9GCXY= +github.com/smartystreets/assertions v0.0.0-20190116191733-b6c0e53d7304/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa h1:E+gaaifzi2xF65PbDmuKI3PhLWY6G5opMLniFq8vmXA= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/spf13/cobra v0.0.0-20160615143614-bc81c21bd0d8/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= From edc95c29a66a0c88c161ae0e4afa0060494fa83f Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 28 Feb 2019 14:49:02 -0500 Subject: [PATCH 07/21] atc: bump migrations Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- ...facts.down.sql => 1551383253_create_worker_artifacts.down.sql} | 0 ...artifacts.up.sql => 1551383253_create_worker_artifacts.up.sql} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename atc/db/migration/migrations/{1550159394_create_worker_artifacts.down.sql => 1551383253_create_worker_artifacts.down.sql} (100%) rename atc/db/migration/migrations/{1550159394_create_worker_artifacts.up.sql => 1551383253_create_worker_artifacts.up.sql} (100%) diff --git a/atc/db/migration/migrations/1550159394_create_worker_artifacts.down.sql b/atc/db/migration/migrations/1551383253_create_worker_artifacts.down.sql similarity index 100% rename from atc/db/migration/migrations/1550159394_create_worker_artifacts.down.sql rename to atc/db/migration/migrations/1551383253_create_worker_artifacts.down.sql diff --git a/atc/db/migration/migrations/1550159394_create_worker_artifacts.up.sql b/atc/db/migration/migrations/1551383253_create_worker_artifacts.up.sql similarity index 100% rename from atc/db/migration/migrations/1550159394_create_worker_artifacts.up.sql rename to atc/db/migration/migrations/1551383253_create_worker_artifacts.up.sql From ba51e4357ab974a6ba4a946fe614ce836df9fc73 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Tue, 12 Feb 2019 12:47:15 -0500 Subject: [PATCH 08/21] atc: fly exec builds get run by the scheduler instead of a calling enginebuild.Resume in the handler *wip note* we are currently hardcoding the engine as 'exec.v2' when creating started builds in the databse. We should probably leave this up to the dbengine to update the engine field if its empty. we should also probably change the engine_metadata to private_plan and reintroduce a metadata field later if needed. concourse/concourse#3240 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/api/builds_test.go | 138 ++++++++++---------------- atc/api/buildserver/create.go | 11 +-- atc/api/pipelines_test.go | 166 +++++++++++--------------------- atc/api/pipelineserver/build.go | 14 +-- atc/db/build.go | 2 +- atc/db/dbfakes/fake_pipeline.go | 78 +++++++++++++++ atc/db/dbfakes/fake_team.go | 78 +++++++++++++++ atc/db/pipeline.go | 55 ++++++++++- atc/db/pipeline_test.go | 71 ++++++++++++++ atc/db/team.go | 52 ++++++++++ atc/db/team_test.go | 71 ++++++++++++++ 11 files changed, 518 insertions(+), 218 deletions(-) diff --git a/atc/api/builds_test.go b/atc/api/builds_test.go index 133bff9aac7..8caf98a10bc 100644 --- a/atc/api/builds_test.go +++ b/atc/api/builds_test.go @@ -7,10 +7,8 @@ import ( "fmt" "io/ioutil" "net/http" - "sync" "time" - "code.cloudfoundry.org/lager" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/api/accessor/accessorfakes" "github.com/concourse/concourse/atc/db" @@ -36,7 +34,6 @@ var _ = Describe("Builds API", func() { var response *http.Response BeforeEach(func() { - fakeaccess = new(accessorfakes.FakeAccess) plan = atc.Plan{ Task: &atc.TaskPlan{ Config: &atc.TaskConfig{ @@ -46,7 +43,6 @@ var _ = Describe("Builds API", func() { }, }, } - fakeaccess.IsAuthenticatedReturns(true) }) JustBeforeEach(func() { @@ -64,52 +60,64 @@ var _ = Describe("Builds API", func() { Expect(err).NotTo(HaveOccurred()) }) - Context("when authorized", func() { + Context("when not authenticated", func() { + BeforeEach(func() { + fakeaccess.IsAuthenticatedReturns(false) + }) + + It("returns 401", func() { + Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("does not trigger a build", func() { + Expect(dbTeam.CreateStartedBuildCallCount()).To(BeZero()) + }) + }) + + Context("when authenticated", func() { BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(true) + fakeaccess.IsAuthenticatedReturns(true) }) - Context("when creating a one-off build succeeds", func() { + Context("when not authorized", func() { BeforeEach(func() { - dbTeam.CreateOneOffBuildStub = func() (db.Build, error) { - Expect(dbTeamFactory.FindTeamCallCount()).To(Equal(1)) - teamName := dbTeamFactory.FindTeamArgsForCall(0) - build.IDReturns(42) - build.NameReturns("1") - build.TeamNameReturns(teamName) - build.StatusReturns(db.BuildStatusStarted) - build.StartTimeReturns(time.Unix(1, 0)) - build.EndTimeReturns(time.Unix(100, 0)) - build.ReapTimeReturns(time.Unix(200, 0)) - return build, nil - } + fakeaccess.IsAuthorizedReturns(false) + }) + + It("returns 403", func() { + Expect(response.StatusCode).To(Equal(http.StatusForbidden)) }) + }) - Context("and building succeeds", func() { - var fakeEngineBuild *enginefakes.FakeBuild - var resumed <-chan struct{} - var blockForever *sync.WaitGroup + Context("when authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(true) + }) + Context("when creating a started build fails", func() { BeforeEach(func() { - fakeEngineBuild = new(enginefakes.FakeBuild) - - blockForever = new(sync.WaitGroup) + dbTeam.CreateStartedBuildReturns(nil, errors.New("oh no!")) + }) - forever := blockForever - forever.Add(1) + It("returns 500 Internal Server Error", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) - r := make(chan struct{}) - resumed = r - fakeEngineBuild.ResumeStub = func(lager.Logger) { - close(r) - forever.Wait() - } + Context("when creating a started build succeeds", func() { + var fakeBuild *dbfakes.FakeBuild - fakeEngine.CreateBuildReturns(fakeEngineBuild, nil) - }) + BeforeEach(func() { + fakeBuild = new(dbfakes.FakeBuild) + fakeBuild.IDReturns(42) + fakeBuild.NameReturns("1") + fakeBuild.TeamNameReturns("some-team") + fakeBuild.StatusReturns("started") + fakeBuild.StartTimeReturns(time.Unix(1, 0)) + fakeBuild.EndTimeReturns(time.Unix(100, 0)) + fakeBuild.ReapTimeReturns(time.Unix(200, 0)) - AfterEach(func() { - blockForever.Done() + dbTeam.CreateStartedBuildReturns(fakeBuild, nil) }) It("returns 201 Created", func() { @@ -120,7 +128,12 @@ var _ = Describe("Builds API", func() { Expect(response.Header.Get("Content-Type")).To(Equal("application/json")) }) - It("creates build for specified team", func() { + It("creates a started build", func() { + Expect(dbTeam.CreateStartedBuildCallCount()).To(Equal(1)) + Expect(dbTeam.CreateStartedBuildArgsForCall(0)).To(Equal(plan)) + }) + + It("returns the created build", func() { body, err := ioutil.ReadAll(response.Body) Expect(err).NotTo(HaveOccurred()) @@ -136,56 +149,11 @@ var _ = Describe("Builds API", func() { }`)) }) - It("creates a one-off build and runs it asynchronously", func() { - Expect(dbTeam.CreateOneOffBuildCallCount()).To(Equal(1)) - - Expect(fakeEngine.CreateBuildCallCount()).To(Equal(1)) - _, oneOffBuild, builtPlan := fakeEngine.CreateBuildArgsForCall(0) - Expect(oneOffBuild).To(Equal(build)) - - Expect(builtPlan).To(Equal(plan)) - - <-resumed - }) - }) - - Context("and building fails", func() { - BeforeEach(func() { - fakeEngine.CreateBuildReturns(nil, errors.New("oh no!")) - }) - - It("returns 500 Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - - Context("when creating a one-off build fails", func() { - BeforeEach(func() { - dbTeam.CreateOneOffBuildReturns(nil, errors.New("oh no!")) - }) - - It("returns 500 Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) }) }) }) - - Context("when not authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(false) - }) - - It("returns 401", func() { - Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - - It("does not trigger a build", func() { - Expect(dbTeam.CreateOneOffBuildCallCount()).To(BeZero()) - Expect(fakeEngine.CreateBuildCallCount()).To(BeZero()) - }) - }) }) + Describe("GET /api/v1/builds", func() { var response *http.Response var queryParams string diff --git a/atc/api/buildserver/create.go b/atc/api/buildserver/create.go index e34397e702a..4ae78c0891c 100644 --- a/atc/api/buildserver/create.go +++ b/atc/api/buildserver/create.go @@ -22,22 +22,13 @@ func (s *Server) CreateBuild(team db.Team) http.Handler { return } - build, err := team.CreateOneOffBuild() + build, err := team.CreateStartedBuild(plan) if err != nil { hLog.Error("failed-to-create-one-off-build", err) w.WriteHeader(http.StatusInternalServerError) return } - engineBuild, err := s.engine.CreateBuild(hLog, build, plan) - if err != nil { - hLog.Error("failed-to-start-build", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - go engineBuild.Resume(hLog) - w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) diff --git a/atc/api/pipelines_test.go b/atc/api/pipelines_test.go index 4c0de5934d5..35831a7f9f4 100644 --- a/atc/api/pipelines_test.go +++ b/atc/api/pipelines_test.go @@ -8,19 +8,15 @@ import ( "io" "io/ioutil" "net/http" - "sync" "time" - "code.cloudfoundry.org/lager" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/api/accessor/accessorfakes" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/algorithm" "github.com/concourse/concourse/atc/db/dbfakes" - "github.com/concourse/concourse/atc/engine/enginefakes" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("Pipelines API", func() { @@ -1663,19 +1659,6 @@ var _ = Describe("Pipelines API", func() { }, }, } - - dbPipeline.CreateOneOffBuildStub = func() (db.Build, error) { - Expect(dbTeamFactory.FindTeamCallCount()).To(Equal(1)) - teamName := dbTeamFactory.FindTeamArgsForCall(0) - build.IDReturns(42) - build.NameReturns("1") - build.TeamNameReturns(teamName) - build.StatusReturns(db.BuildStatusStarted) - build.StartTimeReturns(time.Unix(1, 0)) - build.EndTimeReturns(time.Unix(100, 0)) - build.ReapTimeReturns(time.Unix(200, 0)) - return build, nil - } }) JustBeforeEach(func() { @@ -1691,132 +1674,97 @@ var _ = Describe("Pipelines API", func() { Expect(err).NotTo(HaveOccurred()) }) + Context("when not authenticated", func() { + BeforeEach(func() { + fakeaccess.IsAuthenticatedReturns(false) + }) + + It("returns 401", func() { + Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + + It("does not trigger a build", func() { + Expect(dbPipeline.CreateOneOffBuildCallCount()).To(BeZero()) + }) + }) + Context("when authenticated", func() { BeforeEach(func() { fakeaccess.IsAuthenticatedReturns(true) }) - Context("when requester belongs to the team", func() { + Context("when not authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(false) + }) + + It("returns 403", func() { + Expect(response.StatusCode).To(Equal(http.StatusForbidden)) + }) + }) + + Context("when authorized", func() { BeforeEach(func() { fakeaccess.IsAuthorizedReturns(true) dbTeamFactory.FindTeamReturns(fakeTeam, true, nil) fakeTeam.PipelineReturns(dbPipeline, true, nil) }) - Context("when building succeeds", func() { - var fakeEngineBuild *enginefakes.FakeBuild - var resumed <-chan struct{} - var blockForever *sync.WaitGroup - + Context("when creating a started build fails", func() { BeforeEach(func() { - fakeEngineBuild = new(enginefakes.FakeBuild) - - blockForever = new(sync.WaitGroup) - - forever := blockForever - forever.Add(1) - - r := make(chan struct{}) - resumed = r - fakeEngineBuild.ResumeStub = func(lager.Logger) { - close(r) - forever.Wait() - } - - fakeEngine.CreateBuildReturns(fakeEngineBuild, nil) + dbPipeline.CreateStartedBuildReturns(nil, errors.New("oh no!")) }) - AfterEach(func() { - blockForever.Done() + It("returns 500 Internal Server Error", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) }) + }) - It("constructs teamDB with provided team name", func() { - Expect(dbTeamFactory.FindTeamCallCount()).To(Equal(1)) - Expect(dbTeamFactory.FindTeamArgsForCall(0)).To(Equal("a-team")) - }) + Context("when creating a started build succeeds", func() { + var fakeBuild *dbfakes.FakeBuild - It("injects the proper pipeline", func() { - pipelineName := fakeTeam.PipelineArgsForCall(0) - Expect(pipelineName).To(Equal("a-pipeline")) + BeforeEach(func() { + fakeBuild = new(dbfakes.FakeBuild) + fakeBuild.IDReturns(42) + fakeBuild.NameReturns("1") + fakeBuild.TeamNameReturns("some-team") + fakeBuild.StatusReturns("started") + fakeBuild.StartTimeReturns(time.Unix(1, 0)) + fakeBuild.EndTimeReturns(time.Unix(100, 0)) + fakeBuild.ReapTimeReturns(time.Unix(200, 0)) + + dbPipeline.CreateStartedBuildReturns(fakeBuild, nil) }) It("returns 201 Created", func() { Expect(response.StatusCode).To(Equal(http.StatusCreated)) }) - It("creates build for specified team", func() { + It("returns Content-Type 'application/json'", func() { + Expect(response.Header.Get("Content-Type")).To(Equal("application/json")) + }) + + It("creates a started build", func() { + Expect(dbPipeline.CreateStartedBuildCallCount()).To(Equal(1)) + Expect(dbPipeline.CreateStartedBuildArgsForCall(0)).To(Equal(plan)) + }) + + It("returns the created build", func() { body, err := ioutil.ReadAll(response.Body) Expect(err).NotTo(HaveOccurred()) Expect(body).To(MatchJSON(`{ "id": 42, "name": "1", - "team_name": "a-team", + "team_name": "some-team", "status": "started", "api_url": "/api/v1/builds/42", "start_time": 1, "end_time": 100, "reap_time": 200 - }`)) - }) - - It("creates a one-off build and runs it asynchronously", func() { - Expect(dbPipeline.CreateOneOffBuildCallCount()).To(Equal(1)) - - Expect(fakeEngine.CreateBuildCallCount()).To(Equal(1)) - _, oneOffBuild, builtPlan := fakeEngine.CreateBuildArgsForCall(0) - Expect(oneOffBuild).To(Equal(build)) - - Expect(builtPlan).To(Equal(plan)) - - <-resumed + }`)) }) }) - - Context("and building fails", func() { - BeforeEach(func() { - fakeEngine.CreateBuildReturns(nil, errors.New("oh no!")) - }) - - It("returns 500 Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - - Context("when creating a one-off build fails", func() { - BeforeEach(func() { - dbPipeline.CreateOneOffBuildReturns(nil, errors.New("oh no!")) - }) - - It("returns 500 Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - - Context("when requester does not belong to the team", func() { - BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(false) - }) - - It("returns 403", func() { - Expect(response.StatusCode).To(Equal(http.StatusForbidden)) - }) - }) - }) - - Context("when not authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(false) - }) - - It("returns 401", func() { - Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) - }) - - It("does not trigger a build", func() { - Expect(dbPipeline.CreateOneOffBuildCallCount()).To(BeZero()) - Expect(fakeEngine.CreateBuildCallCount()).To(BeZero()) }) }) }) diff --git a/atc/api/pipelineserver/build.go b/atc/api/pipelineserver/build.go index 62787bb7116..148d6e2bde5 100644 --- a/atc/api/pipelineserver/build.go +++ b/atc/api/pipelineserver/build.go @@ -5,13 +5,12 @@ import ( "net/http" "code.cloudfoundry.org/lager" - "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/api/present" "github.com/concourse/concourse/atc/db" ) -func (s *Server) CreateBuild(pipelineDB db.Pipeline) http.Handler { +func (s *Server) CreateBuild(pipeline db.Pipeline) http.Handler { logger := s.logger.Session("create-build") return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { var plan atc.Plan @@ -22,22 +21,13 @@ func (s *Server) CreateBuild(pipelineDB db.Pipeline) http.Handler { return } - build, err := pipelineDB.CreateOneOffBuild() + build, err := pipeline.CreateStartedBuild(plan) if err != nil { logger.Error("failed-to-create-one-off-build", err) w.WriteHeader(http.StatusInternalServerError) return } - engineBuild, err := s.engine.CreateBuild(logger, build, plan) - if err != nil { - logger.Error("failed-to-start-build", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - go engineBuild.Resume(logger) - w.Header().Set("Content-Type", "application/json") w.WriteHeader(http.StatusCreated) diff --git a/atc/db/build.go b/atc/db/build.go index 3dc182862be..cbcdd6cbe44 100644 --- a/atc/db/build.go +++ b/atc/db/build.go @@ -258,7 +258,7 @@ func (b *build) Start(engine, metadata string, plan atc.Plan) (bool, error) { var startTime time.Time err = psql.Update("builds"). - Set("status", "started"). + Set("status", BuildStatusStarted). Set("start_time", sq.Expr("now()")). Set("engine", engine). Set("engine_metadata", encryptedMetadata). diff --git a/atc/db/dbfakes/fake_pipeline.go b/atc/db/dbfakes/fake_pipeline.go index de107e8732e..3070975ad78 100644 --- a/atc/db/dbfakes/fake_pipeline.go +++ b/atc/db/dbfakes/fake_pipeline.go @@ -106,6 +106,19 @@ type FakePipeline struct { result1 db.Build result2 error } + CreateStartedBuildStub func(atc.Plan) (db.Build, error) + createStartedBuildMutex sync.RWMutex + createStartedBuildArgsForCall []struct { + arg1 atc.Plan + } + createStartedBuildReturns struct { + result1 db.Build + result2 error + } + createStartedBuildReturnsOnCall map[int]struct { + result1 db.Build + result2 error + } DashboardStub func() (db.Dashboard, error) dashboardMutex sync.RWMutex dashboardArgsForCall []struct { @@ -848,6 +861,69 @@ func (fake *FakePipeline) CreateOneOffBuildReturnsOnCall(i int, result1 db.Build }{result1, result2} } +func (fake *FakePipeline) CreateStartedBuild(arg1 atc.Plan) (db.Build, error) { + fake.createStartedBuildMutex.Lock() + ret, specificReturn := fake.createStartedBuildReturnsOnCall[len(fake.createStartedBuildArgsForCall)] + fake.createStartedBuildArgsForCall = append(fake.createStartedBuildArgsForCall, struct { + arg1 atc.Plan + }{arg1}) + fake.recordInvocation("CreateStartedBuild", []interface{}{arg1}) + fake.createStartedBuildMutex.Unlock() + if fake.CreateStartedBuildStub != nil { + return fake.CreateStartedBuildStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.createStartedBuildReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakePipeline) CreateStartedBuildCallCount() int { + fake.createStartedBuildMutex.RLock() + defer fake.createStartedBuildMutex.RUnlock() + return len(fake.createStartedBuildArgsForCall) +} + +func (fake *FakePipeline) CreateStartedBuildCalls(stub func(atc.Plan) (db.Build, error)) { + fake.createStartedBuildMutex.Lock() + defer fake.createStartedBuildMutex.Unlock() + fake.CreateStartedBuildStub = stub +} + +func (fake *FakePipeline) CreateStartedBuildArgsForCall(i int) atc.Plan { + fake.createStartedBuildMutex.RLock() + defer fake.createStartedBuildMutex.RUnlock() + argsForCall := fake.createStartedBuildArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakePipeline) CreateStartedBuildReturns(result1 db.Build, result2 error) { + fake.createStartedBuildMutex.Lock() + defer fake.createStartedBuildMutex.Unlock() + fake.CreateStartedBuildStub = nil + fake.createStartedBuildReturns = struct { + result1 db.Build + result2 error + }{result1, result2} +} + +func (fake *FakePipeline) CreateStartedBuildReturnsOnCall(i int, result1 db.Build, result2 error) { + fake.createStartedBuildMutex.Lock() + defer fake.createStartedBuildMutex.Unlock() + fake.CreateStartedBuildStub = nil + if fake.createStartedBuildReturnsOnCall == nil { + fake.createStartedBuildReturnsOnCall = make(map[int]struct { + result1 db.Build + result2 error + }) + } + fake.createStartedBuildReturnsOnCall[i] = struct { + result1 db.Build + result2 error + }{result1, result2} +} + func (fake *FakePipeline) Dashboard() (db.Dashboard, error) { fake.dashboardMutex.Lock() ret, specificReturn := fake.dashboardReturnsOnCall[len(fake.dashboardArgsForCall)] @@ -2391,6 +2467,8 @@ func (fake *FakePipeline) Invocations() map[string][][]interface{} { defer fake.configVersionMutex.RUnlock() fake.createOneOffBuildMutex.RLock() defer fake.createOneOffBuildMutex.RUnlock() + fake.createStartedBuildMutex.RLock() + defer fake.createStartedBuildMutex.RUnlock() fake.dashboardMutex.RLock() defer fake.dashboardMutex.RUnlock() fake.deleteBuildEventsByBuildIDsMutex.RLock() diff --git a/atc/db/dbfakes/fake_team.go b/atc/db/dbfakes/fake_team.go index 6f1d96f593e..f4b7b80eba7 100644 --- a/atc/db/dbfakes/fake_team.go +++ b/atc/db/dbfakes/fake_team.go @@ -87,6 +87,19 @@ type FakeTeam struct { result1 db.Build result2 error } + CreateStartedBuildStub func(atc.Plan) (db.Build, error) + createStartedBuildMutex sync.RWMutex + createStartedBuildArgsForCall []struct { + arg1 atc.Plan + } + createStartedBuildReturns struct { + result1 db.Build + result2 error + } + createStartedBuildReturnsOnCall map[int]struct { + result1 db.Build + result2 error + } DeleteStub func() error deleteMutex sync.RWMutex deleteArgsForCall []struct { @@ -751,6 +764,69 @@ func (fake *FakeTeam) CreateOneOffBuildReturnsOnCall(i int, result1 db.Build, re }{result1, result2} } +func (fake *FakeTeam) CreateStartedBuild(arg1 atc.Plan) (db.Build, error) { + fake.createStartedBuildMutex.Lock() + ret, specificReturn := fake.createStartedBuildReturnsOnCall[len(fake.createStartedBuildArgsForCall)] + fake.createStartedBuildArgsForCall = append(fake.createStartedBuildArgsForCall, struct { + arg1 atc.Plan + }{arg1}) + fake.recordInvocation("CreateStartedBuild", []interface{}{arg1}) + fake.createStartedBuildMutex.Unlock() + if fake.CreateStartedBuildStub != nil { + return fake.CreateStartedBuildStub(arg1) + } + if specificReturn { + return ret.result1, ret.result2 + } + fakeReturns := fake.createStartedBuildReturns + return fakeReturns.result1, fakeReturns.result2 +} + +func (fake *FakeTeam) CreateStartedBuildCallCount() int { + fake.createStartedBuildMutex.RLock() + defer fake.createStartedBuildMutex.RUnlock() + return len(fake.createStartedBuildArgsForCall) +} + +func (fake *FakeTeam) CreateStartedBuildCalls(stub func(atc.Plan) (db.Build, error)) { + fake.createStartedBuildMutex.Lock() + defer fake.createStartedBuildMutex.Unlock() + fake.CreateStartedBuildStub = stub +} + +func (fake *FakeTeam) CreateStartedBuildArgsForCall(i int) atc.Plan { + fake.createStartedBuildMutex.RLock() + defer fake.createStartedBuildMutex.RUnlock() + argsForCall := fake.createStartedBuildArgsForCall[i] + return argsForCall.arg1 +} + +func (fake *FakeTeam) CreateStartedBuildReturns(result1 db.Build, result2 error) { + fake.createStartedBuildMutex.Lock() + defer fake.createStartedBuildMutex.Unlock() + fake.CreateStartedBuildStub = nil + fake.createStartedBuildReturns = struct { + result1 db.Build + result2 error + }{result1, result2} +} + +func (fake *FakeTeam) CreateStartedBuildReturnsOnCall(i int, result1 db.Build, result2 error) { + fake.createStartedBuildMutex.Lock() + defer fake.createStartedBuildMutex.Unlock() + fake.CreateStartedBuildStub = nil + if fake.createStartedBuildReturnsOnCall == nil { + fake.createStartedBuildReturnsOnCall = make(map[int]struct { + result1 db.Build + result2 error + }) + } + fake.createStartedBuildReturnsOnCall[i] = struct { + result1 db.Build + result2 error + }{result1, result2} +} + func (fake *FakeTeam) Delete() error { fake.deleteMutex.Lock() ret, specificReturn := fake.deleteReturnsOnCall[len(fake.deleteArgsForCall)] @@ -2181,6 +2257,8 @@ func (fake *FakeTeam) Invocations() map[string][][]interface{} { defer fake.containersMutex.RUnlock() fake.createOneOffBuildMutex.RLock() defer fake.createOneOffBuildMutex.RUnlock() + fake.createStartedBuildMutex.RLock() + defer fake.createStartedBuildMutex.RUnlock() fake.deleteMutex.RLock() defer fake.deleteMutex.RUnlock() fake.findCheckContainersMutex.RLock() diff --git a/atc/db/pipeline.go b/atc/db/pipeline.go index 526265c1ac5..508a9b0772c 100644 --- a/atc/db/pipeline.go +++ b/atc/db/pipeline.go @@ -9,11 +9,11 @@ import ( "time" "code.cloudfoundry.org/lager" - sq "github.com/Masterminds/squirrel" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/db/algorithm" "github.com/concourse/concourse/atc/db/lock" + "github.com/concourse/concourse/atc/event" ) type ErrResourceNotFound struct { @@ -50,7 +50,10 @@ type Pipeline interface { GetBuildsWithVersionAsInput(int, int) ([]Build, error) GetBuildsWithVersionAsOutput(int, int) ([]Build, error) Builds(page Page) ([]Build, Pagination, error) + CreateOneOffBuild() (Build, error) + CreateStartedBuild(plan atc.Plan) (Build, error) + GetAllPendingBuilds() (map[string][]Build, error) BuildsWithTime(page Page) ([]Build, Pagination, error) @@ -968,6 +971,56 @@ func (p *pipeline) CreateOneOffBuild() (Build, error) { return build, nil } +func (p *pipeline) CreateStartedBuild(plan atc.Plan) (Build, error) { + tx, err := p.conn.Begin() + if err != nil { + return nil, err + } + + defer Rollback(tx) + + metadata, err := json.Marshal(map[string]interface{}{"plan": plan}) + if err != nil { + return nil, err + } + + encryptedMetadata, nonce, err := p.conn.EncryptionStrategy().Encrypt(metadata) + if err != nil { + return nil, err + } + + build := &build{conn: p.conn, lockFactory: p.lockFactory} + err = createBuild(tx, build, map[string]interface{}{ + "name": sq.Expr("nextval('one_off_name')"), + "pipeline_id": p.id, + "team_id": p.teamID, + "status": BuildStatusStarted, + "start_time": sq.Expr("now()"), + "engine": "exec.v2", + "engine_metadata": encryptedMetadata, + "public_plan": plan.Public(), + "nonce": nonce, + }) + if err != nil { + return nil, err + } + + err = build.saveEvent(tx, event.Status{ + Status: atc.StatusStarted, + Time: build.StartTime().Unix(), + }) + if err != nil { + return nil, err + } + + err = tx.Commit() + if err != nil { + return nil, err + } + + return build, p.conn.Bus().Notify(buildEventsChannel(build.id)) +} + func (p *pipeline) incrementCheckOrderWhenNewerVersion(tx Tx, resourceID int, resourceType string, version string) error { _, err := tx.Exec(` WITH max_checkorder AS ( diff --git a/atc/db/pipeline_test.go b/atc/db/pipeline_test.go index 1f07d8307f9..03ffa307028 100644 --- a/atc/db/pipeline_test.go +++ b/atc/db/pipeline_test.go @@ -1,6 +1,7 @@ package db_test import ( + "strconv" "time" "github.com/concourse/concourse/atc" @@ -1954,6 +1955,76 @@ var _ = Describe("Pipeline", func() { }) }) + Describe("CreateStartedBuild", func() { + var ( + plan atc.Plan + startedBuild db.Build + err error + ) + + BeforeEach(func() { + plan = atc.Plan{ + ID: atc.PlanID("56"), + Get: &atc.GetPlan{ + Type: "some-type", + Name: "some-name", + Resource: "some-resource", + Source: atc.Source{"some": "source"}, + Params: atc.Params{"some": "params"}, + Version: &atc.Version{"some": "version"}, + Tags: atc.Tags{"some-tags"}, + VersionedResourceTypes: atc.VersionedResourceTypes{ + { + ResourceType: atc.ResourceType{ + Name: "some-name", + Source: atc.Source{"some": "source"}, + Type: "some-type", + Privileged: true, + Tags: atc.Tags{"some-tags"}, + }, + Version: atc.Version{"some-resource-type": "version"}, + }, + }, + }, + } + + startedBuild, err = pipeline.CreateStartedBuild(plan) + Expect(err).ToNot(HaveOccurred()) + }) + + It("can create started builds with plans", func() { + Expect(startedBuild.ID()).ToNot(BeZero()) + Expect(startedBuild.JobName()).To(BeZero()) + Expect(startedBuild.PipelineName()).To(Equal("fake-pipeline")) + Expect(startedBuild.Name()).To(Equal(strconv.Itoa(startedBuild.ID()))) + Expect(startedBuild.TeamName()).To(Equal(team.Name())) + Expect(startedBuild.Status()).To(Equal(db.BuildStatusStarted)) + }) + + It("saves the public plan", func() { + found, err := startedBuild.Reload() + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(startedBuild.PublicPlan()).To(Equal(plan.Public())) + }) + + It("creates Start event", func() { + found, err := startedBuild.Reload() + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + + events, err := startedBuild.Events(0) + Expect(err).NotTo(HaveOccurred()) + + defer db.Close(events) + + Expect(events.Next()).To(Equal(envelope(event.Status{ + Status: atc.StatusStarted, + Time: startedBuild.StartTime().Unix(), + }))) + }) + }) + Describe("Resources", func() { var resourceTypes db.ResourceTypes diff --git a/atc/db/team.go b/atc/db/team.go index 40788d4c9f2..02c9a594a14 100644 --- a/atc/db/team.go +++ b/atc/db/team.go @@ -12,6 +12,7 @@ import ( "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db/lock" + "github.com/concourse/concourse/atc/event" "github.com/lib/pq" ) @@ -43,6 +44,8 @@ type Team interface { OrderPipelines([]string) error CreateOneOffBuild() (Build, error) + CreateStartedBuild(plan atc.Plan) (Build, error) + PrivateAndPublicBuilds(Page) ([]Build, Pagination, error) Builds(page Page) ([]Build, Pagination, error) BuildsWithTime(page Page) ([]Build, Pagination, error) @@ -681,6 +684,55 @@ func (t *team) CreateOneOffBuild() (Build, error) { return build, nil } +func (t *team) CreateStartedBuild(plan atc.Plan) (Build, error) { + tx, err := t.conn.Begin() + if err != nil { + return nil, err + } + + defer Rollback(tx) + + metadata, err := json.Marshal(map[string]interface{}{"plan": plan}) + if err != nil { + return nil, err + } + + encryptedMetadata, nonce, err := t.conn.EncryptionStrategy().Encrypt(metadata) + if err != nil { + return nil, err + } + + build := &build{conn: t.conn, lockFactory: t.lockFactory} + err = createBuild(tx, build, map[string]interface{}{ + "name": sq.Expr("nextval('one_off_name')"), + "team_id": t.id, + "status": BuildStatusStarted, + "start_time": sq.Expr("now()"), + "engine": "exec.v2", + "engine_metadata": encryptedMetadata, + "public_plan": plan.Public(), + "nonce": nonce, + }) + if err != nil { + return nil, err + } + + err = build.saveEvent(tx, event.Status{ + Status: atc.StatusStarted, + Time: build.StartTime().Unix(), + }) + if err != nil { + return nil, err + } + + err = tx.Commit() + if err != nil { + return nil, err + } + + return build, t.conn.Bus().Notify(buildEventsChannel(build.id)) +} + func (t *team) PrivateAndPublicBuilds(page Page) ([]Build, Pagination, error) { newBuildsQuery := buildsQuery. Where(sq.Or{sq.Eq{"p.public": true}, sq.Eq{"t.id": t.id}}) diff --git a/atc/db/team_test.go b/atc/db/team_test.go index ab207f43cfb..458543707ac 100644 --- a/atc/db/team_test.go +++ b/atc/db/team_test.go @@ -11,6 +11,7 @@ import ( "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/creds/credsfakes" "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/event" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -1067,6 +1068,76 @@ var _ = Describe("Team", func() { }) }) + Describe("CreateStartedBuild", func() { + var ( + plan atc.Plan + startedBuild db.Build + err error + ) + + BeforeEach(func() { + plan = atc.Plan{ + ID: atc.PlanID("56"), + Get: &atc.GetPlan{ + Type: "some-type", + Name: "some-name", + Resource: "some-resource", + Source: atc.Source{"some": "source"}, + Params: atc.Params{"some": "params"}, + Version: &atc.Version{"some": "version"}, + Tags: atc.Tags{"some-tags"}, + VersionedResourceTypes: atc.VersionedResourceTypes{ + { + ResourceType: atc.ResourceType{ + Name: "some-name", + Source: atc.Source{"some": "source"}, + Type: "some-type", + Privileged: true, + Tags: atc.Tags{"some-tags"}, + }, + Version: atc.Version{"some-resource-type": "version"}, + }, + }, + }, + } + + startedBuild, err = team.CreateStartedBuild(plan) + Expect(err).ToNot(HaveOccurred()) + }) + + It("can create started builds with plans", func() { + Expect(startedBuild.ID()).ToNot(BeZero()) + Expect(startedBuild.JobName()).To(BeZero()) + Expect(startedBuild.PipelineName()).To(BeZero()) + Expect(startedBuild.Name()).To(Equal(strconv.Itoa(startedBuild.ID()))) + Expect(startedBuild.TeamName()).To(Equal(team.Name())) + Expect(startedBuild.Status()).To(Equal(db.BuildStatusStarted)) + }) + + It("saves the public plan", func() { + found, err := startedBuild.Reload() + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + Expect(startedBuild.PublicPlan()).To(Equal(plan.Public())) + }) + + It("creates Start event", func() { + found, err := startedBuild.Reload() + Expect(err).NotTo(HaveOccurred()) + Expect(found).To(BeTrue()) + + events, err := startedBuild.Events(0) + Expect(err).NotTo(HaveOccurred()) + + defer db.Close(events) + + Expect(events.Next()).To(Equal(envelope(event.Status{ + Status: atc.StatusStarted, + Time: startedBuild.StartTime().Unix(), + }))) + }) + }) + Describe("PrivateAndPublicBuilds", func() { Context("when there are no builds", func() { It("returns an empty list of builds", func() { From 5c11d454ec0eb99fd9e9d9364f2518a7b542593f Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Tue, 12 Feb 2019 15:32:09 -0500 Subject: [PATCH 09/21] atc: abort build handler marks build as aborted in db * it no longer relies on the exec engine concourse/concourse#3240 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/api/builds_test.go | 123 ++++++++++++++--------------------- atc/api/buildserver/abort.go | 12 +--- 2 files changed, 52 insertions(+), 83 deletions(-) diff --git a/atc/api/builds_test.go b/atc/api/builds_test.go index 8caf98a10bc..f5b5d3944dc 100644 --- a/atc/api/builds_test.go +++ b/atc/api/builds_test.go @@ -971,74 +971,28 @@ var _ = Describe("Builds API", func() { Expect(err).NotTo(HaveOccurred()) }) + Context("when not authenticated", func() { + BeforeEach(func() { + fakeaccess.IsAuthenticatedReturns(false) + }) + + It("returns 401", func() { + Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + }) + Context("when authenticated", func() { BeforeEach(func() { fakeaccess.IsAuthenticatedReturns(true) }) - Context("when the build can be found", func() { + Context("when looking up the build fails", func() { BeforeEach(func() { - build.TeamNameReturns("some-team") - dbBuildFactory.BuildReturns(build, true, nil) - }) - - Context("when accessing same team's build", func() { - BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(true) - }) - - Context("when the engine returns a build", func() { - var engineBuild *enginefakes.FakeBuild - - BeforeEach(func() { - engineBuild = new(enginefakes.FakeBuild) - fakeEngine.LookupBuildReturns(engineBuild, nil) - }) - - It("aborts the build", func() { - Expect(engineBuild.AbortCallCount()).To(Equal(1)) - }) - - Context("when aborting succeeds", func() { - BeforeEach(func() { - engineBuild.AbortReturns(nil) - }) - - It("returns 204", func() { - Expect(response.StatusCode).To(Equal(http.StatusNoContent)) - }) - }) - - Context("when aborting fails", func() { - BeforeEach(func() { - engineBuild.AbortReturns(errors.New("oh no!")) - }) - - It("returns 500", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - - Context("when the engine returns no build", func() { - BeforeEach(func() { - fakeEngine.LookupBuildReturns(nil, errors.New("oh no!")) - }) - - It("returns Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) + dbBuildFactory.BuildReturns(nil, false, errors.New("nope")) }) - Context("when accessing other team's build", func() { - BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(false) - }) - - It("returns 403", func() { - Expect(response.StatusCode).To(Equal(http.StatusForbidden)) - }) + It("returns 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) }) }) @@ -1047,29 +1001,52 @@ var _ = Describe("Builds API", func() { dbBuildFactory.BuildReturns(nil, false, nil) }) - It("returns Not Found", func() { + It("returns 404", func() { Expect(response.StatusCode).To(Equal(http.StatusNotFound)) }) }) - Context("when calling the database fails", func() { + Context("when the build is found", func() { BeforeEach(func() { - dbBuildFactory.BuildReturns(nil, false, errors.New("nope")) + build.TeamNameReturns("some-team") + dbBuildFactory.BuildReturns(build, true, nil) }) - It("returns Internal Server Error", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + Context("when not authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(false) + }) + + It("returns 403", func() { + Expect(response.StatusCode).To(Equal(http.StatusForbidden)) + }) }) - }) - }) - Context("when not authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(false) - }) + Context("when authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(true) + }) - It("returns 401", func() { - Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + Context("when aborting the build fails", func() { + BeforeEach(func() { + build.MarkAsAbortedReturns(errors.New("nope")) + }) + + It("returns 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when aborting succeeds", func() { + BeforeEach(func() { + build.MarkAsAbortedReturns(nil) + }) + + It("returns 204", func() { + Expect(response.StatusCode).To(Equal(http.StatusNoContent)) + }) + }) + }) }) }) }) diff --git a/atc/api/buildserver/abort.go b/atc/api/buildserver/abort.go index 39d1e90bc44..28f868505e1 100644 --- a/atc/api/buildserver/abort.go +++ b/atc/api/buildserver/abort.go @@ -3,9 +3,8 @@ package buildserver import ( "net/http" - "github.com/concourse/concourse/atc/db" - "code.cloudfoundry.org/lager" + "github.com/concourse/concourse/atc/db" ) func (s *Server) AbortBuild(build db.Build) http.Handler { @@ -14,14 +13,7 @@ func (s *Server) AbortBuild(build db.Build) http.Handler { "build": build.ID(), }) - engineBuild, err := s.engine.LookupBuild(aLog, build) - if err != nil { - aLog.Error("failed-to-lookup-build", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - err = engineBuild.Abort(aLog) + err := build.MarkAsAborted() if err != nil { aLog.Error("failed-to-abort-build", err) w.WriteHeader(http.StatusInternalServerError) From b15eb29a52979a86162a86e7666a12c01d8e5bdb Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Tue, 12 Feb 2019 16:51:53 -0500 Subject: [PATCH 10/21] atc: triggering build from UI does not `TriggerImmediately` * it gets run during the next scheduler tick concourse/concourse#3240 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/api/jobs_test.go | 120 ++++++++++-------------------- atc/api/jobserver/create_build.go | 24 +----- 2 files changed, 42 insertions(+), 102 deletions(-) diff --git a/atc/api/jobs_test.go b/atc/api/jobs_test.go index 0d6899148bc..da61c38e9c1 100644 --- a/atc/api/jobs_test.go +++ b/atc/api/jobs_test.go @@ -9,15 +9,14 @@ import ( "time" "github.com/cloudfoundry/bosh-cli/director/template" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc" "github.com/concourse/concourse/atc/api/accessor/accessorfakes" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/scheduler/schedulerfakes" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var _ = Describe("Jobs API", func() { @@ -1410,18 +1409,11 @@ var _ = Describe("Jobs API", func() { var request *http.Request var response *http.Response - var fakeScheduler *schedulerfakes.FakeBuildScheduler - var fakeResource *dbfakes.FakeResource - var fakeResource2 *dbfakes.FakeResource - BeforeEach(func() { var err error request, err = http.NewRequest("POST", server.URL+"/api/v1/teams/some-team/pipelines/some-pipeline/jobs/some-job/builds", nil) Expect(err).NotTo(HaveOccurred()) - - fakeScheduler = new(schedulerfakes.FakeBuildScheduler) - fakeSchedulerFactory.BuildSchedulerReturns(fakeScheduler) }) JustBeforeEach(func() { @@ -1437,6 +1429,26 @@ var _ = Describe("Jobs API", func() { fakeaccess.IsAuthenticatedReturns(true) }) + Context("when getting the job fails", func() { + BeforeEach(func() { + fakePipeline.JobReturns(nil, false, errors.New("errorrr")) + }) + + It("returns a 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when the job is not found", func() { + BeforeEach(func() { + fakePipeline.JobReturns(nil, false, nil) + }) + + It("returns a 404", func() { + Expect(response.StatusCode).To(Equal(http.StatusNotFound)) + }) + }) + Context("when getting the job succeeds", func() { BeforeEach(func() { fakeJob.NameReturns("some-job") @@ -1448,11 +1460,7 @@ var _ = Describe("Jobs API", func() { fakeJob.ConfigReturns(atc.JobConfig{ Name: "some-job", DisableManualTrigger: true, - Plan: atc.PlanSequence{ - { - Get: "some-input", - }, - }, + Plan: atc.PlanSequence{{Get: "some-input"}}, }) }) @@ -1461,19 +1469,25 @@ var _ = Describe("Jobs API", func() { }) It("does not trigger the build", func() { - Expect(fakeScheduler.TriggerImmediatelyCallCount()).To(Equal(0)) + Expect(fakeJob.CreateBuildCallCount()).To(Equal(0)) }) }) - Context("when getting the job config succeeds", func() { + Context("when manual triggering is enabled", func() { BeforeEach(func() { fakeJob.ConfigReturns(atc.JobConfig{ - Name: "some-job", - Plan: atc.PlanSequence{ - { - Get: "some-input", - }, - }, + Name: "some-job", + DisableManualTrigger: false, + Plan: atc.PlanSequence{{Get: "some-input"}}, + }) + }) + + Context("when triggering the build fails", func() { + BeforeEach(func() { + fakeJob.CreateBuildReturns(nil, errors.New("nopers")) + }) + It("returns a 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) }) }) @@ -1488,26 +1502,12 @@ var _ = Describe("Jobs API", func() { build.StatusReturns(db.BuildStatusStarted) build.StartTimeReturns(time.Unix(1, 0)) build.EndTimeReturns(time.Unix(100, 0)) - fakeScheduler.TriggerImmediatelyReturns(build, nil, nil) - fakeResource = new(dbfakes.FakeResource) - fakeResource.NameReturns("resource-1") - fakeResource.TypeReturns("some-type") - - fakeResource2 = new(dbfakes.FakeResource) - fakeResource2.NameReturns("resource-2") - fakeResource2.TypeReturns("some-other-type") - - fakePipeline.ResourcesReturns(db.Resources{fakeResource, fakeResource2}, nil) + fakeJob.CreateBuildReturns(build, nil) }) - It("triggers using the current config", func() { - Expect(fakeScheduler.TriggerImmediatelyCallCount()).To(Equal(1)) - - _, job, resources, resourceTypes := fakeScheduler.TriggerImmediatelyArgsForCall(0) - Expect(job).To(Equal(fakeJob)) - Expect(resources).To(Equal(db.Resources{fakeResource, fakeResource2})) - Expect(resourceTypes).To(Equal(versionedResourceTypes)) + It("triggers the build", func() { + Expect(fakeJob.CreateBuildCallCount()).To(Equal(1)) }) It("returns 200 OK", func() { @@ -1535,46 +1535,6 @@ var _ = Describe("Jobs API", func() { }`)) }) }) - - Context("when getting the config fails", func() { - BeforeEach(func() { - fakePipeline.ResourcesReturns(db.Resources{}, errors.New("oh no!")) - }) - - It("returns 500", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - - Context("when triggering the build fails", func() { - BeforeEach(func() { - fakeScheduler.TriggerImmediatelyReturns(nil, nil, errors.New("oh no!")) - }) - - It("returns 500", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - }) - - Context("when getting the job fails", func() { - BeforeEach(func() { - fakePipeline.JobReturns(nil, false, errors.New("errorrr")) - }) - - It("returns a 500", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - - Context("when the job is not found", func() { - BeforeEach(func() { - fakePipeline.JobReturns(nil, false, nil) - }) - - It("returns a 404", func() { - Expect(response.StatusCode).To(Equal(http.StatusNotFound)) }) }) }) diff --git a/atc/api/jobserver/create_build.go b/atc/api/jobserver/create_build.go index bea79be85b8..021eb79d05c 100644 --- a/atc/api/jobserver/create_build.go +++ b/atc/api/jobserver/create_build.go @@ -2,7 +2,6 @@ package jobserver import ( "encoding/json" - "fmt" "net/http" "github.com/concourse/concourse/atc/api/present" @@ -33,29 +32,10 @@ func (s *Server) CreateJobBuild(pipeline db.Pipeline) http.Handler { return } - scheduler := s.schedulerFactory.BuildScheduler(pipeline, s.externalURL, s.variablesFactory.NewVariables(pipeline.TeamName(), pipeline.Name())) - - resourceTypes, err := pipeline.ResourceTypes() - if err != nil { - logger.Error("failed-to-get-resource-types", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - versionedResourceTypes := resourceTypes.Deserialize() - - resources, err := pipeline.Resources() - if err != nil { - logger.Error("failed-to-get-resources", err) - w.WriteHeader(http.StatusInternalServerError) - return - } - - build, _, err := scheduler.TriggerImmediately(logger, job, resources, versionedResourceTypes) + build, err := job.CreateBuild() if err != nil { - logger.Error("failed-to-trigger", err) + logger.Error("failed-to-create-job-build", err) w.WriteHeader(http.StatusInternalServerError) - fmt.Fprintf(w, "failed to trigger: %s", err) return } From 490340cc91e116789d204befa619e412db440c7a Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Wed, 13 Feb 2019 10:56:49 -0500 Subject: [PATCH 11/21] atc: list job inputs no longer forces a scheduler tick * this was a weird side effect of a GET request concourse/concourse#3240 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/api/jobs_test.go | 222 +++++++++++++++---------------- atc/api/jobserver/list_inputs.go | 19 +-- testflight/flying_test.go | 27 +++- 3 files changed, 133 insertions(+), 135 deletions(-) diff --git a/atc/api/jobs_test.go b/atc/api/jobs_test.go index da61c38e9c1..f87d97c0953 100644 --- a/atc/api/jobs_test.go +++ b/atc/api/jobs_test.go @@ -14,7 +14,6 @@ import ( "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" - "github.com/concourse/concourse/atc/scheduler/schedulerfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -1550,26 +1549,58 @@ var _ = Describe("Jobs API", func() { Expect(err).NotTo(HaveOccurred()) }) - Context("when authorized", func() { + Context("when not authenticated", func() { + BeforeEach(func() { + fakeaccess.IsAuthenticatedReturns(false) + }) + + It("returns 401", func() { + Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) + }) + }) + + Context("when authenticated", func() { BeforeEach(func() { - fakeaccess.IsAuthorizedReturns(true) fakeaccess.IsAuthenticatedReturns(true) }) - It("looked up the proper pipeline", func() { - Expect(dbTeam.PipelineCallCount()).To(Equal(1)) - pipelineName := dbTeam.PipelineArgsForCall(0) - Expect(pipelineName).To(Equal("some-pipeline")) + Context("when not authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(false) + }) + + It("returns 403", func() { + Expect(response.StatusCode).To(Equal(http.StatusForbidden)) + }) }) - Context("when getting the job succeeds", func() { - var fakeJob *dbfakes.FakeJob + Context("when authorized", func() { + BeforeEach(func() { + fakeaccess.IsAuthorizedReturns(true) + }) - Context("when it contains the requested job", func() { - var fakeScheduler *schedulerfakes.FakeBuildScheduler + Context("when getting the job fails", func() { + BeforeEach(func() { + fakePipeline.JobReturns(nil, false, errors.New("some-error")) + }) + + It("returns 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when the job is not found", func() { + BeforeEach(func() { + fakePipeline.JobReturns(nil, false, nil) + }) + + It("returns 404", func() { + Expect(response.StatusCode).To(Equal(http.StatusNotFound)) + }) + }) + + Context("when getting the job succeeds", func() { BeforeEach(func() { - fakeJob = new(dbfakes.FakeJob) - fakeJob.NameReturns("some-job") fakeJob.ConfigReturns(atc.JobConfig{ Name: "some-job", Plan: atc.PlanSequence{ @@ -1588,31 +1619,60 @@ var _ = Describe("Jobs API", func() { }, }, }) - fakePipeline.JobReturns(fakeJob, true, nil) - fakeScheduler = new(schedulerfakes.FakeBuildScheduler) - fakeSchedulerFactory.BuildSchedulerReturns(fakeScheduler) + fakePipeline.JobReturns(fakeJob, true, nil) + }) - resource1 := new(dbfakes.FakeResource) - resource1.IDReturns(1) - resource1.NameReturns("some-resource") - resource1.TypeReturns("some-type") - resource1.SourceReturns(atc.Source{"some": "source"}) + Context("when getting the resources fails", func() { + BeforeEach(func() { + fakePipeline.ResourcesReturns(nil, errors.New("some-error")) + }) - resource2 := new(dbfakes.FakeResource) - resource1.IDReturns(2) - resource2.NameReturns("some-other-resource") - resource2.TypeReturns("some-other-type") - resource2.SourceReturns(atc.Source{"some": "other-source"}) - fakePipeline.ResourcesReturns([]db.Resource{resource1, resource2}, nil) + It("returns 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) }) - Context("when the input versions for the job can be determined", func() { + Context("when getting the resources succeeds", func() { BeforeEach(func() { - fakeJob.GetNextBuildInputsStub = func() ([]db.BuildInput, bool, error) { - defer GinkgoRecover() - Expect(fakeScheduler.SaveNextInputMappingCallCount()).To(Equal(1)) - return []db.BuildInput{ + resource1 := new(dbfakes.FakeResource) + resource1.IDReturns(1) + resource1.NameReturns("some-resource") + resource1.TypeReturns("some-type") + resource1.SourceReturns(atc.Source{"some": "source"}) + + resource2 := new(dbfakes.FakeResource) + resource1.IDReturns(2) + resource2.NameReturns("some-other-resource") + resource2.TypeReturns("some-other-type") + resource2.SourceReturns(atc.Source{"some": "other-source"}) + + fakePipeline.ResourcesReturns([]db.Resource{resource1, resource2}, nil) + }) + + Context("when getting the input versions for the job fails", func() { + BeforeEach(func() { + fakeJob.GetNextBuildInputsReturns(nil, false, errors.New("oh no!")) + }) + + It("returns 500", func() { + Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) + }) + }) + + Context("when the job has no input versions available", func() { + BeforeEach(func() { + fakeJob.GetNextBuildInputsReturns(nil, false, nil) + }) + + It("returns 404", func() { + Expect(response.StatusCode).To(Equal(http.StatusNotFound)) + }) + }) + + Context("when the job has input versions", func() { + BeforeEach(func() { + inputs := []db.BuildInput{ { Name: "some-input", Version: atc.Version{"some": "version"}, @@ -1623,35 +1683,24 @@ var _ = Describe("Jobs API", func() { Version: atc.Version{"some": "other-version"}, ResourceID: 2, }, - }, true, nil - } - }) + } - It("returns 200 OK", func() { - Expect(response.StatusCode).To(Equal(http.StatusOK)) - }) + fakeJob.GetNextBuildInputsReturns(inputs, true, nil) + }) - It("returns Content-Type 'application/json'", func() { - Expect(response.Header.Get("Content-Type")).To(Equal("application/json")) - }) + It("returns 200 OK", func() { + Expect(response.StatusCode).To(Equal(http.StatusOK)) + }) - It("created the scheduler with the correct fakePipeline and external URL", func() { - actualPipeline, actualExternalURL, actualVariables := fakeSchedulerFactory.BuildSchedulerArgsForCall(0) - Expect(actualPipeline.Name()).To(Equal(fakePipeline.Name())) - Expect(actualExternalURL).To(Equal(externalURL)) - Expect(actualVariables).To(Equal(variables)) - }) + It("returns Content-Type 'application/json'", func() { + Expect(response.Header.Get("Content-Type")).To(Equal("application/json")) + }) - It("determined the inputs with the correct job config", func() { - _, receivedJob, _ := fakeScheduler.SaveNextInputMappingArgsForCall(0) - Expect(receivedJob.Name()).To(Equal(fakeJob.Name())) - }) + It("returns the inputs", func() { + body, err := ioutil.ReadAll(response.Body) + Expect(err).NotTo(HaveOccurred()) - It("returns the inputs", func() { - body, err := ioutil.ReadAll(response.Body) - Expect(err).NotTo(HaveOccurred()) - - Expect(body).To(MatchJSON(`[ + Expect(body).To(MatchJSON(`[ { "name": "some-input", "resource": "some-resource", @@ -1670,71 +1719,10 @@ var _ = Describe("Jobs API", func() { "tags": ["some-tag"] } ]`)) - - }) - - Context("when getting the resources fails", func() { - BeforeEach(func() { - fakePipeline.ResourcesReturns(nil, errors.New("some-error")) - }) - - It("returns 500", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) }) }) }) - - Context("when the job has no input versions available", func() { - BeforeEach(func() { - fakeJob.GetNextBuildInputsReturns(nil, false, nil) - }) - - It("returns 404", func() { - Expect(response.StatusCode).To(Equal(http.StatusNotFound)) - }) - }) - - Context("when the input versions for the job can not be determined", func() { - BeforeEach(func() { - fakeJob.GetNextBuildInputsReturns(nil, false, errors.New("oh no!")) - }) - - It("returns 500", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - }) - }) - - Context("when it does not contain the requested job", func() { - BeforeEach(func() { - fakePipeline.JobReturns(nil, false, nil) - }) - - It("returns 404 Not Found", func() { - Expect(response.StatusCode).To(Equal(http.StatusNotFound)) - }) - }) - - Context("when getting the job fails", func() { - BeforeEach(func() { - fakePipeline.JobReturns(nil, false, errors.New("some-error")) }) - - It("returns 500", func() { - Expect(response.StatusCode).To(Equal(http.StatusInternalServerError)) - }) - }) - - }) - - Context("when not authenticated", func() { - BeforeEach(func() { - fakeaccess.IsAuthenticatedReturns(false) - }) - - It("returns unauthorized", func() { - Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) }) }) }) diff --git a/atc/api/jobserver/list_inputs.go b/atc/api/jobserver/list_inputs.go index 9ae1474d850..cc0c66819b3 100644 --- a/atc/api/jobserver/list_inputs.go +++ b/atc/api/jobserver/list_inputs.go @@ -12,10 +12,9 @@ import ( func (s *Server) ListJobInputs(pipeline db.Pipeline) http.Handler { logger := s.logger.Session("list-job-inputs") return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/json") jobName := r.FormValue(":job_name") - variables := s.variablesFactory.NewVariables(pipeline.TeamName(), pipeline.Name()) - job, found, err := pipeline.Job(jobName) if err != nil { logger.Error("failed-to-get-job", err) @@ -28,8 +27,6 @@ func (s *Server) ListJobInputs(pipeline db.Pipeline) http.Handler { return } - scheduler := s.schedulerFactory.BuildScheduler(pipeline, s.externalURL, variables) - resources, err := pipeline.Resources() if err != nil { logger.Error("failed-to-get-resources", err) @@ -37,12 +34,6 @@ func (s *Server) ListJobInputs(pipeline db.Pipeline) http.Handler { return } - err = scheduler.SaveNextInputMapping(logger, job, resources) - if err != nil { - w.WriteHeader(http.StatusInternalServerError) - return - } - buildInputs, found, err := job.GetNextBuildInputs() if err != nil { logger.Error("failed-to-get-next-build-inputs", err) @@ -56,7 +47,8 @@ func (s *Server) ListJobInputs(pipeline db.Pipeline) http.Handler { } jobInputs := job.Config().Inputs() - presentedBuildInputs := make([]atc.BuildInput, len(buildInputs)) + inputs := make([]atc.BuildInput, len(buildInputs)) + for i, input := range buildInputs { var config atc.JobInput for _, jobInput := range jobInputs { @@ -67,11 +59,10 @@ func (s *Server) ListJobInputs(pipeline db.Pipeline) http.Handler { } resource, _ := resources.Lookup(config.Resource) - presentedBuildInputs[i] = present.BuildInput(input, config, resource) + inputs[i] = present.BuildInput(input, config, resource) } - w.Header().Set("Content-Type", "application/json") - err = json.NewEncoder(w).Encode(presentedBuildInputs) + err = json.NewEncoder(w).Encode(inputs) if err != nil { logger.Error("failed-to-encode-build-inputs", err) w.WriteHeader(http.StatusInternalServerError) diff --git a/testflight/flying_test.go b/testflight/flying_test.go index 3480ce50d33..5c973155a6e 100644 --- a/testflight/flying_test.go +++ b/testflight/flying_test.go @@ -294,6 +294,9 @@ run: By("satisfying the job's passed constraint for the first version") fly("trigger-job", "-w", "-j", pipelineName+"/upstream-job") + By("making sure the second job runs before we use its inputs") + fly("trigger-job", "-w", "-j", pipelineName+"/downstream-job") + By("executing using the first version via -j") execS := flyIn(tmp, "execute", "-c", "task.yml", "-j", pipelineName+"/downstream-job") Expect(execS).To(gbytes.Say("first-version")) @@ -308,6 +311,9 @@ run: By("satisfying the job's passed constraint for the second version") fly("trigger-job", "-w", "-j", pipelineName+"/upstream-job") + By("making sure the second job runs before we use its inputs") + fly("trigger-job", "-w", "-j", pipelineName+"/downstream-job") + By("now executing using the second version via -j") execS = flyIn(tmp, "execute", "-c", "task.yml", "-j", pipelineName+"/downstream-job") Expect(execS).To(gbytes.Say("second-version")) @@ -341,8 +347,6 @@ run: fly("set-pipeline", "-n", "-p", pipelineName, "-c", "fixtures/input-mapping-test.yml", "-v", "task_content="+taskFileContents+"") - fly("check-resource", "-r", pipelineName+"/some-resource", "-f", "version:first-version") - fly("unpause-pipeline", "-p", pipelineName) }) @@ -353,6 +357,9 @@ run: By("satisfying the job's passed constraint for the first version") fly("trigger-job", "-w", "-j", pipelineName+"/upstream-job") + By("making sure the second job runs before we use its inputs") + fly("trigger-job", "-w", "-j", pipelineName+"/downstream-job") + By("executing using the first version via -j") execS := flyIn(tmp, "execute", "-c", "task.yml", "-j", pipelineName+"/downstream-job", "-m", "mapped-resource=some-resource") Expect(execS).To(gbytes.Say("first-version")) @@ -367,6 +374,9 @@ run: By("satisfying the job's passed constraint for the second version") fly("trigger-job", "-w", "-j", pipelineName+"/upstream-job") + By("making sure the second job runs before we use its inputs") + fly("trigger-job", "-w", "-j", pipelineName+"/downstream-job") + By("now executing using the second version via -j") execS = flyIn(tmp, "execute", "-c", "task.yml", "-j", pipelineName+"/downstream-job", "-m", "mapped-resource=some-resource") Expect(execS).To(gbytes.Say("second-version")) @@ -396,8 +406,6 @@ run: fly("set-pipeline", "-n", "-p", pipelineName, "-c", "fixtures/image-resource-test.yml", "-v", "task_content="+taskFileContents+"") - fly("check-resource", "-r", pipelineName+"/some-resource", "-f", "version:first-version") - fly("unpause-pipeline", "-p", pipelineName) }) @@ -408,6 +416,9 @@ run: By("satisfying the job's passed constraint for the first version") fly("trigger-job", "-w", "-j", pipelineName+"/upstream-job") + By("making sure the second job runs before we use its inputs") + fly("trigger-job", "-w", "-j", pipelineName+"/downstream-job") + By("executing using the first version via -j") execS := flyIn(tmp, "execute", "-c", "task.yml", "-j", pipelineName+"/downstream-job", "--image", "some-image") Expect(execS).To(gbytes.Say("first-version")) @@ -422,6 +433,9 @@ run: By("satisfying the job's passed constraint for the second version") fly("trigger-job", "-w", "-j", pipelineName+"/upstream-job") + By("making sure the second job runs before we use its inputs") + fly("trigger-job", "-w", "-j", pipelineName+"/downstream-job") + By("now executing using the second version via -j") execS = flyIn(tmp, "execute", "-c", "task.yml", "-j", pipelineName+"/downstream-job", "--image", "some-image") Expect(execS).To(gbytes.Say("second-version")) @@ -457,6 +471,11 @@ run: }) Context("when -j is specified", func() { + BeforeEach(func() { + By("making sure the job runs before we use its inputs") + fly("trigger-job", "-w", "-j", pipelineName+"/input-test") + }) + It("runs the task without error by infer the pipeline resource types from -j", func() { execS := flyIn(tmp, "execute", "-c", "task.yml", "-j", pipelineName+"/input-test") Expect(execS).To(gbytes.Say("custom-type-version")) From a32c883e70e7a2abe85306a2a4c87344393a3c2d Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Wed, 13 Feb 2019 17:28:39 -0500 Subject: [PATCH 12/21] atc: rename build things * engine -> schema * engine_metadata -> private_plan concourse/concourse#3240 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/api/builds_test.go | 2 +- atc/api/buildserver/plan.go | 2 +- atc/db/build.go | 57 ++-- atc/db/build_factory_test.go | 11 +- atc/db/build_test.go | 12 +- atc/db/dbfakes/fake_build.go | 278 +++++++++--------- atc/db/job_test.go | 20 +- .../migration/fix_build_private_plan_test.go | 123 ++++++++ ...9967_rename_build_engine_metadata.down.sql | 4 + ...159967_rename_build_engine_metadata.up.sql | 4 + .../1550160079_fix_build_private_plan.down.go | 76 +++++ .../1550160079_fix_build_private_plan.up.go | 76 +++++ atc/db/open.go | 6 +- atc/db/pipeline.go | 22 +- atc/db/pipeline_test.go | 6 +- atc/db/team.go | 20 +- atc/db/worker_lifecycle_test.go | 8 +- atc/engine/db_engine.go | 35 +-- atc/engine/db_engine_test.go | 25 +- atc/engine/engine.go | 6 +- atc/engine/enginefakes/fake_engine.go | 128 ++++---- atc/engine/exec_engine.go | 18 +- atc/engine/exec_engine_test.go | 25 +- atc/engine/exec_v1_dummy_engine.go | 8 +- atc/plan.go | 2 +- 25 files changed, 625 insertions(+), 349 deletions(-) create mode 100644 atc/db/migration/fix_build_private_plan_test.go create mode 100755 atc/db/migration/migrations/1550159967_rename_build_engine_metadata.down.sql create mode 100755 atc/db/migration/migrations/1550159967_rename_build_engine_metadata.up.sql create mode 100755 atc/db/migration/migrations/1550160079_fix_build_private_plan.down.go create mode 100755 atc/db/migration/migrations/1550160079_fix_build_private_plan.up.go diff --git a/atc/api/builds_test.go b/atc/api/builds_test.go index f5b5d3944dc..09b41e6bd5e 100644 --- a/atc/api/builds_test.go +++ b/atc/api/builds_test.go @@ -1353,7 +1353,7 @@ var _ = Describe("Builds API", func() { Context("when the build returns a plan", func() { BeforeEach(func() { build.PublicPlanReturns(plan) - build.EngineReturns("some-schema") + build.SchemaReturns("some-schema") }) It("returns OK", func() { diff --git a/atc/api/buildserver/plan.go b/atc/api/buildserver/plan.go index 12417632d9c..3c80652faff 100644 --- a/atc/api/buildserver/plan.go +++ b/atc/api/buildserver/plan.go @@ -14,7 +14,7 @@ func (s *Server) GetBuildPlan(build db.Build) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Set("Content-Type", "application/json") err := json.NewEncoder(w).Encode(atc.PublicBuildPlan{ - Schema: build.Engine(), + Schema: build.Schema(), Plan: build.PublicPlan(), }) if err != nil { diff --git a/atc/db/build.go b/atc/db/build.go index cbcdd6cbe44..974bad2da25 100644 --- a/atc/db/build.go +++ b/atc/db/build.go @@ -42,7 +42,7 @@ const ( BuildStatusErrored BuildStatus = "errored" ) -var buildsQuery = psql.Select("b.id, b.name, b.job_id, b.team_id, b.status, b.manually_triggered, b.scheduled, b.engine, b.engine_metadata, b.public_plan, b.start_time, b.end_time, b.reap_time, j.name, b.pipeline_id, p.name, t.name, b.nonce, b.tracked_by, b.drained"). +var buildsQuery = psql.Select("b.id, b.name, b.job_id, b.team_id, b.status, b.manually_triggered, b.scheduled, b.schema, b.private_plan, b.public_plan, b.start_time, b.end_time, b.reap_time, j.name, b.pipeline_id, p.name, t.name, b.nonce, b.tracked_by, b.drained"). From("builds b"). JoinClause("LEFT OUTER JOIN jobs j ON b.job_id = j.id"). JoinClause("LEFT OUTER JOIN pipelines p ON b.pipeline_id = p.id"). @@ -62,8 +62,8 @@ type Build interface { PipelineName() string TeamID() int TeamName() string - Engine() string - EngineMetadata() string + Schema() string + PrivatePlan() string PublicPlan() *json.RawMessage Status() BuildStatus StartTime() time.Time @@ -82,7 +82,7 @@ type Build interface { Interceptible() (bool, error) Preparation() (BuildPreparation, bool, error) - Start(string, string, atc.Plan) (bool, error) + Start(string, atc.Plan) (bool, error) FinishWithError(cause error) error Finish(BuildStatus) error @@ -127,9 +127,9 @@ type build struct { isManuallyTriggered bool - engine string - engineMetadata string - publicPlan *json.RawMessage + schema string + privatePlan string + publicPlan *json.RawMessage startTime time.Time endTime time.Time @@ -164,8 +164,8 @@ func (b *build) PipelineName() string { return b.pipelineName } func (b *build) TeamID() int { return b.teamID } func (b *build) TeamName() string { return b.teamName } func (b *build) IsManuallyTriggered() bool { return b.isManuallyTriggered } -func (b *build) Engine() string { return b.engine } -func (b *build) EngineMetadata() string { return b.engineMetadata } +func (b *build) Schema() string { return b.schema } +func (b *build) PrivatePlan() string { return b.privatePlan } func (b *build) PublicPlan() *json.RawMessage { return b.publicPlan } func (b *build) StartTime() time.Time { return b.startTime } func (b *build) EndTime() time.Time { return b.endTime } @@ -242,7 +242,7 @@ func (b *build) SetInterceptible(i bool) error { return nil } -func (b *build) Start(engine, metadata string, plan atc.Plan) (bool, error) { +func (b *build) Start(schema string, plan atc.Plan) (bool, error) { tx, err := b.conn.Begin() if err != nil { return false, err @@ -250,7 +250,12 @@ func (b *build) Start(engine, metadata string, plan atc.Plan) (bool, error) { defer Rollback(tx) - encryptedMetadata, nonce, err := b.conn.EncryptionStrategy().Encrypt([]byte(metadata)) + metadata, err := json.Marshal(plan) + if err != nil { + return false, err + } + + encryptedPlan, nonce, err := b.conn.EncryptionStrategy().Encrypt([]byte(metadata)) if err != nil { return false, err } @@ -260,8 +265,8 @@ func (b *build) Start(engine, metadata string, plan atc.Plan) (bool, error) { err = psql.Update("builds"). Set("status", BuildStatusStarted). Set("start_time", sq.Expr("now()")). - Set("engine", engine). - Set("engine_metadata", encryptedMetadata). + Set("schema", schema). + Set("private_plan", encryptedPlan). Set("public_plan", plan.Public()). Set("nonce", nonce). Where(sq.Eq{ @@ -332,7 +337,7 @@ func (b *build) Finish(status BuildStatus) error { Set("status", status). Set("end_time", sq.Expr("now()")). Set("completed", true). - Set("engine_metadata", nil). + Set("private_plan", nil). Set("nonce", nil). Where(sq.Eq{"id": b.id}). Suffix("RETURNING end_time"). @@ -1114,16 +1119,16 @@ func buildEventSeq(buildid int) string { func scanBuild(b *build, row scannable, encryptionStrategy encryption.Strategy) error { var ( - jobID, pipelineID sql.NullInt64 - engine, engineMetadata, jobName, pipelineName, publicPlan, trackedBy sql.NullString - startTime, endTime, reapTime pq.NullTime - nonce sql.NullString - drained bool + jobID, pipelineID sql.NullInt64 + schema, privatePlan, jobName, pipelineName, publicPlan, trackedBy sql.NullString + startTime, endTime, reapTime pq.NullTime + nonce sql.NullString + drained bool status string ) - err := row.Scan(&b.id, &b.name, &jobID, &b.teamID, &status, &b.isManuallyTriggered, &b.scheduled, &engine, &engineMetadata, &publicPlan, &startTime, &endTime, &reapTime, &jobName, &pipelineID, &pipelineName, &b.teamName, &nonce, &trackedBy, &drained) + err := row.Scan(&b.id, &b.name, &jobID, &b.teamID, &status, &b.isManuallyTriggered, &b.scheduled, &schema, &privatePlan, &publicPlan, &startTime, &endTime, &reapTime, &jobName, &pipelineID, &pipelineName, &b.teamName, &nonce, &trackedBy, &drained) if err != nil { return err } @@ -1133,7 +1138,7 @@ func scanBuild(b *build, row scannable, encryptionStrategy encryption.Strategy) b.jobID = int(jobID.Int64) b.pipelineName = pipelineName.String b.pipelineID = int(pipelineID.Int64) - b.engine = engine.String + b.schema = schema.String b.startTime = startTime.Time b.endTime = endTime.Time b.reapTime = reapTime.Time @@ -1141,18 +1146,18 @@ func scanBuild(b *build, row scannable, encryptionStrategy encryption.Strategy) b.drained = drained var ( - noncense *string - decryptedEngineMetadata []byte + noncense *string + decryptedPlan []byte ) if nonce.Valid { noncense = &nonce.String - decryptedEngineMetadata, err = encryptionStrategy.Decrypt(string(engineMetadata.String), noncense) + decryptedPlan, err = encryptionStrategy.Decrypt(string(privatePlan.String), noncense) if err != nil { return err } - b.engineMetadata = string(decryptedEngineMetadata) + b.privatePlan = string(decryptedPlan) } else { - b.engineMetadata = engineMetadata.String + b.privatePlan = privatePlan.String } if publicPlan.Valid { diff --git a/atc/db/build_factory_test.go b/atc/db/build_factory_test.go index 3fc9ae306e8..c3e43afe42c 100644 --- a/atc/db/build_factory_test.go +++ b/atc/db/build_factory_test.go @@ -1,9 +1,8 @@ package db_test import ( - "github.com/concourse/concourse/atc/db" - "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/db" . "github.com/onsi/ginkgo" . "github.com/onsi/ginkgo/extensions/table" . "github.com/onsi/gomega" @@ -198,7 +197,7 @@ var _ = Describe("BuildFactory", func() { Expect(err).NotTo(HaveOccurred()) Expect(i).To(BeTrue()) - _, err = b.Start("exec.v2", `{"so":"meta"}`, atc.Plan{}) + _, err = b.Start("exec.v2", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) err = buildFactory.MarkNonInterceptibleBuilds() @@ -330,7 +329,7 @@ var _ = Describe("BuildFactory", func() { build4DB, err = job.CreateBuild() Expect(err).NotTo(HaveOccurred()) - started, err := build2DB.Start("some-engine", `{"so":"meta"}`, atc.Plan{}) + started, err := build2DB.Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) @@ -382,11 +381,11 @@ var _ = Describe("BuildFactory", func() { _, err = team.CreateOneOffBuild() Expect(err).NotTo(HaveOccurred()) - started, err := build1DB.Start("some-engine", `{"so":"meta"}`, atc.Plan{}) + started, err := build1DB.Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) - started, err = build2DB.Start("some-engine", `{"so":"meta"}`, atc.Plan{}) + started, err = build2DB.Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) }) diff --git a/atc/db/build_test.go b/atc/db/build_test.go index b4934436298..772fe135888 100644 --- a/atc/db/build_test.go +++ b/atc/db/build_test.go @@ -29,7 +29,7 @@ var _ = Describe("Build", func() { It("updates the model", func() { build, err := team.CreateOneOffBuild() Expect(err).NotTo(HaveOccurred()) - started, err := build.Start("engine", `{"meta":"data"}`, atc.Plan{}) + started, err := build.Start("schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) @@ -100,7 +100,7 @@ var _ = Describe("Build", func() { build, err = team.CreateOneOffBuild() Expect(err).NotTo(HaveOccurred()) - started, err := build.Start("engine", `{"meta":"data"}`, plan) + started, err := build.Start("schema", plan) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) }) @@ -196,7 +196,7 @@ var _ = Describe("Build", func() { found, err := build.Reload() Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue()) - Expect(build.EngineMetadata()).To(BeEmpty()) + Expect(build.PrivatePlan()).To(BeEmpty()) }) }) @@ -231,7 +231,7 @@ var _ = Describe("Build", func() { defer db.Close(events) By("emitting a status event when started") - started, err := build.Start("engine", `{"meta":"data"}`, atc.Plan{}) + started, err := build.Start("schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) @@ -675,7 +675,7 @@ var _ = Describe("Build", func() { Context("when the build is started", func() { BeforeEach(func() { - started, err := build.Start("some-engine", `{"meta":"data"}`, atc.Plan{}) + started, err := build.Start("some-schema", atc.Plan{}) Expect(started).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) @@ -773,7 +773,7 @@ var _ = Describe("Build", func() { Context("when the build is started", func() { BeforeEach(func() { - started, err := build.Start("some-engine", `{"meta":"data"}`, atc.Plan{}) + started, err := build.Start("some-schema", atc.Plan{}) Expect(started).To(BeTrue()) Expect(err).NotTo(HaveOccurred()) diff --git a/atc/db/dbfakes/fake_build.go b/atc/db/dbfakes/fake_build.go index 6bd1759a460..a57d530689b 100644 --- a/atc/db/dbfakes/fake_build.go +++ b/atc/db/dbfakes/fake_build.go @@ -89,26 +89,6 @@ type FakeBuild struct { endTimeReturnsOnCall map[int]struct { result1 time.Time } - EngineStub func() string - engineMutex sync.RWMutex - engineArgsForCall []struct { - } - engineReturns struct { - result1 string - } - engineReturnsOnCall map[int]struct { - result1 string - } - EngineMetadataStub func() string - engineMetadataMutex sync.RWMutex - engineMetadataArgsForCall []struct { - } - engineMetadataReturns struct { - result1 string - } - engineMetadataReturnsOnCall map[int]struct { - result1 string - } EventsStub func(uint) (db.EventSource, error) eventsMutex sync.RWMutex eventsArgsForCall []struct { @@ -294,6 +274,16 @@ type FakeBuild struct { result2 bool result3 error } + PrivatePlanStub func() string + privatePlanMutex sync.RWMutex + privatePlanArgsForCall []struct { + } + privatePlanReturns struct { + result1 string + } + privatePlanReturnsOnCall map[int]struct { + result1 string + } PublicPlanStub func() *json.RawMessage publicPlanMutex sync.RWMutex publicPlanArgsForCall []struct { @@ -392,6 +382,16 @@ type FakeBuild struct { result1 bool result2 error } + SchemaStub func() string + schemaMutex sync.RWMutex + schemaArgsForCall []struct { + } + schemaReturns struct { + result1 string + } + schemaReturnsOnCall map[int]struct { + result1 string + } SetDrainedStub func(bool) error setDrainedMutex sync.RWMutex setDrainedArgsForCall []struct { @@ -414,12 +414,11 @@ type FakeBuild struct { setInterceptibleReturnsOnCall map[int]struct { result1 error } - StartStub func(string, string, atc.Plan) (bool, error) + StartStub func(string, atc.Plan) (bool, error) startMutex sync.RWMutex startArgsForCall []struct { arg1 string - arg2 string - arg3 atc.Plan + arg2 atc.Plan } startReturns struct { result1 bool @@ -852,110 +851,6 @@ func (fake *FakeBuild) EndTimeReturnsOnCall(i int, result1 time.Time) { }{result1} } -func (fake *FakeBuild) Engine() string { - fake.engineMutex.Lock() - ret, specificReturn := fake.engineReturnsOnCall[len(fake.engineArgsForCall)] - fake.engineArgsForCall = append(fake.engineArgsForCall, struct { - }{}) - fake.recordInvocation("Engine", []interface{}{}) - fake.engineMutex.Unlock() - if fake.EngineStub != nil { - return fake.EngineStub() - } - if specificReturn { - return ret.result1 - } - fakeReturns := fake.engineReturns - return fakeReturns.result1 -} - -func (fake *FakeBuild) EngineCallCount() int { - fake.engineMutex.RLock() - defer fake.engineMutex.RUnlock() - return len(fake.engineArgsForCall) -} - -func (fake *FakeBuild) EngineCalls(stub func() string) { - fake.engineMutex.Lock() - defer fake.engineMutex.Unlock() - fake.EngineStub = stub -} - -func (fake *FakeBuild) EngineReturns(result1 string) { - fake.engineMutex.Lock() - defer fake.engineMutex.Unlock() - fake.EngineStub = nil - fake.engineReturns = struct { - result1 string - }{result1} -} - -func (fake *FakeBuild) EngineReturnsOnCall(i int, result1 string) { - fake.engineMutex.Lock() - defer fake.engineMutex.Unlock() - fake.EngineStub = nil - if fake.engineReturnsOnCall == nil { - fake.engineReturnsOnCall = make(map[int]struct { - result1 string - }) - } - fake.engineReturnsOnCall[i] = struct { - result1 string - }{result1} -} - -func (fake *FakeBuild) EngineMetadata() string { - fake.engineMetadataMutex.Lock() - ret, specificReturn := fake.engineMetadataReturnsOnCall[len(fake.engineMetadataArgsForCall)] - fake.engineMetadataArgsForCall = append(fake.engineMetadataArgsForCall, struct { - }{}) - fake.recordInvocation("EngineMetadata", []interface{}{}) - fake.engineMetadataMutex.Unlock() - if fake.EngineMetadataStub != nil { - return fake.EngineMetadataStub() - } - if specificReturn { - return ret.result1 - } - fakeReturns := fake.engineMetadataReturns - return fakeReturns.result1 -} - -func (fake *FakeBuild) EngineMetadataCallCount() int { - fake.engineMetadataMutex.RLock() - defer fake.engineMetadataMutex.RUnlock() - return len(fake.engineMetadataArgsForCall) -} - -func (fake *FakeBuild) EngineMetadataCalls(stub func() string) { - fake.engineMetadataMutex.Lock() - defer fake.engineMetadataMutex.Unlock() - fake.EngineMetadataStub = stub -} - -func (fake *FakeBuild) EngineMetadataReturns(result1 string) { - fake.engineMetadataMutex.Lock() - defer fake.engineMetadataMutex.Unlock() - fake.EngineMetadataStub = nil - fake.engineMetadataReturns = struct { - result1 string - }{result1} -} - -func (fake *FakeBuild) EngineMetadataReturnsOnCall(i int, result1 string) { - fake.engineMetadataMutex.Lock() - defer fake.engineMetadataMutex.Unlock() - fake.EngineMetadataStub = nil - if fake.engineMetadataReturnsOnCall == nil { - fake.engineMetadataReturnsOnCall = make(map[int]struct { - result1 string - }) - } - fake.engineMetadataReturnsOnCall[i] = struct { - result1 string - }{result1} -} - func (fake *FakeBuild) Events(arg1 uint) (db.EventSource, error) { fake.eventsMutex.Lock() ret, specificReturn := fake.eventsReturnsOnCall[len(fake.eventsArgsForCall)] @@ -1882,6 +1777,58 @@ func (fake *FakeBuild) PreparationReturnsOnCall(i int, result1 db.BuildPreparati }{result1, result2, result3} } +func (fake *FakeBuild) PrivatePlan() string { + fake.privatePlanMutex.Lock() + ret, specificReturn := fake.privatePlanReturnsOnCall[len(fake.privatePlanArgsForCall)] + fake.privatePlanArgsForCall = append(fake.privatePlanArgsForCall, struct { + }{}) + fake.recordInvocation("PrivatePlan", []interface{}{}) + fake.privatePlanMutex.Unlock() + if fake.PrivatePlanStub != nil { + return fake.PrivatePlanStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.privatePlanReturns + return fakeReturns.result1 +} + +func (fake *FakeBuild) PrivatePlanCallCount() int { + fake.privatePlanMutex.RLock() + defer fake.privatePlanMutex.RUnlock() + return len(fake.privatePlanArgsForCall) +} + +func (fake *FakeBuild) PrivatePlanCalls(stub func() string) { + fake.privatePlanMutex.Lock() + defer fake.privatePlanMutex.Unlock() + fake.PrivatePlanStub = stub +} + +func (fake *FakeBuild) PrivatePlanReturns(result1 string) { + fake.privatePlanMutex.Lock() + defer fake.privatePlanMutex.Unlock() + fake.PrivatePlanStub = nil + fake.privatePlanReturns = struct { + result1 string + }{result1} +} + +func (fake *FakeBuild) PrivatePlanReturnsOnCall(i int, result1 string) { + fake.privatePlanMutex.Lock() + defer fake.privatePlanMutex.Unlock() + fake.PrivatePlanStub = nil + if fake.privatePlanReturnsOnCall == nil { + fake.privatePlanReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.privatePlanReturnsOnCall[i] = struct { + result1 string + }{result1} +} + func (fake *FakeBuild) PublicPlan() *json.RawMessage { fake.publicPlanMutex.Lock() ret, specificReturn := fake.publicPlanReturnsOnCall[len(fake.publicPlanArgsForCall)] @@ -2341,6 +2288,58 @@ func (fake *FakeBuild) ScheduleReturnsOnCall(i int, result1 bool, result2 error) }{result1, result2} } +func (fake *FakeBuild) Schema() string { + fake.schemaMutex.Lock() + ret, specificReturn := fake.schemaReturnsOnCall[len(fake.schemaArgsForCall)] + fake.schemaArgsForCall = append(fake.schemaArgsForCall, struct { + }{}) + fake.recordInvocation("Schema", []interface{}{}) + fake.schemaMutex.Unlock() + if fake.SchemaStub != nil { + return fake.SchemaStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.schemaReturns + return fakeReturns.result1 +} + +func (fake *FakeBuild) SchemaCallCount() int { + fake.schemaMutex.RLock() + defer fake.schemaMutex.RUnlock() + return len(fake.schemaArgsForCall) +} + +func (fake *FakeBuild) SchemaCalls(stub func() string) { + fake.schemaMutex.Lock() + defer fake.schemaMutex.Unlock() + fake.SchemaStub = stub +} + +func (fake *FakeBuild) SchemaReturns(result1 string) { + fake.schemaMutex.Lock() + defer fake.schemaMutex.Unlock() + fake.SchemaStub = nil + fake.schemaReturns = struct { + result1 string + }{result1} +} + +func (fake *FakeBuild) SchemaReturnsOnCall(i int, result1 string) { + fake.schemaMutex.Lock() + defer fake.schemaMutex.Unlock() + fake.SchemaStub = nil + if fake.schemaReturnsOnCall == nil { + fake.schemaReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.schemaReturnsOnCall[i] = struct { + result1 string + }{result1} +} + func (fake *FakeBuild) SetDrained(arg1 bool) error { fake.setDrainedMutex.Lock() ret, specificReturn := fake.setDrainedReturnsOnCall[len(fake.setDrainedArgsForCall)] @@ -2461,18 +2460,17 @@ func (fake *FakeBuild) SetInterceptibleReturnsOnCall(i int, result1 error) { }{result1} } -func (fake *FakeBuild) Start(arg1 string, arg2 string, arg3 atc.Plan) (bool, error) { +func (fake *FakeBuild) Start(arg1 string, arg2 atc.Plan) (bool, error) { fake.startMutex.Lock() ret, specificReturn := fake.startReturnsOnCall[len(fake.startArgsForCall)] fake.startArgsForCall = append(fake.startArgsForCall, struct { arg1 string - arg2 string - arg3 atc.Plan - }{arg1, arg2, arg3}) - fake.recordInvocation("Start", []interface{}{arg1, arg2, arg3}) + arg2 atc.Plan + }{arg1, arg2}) + fake.recordInvocation("Start", []interface{}{arg1, arg2}) fake.startMutex.Unlock() if fake.StartStub != nil { - return fake.StartStub(arg1, arg2, arg3) + return fake.StartStub(arg1, arg2) } if specificReturn { return ret.result1, ret.result2 @@ -2487,17 +2485,17 @@ func (fake *FakeBuild) StartCallCount() int { return len(fake.startArgsForCall) } -func (fake *FakeBuild) StartCalls(stub func(string, string, atc.Plan) (bool, error)) { +func (fake *FakeBuild) StartCalls(stub func(string, atc.Plan) (bool, error)) { fake.startMutex.Lock() defer fake.startMutex.Unlock() fake.StartStub = stub } -func (fake *FakeBuild) StartArgsForCall(i int) (string, string, atc.Plan) { +func (fake *FakeBuild) StartArgsForCall(i int) (string, atc.Plan) { fake.startMutex.RLock() defer fake.startMutex.RUnlock() argsForCall := fake.startArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 + return argsForCall.arg1, argsForCall.arg2 } func (fake *FakeBuild) StartReturns(result1 bool, result2 error) { @@ -2926,10 +2924,6 @@ func (fake *FakeBuild) Invocations() map[string][][]interface{} { defer fake.deleteMutex.RUnlock() fake.endTimeMutex.RLock() defer fake.endTimeMutex.RUnlock() - fake.engineMutex.RLock() - defer fake.engineMutex.RUnlock() - fake.engineMetadataMutex.RLock() - defer fake.engineMetadataMutex.RUnlock() fake.eventsMutex.RLock() defer fake.eventsMutex.RUnlock() fake.finishMutex.RLock() @@ -2964,6 +2958,8 @@ func (fake *FakeBuild) Invocations() map[string][][]interface{} { defer fake.pipelineNameMutex.RUnlock() fake.preparationMutex.RLock() defer fake.preparationMutex.RUnlock() + fake.privatePlanMutex.RLock() + defer fake.privatePlanMutex.RUnlock() fake.publicPlanMutex.RLock() defer fake.publicPlanMutex.RUnlock() fake.reapTimeMutex.RLock() @@ -2980,6 +2976,8 @@ func (fake *FakeBuild) Invocations() map[string][][]interface{} { defer fake.saveOutputMutex.RUnlock() fake.scheduleMutex.RLock() defer fake.scheduleMutex.RUnlock() + fake.schemaMutex.RLock() + defer fake.schemaMutex.RUnlock() fake.setDrainedMutex.RLock() defer fake.setDrainedMutex.RUnlock() fake.setInterceptibleMutex.RLock() diff --git a/atc/db/job_test.go b/atc/db/job_test.go index cc15f3e4023..ac909e85e21 100644 --- a/atc/db/job_test.go +++ b/atc/db/job_test.go @@ -170,14 +170,14 @@ var _ = Describe("Job", func() { nextBuild, err := job.CreateBuild() Expect(err).NotTo(HaveOccurred()) - started, err := nextBuild.Start("some-engine", `{"id":"1"}`, atc.Plan{}) + started, err := nextBuild.Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) otherNextBuild, err := otherJob.CreateBuild() Expect(err).NotTo(HaveOccurred()) - otherStarted, err := otherNextBuild.Start("some-engine", `{"id":"1"}`, atc.Plan{}) + otherStarted, err := otherNextBuild.Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(otherStarted).To(BeTrue()) @@ -196,7 +196,7 @@ var _ = Describe("Job", func() { Expect(next.ID()).To(Equal(nextBuild.ID())) // not anotherRunningBuild Expect(finished.ID()).To(Equal(finishedBuild.ID())) - started, err = anotherRunningBuild.Start("some-engine", `{"meta":"data"}`, atc.Plan{}) + started, err = anotherRunningBuild.Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) @@ -497,7 +497,7 @@ var _ = Describe("Job", func() { startedBuild, err = job.CreateBuild() Expect(err).NotTo(HaveOccurred()) - _, err = startedBuild.Start("", "{}", atc.Plan{}) + _, err = startedBuild.Start("", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) scheduledBuild, err = job.CreateBuild() @@ -1174,18 +1174,18 @@ var _ = Describe("Job", func() { Context("when started", func() { BeforeEach(func() { - started, err := build1DB.Start("some-engine", `{"some":"metadata"}`, atc.Plan{}) + started, err := build1DB.Start("some-schema", atc.Plan{ID: "some-id"}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) }) - It("saves the updated status, and the engine and engine metadata", func() { + It("saves the updated status, and the schema and private plan", func() { found, err := build1DB.Reload() Expect(err).NotTo(HaveOccurred()) Expect(found).To(BeTrue()) Expect(build1DB.Status()).To(Equal(db.BuildStatusStarted)) - Expect(build1DB.Engine()).To(Equal("some-engine")) - Expect(build1DB.EngineMetadata()).To(Equal(`{"some":"metadata"}`)) + Expect(build1DB.Schema()).To(Equal("some-schema")) + Expect(build1DB.PrivatePlan()).To(Equal(`{"id":"some-id"}`)) }) It("saves the build's start time", func() { @@ -1250,7 +1250,7 @@ var _ = Describe("Job", func() { build1, err := job.CreateBuild() Expect(err).NotTo(HaveOccurred()) - started, err := build1.Start("some-engine", `{"some":"metadata"}`, atc.Plan{}) + started, err := build1.Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) }) @@ -1275,7 +1275,7 @@ var _ = Describe("Job", func() { Expect(err).NotTo(HaveOccurred()) Expect(builds2).To(HaveLen(1)) - started, err := builds2[0].Start("some-engine", `{"some":"metadata"}`, atc.Plan{}) + started, err := builds2[0].Start("some-schema", atc.Plan{}) Expect(err).NotTo(HaveOccurred()) Expect(started).To(BeTrue()) diff --git a/atc/db/migration/fix_build_private_plan_test.go b/atc/db/migration/fix_build_private_plan_test.go new file mode 100644 index 00000000000..9b83aa19d45 --- /dev/null +++ b/atc/db/migration/fix_build_private_plan_test.go @@ -0,0 +1,123 @@ +package migration_test + +import ( + "database/sql" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Fix Build Private Plan", func() { + const preMigrationVersion = 1550159967 + const postMigrationVersion = 1550160079 + + var ( + db *sql.DB + ) + + Context("Up", func() { + It("ignores NULL plans", func() { + db = postgresRunner.OpenDBAtVersion(preMigrationVersion) + SetupTeam(db, "some-team", "{}") + SetupBuild(db, "some-build") + db.Close() + + db = postgresRunner.OpenDBAtVersion(postMigrationVersion) + ExpectBuildWithNullPlan(db, "some-build") + db.Close() + }) + + It("removes the 'plan' key", func() { + db = postgresRunner.OpenDBAtVersion(preMigrationVersion) + SetupTeam(db, "some-team", "{}") + SetupBuildWithPlan(db, "some-build", `{"plan":{"some":"plan"}}`) + db.Close() + + db = postgresRunner.OpenDBAtVersion(postMigrationVersion) + ExpectBuildWithPlan(db, "some-build", `{"some":"plan"}`) + db.Close() + }) + + It("updates multiple plans", func() { + db = postgresRunner.OpenDBAtVersion(preMigrationVersion) + SetupTeam(db, "some-team", "{}") + SetupBuildWithPlan(db, "some-build", `{"plan":{"some":"plan"}}`) + SetupBuildWithPlan(db, "some-other-build", `{"plan":{"some":"other-plan"}}`) + db.Close() + + db = postgresRunner.OpenDBAtVersion(postMigrationVersion) + ExpectBuildWithPlan(db, "some-build", `{"some":"plan"}`) + ExpectBuildWithPlan(db, "some-other-build", `{"some":"other-plan"}`) + db.Close() + }) + }) + + Context("Down", func() { + It("ignores NULL plans", func() { + db = postgresRunner.OpenDBAtVersion(postMigrationVersion) + SetupTeam(db, "some-team", "{}") + SetupBuild(db, "some-build") + db.Close() + + db = postgresRunner.OpenDBAtVersion(preMigrationVersion) + ExpectBuildWithNullPlan(db, "some-build") + db.Close() + }) + + It("nests the plan under a 'plan' key", func() { + db = postgresRunner.OpenDBAtVersion(postMigrationVersion) + SetupTeam(db, "some-team", "{}") + SetupBuildWithPlan(db, "some-build", `{"some":"plan"}`) + db.Close() + + db = postgresRunner.OpenDBAtVersion(preMigrationVersion) + ExpectBuildWithPlan(db, "some-build", `{"plan":{"some":"plan"}}`) + db.Close() + }) + + It("updates multiple plans", func() { + db = postgresRunner.OpenDBAtVersion(postMigrationVersion) + SetupTeam(db, "some-team", "{}") + SetupBuildWithPlan(db, "some-build", `{"some":"plan"}`) + SetupBuildWithPlan(db, "some-other-build", `{"some":"other-plan"}`) + db.Close() + + db = postgresRunner.OpenDBAtVersion(preMigrationVersion) + ExpectBuildWithPlan(db, "some-build", `{"plan":{"some":"plan"}}`) + ExpectBuildWithPlan(db, "some-other-build", `{"plan":{"some":"other-plan"}}`) + db.Close() + }) + }) +}) + +func SetupBuild(dbConn *sql.DB, name string) { + _, err := dbConn.Exec("INSERT INTO builds(name, status, team_id) VALUES($1, 'started', 1)", name) + Expect(err).NotTo(HaveOccurred()) +} + +func SetupBuildWithPlan(dbConn *sql.DB, name, plan string) { + _, err := dbConn.Exec("INSERT INTO builds(name, status, team_id, private_plan) VALUES($1, 'started', 1, $2)", name, plan) + Expect(err).NotTo(HaveOccurred()) +} + +func ExpectBuildWithNullPlan(dbConn *sql.DB, name string) { + + plan := fetchBuildPlan(dbConn, name) + + Expect(plan.Valid).To(BeFalse()) +} + +func ExpectBuildWithPlan(dbConn *sql.DB, name, expectedPlan string) { + + plan := fetchBuildPlan(dbConn, name) + + Expect(plan.Valid).To(BeTrue()) + Expect(plan.String).To(Equal(expectedPlan)) +} + +func fetchBuildPlan(dbConn *sql.DB, name string) sql.NullString { + var plan sql.NullString + err := dbConn.QueryRow("SELECT private_plan FROM builds WHERE name = $1", name).Scan(&plan) + Expect(err).NotTo(HaveOccurred()) + return plan +} diff --git a/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.down.sql b/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.down.sql new file mode 100755 index 00000000000..e8f60a83ea3 --- /dev/null +++ b/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.down.sql @@ -0,0 +1,4 @@ +BEGIN; + ALTER TABLE builds RENAME COLUMN schema TO engine; + ALTER TABLE builds RENAME COLUMN private_plan TO engine_metadata; +COMMIT; diff --git a/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.up.sql b/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.up.sql new file mode 100755 index 00000000000..13b66382eb1 --- /dev/null +++ b/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.up.sql @@ -0,0 +1,4 @@ +BEGIN; + ALTER TABLE builds RENAME COLUMN engine TO schema; + ALTER TABLE builds RENAME COLUMN engine_metadata TO private_plan; +COMMIT; diff --git a/atc/db/migration/migrations/1550160079_fix_build_private_plan.down.go b/atc/db/migration/migrations/1550160079_fix_build_private_plan.down.go new file mode 100755 index 00000000000..08254f86485 --- /dev/null +++ b/atc/db/migration/migrations/1550160079_fix_build_private_plan.down.go @@ -0,0 +1,76 @@ +package migrations + +import ( + "database/sql" + "encoding/json" +) + +func (self *migrations) Down_1550160079() error { + + type build struct { + id int + plan sql.NullString + nonce sql.NullString + } + + tx, err := self.DB.Begin() + if err != nil { + return err + } + + defer tx.Rollback() + + rows, err := tx.Query("SELECT id, private_plan, nonce FROM builds WHERE private_plan IS NOT NULL") + if err != nil { + return err + } + + builds := []build{} + for rows.Next() { + + build := build{} + if err = rows.Scan(&build.id, &build.plan, &build.nonce); err != nil { + return err + } + + if build.plan.Valid { + builds = append(builds, build) + } + } + + for _, build := range builds { + + var noncense *string + if build.nonce.Valid { + noncense = &build.nonce.String + } + + decrypted, err := self.Strategy.Decrypt(build.plan.String, noncense) + if err != nil { + return err + } + + var payload map[string]interface{} + err = json.Unmarshal(decrypted, &payload) + if err != nil { + return err + } + + fixed, err := json.Marshal(map[string]interface{}{"plan": payload}) + if err != nil { + return err + } + + encrypted, newnonce, err := self.Strategy.Encrypt(fixed) + if err != nil { + return err + } + + _, err = tx.Exec("UPDATE builds SET private_plan = $1, nonce = $2 WHERE id = $3", encrypted, newnonce, build.id) + if err != nil { + return err + } + } + + return tx.Commit() +} diff --git a/atc/db/migration/migrations/1550160079_fix_build_private_plan.up.go b/atc/db/migration/migrations/1550160079_fix_build_private_plan.up.go new file mode 100755 index 00000000000..d1944d5709b --- /dev/null +++ b/atc/db/migration/migrations/1550160079_fix_build_private_plan.up.go @@ -0,0 +1,76 @@ +package migrations + +import ( + "database/sql" + "encoding/json" +) + +func (self *migrations) Up_1550160079() error { + + type build struct { + id int + plan sql.NullString + nonce sql.NullString + } + + tx, err := self.DB.Begin() + if err != nil { + return err + } + + defer tx.Rollback() + + rows, err := tx.Query("SELECT id, private_plan, nonce FROM builds WHERE private_plan IS NOT NULL") + if err != nil { + return err + } + + builds := []build{} + for rows.Next() { + + build := build{} + if err = rows.Scan(&build.id, &build.plan, &build.nonce); err != nil { + return err + } + + if build.plan.Valid { + builds = append(builds, build) + } + } + + for _, build := range builds { + + var noncense *string + if build.nonce.Valid { + noncense = &build.nonce.String + } + + decrypted, err := self.Strategy.Decrypt(build.plan.String, noncense) + if err != nil { + return err + } + + var payload map[string]interface{} + err = json.Unmarshal(decrypted, &payload) + if err != nil { + return err + } + + fixed, err := json.Marshal(payload["plan"]) + if err != nil { + return err + } + + encrypted, newnonce, err := self.Strategy.Encrypt(fixed) + if err != nil { + return err + } + + _, err = tx.Exec("UPDATE builds SET private_plan = $1, nonce = $2 WHERE id = $3", encrypted, newnonce, build.id) + if err != nil { + return err + } + } + + return tx.Commit() +} diff --git a/atc/db/open.go b/atc/db/open.go index 0e99a5c4f00..8319356c743 100644 --- a/atc/db/open.go +++ b/atc/db/open.go @@ -10,14 +10,12 @@ import ( "time" "code.cloudfoundry.org/lager" - "github.com/Masterminds/squirrel" "github.com/concourse/concourse/atc/db/encryption" "github.com/concourse/concourse/atc/db/lock" "github.com/concourse/concourse/atc/db/migration" - "github.com/lib/pq" - multierror "github.com/hashicorp/go-multierror" + "github.com/lib/pq" ) //go:generate counterfeiter . Conn @@ -121,7 +119,7 @@ var encryptedColumns = map[string]string{ "resources": "config", "jobs": "config", "resource_types": "config", - "builds": "engine_metadata", + "builds": "private_plan", } func encryptPlaintext(logger lager.Logger, sqlDB *sql.DB, key *encryption.Key) error { diff --git a/atc/db/pipeline.go b/atc/db/pipeline.go index 508a9b0772c..ee26c1f8d53 100644 --- a/atc/db/pipeline.go +++ b/atc/db/pipeline.go @@ -979,27 +979,27 @@ func (p *pipeline) CreateStartedBuild(plan atc.Plan) (Build, error) { defer Rollback(tx) - metadata, err := json.Marshal(map[string]interface{}{"plan": plan}) + metadata, err := json.Marshal(plan) if err != nil { return nil, err } - encryptedMetadata, nonce, err := p.conn.EncryptionStrategy().Encrypt(metadata) + encryptedPlan, nonce, err := p.conn.EncryptionStrategy().Encrypt(metadata) if err != nil { return nil, err } build := &build{conn: p.conn, lockFactory: p.lockFactory} err = createBuild(tx, build, map[string]interface{}{ - "name": sq.Expr("nextval('one_off_name')"), - "pipeline_id": p.id, - "team_id": p.teamID, - "status": BuildStatusStarted, - "start_time": sq.Expr("now()"), - "engine": "exec.v2", - "engine_metadata": encryptedMetadata, - "public_plan": plan.Public(), - "nonce": nonce, + "name": sq.Expr("nextval('one_off_name')"), + "pipeline_id": p.id, + "team_id": p.teamID, + "status": BuildStatusStarted, + "start_time": sq.Expr("now()"), + "schema": "exec.v2", + "private_plan": encryptedPlan, + "public_plan": plan.Public(), + "nonce": nonce, }) if err != nil { return nil, err diff --git a/atc/db/pipeline_test.go b/atc/db/pipeline_test.go index 03ffa307028..bed2632695e 100644 --- a/atc/db/pipeline_test.go +++ b/atc/db/pipeline_test.go @@ -1483,7 +1483,7 @@ var _ = Describe("Pipeline", func() { Expect(actualDashboard[0].NextBuild.ID()).To(Equal(firstJobBuild.ID())) By("returning a job's most recent started build") - found, err = firstJobBuild.Start("engine", `{"meta":"data"}`, atc.Plan{}) + found, err = firstJobBuild.Start("some-schema", atc.Plan{ID: "some-id"}) Expect(err).ToNot(HaveOccurred()) Expect(found).To(BeTrue()) @@ -1497,8 +1497,8 @@ var _ = Describe("Pipeline", func() { Expect(actualDashboard[0].Job.Name()).To(Equal(job.Name())) Expect(actualDashboard[0].NextBuild.ID()).To(Equal(firstJobBuild.ID())) Expect(actualDashboard[0].NextBuild.Status()).To(Equal(db.BuildStatusStarted)) - Expect(actualDashboard[0].NextBuild.Engine()).To(Equal("engine")) - Expect(actualDashboard[0].NextBuild.EngineMetadata()).To(Equal(`{"meta":"data"}`)) + Expect(actualDashboard[0].NextBuild.Schema()).To(Equal("some-schema")) + Expect(actualDashboard[0].NextBuild.PrivatePlan()).To(Equal(`{"id":"some-id"}`)) By("returning a job's most recent started build even if there is a newer pending build") job, found, err = pipeline.Job("job-name") diff --git a/atc/db/team.go b/atc/db/team.go index 02c9a594a14..2302e20d11c 100644 --- a/atc/db/team.go +++ b/atc/db/team.go @@ -692,26 +692,26 @@ func (t *team) CreateStartedBuild(plan atc.Plan) (Build, error) { defer Rollback(tx) - metadata, err := json.Marshal(map[string]interface{}{"plan": plan}) + metadata, err := json.Marshal(plan) if err != nil { return nil, err } - encryptedMetadata, nonce, err := t.conn.EncryptionStrategy().Encrypt(metadata) + encryptedPlan, nonce, err := t.conn.EncryptionStrategy().Encrypt(metadata) if err != nil { return nil, err } build := &build{conn: t.conn, lockFactory: t.lockFactory} err = createBuild(tx, build, map[string]interface{}{ - "name": sq.Expr("nextval('one_off_name')"), - "team_id": t.id, - "status": BuildStatusStarted, - "start_time": sq.Expr("now()"), - "engine": "exec.v2", - "engine_metadata": encryptedMetadata, - "public_plan": plan.Public(), - "nonce": nonce, + "name": sq.Expr("nextval('one_off_name')"), + "team_id": t.id, + "status": BuildStatusStarted, + "start_time": sq.Expr("now()"), + "schema": "exec.v2", + "private_plan": encryptedPlan, + "public_plan": plan.Public(), + "nonce": nonce, }) if err != nil { return nil, err diff --git a/atc/db/worker_lifecycle_test.go b/atc/db/worker_lifecycle_test.go index 831f960ddce..6a303e12bcc 100644 --- a/atc/db/worker_lifecycle_test.go +++ b/atc/db/worker_lifecycle_test.go @@ -168,7 +168,7 @@ var _ = Describe("Worker Lifecycle", func() { switch s { case db.BuildStatusPending: case db.BuildStatusStarted: - _, err = dbBuild.Start("exec.v2", "{}", atc.Plan{}) + _, err = dbBuild.Start("exec.v2", atc.Plan{}) Expect(err).ToNot(HaveOccurred()) default: err = dbBuild.Finish(s) @@ -200,7 +200,7 @@ var _ = Describe("Worker Lifecycle", func() { switch s { case db.BuildStatusPending: case db.BuildStatusStarted: - _, err := dbBuild.Start("exec.v2", "{}", atc.Plan{}) + _, err := dbBuild.Start("exec.v2", atc.Plan{}) Expect(err).ToNot(HaveOccurred()) default: err := dbBuild.Finish(s) @@ -412,7 +412,7 @@ var _ = Describe("Worker Lifecycle", func() { switch s { case db.BuildStatusPending: case db.BuildStatusStarted: - _, err := dbBuild.Start("exec.v2", "{}", atc.Plan{}) + _, err := dbBuild.Start("exec.v2", atc.Plan{}) Expect(err).ToNot(HaveOccurred()) default: err := dbBuild.Finish(s) @@ -446,7 +446,7 @@ var _ = Describe("Worker Lifecycle", func() { switch s { case db.BuildStatusPending: case db.BuildStatusStarted: - _, err := dbBuild.Start("exec.v2", "{}", atc.Plan{}) + _, err := dbBuild.Start("exec.v2", atc.Plan{}) Expect(err).ToNot(HaveOccurred()) default: err := dbBuild.Finish(s) diff --git a/atc/engine/db_engine.go b/atc/engine/db_engine.go index 08b4c378fce..b51b7a19fca 100644 --- a/atc/engine/db_engine.go +++ b/atc/engine/db_engine.go @@ -24,11 +24,11 @@ func NewDBEngine(engines Engines, peerURL string) Engine { } type UnknownEngineError struct { - Engine string + Schema string } func (err UnknownEngineError) Error() string { - return fmt.Sprintf("unknown build engine: %s", err.Engine) + return fmt.Sprintf("unknown build engine schema: %s", err.Schema) } type dbEngine struct { @@ -38,7 +38,7 @@ type dbEngine struct { waitGroup *sync.WaitGroup } -func (*dbEngine) Name() string { +func (*dbEngine) Schema() string { return "db" } @@ -50,7 +50,7 @@ func (engine *dbEngine) CreateBuild(logger lager.Logger, build db.Build, plan at return nil, err } - started, err := build.Start(buildEngine.Name(), createdBuild.Metadata(), plan) + started, err := build.Start(buildEngine.Schema(), plan) if err != nil { return nil, err } @@ -146,9 +146,9 @@ func (build *dbBuild) Abort(logger lager.Logger) error { return nil } - buildEngineName := build.build.Engine() - // if there's an engine, there's a real build to abort - if buildEngineName == "" { + schema := build.build.Schema() + // if there's an schema, there's a real build to abort + if schema == "" { // otherwise, CreateBuild had not yet tried to start the build, and so it // will see the conflict when it tries to transition, and abort itself. // @@ -158,10 +158,11 @@ func (build *dbBuild) Abort(logger lager.Logger) error { return build.build.Finish(db.BuildStatusAborted) } - buildEngine, found := build.engines.Lookup(buildEngineName) + buildEngine, found := build.engines.Lookup(schema) if !found { - logger.Error("unknown-engine", nil, lager.Data{"engine": buildEngineName}) - return UnknownEngineError{buildEngineName} + err := UnknownEngineError{schema} + logger.Error("unknown-engine", err, lager.Data{"schema": schema}) + return err } // find the real build to abort... @@ -209,9 +210,9 @@ func (build *dbBuild) Resume(logger lager.Logger) { return } - buildEngineName := build.build.Engine() - if buildEngineName == "" { - logger.Error("build-has-no-engine", err) + schema := build.build.Schema() + if schema == "" { + logger.Error("build-has-no-schema", err) return } @@ -222,11 +223,11 @@ func (build *dbBuild) Resume(logger lager.Logger) { return } - buildEngine, found := build.engines.Lookup(buildEngineName) + buildEngine, found := build.engines.Lookup(schema) if !found { - err := UnknownEngineError{Engine: buildEngineName} - logger.Error("unknown-build-engine", err, lager.Data{ - "engine": buildEngineName, + err := UnknownEngineError{schema} + logger.Error("unknown-engine", err, lager.Data{ + "schema": schema, }) build.finishWithError(logger, err) return diff --git a/atc/engine/db_engine_test.go b/atc/engine/db_engine_test.go index 816a72203b9..ab27f04a96c 100644 --- a/atc/engine/db_engine_test.go +++ b/atc/engine/db_engine_test.go @@ -31,10 +31,10 @@ var _ = Describe("DBEngine", func() { logger = lagertest.NewTestLogger("test") fakeEngineA = new(enginefakes.FakeEngine) - fakeEngineA.NameReturns("fake-engine-a") + fakeEngineA.SchemaReturns("fake-schema-a") fakeEngineB = new(enginefakes.FakeEngine) - fakeEngineB.NameReturns("fake-engine-b") + fakeEngineB.SchemaReturns("fake-schema-b") dbBuild = new(dbfakes.FakeBuild) dbBuild.IDReturns(128) @@ -99,9 +99,8 @@ var _ = Describe("DBEngine", func() { It("starts the build in the database", func() { Expect(dbBuild.StartCallCount()).To(Equal(1)) - engine, metadata, _ := dbBuild.StartArgsForCall(0) - Expect(engine).To(Equal("fake-engine-a")) - Expect(metadata).To(Equal("some-metadata")) + engine, _ := dbBuild.StartArgsForCall(0) + Expect(engine).To(Equal("fake-schema-a")) }) Context("when the build fails to transition to started", func() { @@ -235,7 +234,7 @@ var _ = Describe("DBEngine", func() { Context("when the build is active", func() { BeforeEach(func() { dbBuild.ReloadReturns(true, nil) - dbBuild.EngineReturns("fake-engine-b") + dbBuild.SchemaReturns("fake-schema-b") dbBuild.MarkAsAbortedStub = func() error { Expect(dbBuild.AcquireTrackingLockCallCount()).To(Equal(1)) @@ -339,7 +338,7 @@ var _ = Describe("DBEngine", func() { Context("when the build is not yet active", func() { BeforeEach(func() { dbBuild.ReloadReturns(true, nil) - dbBuild.EngineReturns("") + dbBuild.SchemaReturns("") }) It("succeeds", func() { @@ -463,7 +462,7 @@ var _ = Describe("DBEngine", func() { Context("when the build is active", func() { BeforeEach(func() { - dbBuild.EngineReturns("fake-engine-b") + dbBuild.SchemaReturns("fake-schema-b") dbBuild.IsRunningReturns(true) dbBuild.ReloadReturns(true, nil) }) @@ -636,24 +635,24 @@ var _ = Describe("DBEngine", func() { }) }) - Context("when the build's engine is unknown", func() { + Context("when the build's schema is unknown", func() { BeforeEach(func() { dbBuild.ReloadReturns(true, nil) dbBuild.IsRunningReturns(true) - dbBuild.EngineReturns("bogus") + dbBuild.SchemaReturns("bogus") }) It("marks the build as errored", func() { Expect(dbBuild.FinishWithErrorCallCount()).To(Equal(1)) finishErr := dbBuild.FinishWithErrorArgsForCall(0) - Expect(finishErr).To(Equal(UnknownEngineError{Engine: "bogus"})) + Expect(finishErr).To(Equal(UnknownEngineError{Schema: "bogus"})) }) }) Context("when the build is not yet active", func() { BeforeEach(func() { dbBuild.ReloadReturns(true, nil) - dbBuild.EngineReturns("") + dbBuild.SchemaReturns("") }) It("does not look up the build in the engine", func() { @@ -668,7 +667,7 @@ var _ = Describe("DBEngine", func() { Context("when the build has already finished", func() { BeforeEach(func() { dbBuild.ReloadReturns(true, nil) - dbBuild.EngineReturns("fake-engine-b") + dbBuild.SchemaReturns("fake-schema-b") dbBuild.StatusReturns(db.BuildStatusSucceeded) }) diff --git a/atc/engine/engine.go b/atc/engine/engine.go index a507ef28310..9dd97f45990 100644 --- a/atc/engine/engine.go +++ b/atc/engine/engine.go @@ -9,7 +9,7 @@ import ( //go:generate counterfeiter . Engine type Engine interface { - Name() string + Schema() string CreateBuild(lager.Logger, db.Build, atc.Plan) (Build, error) LookupBuild(lager.Logger, db.Build) (Build, error) ReleaseAll(lager.Logger) @@ -25,9 +25,9 @@ type Build interface { type Engines []Engine -func (engines Engines) Lookup(name string) (Engine, bool) { +func (engines Engines) Lookup(schema string) (Engine, bool) { for _, e := range engines { - if e.Name() == name { + if e.Schema() == schema { return e, true } } diff --git a/atc/engine/enginefakes/fake_engine.go b/atc/engine/enginefakes/fake_engine.go index 809a684fe04..c0c7dbab90a 100644 --- a/atc/engine/enginefakes/fake_engine.go +++ b/atc/engine/enginefakes/fake_engine.go @@ -40,21 +40,21 @@ type FakeEngine struct { result1 engine.Build result2 error } - NameStub func() string - nameMutex sync.RWMutex - nameArgsForCall []struct { - } - nameReturns struct { - result1 string - } - nameReturnsOnCall map[int]struct { - result1 string - } ReleaseAllStub func(lager.Logger) releaseAllMutex sync.RWMutex releaseAllArgsForCall []struct { arg1 lager.Logger } + SchemaStub func() string + schemaMutex sync.RWMutex + schemaArgsForCall []struct { + } + schemaReturns struct { + result1 string + } + schemaReturnsOnCall map[int]struct { + result1 string + } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } @@ -188,58 +188,6 @@ func (fake *FakeEngine) LookupBuildReturnsOnCall(i int, result1 engine.Build, re }{result1, result2} } -func (fake *FakeEngine) Name() string { - fake.nameMutex.Lock() - ret, specificReturn := fake.nameReturnsOnCall[len(fake.nameArgsForCall)] - fake.nameArgsForCall = append(fake.nameArgsForCall, struct { - }{}) - fake.recordInvocation("Name", []interface{}{}) - fake.nameMutex.Unlock() - if fake.NameStub != nil { - return fake.NameStub() - } - if specificReturn { - return ret.result1 - } - fakeReturns := fake.nameReturns - return fakeReturns.result1 -} - -func (fake *FakeEngine) NameCallCount() int { - fake.nameMutex.RLock() - defer fake.nameMutex.RUnlock() - return len(fake.nameArgsForCall) -} - -func (fake *FakeEngine) NameCalls(stub func() string) { - fake.nameMutex.Lock() - defer fake.nameMutex.Unlock() - fake.NameStub = stub -} - -func (fake *FakeEngine) NameReturns(result1 string) { - fake.nameMutex.Lock() - defer fake.nameMutex.Unlock() - fake.NameStub = nil - fake.nameReturns = struct { - result1 string - }{result1} -} - -func (fake *FakeEngine) NameReturnsOnCall(i int, result1 string) { - fake.nameMutex.Lock() - defer fake.nameMutex.Unlock() - fake.NameStub = nil - if fake.nameReturnsOnCall == nil { - fake.nameReturnsOnCall = make(map[int]struct { - result1 string - }) - } - fake.nameReturnsOnCall[i] = struct { - result1 string - }{result1} -} - func (fake *FakeEngine) ReleaseAll(arg1 lager.Logger) { fake.releaseAllMutex.Lock() fake.releaseAllArgsForCall = append(fake.releaseAllArgsForCall, struct { @@ -271,6 +219,58 @@ func (fake *FakeEngine) ReleaseAllArgsForCall(i int) lager.Logger { return argsForCall.arg1 } +func (fake *FakeEngine) Schema() string { + fake.schemaMutex.Lock() + ret, specificReturn := fake.schemaReturnsOnCall[len(fake.schemaArgsForCall)] + fake.schemaArgsForCall = append(fake.schemaArgsForCall, struct { + }{}) + fake.recordInvocation("Schema", []interface{}{}) + fake.schemaMutex.Unlock() + if fake.SchemaStub != nil { + return fake.SchemaStub() + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.schemaReturns + return fakeReturns.result1 +} + +func (fake *FakeEngine) SchemaCallCount() int { + fake.schemaMutex.RLock() + defer fake.schemaMutex.RUnlock() + return len(fake.schemaArgsForCall) +} + +func (fake *FakeEngine) SchemaCalls(stub func() string) { + fake.schemaMutex.Lock() + defer fake.schemaMutex.Unlock() + fake.SchemaStub = stub +} + +func (fake *FakeEngine) SchemaReturns(result1 string) { + fake.schemaMutex.Lock() + defer fake.schemaMutex.Unlock() + fake.SchemaStub = nil + fake.schemaReturns = struct { + result1 string + }{result1} +} + +func (fake *FakeEngine) SchemaReturnsOnCall(i int, result1 string) { + fake.schemaMutex.Lock() + defer fake.schemaMutex.Unlock() + fake.SchemaStub = nil + if fake.schemaReturnsOnCall == nil { + fake.schemaReturnsOnCall = make(map[int]struct { + result1 string + }) + } + fake.schemaReturnsOnCall[i] = struct { + result1 string + }{result1} +} + func (fake *FakeEngine) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() @@ -278,10 +278,10 @@ func (fake *FakeEngine) Invocations() map[string][][]interface{} { defer fake.createBuildMutex.RUnlock() fake.lookupBuildMutex.RLock() defer fake.lookupBuildMutex.RUnlock() - fake.nameMutex.RLock() - defer fake.nameMutex.RUnlock() fake.releaseAllMutex.RLock() defer fake.releaseAllMutex.RUnlock() + fake.schemaMutex.RLock() + defer fake.schemaMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value diff --git a/atc/engine/exec_engine.go b/atc/engine/exec_engine.go index b3855327d7c..93ec90bbcc9 100644 --- a/atc/engine/exec_engine.go +++ b/atc/engine/exec_engine.go @@ -14,11 +14,9 @@ import ( "github.com/concourse/concourse/atc/exec" ) -type execMetadata struct { - Plan atc.Plan -} +type execMetadata atc.Plan -const execEngineName = "exec.v2" +const execEngineSchema = "exec.v2" type execEngine struct { factory exec.Factory @@ -44,8 +42,8 @@ func NewExecEngine( } } -func (engine *execEngine) Name() string { - return execEngineName +func (engine *execEngine) Schema() string { + return execEngineSchema } func (engine *execEngine) CreateBuild(logger lager.Logger, build db.Build, plan atc.Plan) (Build, error) { @@ -58,9 +56,7 @@ func (engine *execEngine) CreateBuild(logger lager.Logger, build db.Build, plan factory: engine.factory, delegate: engine.delegateFactory.Delegate(build), - metadata: execMetadata{ - Plan: plan, - }, + metadata: execMetadata(plan), ctx: ctx, cancel: cancel, @@ -74,7 +70,7 @@ func (engine *execEngine) LookupBuild(logger lager.Logger, build db.Build) (Buil ctx, cancel := context.WithCancel(context.Background()) var metadata execMetadata - err := json.Unmarshal([]byte(build.EngineMetadata()), &metadata) + err := json.Unmarshal([]byte(build.PrivatePlan()), &metadata) if err != nil { cancel() logger.Error("invalid-metadata", err) @@ -145,7 +141,7 @@ func (build *execBuild) Abort(lager.Logger) error { } func (build *execBuild) Resume(logger lager.Logger) { - step := build.buildStep(logger, build.metadata.Plan) + step := build.buildStep(logger, atc.Plan(build.metadata)) runCtx := lagerctx.NewContext(build.ctx, logger) diff --git a/atc/engine/exec_engine_test.go b/atc/engine/exec_engine_test.go index 31d794d57b8..bf780ebbb9c 100644 --- a/atc/engine/exec_engine_test.go +++ b/atc/engine/exec_engine_test.go @@ -8,7 +8,6 @@ import ( "github.com/concourse/concourse/atc/engine" "github.com/concourse/concourse/atc/engine/enginefakes" "github.com/concourse/concourse/atc/exec/execfakes" - . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -573,18 +572,16 @@ var _ = Describe("ExecEngine", func() { Context("when the build has a get step", func() { BeforeEach(func() { - dbBuild.EngineMetadataReturns(`{ - "Plan": { - "id": "47", - "attempts": [1], - "get": { - "name": "some-get", - "resource": "some-input-resource", - "type": "get", - "source": {"some": "source"}, - "params": {"some": "params"}, - "pipeline_id": 2222 - } + dbBuild.PrivatePlanReturns(`{ + "id": "47", + "attempts": [1], + "get": { + "name": "some-get", + "resource": "some-input-resource", + "type": "get", + "source": {"some": "source"}, + "params": {"some": "params"}, + "pipeline_id": 2222 } }`, ) @@ -631,7 +628,7 @@ var _ = Describe("ExecEngine", func() { Context("when engine metadata is empty", func() { BeforeEach(func() { - dbBuild.EngineMetadataReturns("{}") + dbBuild.PrivatePlanReturns("{}") fakeDelegate := new(enginefakes.FakeBuildDelegate) fakeDelegateFactory.DelegateReturns(fakeDelegate) diff --git a/atc/engine/exec_v1_dummy_engine.go b/atc/engine/exec_v1_dummy_engine.go index 89d3e8df28f..d95f0e5fc6d 100644 --- a/atc/engine/exec_v1_dummy_engine.go +++ b/atc/engine/exec_v1_dummy_engine.go @@ -10,14 +10,14 @@ import ( type execV1DummyEngine struct{} -const execV1DummyEngineName = "exec.v1" +const execV1DummyEngineSchema = "exec.v1" func NewExecV1DummyEngine() Engine { return execV1DummyEngine{} } -func (execV1DummyEngine) Name() string { - return execV1DummyEngineName +func (execV1DummyEngine) Schema() string { + return execV1DummyEngineSchema } func (execV1DummyEngine) CreateBuild(logger lager.Logger, build db.Build, plan atc.Plan) (Build, error) { @@ -40,7 +40,7 @@ func (execV1DummyBuild) Metadata() string { func (execV1DummyBuild) PublicPlan(lager.Logger) (atc.PublicBuildPlan, error) { return atc.PublicBuildPlan{ - Schema: execV1DummyEngineName, + Schema: execV1DummyEngineSchema, Plan: nil, }, nil } diff --git a/atc/plan.go b/atc/plan.go index 64a4e2c5272..f24486ca627 100644 --- a/atc/plan.go +++ b/atc/plan.go @@ -9,7 +9,7 @@ type Plan struct { Get *GetPlan `json:"get,omitempty"` Put *PutPlan `json:"put,omitempty"` Task *TaskPlan `json:"task,omitempty"` - OnAbort *OnAbortPlan `json:"on_abort,ommitempty"` + OnAbort *OnAbortPlan `json:"on_abort,omitempty"` Ensure *EnsurePlan `json:"ensure,omitempty"` OnSuccess *OnSuccessPlan `json:"on_success,omitempty"` OnFailure *OnFailurePlan `json:"on_failure,omitempty"` From 4ae21e3a8b7edafcbdde9b83ae3738fa0316e884 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 14 Feb 2019 10:40:15 -0500 Subject: [PATCH 13/21] atc: api cleanse * remove all the scheduler/engine deps from the api * also remove TriggerImmediately and SaveNextInputMappings from the scheduler concourse/concourse#3240 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/api/api_suite_test.go | 23 +-- atc/api/builds_test.go | 6 - atc/api/buildserver/server.go | 4 - atc/api/containers_test.go | 4 - atc/api/handler.go | 12 +- .../jobserverfakes/fake_scheduler_factory.go | 117 ------------ atc/api/jobserver/server.go | 10 -- atc/api/pipelineserver/server.go | 4 - atc/api/workers_test.go | 7 +- atc/api/workerserver/server.go | 4 - atc/atccmd/command.go | 37 ---- atc/scheduler/runner.go | 9 - atc/scheduler/scheduler.go | 51 ------ atc/scheduler/scheduler_test.go | 156 +--------------- .../schedulerfakes/fake_build_scheduler.go | 166 ------------------ 15 files changed, 12 insertions(+), 598 deletions(-) delete mode 100644 atc/api/jobserver/jobserverfakes/fake_scheduler_factory.go diff --git a/atc/api/api_suite_test.go b/atc/api/api_suite_test.go index 2e132c449f8..eee0047f6a8 100644 --- a/atc/api/api_suite_test.go +++ b/atc/api/api_suite_test.go @@ -10,25 +10,21 @@ import ( "code.cloudfoundry.org/lager" "code.cloudfoundry.org/lager/lagertest" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" - "github.com/concourse/concourse/atc/api" "github.com/concourse/concourse/atc/api/accessor" + "github.com/concourse/concourse/atc/api/accessor/accessorfakes" "github.com/concourse/concourse/atc/api/auth" + "github.com/concourse/concourse/atc/api/containerserver/containerserverfakes" + "github.com/concourse/concourse/atc/api/resourceserver/resourceserverfakes" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/creds/credsfakes" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" "github.com/concourse/concourse/atc/gc/gcfakes" - - "github.com/concourse/concourse/atc/api/accessor/accessorfakes" - "github.com/concourse/concourse/atc/api/containerserver/containerserverfakes" - "github.com/concourse/concourse/atc/api/jobserver/jobserverfakes" - "github.com/concourse/concourse/atc/api/resourceserver/resourceserverfakes" - "github.com/concourse/concourse/atc/engine/enginefakes" "github.com/concourse/concourse/atc/worker/workerfakes" "github.com/concourse/concourse/atc/wrappa" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) var ( @@ -36,9 +32,7 @@ var ( externalURL = "https://example.com" - fakeEngine *enginefakes.FakeEngine fakeWorkerClient *workerfakes.FakeClient - fakeWorkerProvider *workerfakes.FakeWorkerProvider fakeVolumeRepository *dbfakes.FakeVolumeRepository fakeContainerRepository *dbfakes.FakeContainerRepository fakeDestroyer *gcfakes.FakeDestroyer @@ -54,7 +48,6 @@ var ( build *dbfakes.FakeBuild dbBuildFactory *dbfakes.FakeBuildFactory dbTeam *dbfakes.FakeTeam - fakeSchedulerFactory *jobserverfakes.FakeSchedulerFactory fakeScannerFactory *resourceserverfakes.FakeScannerFactory fakeVariablesFactory *credsfakes.FakeVariablesFactory credsManagers creds.Managers @@ -121,11 +114,8 @@ var _ = BeforeEach(func() { drain = make(chan struct{}) - fakeEngine = new(enginefakes.FakeEngine) fakeWorkerClient = new(workerfakes.FakeClient) - fakeWorkerProvider = new(workerfakes.FakeWorkerProvider) - fakeSchedulerFactory = new(jobserverfakes.FakeSchedulerFactory) fakeScannerFactory = new(resourceserverfakes.FakeScannerFactory) fakeVolumeRepository = new(dbfakes.FakeVolumeRepository) @@ -186,11 +176,8 @@ var _ = BeforeEach(func() { constructedEventHandler.Construct, drain, - fakeEngine, fakeWorkerClient, - fakeWorkerProvider, - fakeSchedulerFactory, fakeScannerFactory, sink, diff --git a/atc/api/builds_test.go b/atc/api/builds_test.go index 09b41e6bd5e..b13a89ff729 100644 --- a/atc/api/builds_test.go +++ b/atc/api/builds_test.go @@ -13,7 +13,6 @@ import ( "github.com/concourse/concourse/atc/api/accessor/accessorfakes" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" - "github.com/concourse/concourse/atc/engine/enginefakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -1282,15 +1281,10 @@ var _ = Describe("Builds API", func() { }) Context("when the build is found", func() { - var engineBuild *enginefakes.FakeBuild - BeforeEach(func() { build.JobNameReturns("job1") build.TeamNameReturns("some-team") dbBuildFactory.BuildReturns(build, true, nil) - - engineBuild = new(enginefakes.FakeBuild) - fakeEngine.LookupBuildReturns(engineBuild, nil) }) Context("when authenticated, but not authorized", func() { diff --git a/atc/api/buildserver/server.go b/atc/api/buildserver/server.go index b8c1041e37a..fe7837cf7e1 100644 --- a/atc/api/buildserver/server.go +++ b/atc/api/buildserver/server.go @@ -6,7 +6,6 @@ import ( "code.cloudfoundry.org/lager" "github.com/concourse/concourse/atc/api/auth" "github.com/concourse/concourse/atc/db" - "github.com/concourse/concourse/atc/engine" ) type EventHandlerFactory func(lager.Logger, db.Build) http.Handler @@ -17,7 +16,6 @@ type Server struct { externalURL string peerURL string - engine engine.Engine teamFactory db.TeamFactory buildFactory db.BuildFactory eventHandlerFactory EventHandlerFactory @@ -29,7 +27,6 @@ func NewServer( logger lager.Logger, externalURL string, peerURL string, - engine engine.Engine, teamFactory db.TeamFactory, buildFactory db.BuildFactory, eventHandlerFactory EventHandlerFactory, @@ -41,7 +38,6 @@ func NewServer( externalURL: externalURL, peerURL: peerURL, - engine: engine, teamFactory: teamFactory, buildFactory: buildFactory, eventHandlerFactory: eventHandlerFactory, diff --git a/atc/api/containers_test.go b/atc/api/containers_test.go index 358a2dd1510..fb0155ecb59 100644 --- a/atc/api/containers_test.go +++ b/atc/api/containers_test.go @@ -974,10 +974,6 @@ var _ = Describe("Containers API", func() { It("returns 401 Unauthorized", func() { Expect(response.StatusCode).To(Equal(http.StatusUnauthorized)) }) - - It("does not hijack the build", func() { - Expect(fakeEngine.LookupBuildCallCount()).To(BeZero()) - }) }) }) diff --git a/atc/api/handler.go b/atc/api/handler.go index dc3b4e3ac9b..a502949c91f 100644 --- a/atc/api/handler.go +++ b/atc/api/handler.go @@ -23,7 +23,6 @@ import ( "github.com/concourse/concourse/atc/api/workerserver" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" - "github.com/concourse/concourse/atc/engine" "github.com/concourse/concourse/atc/gc" "github.com/concourse/concourse/atc/mainredirect" "github.com/concourse/concourse/atc/worker" @@ -53,11 +52,8 @@ func NewHandler( eventHandlerFactory buildserver.EventHandlerFactory, drain <-chan struct{}, - engine engine.Engine, workerClient worker.Client, - workerProvider worker.WorkerProvider, - schedulerFactory jobserver.SchedulerFactory, scannerFactory resourceserver.ScannerFactory, sink *lager.ReconfigurableSink, @@ -81,14 +77,14 @@ func NewHandler( buildHandlerFactory := buildserver.NewScopedHandlerFactory(logger) teamHandlerFactory := NewTeamScopedHandlerFactory(logger, dbTeamFactory) - buildServer := buildserver.NewServer(logger, externalURL, peerURL, engine, dbTeamFactory, dbBuildFactory, eventHandlerFactory, drain) - jobServer := jobserver.NewServer(logger, schedulerFactory, externalURL, variablesFactory, dbJobFactory) + buildServer := buildserver.NewServer(logger, externalURL, peerURL, dbTeamFactory, dbBuildFactory, eventHandlerFactory, drain) + jobServer := jobserver.NewServer(logger, externalURL, variablesFactory, dbJobFactory) resourceServer := resourceserver.NewServer(logger, scannerFactory, variablesFactory, dbResourceFactory, dbResourceConfigFactory) versionServer := versionserver.NewServer(logger, externalURL) - pipelineServer := pipelineserver.NewServer(logger, dbTeamFactory, dbPipelineFactory, externalURL, engine) + pipelineServer := pipelineserver.NewServer(logger, dbTeamFactory, dbPipelineFactory, externalURL) configServer := configserver.NewServer(logger, dbTeamFactory, variablesFactory) ccServer := ccserver.NewServer(logger, dbTeamFactory, externalURL) - workerServer := workerserver.NewServer(logger, dbTeamFactory, dbWorkerFactory, workerProvider) + workerServer := workerserver.NewServer(logger, dbTeamFactory, dbWorkerFactory) logLevelServer := loglevelserver.NewServer(logger, sink) cliServer := cliserver.NewServer(logger, absCLIDownloadsDir) containerServer := containerserver.NewServer(logger, workerClient, variablesFactory, interceptTimeoutFactory, containerRepository, destroyer) diff --git a/atc/api/jobserver/jobserverfakes/fake_scheduler_factory.go b/atc/api/jobserver/jobserverfakes/fake_scheduler_factory.go deleted file mode 100644 index bcdb673882d..00000000000 --- a/atc/api/jobserver/jobserverfakes/fake_scheduler_factory.go +++ /dev/null @@ -1,117 +0,0 @@ -// Code generated by counterfeiter. DO NOT EDIT. -package jobserverfakes - -import ( - sync "sync" - - jobserver "github.com/concourse/concourse/atc/api/jobserver" - creds "github.com/concourse/concourse/atc/creds" - db "github.com/concourse/concourse/atc/db" - scheduler "github.com/concourse/concourse/atc/scheduler" -) - -type FakeSchedulerFactory struct { - BuildSchedulerStub func(db.Pipeline, string, creds.Variables) scheduler.BuildScheduler - buildSchedulerMutex sync.RWMutex - buildSchedulerArgsForCall []struct { - arg1 db.Pipeline - arg2 string - arg3 creds.Variables - } - buildSchedulerReturns struct { - result1 scheduler.BuildScheduler - } - buildSchedulerReturnsOnCall map[int]struct { - result1 scheduler.BuildScheduler - } - invocations map[string][][]interface{} - invocationsMutex sync.RWMutex -} - -func (fake *FakeSchedulerFactory) BuildScheduler(arg1 db.Pipeline, arg2 string, arg3 creds.Variables) scheduler.BuildScheduler { - fake.buildSchedulerMutex.Lock() - ret, specificReturn := fake.buildSchedulerReturnsOnCall[len(fake.buildSchedulerArgsForCall)] - fake.buildSchedulerArgsForCall = append(fake.buildSchedulerArgsForCall, struct { - arg1 db.Pipeline - arg2 string - arg3 creds.Variables - }{arg1, arg2, arg3}) - fake.recordInvocation("BuildScheduler", []interface{}{arg1, arg2, arg3}) - fake.buildSchedulerMutex.Unlock() - if fake.BuildSchedulerStub != nil { - return fake.BuildSchedulerStub(arg1, arg2, arg3) - } - if specificReturn { - return ret.result1 - } - fakeReturns := fake.buildSchedulerReturns - return fakeReturns.result1 -} - -func (fake *FakeSchedulerFactory) BuildSchedulerCallCount() int { - fake.buildSchedulerMutex.RLock() - defer fake.buildSchedulerMutex.RUnlock() - return len(fake.buildSchedulerArgsForCall) -} - -func (fake *FakeSchedulerFactory) BuildSchedulerCalls(stub func(db.Pipeline, string, creds.Variables) scheduler.BuildScheduler) { - fake.buildSchedulerMutex.Lock() - defer fake.buildSchedulerMutex.Unlock() - fake.BuildSchedulerStub = stub -} - -func (fake *FakeSchedulerFactory) BuildSchedulerArgsForCall(i int) (db.Pipeline, string, creds.Variables) { - fake.buildSchedulerMutex.RLock() - defer fake.buildSchedulerMutex.RUnlock() - argsForCall := fake.buildSchedulerArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - -func (fake *FakeSchedulerFactory) BuildSchedulerReturns(result1 scheduler.BuildScheduler) { - fake.buildSchedulerMutex.Lock() - defer fake.buildSchedulerMutex.Unlock() - fake.BuildSchedulerStub = nil - fake.buildSchedulerReturns = struct { - result1 scheduler.BuildScheduler - }{result1} -} - -func (fake *FakeSchedulerFactory) BuildSchedulerReturnsOnCall(i int, result1 scheduler.BuildScheduler) { - fake.buildSchedulerMutex.Lock() - defer fake.buildSchedulerMutex.Unlock() - fake.BuildSchedulerStub = nil - if fake.buildSchedulerReturnsOnCall == nil { - fake.buildSchedulerReturnsOnCall = make(map[int]struct { - result1 scheduler.BuildScheduler - }) - } - fake.buildSchedulerReturnsOnCall[i] = struct { - result1 scheduler.BuildScheduler - }{result1} -} - -func (fake *FakeSchedulerFactory) Invocations() map[string][][]interface{} { - fake.invocationsMutex.RLock() - defer fake.invocationsMutex.RUnlock() - fake.buildSchedulerMutex.RLock() - defer fake.buildSchedulerMutex.RUnlock() - copiedInvocations := map[string][][]interface{}{} - for key, value := range fake.invocations { - copiedInvocations[key] = value - } - return copiedInvocations -} - -func (fake *FakeSchedulerFactory) recordInvocation(key string, args []interface{}) { - fake.invocationsMutex.Lock() - defer fake.invocationsMutex.Unlock() - if fake.invocations == nil { - fake.invocations = map[string][][]interface{}{} - } - if fake.invocations[key] == nil { - fake.invocations[key] = [][]interface{}{} - } - fake.invocations[key] = append(fake.invocations[key], args) -} - -var _ jobserver.SchedulerFactory = new(FakeSchedulerFactory) diff --git a/atc/api/jobserver/server.go b/atc/api/jobserver/server.go index 69ba685ac43..b09d56790b7 100644 --- a/atc/api/jobserver/server.go +++ b/atc/api/jobserver/server.go @@ -5,19 +5,11 @@ import ( "github.com/concourse/concourse/atc/api/auth" "github.com/concourse/concourse/atc/creds" "github.com/concourse/concourse/atc/db" - "github.com/concourse/concourse/atc/scheduler" ) -//go:generate counterfeiter . SchedulerFactory - -type SchedulerFactory interface { - BuildScheduler(db.Pipeline, string, creds.Variables) scheduler.BuildScheduler -} - type Server struct { logger lager.Logger - schedulerFactory SchedulerFactory externalURL string rejector auth.Rejector variablesFactory creds.VariablesFactory @@ -26,14 +18,12 @@ type Server struct { func NewServer( logger lager.Logger, - schedulerFactory SchedulerFactory, externalURL string, variablesFactory creds.VariablesFactory, jobFactory db.JobFactory, ) *Server { return &Server{ logger: logger, - schedulerFactory: schedulerFactory, externalURL: externalURL, rejector: auth.UnauthorizedRejector{}, variablesFactory: variablesFactory, diff --git a/atc/api/pipelineserver/server.go b/atc/api/pipelineserver/server.go index 3e26ccaacef..5c210ace62c 100644 --- a/atc/api/pipelineserver/server.go +++ b/atc/api/pipelineserver/server.go @@ -4,7 +4,6 @@ import ( "code.cloudfoundry.org/lager" "github.com/concourse/concourse/atc/api/auth" "github.com/concourse/concourse/atc/db" - "github.com/concourse/concourse/atc/engine" ) type Server struct { @@ -12,7 +11,6 @@ type Server struct { teamFactory db.TeamFactory rejector auth.Rejector pipelineFactory db.PipelineFactory - engine engine.Engine externalURL string } @@ -21,7 +19,6 @@ func NewServer( teamFactory db.TeamFactory, pipelineFactory db.PipelineFactory, externalURL string, - engine engine.Engine, ) *Server { return &Server{ logger: logger, @@ -29,6 +26,5 @@ func NewServer( rejector: auth.UnauthorizedRejector{}, pipelineFactory: pipelineFactory, externalURL: externalURL, - engine: engine, } } diff --git a/atc/api/workers_test.go b/atc/api/workers_test.go index 81094743d6f..821c1a0a276 100644 --- a/atc/api/workers_test.go +++ b/atc/api/workers_test.go @@ -12,7 +12,6 @@ import ( "github.com/concourse/concourse/atc/api/accessor/accessorfakes" "github.com/concourse/concourse/atc/db" "github.com/concourse/concourse/atc/db/dbfakes" - "github.com/concourse/concourse/atc/worker/workerfakes" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" ) @@ -135,8 +134,7 @@ var _ = Describe("Workers API", func() { ttl string certsPath string - response *http.Response - fakeGardenWorker *workerfakes.FakeWorker + response *http.Response ) BeforeEach(func() { @@ -162,9 +160,6 @@ var _ = Describe("Workers API", func() { ttl = "30s" fakeaccess.IsAuthorizedReturns(true) fakeaccess.IsSystemReturns(true) - - fakeGardenWorker = new(workerfakes.FakeWorker) - fakeWorkerProvider.NewGardenWorkerReturns(fakeGardenWorker) }) JustBeforeEach(func() { diff --git a/atc/api/workerserver/server.go b/atc/api/workerserver/server.go index 6d252dc4721..1d2f5320bc3 100644 --- a/atc/api/workerserver/server.go +++ b/atc/api/workerserver/server.go @@ -3,7 +3,6 @@ package workerserver import ( "code.cloudfoundry.org/lager" "github.com/concourse/concourse/atc/db" - "github.com/concourse/concourse/atc/worker" ) type Server struct { @@ -11,20 +10,17 @@ type Server struct { teamFactory db.TeamFactory dbWorkerFactory db.WorkerFactory - workerProvider worker.WorkerProvider } func NewServer( logger lager.Logger, teamFactory db.TeamFactory, dbWorkerFactory db.WorkerFactory, - workerProvider worker.WorkerProvider, ) *Server { return &Server{ logger: logger, teamFactory: teamFactory, dbWorkerFactory: dbWorkerFactory, - workerProvider: workerProvider, } } diff --git a/atc/atccmd/command.go b/atc/atccmd/command.go index 1d9a4e28374..61657e374cd 100644 --- a/atc/atccmd/command.go +++ b/atc/atccmd/command.go @@ -539,41 +539,13 @@ func (cmd *RunCommand) constructAPIMembers( pool := worker.NewPool(workerProvider) workerClient := worker.NewClient(pool, workerProvider) - defaultLimits, err := cmd.parseDefaultLimits() - if err != nil { - return nil, err - } - variablesFactory, err := cmd.variablesFactory(logger) if err != nil { return nil, err } - buildContainerStrategy := cmd.chooseBuildContainerStrategy() checkContainerStrategy := worker.NewRandomPlacementStrategy() - engine := cmd.constructEngine( - pool, - workerClient, - resourceFetcher, - dbResourceCacheFactory, - dbResourceConfigFactory, - variablesFactory, - defaultLimits, - buildContainerStrategy, - resourceFactory, - ) - - radarSchedulerFactory := pipelines.NewRadarSchedulerFactory( - pool, - resourceFactory, - dbResourceConfigFactory, - cmd.ResourceTypeCheckingInterval, - cmd.ResourceCheckingInterval, - engine, - checkContainerStrategy, - ) - radarScannerFactory := radar.NewScannerFactory( pool, resourceFactory, @@ -608,11 +580,8 @@ func (cmd *RunCommand) constructAPIMembers( gcContainerDestroyer, dbBuildFactory, dbResourceConfigFactory, - engine, workerClient, - workerProvider, drain, - radarSchedulerFactory, radarScannerFactory, variablesFactory, credsManagers, @@ -1308,11 +1277,8 @@ func (cmd *RunCommand) constructAPIHandler( gcContainerDestroyer gc.Destroyer, dbBuildFactory db.BuildFactory, resourceConfigFactory db.ResourceConfigFactory, - engine engine.Engine, workerClient worker.Client, - workerProvider worker.WorkerProvider, drain <-chan struct{}, - radarSchedulerFactory pipelines.RadarSchedulerFactory, radarScannerFactory radar.ScannerFactory, variablesFactory creds.VariablesFactory, credsManagers creds.Managers, @@ -1356,10 +1322,7 @@ func (cmd *RunCommand) constructAPIHandler( buildserver.NewEventHandler, drain, - engine, workerClient, - workerProvider, - radarSchedulerFactory, radarScannerFactory, reconfigurableSink, diff --git a/atc/scheduler/runner.go b/atc/scheduler/runner.go index 64957b464d1..e17c45e1266 100644 --- a/atc/scheduler/runner.go +++ b/atc/scheduler/runner.go @@ -22,15 +22,6 @@ type BuildScheduler interface { resources db.Resources, resourceTypes atc.VersionedResourceTypes, ) (map[string]time.Duration, error) - - TriggerImmediately( - logger lager.Logger, - job db.Job, - resources db.Resources, - resourceTypes atc.VersionedResourceTypes, - ) (db.Build, Waiter, error) - - SaveNextInputMapping(logger lager.Logger, job db.Job, resource db.Resources) error } var errPipelineRemoved = errors.New("pipeline removed") diff --git a/atc/scheduler/scheduler.go b/atc/scheduler/scheduler.go index 496edaa17bc..b63bd3eaf5f 100644 --- a/atc/scheduler/scheduler.go +++ b/atc/scheduler/scheduler.go @@ -1,7 +1,6 @@ package scheduler import ( - "sync" "time" "code.cloudfoundry.org/lager" @@ -95,53 +94,3 @@ func (s *Scheduler) ensurePendingBuildExists( return nil } - -type Waiter interface { - Wait() -} - -func (s *Scheduler) TriggerImmediately( - logger lager.Logger, - job db.Job, - resources db.Resources, - resourceTypes atc.VersionedResourceTypes, -) (db.Build, Waiter, error) { - logger = logger.Session("trigger-immediately", lager.Data{"job_name": job.Name()}) - - build, err := job.CreateBuild() - if err != nil { - logger.Error("failed-to-create-job-build", err) - return nil, nil, err - } - wg := new(sync.WaitGroup) - wg.Add(1) - - go func() { - defer wg.Done() - - nextPendingBuilds, err := job.GetPendingBuilds() - if err != nil { - logger.Error("failed-to-get-next-pending-build-for-job", err) - return - } - - err = s.BuildStarter.TryStartPendingBuildsForJob(logger, job, resources, resourceTypes, nextPendingBuilds) - if err != nil { - logger.Error("failed-to-start-next-pending-build-for-job", err, lager.Data{"job-name": job.Name()}) - return - } - }() - - return build, wg, nil -} - -func (s *Scheduler) SaveNextInputMapping(logger lager.Logger, job db.Job, resources db.Resources) error { - versions, err := s.Pipeline.LoadVersionsDB() - if err != nil { - logger.Error("failed-to-load-versions-db", err) - return err - } - - _, err = s.InputMapper.SaveNextInputMapping(logger, versions, job, resources) - return err -} diff --git a/atc/scheduler/scheduler_test.go b/atc/scheduler/scheduler_test.go index a838a9411f8..ce663e7094e 100644 --- a/atc/scheduler/scheduler_test.go +++ b/atc/scheduler/scheduler_test.go @@ -81,7 +81,8 @@ var _ = Describe("Scheduler", func() { JustBeforeEach(func() { versionsDB = &algorithm.VersionsDB{JobIDs: map[string]int{"j1": 1}} - var waiter Waiter + var waiter interface{ Wait() } + _, scheduleErr = scheduler.Schedule( lagertest.NewTestLogger("test"), versionsDB, @@ -252,157 +253,4 @@ var _ = Describe("Scheduler", func() { }) }) }) - - Describe("TriggerImmediately", func() { - var ( - fakeJob *dbfakes.FakeJob - fakeResource *dbfakes.FakeResource - triggerErr error - nextPendingBuilds []db.Build - ) - - BeforeEach(func() { - fakeJob = new(dbfakes.FakeJob) - fakeJob.NameReturns("some-job") - fakeJob.ConfigReturns(atc.JobConfig{Plan: atc.PlanSequence{{Get: "input-1"}, {Get: "input-2"}}}) - - fakeResource = new(dbfakes.FakeResource) - fakeResource.NameReturns("some-resource") - }) - - JustBeforeEach(func() { - var waiter Waiter - _, waiter, triggerErr = scheduler.TriggerImmediately( - lagertest.NewTestLogger("test"), - fakeJob, - db.Resources{fakeResource}, - atc.VersionedResourceTypes{ - { - ResourceType: atc.ResourceType{Name: "some-resource-type"}, - Version: atc.Version{"some": "version"}, - }, - }, - ) - if waiter != nil { - waiter.Wait() - } - }) - - Context("when creating the build fails", func() { - BeforeEach(func() { - fakeJob.CreateBuildReturns(nil, disaster) - }) - - It("returns the error", func() { - Expect(triggerErr).To(Equal(disaster)) - }) - }) - - Context("when creating the build succeeds", func() { - var createdBuild *dbfakes.FakeBuild - - BeforeEach(func() { - createdBuild = new(dbfakes.FakeBuild) - createdBuild.IsManuallyTriggeredReturns(true) - fakeJob.CreateBuildReturns(createdBuild, nil) - }) - - It("tried to create a build for the right job", func() { - Expect(fakeJob.CreateBuildCallCount()).To(Equal(1)) - }) - - Context("when get pending builds for job fails", func() { - BeforeEach(func() { - fakeJob.GetPendingBuildsReturns(nil, disaster) - }) - - It("does not try to start pending builds for job", func() { - Expect(fakeBuildStarter.TryStartPendingBuildsForJobCallCount()).To(Equal(0)) - }) - }) - - Context("when get pending builds for job succeeds", func() { - BeforeEach(func() { - nextPendingBuilds = []db.Build{new(dbfakes.FakeBuild)} - fakeJob.GetPendingBuildsReturns(nextPendingBuilds, nil) - }) - - It("tried to get pending builds for the right job", func() { - Expect(fakeJob.GetPendingBuildsCallCount()).To(Equal(1)) - }) - - Context("when trying to start pending builds succeeds", func() { - BeforeEach(func() { - fakeBuildStarter.TryStartPendingBuildsForJobReturns(nil) - }) - - It("tries to start builds for the right job", func() { - Expect(fakeBuildStarter.TryStartPendingBuildsForJobCallCount()).To(Equal(1)) - _, _, _, _, b := fakeBuildStarter.TryStartPendingBuildsForJobArgsForCall(0) - Expect(b).To(Equal(nextPendingBuilds)) - }) - }) - }) - }) - }) - - Describe("SaveNextInputMapping", func() { - var saveErr error - var fakeJob *dbfakes.FakeJob - - JustBeforeEach(func() { - fakeJob = new(dbfakes.FakeJob) - fakeJob.NameReturns("some-job") - - saveErr = scheduler.SaveNextInputMapping(lagertest.NewTestLogger("test"), fakeJob, db.Resources{}) - }) - - Context("when loading the versions DB fails", func() { - BeforeEach(func() { - fakePipeline.LoadVersionsDBReturns(nil, disaster) - }) - - It("returns the error", func() { - Expect(saveErr).To(Equal(disaster)) - }) - }) - - Context("when loading the versions DB succeeds", func() { - var versionsDB *algorithm.VersionsDB - - BeforeEach(func() { - versionsDB = &algorithm.VersionsDB{JobIDs: map[string]int{"j1": 1}} - fakePipeline.LoadVersionsDBReturns(versionsDB, nil) - }) - - Context("when saving the next input mapping fails", func() { - BeforeEach(func() { - fakeInputMapper.SaveNextInputMappingReturns(nil, disaster) - }) - - It("returns the error", func() { - Expect(saveErr).To(Equal(disaster)) - }) - - It("saved the next input mapping for the right job and versions", func() { - Expect(fakeInputMapper.SaveNextInputMappingCallCount()).To(Equal(1)) - _, actualVersionsDB, actualJob, _ := fakeInputMapper.SaveNextInputMappingArgsForCall(0) - Expect(actualVersionsDB).To(Equal(versionsDB)) - Expect(actualJob.Name()).To(Equal(fakeJob.Name())) - }) - }) - - Context("when saving the next input mapping succeeds", func() { - BeforeEach(func() { - fakeInputMapper.SaveNextInputMappingReturns(algorithm.InputMapping{ - "some-input": algorithm.InputVersion{VersionID: 1, ResourceID: 11, FirstOccurrence: true}, - }, nil) - }) - - It("returns no error", func() { - Expect(saveErr).NotTo(HaveOccurred()) - }) - }) - }) - }) }) diff --git a/atc/scheduler/schedulerfakes/fake_build_scheduler.go b/atc/scheduler/schedulerfakes/fake_build_scheduler.go index a5bee98c345..4453cea8a0f 100644 --- a/atc/scheduler/schedulerfakes/fake_build_scheduler.go +++ b/atc/scheduler/schedulerfakes/fake_build_scheduler.go @@ -13,19 +13,6 @@ import ( ) type FakeBuildScheduler struct { - SaveNextInputMappingStub func(lager.Logger, db.Job, db.Resources) error - saveNextInputMappingMutex sync.RWMutex - saveNextInputMappingArgsForCall []struct { - arg1 lager.Logger - arg2 db.Job - arg3 db.Resources - } - saveNextInputMappingReturns struct { - result1 error - } - saveNextInputMappingReturnsOnCall map[int]struct { - result1 error - } ScheduleStub func(lager.Logger, *algorithm.VersionsDB, []db.Job, db.Resources, atc.VersionedResourceTypes) (map[string]time.Duration, error) scheduleMutex sync.RWMutex scheduleArgsForCall []struct { @@ -43,90 +30,10 @@ type FakeBuildScheduler struct { result1 map[string]time.Duration result2 error } - TriggerImmediatelyStub func(lager.Logger, db.Job, db.Resources, atc.VersionedResourceTypes) (db.Build, scheduler.Waiter, error) - triggerImmediatelyMutex sync.RWMutex - triggerImmediatelyArgsForCall []struct { - arg1 lager.Logger - arg2 db.Job - arg3 db.Resources - arg4 atc.VersionedResourceTypes - } - triggerImmediatelyReturns struct { - result1 db.Build - result2 scheduler.Waiter - result3 error - } - triggerImmediatelyReturnsOnCall map[int]struct { - result1 db.Build - result2 scheduler.Waiter - result3 error - } invocations map[string][][]interface{} invocationsMutex sync.RWMutex } -func (fake *FakeBuildScheduler) SaveNextInputMapping(arg1 lager.Logger, arg2 db.Job, arg3 db.Resources) error { - fake.saveNextInputMappingMutex.Lock() - ret, specificReturn := fake.saveNextInputMappingReturnsOnCall[len(fake.saveNextInputMappingArgsForCall)] - fake.saveNextInputMappingArgsForCall = append(fake.saveNextInputMappingArgsForCall, struct { - arg1 lager.Logger - arg2 db.Job - arg3 db.Resources - }{arg1, arg2, arg3}) - fake.recordInvocation("SaveNextInputMapping", []interface{}{arg1, arg2, arg3}) - fake.saveNextInputMappingMutex.Unlock() - if fake.SaveNextInputMappingStub != nil { - return fake.SaveNextInputMappingStub(arg1, arg2, arg3) - } - if specificReturn { - return ret.result1 - } - fakeReturns := fake.saveNextInputMappingReturns - return fakeReturns.result1 -} - -func (fake *FakeBuildScheduler) SaveNextInputMappingCallCount() int { - fake.saveNextInputMappingMutex.RLock() - defer fake.saveNextInputMappingMutex.RUnlock() - return len(fake.saveNextInputMappingArgsForCall) -} - -func (fake *FakeBuildScheduler) SaveNextInputMappingCalls(stub func(lager.Logger, db.Job, db.Resources) error) { - fake.saveNextInputMappingMutex.Lock() - defer fake.saveNextInputMappingMutex.Unlock() - fake.SaveNextInputMappingStub = stub -} - -func (fake *FakeBuildScheduler) SaveNextInputMappingArgsForCall(i int) (lager.Logger, db.Job, db.Resources) { - fake.saveNextInputMappingMutex.RLock() - defer fake.saveNextInputMappingMutex.RUnlock() - argsForCall := fake.saveNextInputMappingArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3 -} - -func (fake *FakeBuildScheduler) SaveNextInputMappingReturns(result1 error) { - fake.saveNextInputMappingMutex.Lock() - defer fake.saveNextInputMappingMutex.Unlock() - fake.SaveNextInputMappingStub = nil - fake.saveNextInputMappingReturns = struct { - result1 error - }{result1} -} - -func (fake *FakeBuildScheduler) SaveNextInputMappingReturnsOnCall(i int, result1 error) { - fake.saveNextInputMappingMutex.Lock() - defer fake.saveNextInputMappingMutex.Unlock() - fake.SaveNextInputMappingStub = nil - if fake.saveNextInputMappingReturnsOnCall == nil { - fake.saveNextInputMappingReturnsOnCall = make(map[int]struct { - result1 error - }) - } - fake.saveNextInputMappingReturnsOnCall[i] = struct { - result1 error - }{result1} -} - func (fake *FakeBuildScheduler) Schedule(arg1 lager.Logger, arg2 *algorithm.VersionsDB, arg3 []db.Job, arg4 db.Resources, arg5 atc.VersionedResourceTypes) (map[string]time.Duration, error) { var arg3Copy []db.Job if arg3 != nil { @@ -199,84 +106,11 @@ func (fake *FakeBuildScheduler) ScheduleReturnsOnCall(i int, result1 map[string] }{result1, result2} } -func (fake *FakeBuildScheduler) TriggerImmediately(arg1 lager.Logger, arg2 db.Job, arg3 db.Resources, arg4 atc.VersionedResourceTypes) (db.Build, scheduler.Waiter, error) { - fake.triggerImmediatelyMutex.Lock() - ret, specificReturn := fake.triggerImmediatelyReturnsOnCall[len(fake.triggerImmediatelyArgsForCall)] - fake.triggerImmediatelyArgsForCall = append(fake.triggerImmediatelyArgsForCall, struct { - arg1 lager.Logger - arg2 db.Job - arg3 db.Resources - arg4 atc.VersionedResourceTypes - }{arg1, arg2, arg3, arg4}) - fake.recordInvocation("TriggerImmediately", []interface{}{arg1, arg2, arg3, arg4}) - fake.triggerImmediatelyMutex.Unlock() - if fake.TriggerImmediatelyStub != nil { - return fake.TriggerImmediatelyStub(arg1, arg2, arg3, arg4) - } - if specificReturn { - return ret.result1, ret.result2, ret.result3 - } - fakeReturns := fake.triggerImmediatelyReturns - return fakeReturns.result1, fakeReturns.result2, fakeReturns.result3 -} - -func (fake *FakeBuildScheduler) TriggerImmediatelyCallCount() int { - fake.triggerImmediatelyMutex.RLock() - defer fake.triggerImmediatelyMutex.RUnlock() - return len(fake.triggerImmediatelyArgsForCall) -} - -func (fake *FakeBuildScheduler) TriggerImmediatelyCalls(stub func(lager.Logger, db.Job, db.Resources, atc.VersionedResourceTypes) (db.Build, scheduler.Waiter, error)) { - fake.triggerImmediatelyMutex.Lock() - defer fake.triggerImmediatelyMutex.Unlock() - fake.TriggerImmediatelyStub = stub -} - -func (fake *FakeBuildScheduler) TriggerImmediatelyArgsForCall(i int) (lager.Logger, db.Job, db.Resources, atc.VersionedResourceTypes) { - fake.triggerImmediatelyMutex.RLock() - defer fake.triggerImmediatelyMutex.RUnlock() - argsForCall := fake.triggerImmediatelyArgsForCall[i] - return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 -} - -func (fake *FakeBuildScheduler) TriggerImmediatelyReturns(result1 db.Build, result2 scheduler.Waiter, result3 error) { - fake.triggerImmediatelyMutex.Lock() - defer fake.triggerImmediatelyMutex.Unlock() - fake.TriggerImmediatelyStub = nil - fake.triggerImmediatelyReturns = struct { - result1 db.Build - result2 scheduler.Waiter - result3 error - }{result1, result2, result3} -} - -func (fake *FakeBuildScheduler) TriggerImmediatelyReturnsOnCall(i int, result1 db.Build, result2 scheduler.Waiter, result3 error) { - fake.triggerImmediatelyMutex.Lock() - defer fake.triggerImmediatelyMutex.Unlock() - fake.TriggerImmediatelyStub = nil - if fake.triggerImmediatelyReturnsOnCall == nil { - fake.triggerImmediatelyReturnsOnCall = make(map[int]struct { - result1 db.Build - result2 scheduler.Waiter - result3 error - }) - } - fake.triggerImmediatelyReturnsOnCall[i] = struct { - result1 db.Build - result2 scheduler.Waiter - result3 error - }{result1, result2, result3} -} - func (fake *FakeBuildScheduler) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.saveNextInputMappingMutex.RLock() - defer fake.saveNextInputMappingMutex.RUnlock() fake.scheduleMutex.RLock() defer fake.scheduleMutex.RUnlock() - fake.triggerImmediatelyMutex.RLock() - defer fake.triggerImmediatelyMutex.RUnlock() copiedInvocations := map[string][][]interface{}{} for key, value := range fake.invocations { copiedInvocations[key] = value From 6956b1602efae1fa7e15bd2f5d428ca1804145bb Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 14 Feb 2019 15:22:45 -0500 Subject: [PATCH 14/21] atc: remove peer url * because we don't need it anymore concourse/concourse#3267 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/api/api_suite_test.go | 4 ---- atc/api/buildserver/server.go | 3 --- atc/api/handler.go | 3 +-- atc/atccmd/command.go | 17 ++++---------- atc/db/build.go | 42 ++++++----------------------------- atc/db/build_test.go | 20 ----------------- atc/engine/db_engine.go | 13 +---------- atc/engine/db_engine_test.go | 19 +--------------- cmd/concourse/web.go | 4 ++-- 9 files changed, 16 insertions(+), 109 deletions(-) diff --git a/atc/api/api_suite_test.go b/atc/api/api_suite_test.go index eee0047f6a8..b50397ec539 100644 --- a/atc/api/api_suite_test.go +++ b/atc/api/api_suite_test.go @@ -53,7 +53,6 @@ var ( credsManagers creds.Managers interceptTimeoutFactory *containerserverfakes.FakeInterceptTimeoutFactory interceptTimeout *containerserverfakes.FakeInterceptTimeout - peerURL string drain chan struct{} expire time.Duration isTLSEnabled bool @@ -110,8 +109,6 @@ var _ = BeforeEach(func() { dbWorkerFactory = new(dbfakes.FakeWorkerFactory) dbWorkerLifecycle = new(dbfakes.FakeWorkerLifecycle) - peerURL = "http://127.0.0.1:1234" - drain = make(chan struct{}) fakeWorkerClient = new(workerfakes.FakeClient) @@ -172,7 +169,6 @@ var _ = BeforeEach(func() { dbBuildFactory, dbResourceConfigFactory, - peerURL, constructedEventHandler.Construct, drain, diff --git a/atc/api/buildserver/server.go b/atc/api/buildserver/server.go index fe7837cf7e1..31c8c8c1e1c 100644 --- a/atc/api/buildserver/server.go +++ b/atc/api/buildserver/server.go @@ -14,7 +14,6 @@ type Server struct { logger lager.Logger externalURL string - peerURL string teamFactory db.TeamFactory buildFactory db.BuildFactory @@ -26,7 +25,6 @@ type Server struct { func NewServer( logger lager.Logger, externalURL string, - peerURL string, teamFactory db.TeamFactory, buildFactory db.BuildFactory, eventHandlerFactory EventHandlerFactory, @@ -36,7 +34,6 @@ func NewServer( logger: logger, externalURL: externalURL, - peerURL: peerURL, teamFactory: teamFactory, buildFactory: buildFactory, diff --git a/atc/api/handler.go b/atc/api/handler.go index a502949c91f..3f8ed862015 100644 --- a/atc/api/handler.go +++ b/atc/api/handler.go @@ -48,7 +48,6 @@ func NewHandler( dbBuildFactory db.BuildFactory, dbResourceConfigFactory db.ResourceConfigFactory, - peerURL string, eventHandlerFactory buildserver.EventHandlerFactory, drain <-chan struct{}, @@ -77,7 +76,7 @@ func NewHandler( buildHandlerFactory := buildserver.NewScopedHandlerFactory(logger) teamHandlerFactory := NewTeamScopedHandlerFactory(logger, dbTeamFactory) - buildServer := buildserver.NewServer(logger, externalURL, peerURL, dbTeamFactory, dbBuildFactory, eventHandlerFactory, drain) + buildServer := buildserver.NewServer(logger, externalURL, dbTeamFactory, dbBuildFactory, eventHandlerFactory, drain) jobServer := jobserver.NewServer(logger, externalURL, variablesFactory, dbJobFactory) resourceServer := resourceserver.NewServer(logger, scannerFactory, variablesFactory, dbResourceFactory, dbResourceConfigFactory) versionServer := versionserver.NewServer(logger, externalURL) diff --git a/atc/atccmd/command.go b/atc/atccmd/command.go index 61657e374cd..250862d7c29 100644 --- a/atc/atccmd/command.go +++ b/atc/atccmd/command.go @@ -87,7 +87,6 @@ type RunCommand struct { TLSKey flag.File `long:"tls-key" description:"File containing an RSA private key, used to encrypt HTTPS traffic."` ExternalURL flag.URL `long:"external-url" description:"URL used to reach any ATC from the outside world."` - PeerURL flag.URL `long:"peer-url" description:"URL used to reach this ATC from other ATCs in the cluster."` Postgres flag.PostgresConfig `group:"PostgreSQL Configuration" namespace:"postgres"` @@ -166,13 +165,6 @@ type RunCommand struct { } `group:"Authentication"` } -func (cmd *RunCommand) PeerURLOrDefault() flag.URL { - if cmd.PeerURL.URL == nil { - cmd.PeerURL = cmd.defaultURL() - } - return cmd.PeerURL -} - var HelpError = errors.New("must specify one of `--current-db-version`, `--supported-db-version`, or `--migrate-db-to-version`") type Migration struct { @@ -328,7 +320,7 @@ func (cmd *RunCommand) Execute(args []string) error { func (cmd *RunCommand) Runner(positionalArguments []string) (ifrit.Runner, error) { if cmd.ExternalURL.URL == nil { - cmd.ExternalURL = cmd.defaultURL() + cmd.ExternalURL = cmd.DefaultURL() } if len(positionalArguments) != 0 { @@ -969,7 +961,7 @@ func (cmd *RunCommand) skyHttpClient() (*http.Client, error) { RoundTripper: httpClient.Transport, SourceHost: cmd.ExternalURL.URL.Host, - TargetURL: cmd.defaultURL().URL, + TargetURL: cmd.DefaultURL().URL, } return httpClient, nil @@ -1034,7 +1026,7 @@ func (cmd *RunCommand) defaultBindIP() net.IP { return net.ParseIP(URL) } -func (cmd *RunCommand) defaultURL() flag.URL { +func (cmd *RunCommand) DefaultURL() flag.URL { return flag.URL{ URL: &url.URL{ Scheme: "http", @@ -1230,7 +1222,7 @@ func (cmd *RunCommand) constructEngine( execV1Engine := engine.NewExecV1DummyEngine() - return engine.NewDBEngine(engine.Engines{execV2Engine, execV1Engine}, cmd.PeerURLOrDefault().String()) + return engine.NewDBEngine(engine.Engines{execV2Engine, execV1Engine}) } func (cmd *RunCommand) constructHTTPHandler( @@ -1318,7 +1310,6 @@ func (cmd *RunCommand) constructAPIHandler( dbBuildFactory, resourceConfigFactory, - cmd.PeerURLOrDefault().String(), buildserver.NewEventHandler, drain, diff --git a/atc/db/build.go b/atc/db/build.go index 974bad2da25..073828dce65 100644 --- a/atc/db/build.go +++ b/atc/db/build.go @@ -42,7 +42,7 @@ const ( BuildStatusErrored BuildStatus = "errored" ) -var buildsQuery = psql.Select("b.id, b.name, b.job_id, b.team_id, b.status, b.manually_triggered, b.scheduled, b.schema, b.private_plan, b.public_plan, b.start_time, b.end_time, b.reap_time, j.name, b.pipeline_id, p.name, t.name, b.nonce, b.tracked_by, b.drained"). +var buildsQuery = psql.Select("b.id, b.name, b.job_id, b.team_id, b.status, b.manually_triggered, b.scheduled, b.schema, b.private_plan, b.public_plan, b.start_time, b.end_time, b.reap_time, j.name, b.pipeline_id, p.name, t.name, b.nonce, b.drained"). From("builds b"). JoinClause("LEFT OUTER JOIN jobs j ON b.job_id = j.id"). JoinClause("LEFT OUTER JOIN pipelines p ON b.pipeline_id = p.id"). @@ -69,7 +69,6 @@ type Build interface { StartTime() time.Time EndTime() time.Time ReapTime() time.Time - Tracker() string IsManuallyTriggered() bool IsScheduled() bool IsRunning() bool @@ -77,7 +76,6 @@ type Build interface { Reload() (bool, error) AcquireTrackingLock(logger lager.Logger, interval time.Duration) (lock.Lock, bool, error) - TrackedBy(peerURL string) error Interceptible() (bool, error) Preparation() (BuildPreparation, bool, error) @@ -135,8 +133,6 @@ type build struct { endTime time.Time reapTime time.Time - trackedBy string - conn Conn lockFactory lock.LockFactory drained bool @@ -171,7 +167,6 @@ func (b *build) StartTime() time.Time { return b.startTime } func (b *build) EndTime() time.Time { return b.endTime } func (b *build) ReapTime() time.Time { return b.reapTime } func (b *build) Status() BuildStatus { return b.status } -func (b *build) Tracker() string { return b.trackedBy } func (b *build) IsScheduled() bool { return b.scheduled } func (b *build) IsDrained() bool { return b.drained } @@ -557,28 +552,6 @@ func (b *build) AcquireTrackingLock(logger lager.Logger, interval time.Duration) return lock, true, nil } -func (b *build) TrackedBy(peerURL string) error { - rows, err := psql.Update("builds"). - Set("tracked_by", peerURL). - Where(sq.Eq{"id": b.id}). - RunWith(b.conn). - Exec() - if err != nil { - return err - } - - affected, err := rows.RowsAffected() - if err != nil { - return err - } - - if affected == 0 { - return ErrBuildDisappeared - } - - return nil -} - func (b *build) Preparation() (BuildPreparation, bool, error) { if b.jobID == 0 || b.status != BuildStatusPending { return BuildPreparation{ @@ -1119,16 +1092,16 @@ func buildEventSeq(buildid int) string { func scanBuild(b *build, row scannable, encryptionStrategy encryption.Strategy) error { var ( - jobID, pipelineID sql.NullInt64 - schema, privatePlan, jobName, pipelineName, publicPlan, trackedBy sql.NullString - startTime, endTime, reapTime pq.NullTime - nonce sql.NullString - drained bool + jobID, pipelineID sql.NullInt64 + schema, privatePlan, jobName, pipelineName, publicPlan sql.NullString + startTime, endTime, reapTime pq.NullTime + nonce sql.NullString + drained bool status string ) - err := row.Scan(&b.id, &b.name, &jobID, &b.teamID, &status, &b.isManuallyTriggered, &b.scheduled, &schema, &privatePlan, &publicPlan, &startTime, &endTime, &reapTime, &jobName, &pipelineID, &pipelineName, &b.teamName, &nonce, &trackedBy, &drained) + err := row.Scan(&b.id, &b.name, &jobID, &b.teamID, &status, &b.isManuallyTriggered, &b.scheduled, &schema, &privatePlan, &publicPlan, &startTime, &endTime, &reapTime, &jobName, &pipelineID, &pipelineName, &b.teamName, &nonce, &drained) if err != nil { return err } @@ -1142,7 +1115,6 @@ func scanBuild(b *build, row scannable, encryptionStrategy encryption.Strategy) b.startTime = startTime.Time b.endTime = endTime.Time b.reapTime = reapTime.Time - b.trackedBy = trackedBy.String b.drained = drained var ( diff --git a/atc/db/build_test.go b/atc/db/build_test.go index 772fe135888..9fb9f9f53a5 100644 --- a/atc/db/build_test.go +++ b/atc/db/build_test.go @@ -137,26 +137,6 @@ var _ = Describe("Build", func() { }) }) - Describe("TrackedBy", func() { - var build db.Build - - BeforeEach(func() { - var err error - build, err = team.CreateOneOffBuild() - Expect(err).NotTo(HaveOccurred()) - }) - - It("updates build status", func() { - Expect(build.TrackedBy("http://1.2.3.4:8080")).To(Succeed()) - - found, err := build.Reload() - Expect(err).NotTo(HaveOccurred()) - Expect(found).To(BeTrue()) - - Expect(build.Tracker()).To(Equal("http://1.2.3.4:8080")) - }) - }) - Describe("Finish", func() { var build db.Build BeforeEach(func() { diff --git a/atc/engine/db_engine.go b/atc/engine/db_engine.go index b51b7a19fca..8ee9b7911c7 100644 --- a/atc/engine/db_engine.go +++ b/atc/engine/db_engine.go @@ -14,10 +14,9 @@ import ( const trackLockDuration = time.Minute -func NewDBEngine(engines Engines, peerURL string) Engine { +func NewDBEngine(engines Engines) Engine { return &dbEngine{ engines: engines, - peerURL: peerURL, releaseCh: make(chan struct{}), waitGroup: new(sync.WaitGroup), } @@ -33,7 +32,6 @@ func (err UnknownEngineError) Error() string { type dbEngine struct { engines Engines - peerURL string releaseCh chan struct{} waitGroup *sync.WaitGroup } @@ -61,7 +59,6 @@ func (engine *dbEngine) CreateBuild(logger lager.Logger, build db.Build, plan at return &dbBuild{ engines: engine.engines, - peerURL: engine.peerURL, releaseCh: engine.releaseCh, waitGroup: engine.waitGroup, build: build, @@ -71,7 +68,6 @@ func (engine *dbEngine) CreateBuild(logger lager.Logger, build db.Build, plan at func (engine *dbEngine) LookupBuild(logger lager.Logger, build db.Build) (Build, error) { return &dbBuild{ engines: engine.engines, - peerURL: engine.peerURL, releaseCh: engine.releaseCh, waitGroup: engine.waitGroup, build: build, @@ -96,7 +92,6 @@ func (engine *dbEngine) ReleaseAll(logger lager.Logger) { type dbBuild struct { engines Engines - peerURL string releaseCh chan struct{} build db.Build waitGroup *sync.WaitGroup @@ -193,12 +188,6 @@ func (build *dbBuild) Resume(logger lager.Logger) { defer lock.Release() - err = build.build.TrackedBy(build.peerURL) - if err != nil { - logger.Error("failed-to-update-build-tracker", err) - return - } - found, err := build.build.Reload() if err != nil { logger.Error("failed-to-load-build-from-db", err) diff --git a/atc/engine/db_engine_test.go b/atc/engine/db_engine_test.go index ab27f04a96c..54aadc5e14f 100644 --- a/atc/engine/db_engine_test.go +++ b/atc/engine/db_engine_test.go @@ -39,7 +39,7 @@ var _ = Describe("DBEngine", func() { dbBuild = new(dbfakes.FakeBuild) dbBuild.IDReturns(128) - dbEngine = NewDBEngine(Engines{fakeEngineA, fakeEngineB}, "http://10.2.3.4:8080") + dbEngine = NewDBEngine(Engines{fakeEngineA, fakeEngineB}) }) Describe("CreateBuild", func() { @@ -455,11 +455,6 @@ var _ = Describe("DBEngine", func() { dbBuild.AcquireTrackingLockReturns(fakeLock, true, nil) }) - It("updates the tracking information on the build", func() { - Expect(dbBuild.TrackedByCallCount()).To(Equal(1)) - Expect(dbBuild.TrackedByArgsForCall(0)).To(Equal("http://10.2.3.4:8080")) - }) - Context("when the build is active", func() { BeforeEach(func() { dbBuild.SchemaReturns("fake-schema-b") @@ -467,18 +462,6 @@ var _ = Describe("DBEngine", func() { dbBuild.ReloadReturns(true, nil) }) - Context("when updating the tracking information fails", func() { - disaster := errors.New("oh no") - - BeforeEach(func() { - dbBuild.TrackedByReturns(disaster) - }) - - It("does not resume the build", func() { - Expect(fakeEngineB.LookupBuildCallCount()).To(BeZero()) - }) - }) - Context("when the engine build exists", func() { var realBuild *enginefakes.FakeBuild diff --git a/cmd/concourse/web.go b/cmd/concourse/web.go index 5b02933266c..84cb0a7f943 100644 --- a/cmd/concourse/web.go +++ b/cmd/concourse/web.go @@ -87,10 +87,10 @@ func (cmd *WebCommand) populateTSAFlagsFromATCFlags() error { } if len(cmd.TSACommand.ATCURLs) == 0 { - cmd.TSACommand.ATCURLs = []flag.URL{cmd.RunCommand.PeerURLOrDefault()} + cmd.TSACommand.ATCURLs = []flag.URL{cmd.RunCommand.DefaultURL()} } - host, _, err := net.SplitHostPort(cmd.RunCommand.PeerURLOrDefault().URL.Host) + host, _, err := net.SplitHostPort(cmd.RunCommand.DefaultURL().URL.Host) if err != nil { return err } From ccb2c4640d92dbe4a9e68aaf85d504b6de07cedf Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 28 Feb 2019 14:37:04 -0500 Subject: [PATCH 15/21] tsa: default peer ip to 127.0.0.1 concourse/concourse#3267 Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- cmd/concourse/web.go | 8 -------- tsa/tsacmd/command.go | 6 ++---- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/cmd/concourse/web.go b/cmd/concourse/web.go index 84cb0a7f943..c0b2eac9982 100644 --- a/cmd/concourse/web.go +++ b/cmd/concourse/web.go @@ -4,7 +4,6 @@ import ( "crypto/rand" "crypto/rsa" "fmt" - "net" "os" "github.com/concourse/concourse/atc/atccmd" @@ -90,12 +89,5 @@ func (cmd *WebCommand) populateTSAFlagsFromATCFlags() error { cmd.TSACommand.ATCURLs = []flag.URL{cmd.RunCommand.DefaultURL()} } - host, _, err := net.SplitHostPort(cmd.RunCommand.DefaultURL().URL.Host) - if err != nil { - return err - } - - cmd.TSACommand.PeerIP = host - return nil } diff --git a/tsa/tsacmd/command.go b/tsa/tsacmd/command.go index 3bffb677cc6..9262a37a121 100644 --- a/tsa/tsacmd/command.go +++ b/tsa/tsacmd/command.go @@ -9,21 +9,19 @@ import ( "time" "code.cloudfoundry.org/lager" - - "golang.org/x/crypto/ssh" - "github.com/concourse/concourse/tsa" "github.com/concourse/flag" "github.com/tedsuo/ifrit" "github.com/tedsuo/ifrit/grouper" "github.com/tedsuo/ifrit/http_server" "github.com/tedsuo/ifrit/sigmon" + "golang.org/x/crypto/ssh" ) type TSACommand struct { Logger flag.Lager - PeerIP string `long:"peer-ip" required:"true" description:"IP address of this TSA, reachable by the ATCs. Used for forwarded worker addresses."` + PeerIP string `long:"peer-ip" default:"127.0.0.1" description:"IP address of this TSA, reachable by the ATCs. Used for forwarded worker addresses."` BindIP flag.IP `long:"bind-ip" default:"0.0.0.0" description:"IP address on which to listen for SSH."` BindPort uint16 `long:"bind-port" default:"2222" description:"Port on which to listen for SSH."` From 0791d27cef71fb132417d4a6794e02d7562cf160 Mon Sep 17 00:00:00 2001 From: Josh Winters Date: Thu, 28 Feb 2019 15:14:40 -0500 Subject: [PATCH 16/21] atc: bump migrations Signed-off-by: Josh Winters Co-authored-by: Mark Huang --- atc/db/migration/fix_build_private_plan_test.go | 4 ++-- ...n.sql => 1551384519_rename_build_engine_metadata.down.sql} | 0 ....up.sql => 1551384519_rename_build_engine_metadata.up.sql} | 0 ...plan.down.go => 1551384520_fix_build_private_plan.down.go} | 0 ...ate_plan.up.go => 1551384520_fix_build_private_plan.up.go} | 0 5 files changed, 2 insertions(+), 2 deletions(-) rename atc/db/migration/migrations/{1550159967_rename_build_engine_metadata.down.sql => 1551384519_rename_build_engine_metadata.down.sql} (100%) rename atc/db/migration/migrations/{1550159967_rename_build_engine_metadata.up.sql => 1551384519_rename_build_engine_metadata.up.sql} (100%) rename atc/db/migration/migrations/{1550160079_fix_build_private_plan.down.go => 1551384520_fix_build_private_plan.down.go} (100%) rename atc/db/migration/migrations/{1550160079_fix_build_private_plan.up.go => 1551384520_fix_build_private_plan.up.go} (100%) diff --git a/atc/db/migration/fix_build_private_plan_test.go b/atc/db/migration/fix_build_private_plan_test.go index 9b83aa19d45..2f73b613ae1 100644 --- a/atc/db/migration/fix_build_private_plan_test.go +++ b/atc/db/migration/fix_build_private_plan_test.go @@ -8,8 +8,8 @@ import ( ) var _ = Describe("Fix Build Private Plan", func() { - const preMigrationVersion = 1550159967 - const postMigrationVersion = 1550160079 + const preMigrationVersion = 1551384519 + const postMigrationVersion = 1551384520 var ( db *sql.DB diff --git a/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.down.sql b/atc/db/migration/migrations/1551384519_rename_build_engine_metadata.down.sql similarity index 100% rename from atc/db/migration/migrations/1550159967_rename_build_engine_metadata.down.sql rename to atc/db/migration/migrations/1551384519_rename_build_engine_metadata.down.sql diff --git a/atc/db/migration/migrations/1550159967_rename_build_engine_metadata.up.sql b/atc/db/migration/migrations/1551384519_rename_build_engine_metadata.up.sql similarity index 100% rename from atc/db/migration/migrations/1550159967_rename_build_engine_metadata.up.sql rename to atc/db/migration/migrations/1551384519_rename_build_engine_metadata.up.sql diff --git a/atc/db/migration/migrations/1550160079_fix_build_private_plan.down.go b/atc/db/migration/migrations/1551384520_fix_build_private_plan.down.go similarity index 100% rename from atc/db/migration/migrations/1550160079_fix_build_private_plan.down.go rename to atc/db/migration/migrations/1551384520_fix_build_private_plan.down.go diff --git a/atc/db/migration/migrations/1550160079_fix_build_private_plan.up.go b/atc/db/migration/migrations/1551384520_fix_build_private_plan.up.go similarity index 100% rename from atc/db/migration/migrations/1550160079_fix_build_private_plan.up.go rename to atc/db/migration/migrations/1551384520_fix_build_private_plan.up.go From fbd552826b06ae850c817b7e162a03731dc96899 Mon Sep 17 00:00:00 2001 From: Joshua Winters Date: Wed, 13 Mar 2019 10:57:02 -0400 Subject: [PATCH 17/21] atc: make the get artifact API members only concourse/concourse#3307 Signed-off-by: Joshua Winters --- atc/api/accessor/accessor.go | 2 +- atc/api/accessor/accessor_test.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/atc/api/accessor/accessor.go b/atc/api/accessor/accessor.go index f6d43b793fb..e1deba3d803 100644 --- a/atc/api/accessor/accessor.go +++ b/atc/api/accessor/accessor.go @@ -198,6 +198,6 @@ var requiredRoles = map[string]string{ atc.DestroyTeam: "owner", atc.ListTeamBuilds: "viewer", atc.CreateArtifact: "member", - atc.GetArtifact: "viewer", + atc.GetArtifact: "member", atc.ListBuildArtifacts: "viewer", } diff --git a/atc/api/accessor/accessor_test.go b/atc/api/accessor/accessor_test.go index 2bddc4a9aab..f517bfeea90 100644 --- a/atc/api/accessor/accessor_test.go +++ b/atc/api/accessor/accessor_test.go @@ -656,7 +656,7 @@ var _ = Describe("Accessor", func() { Entry("owner :: "+atc.GetArtifact, atc.GetArtifact, "owner", true), Entry("member :: "+atc.GetArtifact, atc.GetArtifact, "member", true), - Entry("viewer :: "+atc.GetArtifact, atc.GetArtifact, "viewer", true), + Entry("viewer :: "+atc.GetArtifact, atc.GetArtifact, "viewer", false), Entry("owner :: "+atc.ListBuildArtifacts, atc.ListBuildArtifacts, "owner", true), Entry("member :: "+atc.ListBuildArtifacts, atc.ListBuildArtifacts, "member", true), From 9981d9077ba71f29f37cb7d8008f2f27f398ec32 Mon Sep 17 00:00:00 2001 From: Joshua Winters Date: Wed, 13 Mar 2019 13:25:54 -0400 Subject: [PATCH 18/21] fly: add back upload/download progress concourse/concourse#3307 Signed-off-by: Joshua Winters --- fly/commands/internal/executehelpers/downloads.go | 9 ++++++++- .../commands/internal/executehelpers}/progress.go | 2 +- fly/commands/internal/executehelpers/uploads.go | 8 +++++++- 3 files changed, 16 insertions(+), 3 deletions(-) rename {atc/exec => fly/commands/internal/executehelpers}/progress.go (95%) diff --git a/fly/commands/internal/executehelpers/downloads.go b/fly/commands/internal/executehelpers/downloads.go index f41d1ad9428..018983583c2 100644 --- a/fly/commands/internal/executehelpers/downloads.go +++ b/fly/commands/internal/executehelpers/downloads.go @@ -1,11 +1,18 @@ package executehelpers import ( + "os" + "github.com/concourse/concourse/go-concourse/concourse" "github.com/concourse/go-archive/tgzfs" ) func Download(team concourse.Team, artifactID int, path string) error { + pb := progress(path+":", os.Stdout) + + pb.Start() + defer pb.Finish() + out, err := team.GetArtifact(artifactID) if err != nil { return err @@ -13,5 +20,5 @@ func Download(team concourse.Team, artifactID int, path string) error { defer out.Close() - return tgzfs.Extract(out, path) + return tgzfs.Extract(pb.NewProxyReader(out), path) } diff --git a/atc/exec/progress.go b/fly/commands/internal/executehelpers/progress.go similarity index 95% rename from atc/exec/progress.go rename to fly/commands/internal/executehelpers/progress.go index 482797d67cc..43400ffa35b 100644 --- a/atc/exec/progress.go +++ b/fly/commands/internal/executehelpers/progress.go @@ -1,4 +1,4 @@ -package exec +package executehelpers import ( "io" diff --git a/fly/commands/internal/executehelpers/uploads.go b/fly/commands/internal/executehelpers/uploads.go index d82c73f63ae..4ace241611b 100644 --- a/fly/commands/internal/executehelpers/uploads.go +++ b/fly/commands/internal/executehelpers/uploads.go @@ -5,6 +5,7 @@ import ( "bytes" "errors" "io" + "os" "os/exec" "github.com/concourse/concourse/atc" @@ -25,7 +26,12 @@ func Upload(team concourse.Team, path string, includeIgnored bool) (atc.WorkerAr archiveWriter.CloseWithError(tgzfs.Compress(archiveWriter, path, files...)) }() - return team.CreateArtifact(archiveStream) + pb := progress(path+":", os.Stdout) + + pb.Start() + defer pb.Finish() + + return team.CreateArtifact(pb.NewProxyReader(archiveStream)) } func getFiles(dir string, includeIgnored bool) []string { From a04753b8dd3f89ea7c8ad1ac867e8bd491d32608 Mon Sep 17 00:00:00 2001 From: Joshua Winters Date: Wed, 13 Mar 2019 14:13:48 -0400 Subject: [PATCH 19/21] atc: register output artifacts in step * instead of for every task concourse/concourse#3307 Signed-off-by: Joshua Winters --- atc/engine/build_step.go | 14 +- atc/engine/exec_engine.go | 8 +- ...rtifact_step.go => artifact_input_step.go} | 17 +- ...ep_test.go => artifact_input_step_test.go} | 6 +- atc/exec/artifact_output_step.go | 72 ++++++++ atc/exec/artifact_output_step_test.go | 108 ++++++++++++ atc/exec/execfakes/fake_factory.go | 159 +++++++++++++----- atc/exec/factory.go | 9 +- atc/exec/garden_factory.go | 13 +- atc/exec/task_step.go | 5 - atc/exec/task_step_test.go | 18 -- atc/plan.go | 4 +- atc/plan_factory.go | 4 +- atc/public_plan.go | 8 +- atc/public_plan_test.go | 4 +- .../internal/executehelpers/inputs.go | 2 +- .../execute_multiple_inputs_test.go | 4 +- .../execute_overriding_inputs_test.go | 2 +- fly/integration/execute_test.go | 2 +- fly/integration/execute_with_outputs_test.go | 2 +- go.mod | 1 + go.sum | 2 + 22 files changed, 367 insertions(+), 97 deletions(-) rename atc/exec/{user_artifact_step.go => artifact_input_step.go} (71%) rename atc/exec/{user_artifact_step_test.go => artifact_input_step_test.go} (94%) create mode 100644 atc/exec/artifact_output_step.go create mode 100644 atc/exec/artifact_output_step_test.go diff --git a/atc/engine/build_step.go b/atc/engine/build_step.go index 9bc7a0e5c29..9f1e981b2d6 100644 --- a/atc/engine/build_step.go +++ b/atc/engine/build_step.go @@ -157,9 +157,19 @@ func (build *execBuild) buildRetryStep(logger lager.Logger, plan atc.Plan) exec. return exec.Retry(steps...) } -func (build *execBuild) buildUserArtifactStep(logger lager.Logger, plan atc.Plan) exec.Step { +func (build *execBuild) buildArtifactInputStep(logger lager.Logger, plan atc.Plan) exec.Step { - return build.factory.ArtifactStep( + return build.factory.ArtifactInputStep( + logger, + plan, + build.dbBuild, + build.delegate.BuildStepDelegate(plan.ID), + ) +} + +func (build *execBuild) buildArtifactOutputStep(logger lager.Logger, plan atc.Plan) exec.Step { + + return build.factory.ArtifactOutputStep( logger, plan, build.dbBuild, diff --git a/atc/engine/exec_engine.go b/atc/engine/exec_engine.go index 93ec90bbcc9..9ca1205a053 100644 --- a/atc/engine/exec_engine.go +++ b/atc/engine/exec_engine.go @@ -223,8 +223,12 @@ func (build *execBuild) buildStep(logger lager.Logger, plan atc.Plan) exec.Step return build.buildRetryStep(logger, plan) } - if plan.UserArtifact != nil { - return build.buildUserArtifactStep(logger, plan) + if plan.ArtifactInput != nil { + return build.buildArtifactInputStep(logger, plan) + } + + if plan.ArtifactOutput != nil { + return build.buildArtifactOutputStep(logger, plan) } return exec.IdentityStep{} diff --git a/atc/exec/user_artifact_step.go b/atc/exec/artifact_input_step.go similarity index 71% rename from atc/exec/user_artifact_step.go rename to atc/exec/artifact_input_step.go index 57659290951..210f5fa43da 100644 --- a/atc/exec/user_artifact_step.go +++ b/atc/exec/artifact_input_step.go @@ -18,7 +18,7 @@ func (e ArtifactVolumeNotFoundErr) Error() string { return fmt.Sprintf("volume for worker artifact '%s' not found", e) } -type ArtifactStep struct { +type ArtifactInputStep struct { plan atc.Plan build db.Build workerClient worker.Client @@ -26,8 +26,8 @@ type ArtifactStep struct { succeeded bool } -func NewArtifactStep(plan atc.Plan, build db.Build, workerClient worker.Client, delegate BuildStepDelegate) Step { - return &ArtifactStep{ +func NewArtifactInputStep(plan atc.Plan, build db.Build, workerClient worker.Client, delegate BuildStepDelegate) Step { + return &ArtifactInputStep{ plan: plan, build: build, workerClient: workerClient, @@ -35,12 +35,12 @@ func NewArtifactStep(plan atc.Plan, build db.Build, workerClient worker.Client, } } -func (step *ArtifactStep) Run(ctx context.Context, state RunState) error { +func (step *ArtifactInputStep) Run(ctx context.Context, state RunState) error { logger := lagerctx.FromContext(ctx).WithData(lager.Data{ "plan-id": step.plan.ID, }) - buildArtifact, err := step.build.Artifact(step.plan.UserArtifact.ArtifactID) + buildArtifact, err := step.build.Artifact(step.plan.ArtifactInput.ArtifactID) if err != nil { return err } @@ -64,17 +64,18 @@ func (step *ArtifactStep) Run(ctx context.Context, state RunState) error { } logger.Info("register-artifact-source", lager.Data{ - "handle": workerVolume.Handle(), + "artifact_id": buildArtifact.ID(), + "handle": workerVolume.Handle(), }) source := NewTaskArtifactSource(workerVolume) - state.Artifacts().RegisterSource(artifact.Name(step.plan.UserArtifact.Name), source) + state.Artifacts().RegisterSource(artifact.Name(step.plan.ArtifactInput.Name), source) step.succeeded = true return nil } -func (step *ArtifactStep) Succeeded() bool { +func (step *ArtifactInputStep) Succeeded() bool { return step.succeeded } diff --git a/atc/exec/user_artifact_step_test.go b/atc/exec/artifact_input_step_test.go similarity index 94% rename from atc/exec/user_artifact_step_test.go rename to atc/exec/artifact_input_step_test.go index 5e935d89b85..0b67290e819 100644 --- a/atc/exec/user_artifact_step_test.go +++ b/atc/exec/artifact_input_step_test.go @@ -14,7 +14,7 @@ import ( . "github.com/onsi/gomega" ) -var _ = Describe("ArtifactStep", func() { +var _ = Describe("ArtifactInputStep", func() { var ( ctx context.Context cancel func() @@ -46,9 +46,9 @@ var _ = Describe("ArtifactStep", func() { }) JustBeforeEach(func() { - plan = atc.Plan{UserArtifact: &atc.UserArtifactPlan{0, "some-name"}} + plan = atc.Plan{ArtifactInput: &atc.ArtifactInputPlan{0, "some-name"}} - step = exec.NewArtifactStep(plan, fakeBuild, fakeWorkerClient, delegate) + step = exec.NewArtifactInputStep(plan, fakeBuild, fakeWorkerClient, delegate) stepErr = step.Run(ctx, state) }) diff --git a/atc/exec/artifact_output_step.go b/atc/exec/artifact_output_step.go new file mode 100644 index 00000000000..104304335f5 --- /dev/null +++ b/atc/exec/artifact_output_step.go @@ -0,0 +1,72 @@ +package exec + +import ( + "context" + "fmt" + + "code.cloudfoundry.org/lager" + "code.cloudfoundry.org/lager/lagerctx" + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/db" + "github.com/concourse/concourse/atc/exec/artifact" + "github.com/concourse/concourse/atc/worker" +) + +type ArtifactNotFoundErr string + +func (e ArtifactNotFoundErr) Error() string { + return fmt.Sprintf("artifact '%s' not found", e) +} + +type ArtifactOutputStep struct { + plan atc.Plan + build db.Build + workerClient worker.Client + delegate BuildStepDelegate + succeeded bool +} + +func NewArtifactOutputStep(plan atc.Plan, build db.Build, workerClient worker.Client, delegate BuildStepDelegate) Step { + return &ArtifactOutputStep{ + plan: plan, + build: build, + workerClient: workerClient, + delegate: delegate, + } +} + +func (step *ArtifactOutputStep) Run(ctx context.Context, state RunState) error { + logger := lagerctx.FromContext(ctx).WithData(lager.Data{ + "plan-id": step.plan.ID, + }) + + outputName := step.plan.ArtifactOutput.Name + + source, found := state.Artifacts().SourceFor(artifact.Name(outputName)) + if !found { + return ArtifactNotFoundErr(outputName) + } + + volume, ok := source.(worker.Volume) + if !ok { + return ArtifactNotFoundErr(outputName) + } + + artifact, err := volume.InitializeArtifact(outputName, step.build.ID()) + if err != nil { + return err + } + + logger.Info("initialize-artifact-from-source", lager.Data{ + "handle": volume.Handle(), + "artifact_id": artifact.ID(), + }) + + step.succeeded = true + + return nil +} + +func (step *ArtifactOutputStep) Succeeded() bool { + return step.succeeded +} diff --git a/atc/exec/artifact_output_step_test.go b/atc/exec/artifact_output_step_test.go new file mode 100644 index 00000000000..de6a71515f7 --- /dev/null +++ b/atc/exec/artifact_output_step_test.go @@ -0,0 +1,108 @@ +package exec_test + +import ( + "context" + "errors" + "io/ioutil" + + "github.com/concourse/concourse/atc" + "github.com/concourse/concourse/atc/db/dbfakes" + "github.com/concourse/concourse/atc/exec" + "github.com/concourse/concourse/atc/exec/artifact" + "github.com/concourse/concourse/atc/exec/execfakes" + "github.com/concourse/concourse/atc/worker/workerfakes" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("ArtifactOutputStep", func() { + var ( + ctx context.Context + cancel func() + + state exec.RunState + delegate *execfakes.FakeBuildStepDelegate + + step exec.Step + stepErr error + plan atc.Plan + fakeBuild *dbfakes.FakeBuild + fakeWorkerClient *workerfakes.FakeClient + ) + + BeforeEach(func() { + ctx, cancel = context.WithCancel(context.Background()) + + state = exec.NewRunState() + + delegate = new(execfakes.FakeBuildStepDelegate) + delegate.StdoutReturns(ioutil.Discard) + + fakeBuild = new(dbfakes.FakeBuild) + fakeWorkerClient = new(workerfakes.FakeClient) + }) + + AfterEach(func() { + cancel() + }) + + JustBeforeEach(func() { + plan = atc.Plan{ArtifactOutput: &atc.ArtifactOutputPlan{"some-name"}} + + step = exec.NewArtifactOutputStep(plan, fakeBuild, fakeWorkerClient, delegate) + stepErr = step.Run(ctx, state) + }) + + Context("when the source does not exist", func() { + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) + }) + + Context("when the source exists", func() { + Context("when the source is not a worker.Volume", func() { + BeforeEach(func() { + fakeSource := new(workerfakes.FakeArtifactSource) + state.Artifacts().RegisterSource(artifact.Name("some-name"), fakeSource) + }) + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) + }) + + Context("when the source is a worker.Volume", func() { + var fakeWorkerVolume *workerfakes.FakeVolume + + BeforeEach(func() { + fakeWorkerVolume = new(workerfakes.FakeVolume) + fakeWorkerVolume.HandleReturns("handle") + + source := exec.NewTaskArtifactSource(fakeWorkerVolume) + state.Artifacts().RegisterSource(artifact.Name("some-name"), source) + }) + + Context("when initializing the artifact fails", func() { + BeforeEach(func() { + fakeWorkerVolume.InitializeArtifactReturns(nil, errors.New("nope")) + }) + It("returns the error", func() { + Expect(stepErr).To(HaveOccurred()) + }) + }) + + Context("when initializing the artifact succeeds", func() { + var fakeWorkerArtifact *dbfakes.FakeWorkerArtifact + + BeforeEach(func() { + fakeWorkerArtifact = new(dbfakes.FakeWorkerArtifact) + fakeWorkerArtifact.IDReturns(0) + + fakeWorkerVolume.InitializeArtifactReturns(fakeWorkerArtifact, nil) + }) + It("succeeds", func() { + Expect(step.Succeeded()).To(BeTrue()) + }) + }) + }) + }) +}) diff --git a/atc/exec/execfakes/fake_factory.go b/atc/exec/execfakes/fake_factory.go index fdb0ddc9597..569fdac5c6c 100644 --- a/atc/exec/execfakes/fake_factory.go +++ b/atc/exec/execfakes/fake_factory.go @@ -11,18 +11,32 @@ import ( ) type FakeFactory struct { - ArtifactStepStub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step - artifactStepMutex sync.RWMutex - artifactStepArgsForCall []struct { + ArtifactInputStepStub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step + artifactInputStepMutex sync.RWMutex + artifactInputStepArgsForCall []struct { arg1 lager.Logger arg2 atc.Plan arg3 db.Build arg4 exec.BuildStepDelegate } - artifactStepReturns struct { + artifactInputStepReturns struct { result1 exec.Step } - artifactStepReturnsOnCall map[int]struct { + artifactInputStepReturnsOnCall map[int]struct { + result1 exec.Step + } + ArtifactOutputStepStub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step + artifactOutputStepMutex sync.RWMutex + artifactOutputStepArgsForCall []struct { + arg1 lager.Logger + arg2 atc.Plan + arg3 db.Build + arg4 exec.BuildStepDelegate + } + artifactOutputStepReturns struct { + result1 exec.Step + } + artifactOutputStepReturnsOnCall map[int]struct { result1 exec.Step } GetStub func(lager.Logger, atc.Plan, db.Build, exec.StepMetadata, db.ContainerMetadata, exec.GetDelegate) exec.Step @@ -76,65 +90,128 @@ type FakeFactory struct { invocationsMutex sync.RWMutex } -func (fake *FakeFactory) ArtifactStep(arg1 lager.Logger, arg2 atc.Plan, arg3 db.Build, arg4 exec.BuildStepDelegate) exec.Step { - fake.artifactStepMutex.Lock() - ret, specificReturn := fake.artifactStepReturnsOnCall[len(fake.artifactStepArgsForCall)] - fake.artifactStepArgsForCall = append(fake.artifactStepArgsForCall, struct { +func (fake *FakeFactory) ArtifactInputStep(arg1 lager.Logger, arg2 atc.Plan, arg3 db.Build, arg4 exec.BuildStepDelegate) exec.Step { + fake.artifactInputStepMutex.Lock() + ret, specificReturn := fake.artifactInputStepReturnsOnCall[len(fake.artifactInputStepArgsForCall)] + fake.artifactInputStepArgsForCall = append(fake.artifactInputStepArgsForCall, struct { + arg1 lager.Logger + arg2 atc.Plan + arg3 db.Build + arg4 exec.BuildStepDelegate + }{arg1, arg2, arg3, arg4}) + fake.recordInvocation("ArtifactInputStep", []interface{}{arg1, arg2, arg3, arg4}) + fake.artifactInputStepMutex.Unlock() + if fake.ArtifactInputStepStub != nil { + return fake.ArtifactInputStepStub(arg1, arg2, arg3, arg4) + } + if specificReturn { + return ret.result1 + } + fakeReturns := fake.artifactInputStepReturns + return fakeReturns.result1 +} + +func (fake *FakeFactory) ArtifactInputStepCallCount() int { + fake.artifactInputStepMutex.RLock() + defer fake.artifactInputStepMutex.RUnlock() + return len(fake.artifactInputStepArgsForCall) +} + +func (fake *FakeFactory) ArtifactInputStepCalls(stub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step) { + fake.artifactInputStepMutex.Lock() + defer fake.artifactInputStepMutex.Unlock() + fake.ArtifactInputStepStub = stub +} + +func (fake *FakeFactory) ArtifactInputStepArgsForCall(i int) (lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) { + fake.artifactInputStepMutex.RLock() + defer fake.artifactInputStepMutex.RUnlock() + argsForCall := fake.artifactInputStepArgsForCall[i] + return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 +} + +func (fake *FakeFactory) ArtifactInputStepReturns(result1 exec.Step) { + fake.artifactInputStepMutex.Lock() + defer fake.artifactInputStepMutex.Unlock() + fake.ArtifactInputStepStub = nil + fake.artifactInputStepReturns = struct { + result1 exec.Step + }{result1} +} + +func (fake *FakeFactory) ArtifactInputStepReturnsOnCall(i int, result1 exec.Step) { + fake.artifactInputStepMutex.Lock() + defer fake.artifactInputStepMutex.Unlock() + fake.ArtifactInputStepStub = nil + if fake.artifactInputStepReturnsOnCall == nil { + fake.artifactInputStepReturnsOnCall = make(map[int]struct { + result1 exec.Step + }) + } + fake.artifactInputStepReturnsOnCall[i] = struct { + result1 exec.Step + }{result1} +} + +func (fake *FakeFactory) ArtifactOutputStep(arg1 lager.Logger, arg2 atc.Plan, arg3 db.Build, arg4 exec.BuildStepDelegate) exec.Step { + fake.artifactOutputStepMutex.Lock() + ret, specificReturn := fake.artifactOutputStepReturnsOnCall[len(fake.artifactOutputStepArgsForCall)] + fake.artifactOutputStepArgsForCall = append(fake.artifactOutputStepArgsForCall, struct { arg1 lager.Logger arg2 atc.Plan arg3 db.Build arg4 exec.BuildStepDelegate }{arg1, arg2, arg3, arg4}) - fake.recordInvocation("ArtifactStep", []interface{}{arg1, arg2, arg3, arg4}) - fake.artifactStepMutex.Unlock() - if fake.ArtifactStepStub != nil { - return fake.ArtifactStepStub(arg1, arg2, arg3, arg4) + fake.recordInvocation("ArtifactOutputStep", []interface{}{arg1, arg2, arg3, arg4}) + fake.artifactOutputStepMutex.Unlock() + if fake.ArtifactOutputStepStub != nil { + return fake.ArtifactOutputStepStub(arg1, arg2, arg3, arg4) } if specificReturn { return ret.result1 } - fakeReturns := fake.artifactStepReturns + fakeReturns := fake.artifactOutputStepReturns return fakeReturns.result1 } -func (fake *FakeFactory) ArtifactStepCallCount() int { - fake.artifactStepMutex.RLock() - defer fake.artifactStepMutex.RUnlock() - return len(fake.artifactStepArgsForCall) +func (fake *FakeFactory) ArtifactOutputStepCallCount() int { + fake.artifactOutputStepMutex.RLock() + defer fake.artifactOutputStepMutex.RUnlock() + return len(fake.artifactOutputStepArgsForCall) } -func (fake *FakeFactory) ArtifactStepCalls(stub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step) { - fake.artifactStepMutex.Lock() - defer fake.artifactStepMutex.Unlock() - fake.ArtifactStepStub = stub +func (fake *FakeFactory) ArtifactOutputStepCalls(stub func(lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) exec.Step) { + fake.artifactOutputStepMutex.Lock() + defer fake.artifactOutputStepMutex.Unlock() + fake.ArtifactOutputStepStub = stub } -func (fake *FakeFactory) ArtifactStepArgsForCall(i int) (lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) { - fake.artifactStepMutex.RLock() - defer fake.artifactStepMutex.RUnlock() - argsForCall := fake.artifactStepArgsForCall[i] +func (fake *FakeFactory) ArtifactOutputStepArgsForCall(i int) (lager.Logger, atc.Plan, db.Build, exec.BuildStepDelegate) { + fake.artifactOutputStepMutex.RLock() + defer fake.artifactOutputStepMutex.RUnlock() + argsForCall := fake.artifactOutputStepArgsForCall[i] return argsForCall.arg1, argsForCall.arg2, argsForCall.arg3, argsForCall.arg4 } -func (fake *FakeFactory) ArtifactStepReturns(result1 exec.Step) { - fake.artifactStepMutex.Lock() - defer fake.artifactStepMutex.Unlock() - fake.ArtifactStepStub = nil - fake.artifactStepReturns = struct { +func (fake *FakeFactory) ArtifactOutputStepReturns(result1 exec.Step) { + fake.artifactOutputStepMutex.Lock() + defer fake.artifactOutputStepMutex.Unlock() + fake.ArtifactOutputStepStub = nil + fake.artifactOutputStepReturns = struct { result1 exec.Step }{result1} } -func (fake *FakeFactory) ArtifactStepReturnsOnCall(i int, result1 exec.Step) { - fake.artifactStepMutex.Lock() - defer fake.artifactStepMutex.Unlock() - fake.ArtifactStepStub = nil - if fake.artifactStepReturnsOnCall == nil { - fake.artifactStepReturnsOnCall = make(map[int]struct { +func (fake *FakeFactory) ArtifactOutputStepReturnsOnCall(i int, result1 exec.Step) { + fake.artifactOutputStepMutex.Lock() + defer fake.artifactOutputStepMutex.Unlock() + fake.ArtifactOutputStepStub = nil + if fake.artifactOutputStepReturnsOnCall == nil { + fake.artifactOutputStepReturnsOnCall = make(map[int]struct { result1 exec.Step }) } - fake.artifactStepReturnsOnCall[i] = struct { + fake.artifactOutputStepReturnsOnCall[i] = struct { result1 exec.Step }{result1} } @@ -336,8 +413,10 @@ func (fake *FakeFactory) TaskReturnsOnCall(i int, result1 exec.Step) { func (fake *FakeFactory) Invocations() map[string][][]interface{} { fake.invocationsMutex.RLock() defer fake.invocationsMutex.RUnlock() - fake.artifactStepMutex.RLock() - defer fake.artifactStepMutex.RUnlock() + fake.artifactInputStepMutex.RLock() + defer fake.artifactInputStepMutex.RUnlock() + fake.artifactOutputStepMutex.RLock() + defer fake.artifactOutputStepMutex.RUnlock() fake.getMutex.RLock() defer fake.getMutex.RUnlock() fake.putMutex.RLock() diff --git a/atc/exec/factory.go b/atc/exec/factory.go index 951fe9703bd..f40e189636f 100644 --- a/atc/exec/factory.go +++ b/atc/exec/factory.go @@ -41,7 +41,14 @@ type Factory interface { TaskDelegate, ) Step - ArtifactStep( + ArtifactInputStep( + lager.Logger, + atc.Plan, + db.Build, + BuildStepDelegate, + ) Step + + ArtifactOutputStep( lager.Logger, atc.Plan, db.Build, diff --git a/atc/exec/garden_factory.go b/atc/exec/garden_factory.go index 1655187dd08..d248f27ea76 100644 --- a/atc/exec/garden_factory.go +++ b/atc/exec/garden_factory.go @@ -209,13 +209,22 @@ func (factory *gardenFactory) Task( return LogError(taskStep, delegate) } -func (factory *gardenFactory) ArtifactStep( +func (factory *gardenFactory) ArtifactInputStep( logger lager.Logger, plan atc.Plan, build db.Build, delegate BuildStepDelegate, ) Step { - return NewArtifactStep(plan, build, factory.client, delegate) + return NewArtifactInputStep(plan, build, factory.client, delegate) +} + +func (factory *gardenFactory) ArtifactOutputStep( + logger lager.Logger, + plan atc.Plan, + build db.Build, + delegate BuildStepDelegate, +) Step { + return NewArtifactOutputStep(plan, build, factory.client, delegate) } func (factory *gardenFactory) taskWorkingDirectory(sourceName artifact.Name) string { diff --git a/atc/exec/task_step.go b/atc/exec/task_step.go index 6e34fec5209..0887d65cf29 100644 --- a/atc/exec/task_step.go +++ b/atc/exec/task_step.go @@ -453,11 +453,6 @@ func (action *TaskStep) registerOutputs(logger lager.Logger, repository *artifac if filepath.Clean(mount.MountPath) == filepath.Clean(outputPath) { source := NewTaskArtifactSource(mount.Volume) repository.RegisterSource(artifact.Name(outputName), source) - - _, err := mount.Volume.InitializeArtifact(outputName, action.buildID) - if err != nil { - return err - } } } } diff --git a/atc/exec/task_step_test.go b/atc/exec/task_step_test.go index b4498d87b1a..b9d5e9c814f 100644 --- a/atc/exec/task_step_test.go +++ b/atc/exec/task_step_test.go @@ -326,7 +326,6 @@ var _ = Describe("TaskStep", func() { Expect(actualResourceTypes).To(Equal(resourceTypes)) }) - }) Context("when an exit status is already saved off", func() { @@ -429,23 +428,6 @@ var _ = Describe("TaskStep", func() { sourceMap := repo.AsMap() Expect(sourceMap).To(ConsistOf(artifactSource1, artifactSource2, artifactSource3)) }) - - It("initializes the outputs as artifacts", func() { - Expect(fakeVolume1.InitializeArtifactCallCount()).To(Equal(1)) - outputName, id := fakeVolume1.InitializeArtifactArgsForCall(0) - Expect(outputName).To(Equal("some-output")) - Expect(id).To(Equal(buildID)) - - Expect(fakeVolume2.InitializeArtifactCallCount()).To(Equal(1)) - outputName, id = fakeVolume2.InitializeArtifactArgsForCall(0) - Expect(outputName).To(Equal("some-other-output")) - Expect(id).To(Equal(buildID)) - - Expect(fakeVolume3.InitializeArtifactCallCount()).To(Equal(1)) - outputName, id = fakeVolume3.InitializeArtifactArgsForCall(0) - Expect(outputName).To(Equal("some-trailing-slash-output")) - Expect(id).To(Equal(buildID)) - }) }) }) diff --git a/atc/plan.go b/atc/plan.go index f24486ca627..14f7f2e2fef 100644 --- a/atc/plan.go +++ b/atc/plan.go @@ -18,7 +18,7 @@ type Plan struct { Retry *RetryPlan `json:"retry,omitempty"` // used for 'fly execute' - UserArtifact *UserArtifactPlan `json:"user_artifact,omitempty"` + ArtifactInput *ArtifactInputPlan `json:"artifact_input,omitempty"` ArtifactOutput *ArtifactOutputPlan `json:"artifact_output,omitempty"` // deprecated, kept for backwards compatibility to be able to show old builds @@ -27,7 +27,7 @@ type Plan struct { type PlanID string -type UserArtifactPlan struct { +type ArtifactInputPlan struct { ArtifactID int `json:"artifact_id"` Name string `json:"name"` } diff --git a/atc/plan_factory.go b/atc/plan_factory.go index 93b944c4f88..4c0a2327763 100644 --- a/atc/plan_factory.go +++ b/atc/plan_factory.go @@ -49,8 +49,8 @@ func (factory PlanFactory) NewPlan(step Step) Plan { plan.Timeout = &t case RetryPlan: plan.Retry = &t - case UserArtifactPlan: - plan.UserArtifact = &t + case ArtifactInputPlan: + plan.ArtifactInput = &t case ArtifactOutputPlan: plan.ArtifactOutput = &t default: diff --git a/atc/public_plan.go b/atc/public_plan.go index ba1be1f5abc..13da5c77e23 100644 --- a/atc/public_plan.go +++ b/atc/public_plan.go @@ -19,7 +19,7 @@ func (plan Plan) Public() *json.RawMessage { DependentGet *json.RawMessage `json:"dependent_get,omitempty"` Timeout *json.RawMessage `json:"timeout,omitempty"` Retry *json.RawMessage `json:"retry,omitempty"` - UserArtifact *json.RawMessage `json:"user_artifact,omitempty"` + ArtifactInput *json.RawMessage `json:"artifact_input,omitempty"` ArtifactOutput *json.RawMessage `json:"artifact_output,omitempty"` } @@ -73,8 +73,8 @@ func (plan Plan) Public() *json.RawMessage { public.Retry = plan.Retry.Public() } - if plan.UserArtifact != nil { - public.UserArtifact = plan.UserArtifact.Public() + if plan.ArtifactInput != nil { + public.ArtifactInput = plan.ArtifactInput.Public() } if plan.ArtifactOutput != nil { @@ -224,7 +224,7 @@ func (plan RetryPlan) Public() *json.RawMessage { return enc(public) } -func (plan UserArtifactPlan) Public() *json.RawMessage { +func (plan ArtifactInputPlan) Public() *json.RawMessage { return enc(plan) } diff --git a/atc/public_plan_test.go b/atc/public_plan_test.go index c1c73bd8f49..fd9222798f9 100644 --- a/atc/public_plan_test.go +++ b/atc/public_plan_test.go @@ -283,7 +283,7 @@ var _ = Describe("Plan", func() { atc.Plan{ ID: "31", - UserArtifact: &atc.UserArtifactPlan{ + ArtifactInput: &atc.ArtifactInputPlan{ ArtifactID: 17, Name: "some-name", }, @@ -501,7 +501,7 @@ var _ = Describe("Plan", func() { }, { "id": "31", - "user_artifact": { + "artifact_input": { "artifact_id": 17, "name" : "some-name" } diff --git a/fly/commands/internal/executehelpers/inputs.go b/fly/commands/internal/executehelpers/inputs.go index b824dbd13d4..70416c475ef 100644 --- a/fly/commands/internal/executehelpers/inputs.go +++ b/fly/commands/internal/executehelpers/inputs.go @@ -152,7 +152,7 @@ func GenerateLocalInputs( kvMap[inputName] = Input{ Name: inputName, Path: absPath, - Plan: fact.NewPlan(atc.UserArtifactPlan{ + Plan: fact.NewPlan(atc.ArtifactInputPlan{ ArtifactID: artifact.ID, Name: inputName, }), diff --git a/fly/integration/execute_multiple_inputs_test.go b/fly/integration/execute_multiple_inputs_test.go index 1a91fb891a2..27b00bf3cab 100644 --- a/fly/integration/execute_multiple_inputs_test.go +++ b/fly/integration/execute_multiple_inputs_test.go @@ -87,11 +87,11 @@ run: expectedPlan = planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ - planFactory.NewPlan(atc.UserArtifactPlan{ + planFactory.NewPlan(atc.ArtifactInputPlan{ ArtifactID: 125, Name: "some-input", }), - planFactory.NewPlan(atc.UserArtifactPlan{ + planFactory.NewPlan(atc.ArtifactInputPlan{ ArtifactID: 125, Name: "some-other-input", }), diff --git a/fly/integration/execute_overriding_inputs_test.go b/fly/integration/execute_overriding_inputs_test.go index 91205112505..2c148f8d4d7 100644 --- a/fly/integration/execute_overriding_inputs_test.go +++ b/fly/integration/execute_overriding_inputs_test.go @@ -85,7 +85,7 @@ run: expectedPlan = planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ - planFactory.NewPlan(atc.UserArtifactPlan{ + planFactory.NewPlan(atc.ArtifactInputPlan{ ArtifactID: 125, Name: "some-input", }), diff --git a/fly/integration/execute_test.go b/fly/integration/execute_test.go index 30fab35f1b7..ba9e3430130 100644 --- a/fly/integration/execute_test.go +++ b/fly/integration/execute_test.go @@ -87,7 +87,7 @@ run: expectedPlan = planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ - planFactory.NewPlan(atc.UserArtifactPlan{ + planFactory.NewPlan(atc.ArtifactInputPlan{ ArtifactID: 125, Name: filepath.Base(buildDir), }), diff --git a/fly/integration/execute_with_outputs_test.go b/fly/integration/execute_with_outputs_test.go index 905e80d27a4..0b1f944496f 100644 --- a/fly/integration/execute_with_outputs_test.go +++ b/fly/integration/execute_with_outputs_test.go @@ -92,7 +92,7 @@ run: expectedPlan = planFactory.NewPlan(atc.EnsurePlan{ Step: planFactory.NewPlan(atc.DoPlan{ planFactory.NewPlan(atc.AggregatePlan{ - planFactory.NewPlan(atc.UserArtifactPlan{ + planFactory.NewPlan(atc.ArtifactInputPlan{ Name: filepath.Base(buildDir), ArtifactID: 125, }), diff --git a/go.mod b/go.mod index 87b730f60f4..f5aff798aad 100644 --- a/go.mod +++ b/go.mod @@ -147,6 +147,7 @@ require ( github.com/lib/pq v0.0.0-20181016162627-9eb73efc1fcc github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 // indirect github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b // indirect + github.com/mattes/migrate v3.0.1+incompatible github.com/mattn/go-colorable v0.1.1 github.com/mattn/go-isatty v0.0.6 github.com/mattn/go-runewidth v0.0.3 // indirect diff --git a/go.sum b/go.sum index 247c00fed8c..974d714a5a7 100644 --- a/go.sum +++ b/go.sum @@ -380,6 +380,8 @@ github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329 h1:2gxZ0XQIU/5z3Z3 github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b h1:v29yPGHhOqw7VHEnTeQFAth3SsBrmwc8JfuhNY0G34k= github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b/go.mod h1:5MWrJXKRQyhQdUCF+vu6U5c4nQpg70vW3eHaU0/AYbU= +github.com/mattes/migrate v3.0.1+incompatible h1:PhAZP82Vqejw8JZLF4U5UkLGzEVaCnbtJpB6DONcDow= +github.com/mattes/migrate v3.0.1+incompatible/go.mod h1:LJcqgpj1jQoxv3m2VXd3drv0suK5CbN/RCX7MXwgnVI= github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0XgOg= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= From 6828e5c77f51ecdfbbb5c986fe64e23ad821748e Mon Sep 17 00:00:00 2001 From: Joshua Winters Date: Wed, 13 Mar 2019 14:42:24 -0400 Subject: [PATCH 20/21] atc: double status ALL THE WAY concourse/concourse#3307 Signed-off-by: Joshua Winters --- atc/api/artifactserver/get.go | 1 - 1 file changed, 1 deletion(-) diff --git a/atc/api/artifactserver/get.go b/atc/api/artifactserver/get.go index 44d5d36ea78..efb07e0d6c1 100644 --- a/atc/api/artifactserver/get.go +++ b/atc/api/artifactserver/get.go @@ -60,7 +60,6 @@ func (s *Server) GetArtifact(team db.Team) http.Handler { _, err = io.Copy(w, reader) if err != nil { logger.Error("failed-to-encode-artifact", err) - w.WriteHeader(http.StatusInternalServerError) } }) } From 1afb5a60d5b84e1a52ff585f57651182cf40e986 Mon Sep 17 00:00:00 2001 From: Alex Suraci Date: Fri, 15 Mar 2019 13:07:33 -0400 Subject: [PATCH 21/21] fly: make upload/download progress clearer Signed-off-by: Alex Suraci --- fly/commands/internal/executehelpers/downloads.go | 2 +- fly/commands/internal/executehelpers/inputs.go | 2 +- fly/commands/internal/executehelpers/outputs.go | 8 +------- fly/commands/internal/executehelpers/uploads.go | 2 +- 4 files changed, 4 insertions(+), 10 deletions(-) diff --git a/fly/commands/internal/executehelpers/downloads.go b/fly/commands/internal/executehelpers/downloads.go index 018983583c2..b813f005185 100644 --- a/fly/commands/internal/executehelpers/downloads.go +++ b/fly/commands/internal/executehelpers/downloads.go @@ -8,7 +8,7 @@ import ( ) func Download(team concourse.Team, artifactID int, path string) error { - pb := progress(path+":", os.Stdout) + pb := progress("downloading to "+path+":", os.Stdout) pb.Start() defer pb.Finish() diff --git a/fly/commands/internal/executehelpers/inputs.go b/fly/commands/internal/executehelpers/inputs.go index 70416c475ef..91cc949650f 100644 --- a/fly/commands/internal/executehelpers/inputs.go +++ b/fly/commands/internal/executehelpers/inputs.go @@ -49,7 +49,7 @@ func DetermineInputs( localInputMappings = append(localInputMappings, flaghelpers.InputPairFlag{ Name: filepath.Base(wd), - Path: wd, + Path: ".", }) } diff --git a/fly/commands/internal/executehelpers/outputs.go b/fly/commands/internal/executehelpers/outputs.go index d4d3f140e70..30fc3a86a14 100644 --- a/fly/commands/internal/executehelpers/outputs.go +++ b/fly/commands/internal/executehelpers/outputs.go @@ -2,7 +2,6 @@ package executehelpers import ( "fmt" - "path/filepath" "github.com/concourse/concourse/atc" "github.com/concourse/concourse/fly/commands/internal/flaghelpers" @@ -35,14 +34,9 @@ func DetermineOutputs( return nil, fmt.Errorf("unknown output '%s'", outputName) } - absPath, err := filepath.Abs(i.Path) - if err != nil { - return nil, err - } - outputs = append(outputs, Output{ Name: outputName, - Path: absPath, + Path: i.Path, Plan: fact.NewPlan(atc.ArtifactOutputPlan{ Name: outputName, }), diff --git a/fly/commands/internal/executehelpers/uploads.go b/fly/commands/internal/executehelpers/uploads.go index 4ace241611b..1c2b440d139 100644 --- a/fly/commands/internal/executehelpers/uploads.go +++ b/fly/commands/internal/executehelpers/uploads.go @@ -26,7 +26,7 @@ func Upload(team concourse.Team, path string, includeIgnored bool) (atc.WorkerAr archiveWriter.CloseWithError(tgzfs.Compress(archiveWriter, path, files...)) }() - pb := progress(path+":", os.Stdout) + pb := progress("uploading "+path+":", os.Stdout) pb.Start() defer pb.Finish()