From 8dca4254dd2e6f172298e1c22e7328409e9d9846 Mon Sep 17 00:00:00 2001 From: Tyler Roscoe Date: Fri, 12 Apr 2019 18:20:23 -0600 Subject: [PATCH] Delete sync-images. It moved to gpii-version-updater. --- shared/rakefiles/sync-images/Gemfile | 5 - shared/rakefiles/sync-images/Gemfile.lock | 34 ---- shared/rakefiles/sync-images/Rakefile | 6 - .../sync-images/spec/sync_images_spec.rb | 177 ------------------ shared/rakefiles/sync-images/sync_images.rb | 103 ---------- 5 files changed, 325 deletions(-) delete mode 100644 shared/rakefiles/sync-images/Gemfile delete mode 100644 shared/rakefiles/sync-images/Gemfile.lock delete mode 100644 shared/rakefiles/sync-images/Rakefile delete mode 100644 shared/rakefiles/sync-images/spec/sync_images_spec.rb delete mode 100755 shared/rakefiles/sync-images/sync_images.rb diff --git a/shared/rakefiles/sync-images/Gemfile b/shared/rakefiles/sync-images/Gemfile deleted file mode 100644 index b8267bfa..00000000 --- a/shared/rakefiles/sync-images/Gemfile +++ /dev/null @@ -1,5 +0,0 @@ -source 'https://rubygems.org' - -gem "docker-api" -gem "rake" -gem "rspec" diff --git a/shared/rakefiles/sync-images/Gemfile.lock b/shared/rakefiles/sync-images/Gemfile.lock deleted file mode 100644 index 1e6996e5..00000000 --- a/shared/rakefiles/sync-images/Gemfile.lock +++ /dev/null @@ -1,34 +0,0 @@ -GEM - remote: https://rubygems.org/ - specs: - diff-lcs (1.3) - docker-api (1.34.2) - excon (>= 0.47.0) - multi_json - excon (0.62.0) - multi_json (1.13.1) - rake (12.3.2) - rspec (3.8.0) - rspec-core (~> 3.8.0) - rspec-expectations (~> 3.8.0) - rspec-mocks (~> 3.8.0) - rspec-core (3.8.0) - rspec-support (~> 3.8.0) - rspec-expectations (3.8.2) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-mocks (3.8.0) - diff-lcs (>= 1.2.0, < 2.0) - rspec-support (~> 3.8.0) - rspec-support (3.8.0) - -PLATFORMS - ruby - -DEPENDENCIES - docker-api - rake - rspec - -BUNDLED WITH - 2.0.1 diff --git a/shared/rakefiles/sync-images/Rakefile b/shared/rakefiles/sync-images/Rakefile deleted file mode 100644 index 71b63306..00000000 --- a/shared/rakefiles/sync-images/Rakefile +++ /dev/null @@ -1,6 +0,0 @@ -desc "Run tests" -task :default do - sh "bundle exec rspec" -end - -# vim: set et ts=2 sw=2: diff --git a/shared/rakefiles/sync-images/spec/sync_images_spec.rb b/shared/rakefiles/sync-images/spec/sync_images_spec.rb deleted file mode 100644 index 88b7e621..00000000 --- a/shared/rakefiles/sync-images/spec/sync_images_spec.rb +++ /dev/null @@ -1,177 +0,0 @@ -require "./sync_images.rb" - -describe SyncImages do - - # It is not necessary or desirable to test File.read or YAML.load, but this - # validates some plumbing. - it "load_config returns parsed yaml" do - fake_yaml = "foo: bar" - allow(File).to receive(:read).with(SyncImages::CONFIG_FILE).and_return(fake_yaml) - - expected = {"foo" => "bar"} - actual = SyncImages.load_config() - expect(actual).to eq(expected) - end - - it "process_config calls process_image on each image" do - fake_config = { - "dataloader" => { - "upstream_image" => "gpii/universal:latest", - }, - "flowmanager" => { - "upstream_image" => "gpii/universal:latest", - }, - } - - allow(SyncImages).to receive(:process_image) - allow(SyncImages).to receive(:write_new_config) - - SyncImages.process_config(fake_config) - - expect(SyncImages).to have_received(:process_image).with("dataloader", "gpii/universal:latest") - expect(SyncImages).to have_received(:process_image).with("flowmanager", "gpii/universal:latest") - end - - it "process_config writes new config" do - # Keys are out of lexical order to test that they get sorted at the end - # (and thus get the shas in the right order). - fake_config = { - "flowmanager" => { - "upstream_image" => "gpii/universal:latest", - }, - "dataloader" => { - "upstream_image" => "gpii/universal:latest", - }, - } - fake_new_image_name = "#{SyncImages::REGISTRY_URL}/gpii/universal" - fake_sha_1 = "sha256:c0ffee" - fake_sha_2 = "sha256:50da" - fake_tag = "latest" - expected_config = { - "dataloader" => { - "upstream_image" => "gpii/universal:latest", - "image" => fake_new_image_name, - "sha" => fake_sha_1, - "tag" => fake_tag, - }, - "flowmanager" => { - "upstream_image" => "gpii/universal:latest", - "image" => fake_new_image_name, - "sha" => fake_sha_2, - "tag" => fake_tag, - }, - } - - allow(SyncImages).to receive(:process_image).and_return( - [fake_new_image_name, fake_sha_1, fake_tag], - [fake_new_image_name, fake_sha_2, fake_tag], - ) - allow(SyncImages).to receive(:write_new_config) - - SyncImages.process_config(fake_config) - - expect(SyncImages).to have_received(:write_new_config).with(expected_config) - end - - it "process_image calls helpers on image" do - fake_component = "fake_component" - fake_image = "fake Docker::Image object" - fake_image_name = "fake_org/fake_img:fake_tag" - fake_new_image_name = "#{SyncImages::REGISTRY_URL}/#{fake_image_name}" - fake_new_image_name_without_tag = "#{SyncImages::REGISTRY_URL}/fake_org/fake_img" - fake_sha = "sha256:c0ffee" - fake_tag = "fake_tag" - - allow(SyncImages).to receive(:pull_image).and_return(fake_image) - allow(SyncImages).to receive(:retag_image).and_return(fake_new_image_name) - allow(SyncImages).to receive(:get_sha_from_image).and_return(fake_sha) - allow(SyncImages).to receive(:push_image) - - actual = SyncImages.process_image(fake_component, fake_image_name) - - expect(SyncImages).to have_received(:pull_image).with(fake_image_name) - expect(SyncImages).to have_received(:retag_image).with(fake_image, fake_image_name) - expect(SyncImages).to have_received(:get_sha_from_image).with(fake_image) - expect(SyncImages).to have_received(:push_image).with(fake_image, fake_new_image_name) - expect(actual).to eq([fake_new_image_name_without_tag, fake_sha, fake_tag]) - end - - it "pull_image pulls image" do - fake_image_name = "fake_org/fake_img:fake_tag" - fake_image = "fake docker image object" - allow(Docker::Image).to receive(:create).and_return(fake_image) - actual = SyncImages.pull_image(fake_image_name) - expect(actual).to eq(fake_image) - expect(Docker::Image).to have_received(:create).with({"fromImage" => fake_image_name}, creds: {}) - end - - it "get_sha_from_image gets sha" do - fake_image = double(Docker::Image) - fake_sha = "sha256:c0ffee" - allow(fake_image).to receive(:info).and_return({ - "RepoDigests" => [ - "fake_org/fake_img@#{fake_sha}", - "another_org/another_img@sha256:50da", - ] - }) - actual = SyncImages.get_sha_from_image(fake_image) - expect(actual).to eq(fake_sha) - end - - it "get_sha_from_image explodes when RepoDigests is empty" do - fake_image = double(Docker::Image) - allow(fake_image).to receive(:info).and_return({ - "RepoDigests" => [], - }) - expect { SyncImages.get_sha_from_image(fake_image) }.to raise_error(ArgumentError, /Could not find sha!/) - end - - it "retag_image retags iamge" do - fake_image = double(Docker::Image) - fake_image_name = "fake_org/fake_img:fake_tag" - fake_new_image_name = "#{SyncImages::REGISTRY_URL}/#{fake_image_name}" - - allow(fake_image).to receive(:tag) - actual = SyncImages.retag_image(fake_image, fake_image_name) - expect(fake_image).to have_received(:tag).with({"repo" => fake_new_image_name}) - expect(actual).to eq(fake_new_image_name) - end - - it "push_image pushes image" do - fake_image = double(Docker::Image) - fake_new_image_name = "fake_registry/fake_org/fake_img:fake_tag" - allow(fake_image).to receive(:push) - SyncImages.push_image(fake_image, fake_new_image_name) - expect(fake_image).to have_received(:push).with(nil, "repo_tag": fake_new_image_name) - end - - it "push_image explodes if push output contains error" do - fake_image = double(Docker::Image) - fake_new_image_name = "fake_registry/fake_org/fake_img:fake_tag" - # Based on real output when I accidentally mismatched credentials and - # registry. - fake_output = [ - '{"status":"The push refers to repository [docker.io/library/fake_img]"}', - '{"status":"Preparing","progressDetail":{},"id":"123456789abc"}', - '{"errorDetail":{"message":"unauthorized: incorrect username or password"},"error":"unauthorized: incorrect username or password"}', - ] - allow(fake_image).to receive(:push).and_yield(fake_output[0]).and_yield(fake_output[1]).and_yield(fake_output[2]) - expect { SyncImages.push_image(fake_image, fake_new_image_name) }.to raise_error(ArgumentError, /Found error message in output/) - end - - # It is not necessary or desirable to test File.write or YAML.dump, but this - # validates some plumbing. - it "write_new_config dumps and writes yaml" do - fake_config = { - "foo" => "bar", - } - buffer = StringIO.new() - allow(File).to receive(:open).and_yield(buffer) - SyncImages.write_new_config(fake_config) - expect(buffer.string).to eq("---\nfoo: bar\n") - end - -end - - -# vim: set et ts=2 sw=2: diff --git a/shared/rakefiles/sync-images/sync_images.rb b/shared/rakefiles/sync-images/sync_images.rb deleted file mode 100755 index aa73ddb7..00000000 --- a/shared/rakefiles/sync-images/sync_images.rb +++ /dev/null @@ -1,103 +0,0 @@ -#!/usr/bin/env ruby - - -require "docker-api" -require "yaml" - -class SyncImages - - CONFIG_FILE = "../../versions.yml" - CREDS_FILE = "./creds.json" - REGISTRY_URL = "gcr.io/gpii2test-common-stg" - - def self.load_config() - return YAML.load(File.read(SyncImages::CONFIG_FILE)) - end - - def self.login() - puts "Logging in with credentials from #{SyncImages::CREDS_FILE}..." - creds = File.read(SyncImages::CREDS_FILE) - Docker.authenticate!( - "username" => "_json_key", - "password" => creds, - "serveraddress" => "https://gcr.io", - ) - end - - def self.process_config(config) - config.keys.sort.each do |component| - image_name = config[component]["upstream_image"] - (new_image_name, sha, tag) = self.process_image(component, image_name) - config[component]["image"] = new_image_name - config[component]["sha"] = sha - config[component]["tag"] = tag - end - self.write_new_config(config) - end - - def self.process_image(component, image_name) - image = self.pull_image(image_name) - new_image_name = self.retag_image(image, image_name) - new_image_name_without_tag, tag = Docker::Util.parse_repo_tag(new_image_name) - sha = self.get_sha_from_image(image) - self.push_image(image, new_image_name) - - return [new_image_name_without_tag, sha, tag] - end - - def self.pull_image(image_name) - puts "Pulling #{image_name}..." - image = Docker::Image.create({"fromImage" => image_name}, creds: {}) - return image - end - - def self.get_sha_from_image(image) - sha = nil - begin - image_with_sha = image.info["RepoDigests"][0] - sha = image_with_sha.split('@')[1] - rescue - raise ArgumentError, "Could not find sha! image.info was #{image.info}" - end - puts "Got image with sha #{sha}..." - return sha - end - - def self.retag_image(image, image_name) - new_image_name = "#{SyncImages::REGISTRY_URL}/#{image_name}" - puts "Retagging #{image_name} as #{new_image_name}..." - image.tag("repo" => new_image_name) - return new_image_name - end - - def self.push_image(image, new_image_name) - puts "Pushing #{new_image_name}..." - # Docker.push collects output from the API call via 'response_block()', a - # kind of callback function. Docker.push ignores errors and discards - # output, though the output is available to a block passed to Docker.push. - # Hence, we use a block to look for errors and explode if we find one. - image.push(nil, repo_tag: new_image_name) do |output_line| - puts "...output from push: #{output_line}" - if output_line.include? '"error":' - raise ArgumentError, "Found error message in output (see above)!" - end - end - end - - def self.write_new_config(config) - File.open(CONFIG_FILE, "w") do |f| - f.write(YAML.dump(config)) - end - end - -end - - -def main() - config = SyncImages.load_config() - SyncImages.login() - SyncImages.process_config(config) -end - - -# vim: et ts=2 sw=2: