background: implemented garbage collector

Implemented a new background task: garbage collector. This task will
clean up old tags. It is disabled by default and it can be further
configured under the `delete.garbage_collector` option.

Fixes #1479

Signed-off-by: Miquel Sabaté Solà <msabate@suse.com>

bin/background.rb

@@ -11,14 +11,16 @@
 # The DB is up, now let's define the different background jobs as classes.
 #
 
+require "portus/background/garbage_collector"
 require "portus/background/registry"
 require "portus/background/security_scanning"
 require "portus/background/sync"
 
 they = [
   ::Portus::Background::Registry.new,
   ::Portus::Background::SecurityScanning.new,
-  ::Portus::Background::Sync.new
+  ::Portus::Background::Sync.new,
+  ::Portus::Background::GarbageCollector.new
 ].select(&:enabled?)
 
 values = they.map { |v| "'#{v}'" }.join(", ")

config/config.yml

@@ -42,6 +42,26 @@ delete:
   # Allow contributors to delete images and tags
   contributors: false
 
+  # The garbage collector will run when enabled in the background process, and
+  # it will only delete tags matching the given conditions.
+  garbage_collector:
+    enabled: false
+
+    # Remove images older than a specific value. This value is interpreted as
+    # the number of days.
+    older_than: 30
+
+    # Provide a string containing a regular expression. If you provide a
+    # valid regular expression, garbage collector will only be applied into tags
+    # matching a given name.
+    #
+    # Valid values might be:
+    #   - "jenkins": if you anticipate that you will always have a tag with a
+    #     specific name, you can simply use that.
+    #   - "build-\\d+": your tag follows a format like "build-1234" (note that
+    #     we need to specify "\\d" and not just "\d").
+    tag: ""
+
 # LDAP support. If enabled, then only users of the specified LDAP server will
 # be able to use Portus. Take a look at the documentation of LDAP support in our
 # online docs: http://port.us.org/features/2_LDAP-support.html.

lib/portus/background/garbage_collector.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Portus
+  module Background
+    # GarbageCollector cleans up the registry from old tags. The behavior of
+    # this task depends on the `delete.garbage_collector` configuration option.
+    class GarbageCollector
+      def initialize
+        @tags = nil
+      end
+
+      def sleep_value
+        60
+      end
+
+      def work?
+        return false unless enabled?
+        @tags = tags_to_be_collected
+        @tags.any?
+      end
+
+      def enabled?
+        APP_CONFIG.enabled?("delete.garbage_collector")
+      end
+
+      def disable?
+        false
+      end
+
+      def execute!
+        @tags ||= tags_to_be_collected
+        service = ::Tags::DestroyService.new(User.find_by(username: "portus"))
+
+        @tags.each do |tag|
+          next if service.execute(tag)
+          Rails.logger.tagged(:garbage_collector) { Rails.logger.warn(service.error.to_s) }
+        end
+      end
+
+      def to_s
+        "Garbage collector"
+      end
+
+      protected
+
+      def tags_to_be_collected
+        tags = Tag.where(marked: false).where("updated_at < ?", older_than)
+        return tags if APP_CONFIG["delete"]["garbage_collector"]["tag"].blank?
+
+        rx = tag_regexp
+        tags.select { |t| t.name.match(rx) }
+      end
+
+      def older_than
+        APP_CONFIG["delete"]["garbage_collector"]["older_than"].to_i.days.ago
+      end
+
+      def tag_regexp
+        Regexp.new(APP_CONFIG["delete"]["garbage_collector"]["tag"])
+      end
+    end
+  end
+end

spec/lib/portus/background/garbage_collector_spec.rb

@@ -0,0 +1,128 @@
+# frozen_string_literal: true
+
+describe ::Portus::Background::GarbageCollector do
+  let(:old_tag)    { (APP_CONFIG["delete"]["garbage_collector"]["older_than"].to_i + 10).days.ago }
+  let(:recent_tag) { (APP_CONFIG["delete"]["garbage_collector"]["older_than"].to_i - 10).days.ago }
+
+  before do
+    APP_CONFIG["delete"]["garbage_collector"]["enabled"] = true
+  end
+
+  it "returns the proper value for sleep_value" do
+    expect(subject.sleep_value).to eq 60
+  end
+
+  it "should never be disabled after being enabled" do
+    expect(subject.disable?).to be_falsey
+  end
+
+  it "returns the proper value for to_s" do
+    expect(subject.to_s).to eq "Garbage collector"
+  end
+
+  describe "#enabled?" do
+    it "is marked as enabled" do
+      expect(subject.enabled?).to be_truthy
+    end
+
+    it "is marked as disabled" do
+      APP_CONFIG["delete"]["garbage_collector"]["enabled"] = false
+      expect(subject.enabled?).to be_falsey
+    end
+  end
+
+  describe "#work?" do
+    it "returns false if the feature is disabled entirely" do
+      APP_CONFIG["delete"]["garbage_collector"]["enabled"] = false
+      expect(subject.work?).to be_falsey
+    end
+
+    it "returns false if there are no tags matching the given expectations" do
+      allow_any_instance_of(::Portus::Background::GarbageCollector).to(
+        receive(:tags_to_be_collected).and_return([])
+      )
+      expect(subject.work?).to be_falsey
+    end
+
+    it "returns true if there are tags available to be updated" do
+      allow_any_instance_of(::Portus::Background::GarbageCollector).to(
+        receive(:tags_to_be_collected).and_return(["tag"])
+      )
+      expect(subject.work?).to be_truthy
+    end
+  end
+
+  describe "#tags_to_be_collected" do
+    let!(:registry)   { create(:registry, hostname: "registry.test.lan") }
+    let!(:user)       { create(:admin) }
+    let!(:repository) { create(:repository, namespace: registry.global_namespace, name: "repo") }
+
+    it "returns an empty collection if there are no tags" do
+      tags = subject.send(:tags_to_be_collected)
+      expect(tags).to be_empty
+    end
+
+    it "exists a tag but it's considered recent" do
+      create(:tag, name: "tag", repository: repository, updated_at: recent_tag)
+      tags = subject.send(:tags_to_be_collected)
+      expect(tags).to be_empty
+    end
+
+    it "ignores tags which are marked" do
+      create(:tag, name: "tag", repository: repository, updated_at: old_tag, marked: true)
+      tags = subject.send(:tags_to_be_collected)
+      expect(tags).to be_empty
+    end
+
+    it "exists a tag which is older than expected" do
+      create(:tag, name: "tag", repository: repository, updated_at: old_tag)
+      tags = subject.send(:tags_to_be_collected)
+      expect(tags.size).to eq 1
+    end
+
+    it "exists a tag which is older than expected but the name does not match" do
+      APP_CONFIG["delete"]["garbage_collector"]["tag"] = "build-\d+"
+
+      create(:tag, name: "tag", repository: repository, updated_at: old_tag)
+      tags = subject.send(:tags_to_be_collected)
+      expect(tags.size).to eq 0
+    end
+
+    it "exists a tag which is older and with a proper name" do
+      APP_CONFIG["delete"]["garbage_collector"]["tag"] = "^build-\\d+$"
+
+      create(:tag, name: "build-1234", repository: repository, updated_at: old_tag)
+      tags = subject.send(:tags_to_be_collected)
+      expect(tags.size).to eq 1
+    end
+  end
+
+  describe "#execute!" do
+    let!(:registry)   { create(:registry, hostname: "registry.test.lan") }
+    let!(:portus)     { create(:admin, username: "portus") }
+    let!(:repository) { create(:repository, namespace: registry.global_namespace, name: "repo") }
+
+    it "removes tags" do
+      allow_any_instance_of(Tag).to(receive(:fetch_digest).and_return("1234"))
+      allow_any_instance_of(::Portus::RegistryClient).to(receive(:delete).and_return(true))
+
+      create(:tag, name: "tag", digest: "1234", repository: repository, updated_at: old_tag)
+      expect do
+        subject.execute!
+      end.to(change { Tag.all.count }.from(1).to(0))
+    end
+
+    it "skips tags which could not be removed for whatever reason" do
+      allow_any_instance_of(Tag).to(
+        receive(:fetch_digest) { |tag| tag.digest == "wrong" ? "" : tag.digest }
+      )
+      allow_any_instance_of(::Portus::RegistryClient).to(receive(:delete).and_return(true))
+
+      expect(Rails.logger).to(receive(:warn).with("Could not remove <strong>tag2</strong> tag"))
+
+      create(:tag, name: "tag1", digest: "1234", repository: repository, updated_at: old_tag)
+      create(:tag, name: "tag2", digest: "wrong", repository: repository, updated_at: old_tag)
+      expect { subject.execute! }.to(change { Tag.all.count }.from(2).to(1))
+    end
+  end
+end

spec/spec_helper.rb

@@ -96,8 +96,13 @@
     }
 
     APP_CONFIG["delete"] = {
-      "enabled"      => false,
-      "contributors" => false
+      "enabled"           => false,
+      "contributors"      => false,
+      "garbage_collector" => {
+        "enabled"    => false,
+        "older_than" => 30,
+        "tag"        => ""
+      }
     }
 
     APP_CONFIG["pagination"] = {