Bug 1798065 - GCP: Move bloomfilter whitelist files and rate_limit fi…

…les from data/ to the DB
mozilla-bteam · Nov 9, 2022 · 1d19fda · 1d19fda
1 parent addae7c
commit 1d19fda
Show file tree

Hide file tree

Showing 5 changed files with 62 additions and 21 deletions.
diff --git a/Bugzilla/Bloomfilter.pm b/Bugzilla/Bloomfilter.pm
@@ -12,9 +12,9 @@ use strict;
 use warnings;
 
 use Bugzilla::Constants;
+
 use Algorithm::BloomFilter;
 use Mojo::File qw(path);
-use File::Spec::Functions qw(catfile);
 
 sub _new_bloom_filter {
   my ($n) = @_;
@@ -24,34 +24,47 @@ sub _new_bloom_filter {
   return Algorithm::BloomFilter->new($m, $k);
 }
 
-sub _file {
-  my ($name, $type) = @_;
-
-  my $datadir = bz_locations->{datadir};
-
-  return path(catfile($datadir, "$name.$type"));
-}
-
 sub populate {
-  my ($class, $name) = @_;
+  my ($class, $name, $file) = @_;
+  my $dbh       = Bugzilla->dbh;
   my $memcached = Bugzilla->memcached;
-  my @items     = split(/\n/, _file($name, 'list')->slurp);
-  my $filter    = _new_bloom_filter(@items + 0);
 
-  $filter->add($_) foreach @items;
-  _file($name, 'bloom')->spurt($filter->serialize);
+  # Load values from file, one per row
+  my @values = split /\n/, path($file)->slurp;
+
+  # Put items in database
+  foreach my $value (@values) {
+    my $exists
+      = $dbh->selectrow_array(
+      'SELECT value FROM bloomfilter_values WHERE name = ? AND value = ?',
+      undef, $name, $value);
+    if (!$exists) {
+      $dbh->do('INSERT INTO bloomfilter_values (name, value) VALUES (?, ?)',
+        undef, $name, $value);
+    }
+  }
+
   $memcached->clear_bloomfilter({name => $name});
 }
 
 sub lookup {
   my ($class, $name) = @_;
   my $memcached   = Bugzilla->memcached;
-  my $file        = _file($name, 'bloom');
   my $filter_data = $memcached->get_bloomfilter({name => $name});
 
-  if (!$filter_data && -f $file) {
-    $filter_data = $file->slurp;
-    $memcached->set_bloomfilter({name => $name, filter => $filter_data});
+  if (!$filter_data) {
+
+    # Read filter values from database
+    my $values
+      = Bugzilla->dbh->selectcol_arrayref(
+      "SELECT value FROM bloomfilter_values WHERE name = ?",
+      undef, $name);
+    if (@$values) {
+      my $filter = _new_bloom_filter(@$values + 0);
+      $filter->add($_) foreach @$values;
+      $filter_data = $filter->serialize;
+      $memcached->set_bloomfilter({name => $name, filter => $filter_data});
+    }
   }
 
   return Algorithm::BloomFilter->deserialize($filter_data) if $filter_data;

diff --git a/Bugzilla/DB/Schema.pm b/Bugzilla/DB/Schema.pm
@@ -1956,8 +1956,18 @@ use constant ABSTRACT_SCHEMA => {
       name  => {TYPE => 'VARCHAR(64)',   NOTNULL => 1},
       value => {TYPE => 'VARCHAR(4000)', NOTNULL => 1},
     ],
-  }
+  },
 
+  # Bloomfilter Values Table
+  # ------------------------
+
+  bloomfilter_values => {
+    FIELDS => [
+      id    => {TYPE => 'MEDIUMSERIAL',  NOTNULL => 1, PRIMARYKEY => 1},
+      name  => {TYPE => 'VARCHAR(64)',   NOTNULL => 1},
+      value => {TYPE => 'VARCHAR(255)', NOTNULL => 1},
+    ],
+  }
 };
 
 # Foreign Keys are added in Bugzilla::DB::bz_add_field_tables

diff --git a/extensions/EditTable/Extension.pm b/extensions/EditTable/Extension.pm
@@ -48,7 +48,17 @@ our $VERSION = '1';
 # },
 
 sub EDITABLE_TABLES {
-  my $tables = {};
+  my $tables = {
+    bloomfilter_values => {
+      id_field => 'id',
+      order_by => 'name',
+      blurb => 'List of Bloomfilter values used for features such as rate limiting.',
+      group => 'admin',
+      post_commit => sub {
+        Bugzilla->memcached->clear_bloomfilter();
+      }
+    }
+  };
   Bugzilla::Hook::process("editable_tables", {tables => $tables});
   return $tables;
 }
@@ -122,6 +132,9 @@ sub page_before_template {
         }
       }
 
+      # Run any post commit action if defined
+      $table->{post_commit}->() if $table->{post_commit};
+
       $dbh->bz_commit_transaction;
       $vars->{updated} = 1;
       $edits = [];

diff --git a/scripts/bloomfilter-populate.pl b/scripts/bloomfilter-populate.pl
@@ -15,5 +15,6 @@
 Bugzilla->usage_mode(USAGE_MODE_CMDLINE);
 
 my $name = shift @ARGV or die "usage: $0 \$name\n";
-Bugzilla::Bloomfilter->populate($name);
+my $file = shift @ARGV or die "usage: $0 \$file\n";
 
+Bugzilla::Bloomfilter->populate($name, $file);
diff --git a/template/en/default/admin/admin.html.tmpl b/template/en/default/admin/admin.html.tmpl
@@ -99,6 +99,10 @@
         <dt id="new_release" class="[% class %]"><a href="[% basepath FILTER none %]admin/new_release">New Firefox Release</a></dt>
         <dd class="[% class %]">Add new version and milestone for a new release of Firefox</dd>
 
+        [% class = user.in_group('admin') ? "" : "forbidden" %]
+        <dt id="bloomfilter" class="[% class %]"><a href="[% basepath FILTER none %]page.cgi?id=edit_table.html&table=bloomfilter_values">Bloomfilter Values</a></dt>
+        <dd class="[% class %]">Add or remove values from the Bloomfilter table.</dd>
+
         [% Hook.process('end_links_left') %]
       </dl>
     </td>