Bug 1877578 - Update sitemap extension and mining chart code to uploa…

…d data to google cloud storage instead of aws s3
mozilla-bteam · Feb 22, 2024 · 1d3ae24 · 1d3ae24
1 parent bd6212c
commit 1d3ae24
Show file tree

Hide file tree

Showing 10 changed files with 81 additions and 83 deletions.
diff --git a/Bugzilla/Config/Reports.pm b/Bugzilla/Config/Reports.pm
@@ -18,40 +18,17 @@ our $sortkey = 1100;
 sub get_param_list {
   my $class      = shift;
   my @param_list = (
-    {
-      name    => 'report_secbugs_active',
-      type    => 'b',
-      default => 1,
-    },
+    {name => 'report_secbugs_active', type => 'b', default => 1,},
     {
       name    => 'report_secbugs_emails',
       type    => 't',
       default => 'bugzilla-admin@mozilla.org'
     },
-    {
-      name    => 'report_secbugs_teams',
-      type    => 'l',
-      default => '{}',
-    },
-    {
-      name    => 's3_mining_enabled',
-      type    => 'b',
-      default => 0,
-    },
-    {
-      name    => 's3_mining_access_key_id',
-      type    => 't',
-      default => '',
-    },
-    {
-      name    => 's3_mining_secret_access_key',
-      type    => 't',
-      default => '',
-    },
-    {
-      name    => 's3_mining_bucket',
-      type    => 't',
-      default => '',
-    }
+    {name => 'report_secbugs_teams',          type => 'l', default => '{}',},
+    {name => 'mining_enabled',                type => 'b', default => 0,},
+    {name => 'mining_google_bucket',          type => 't', default => '',},
+    {name => 'mining_google_host',            type => 't', default => '',},
+    {name => 'mining_google_service_account', type => 't', default => '',},
+
   );
 }
diff --git a/Bugzilla/Report/Net.pm b/Bugzilla/Report/Net.pm
@@ -13,34 +13,32 @@ use Moo;
 use Bugzilla;
 use Bugzilla::Error;
 use Bugzilla::Net::Google;
-use Bugzilla::Net::S3;
 
 has driver     => (is => 'lazy');
 has is_enabled => (is => 'lazy');
 
 sub _build_driver {
   my $self = shift;
 
-  my $driver = Bugzilla::Net::S3->new({
-    client_id  => Bugzilla->params->{s3_mining_access_key_id},
-    secret_key => Bugzilla->params->{s3_mining_secret_access_key},
-    bucket     => Bugzilla->params->{s3_mining_bucket},
-    host       => Bugzilla->params->{aws_host},
-    secure     => 1,
-    retry      => 1,
+  my $driver = Bugzilla::Net::Google->new({
+    bucket          => Bugzilla->params->{mining_google_bucket},
+    host            => Bugzilla->params->{mining_google_host},
+    service_account => Bugzilla->params->{mining_google_service_account},
+    secure          => 1,
+    retry           => 1,
   });
 
   return $driver;
 }
 
 sub _build_is_enabled {
-  return Bugzilla->params->{s3_mining_enabled} ? 1 : 0;
+  return Bugzilla->params->{mining_enabled} ? 1 : 0;
 }
 
 sub set_data {
   my ($self, $product, $data) = @_;
   unless ($self->driver->add_key($product, $data)) {
-    warn "Failed to add data for product $product to S3: "
+    warn "Failed to add data for product $product to net storage: "
       . $self->driver->error_string . "\n";
     ThrowCodeError('net_mining_add_failed',
       {product => $product, reason => $self->driver->error_string});
@@ -52,7 +50,7 @@ sub get_data {
   my ($self, $product) = @_;
   my $data = $self->driver->get_key($product);
   if (!$data) {
-    warn "Failed to retrieve data for product $product from S3: "
+    warn "Failed to retrieve data for product $product from net storage: "
       . $self->driver->error_string . "\n";
     ThrowCodeError('net_mining_get_failed',
       {product => $product, reason => $self->driver->error_string});
@@ -63,7 +61,7 @@ sub get_data {
 sub remove_data {
   my ($self, $product) = @_;
   $self->driver->delete_key($product)
-    or warn "Failed to remove data for product $product from S3: "
+    or warn "Failed to remove data for product $product from net storage: "
     . $self->driver->error_string . "\n";
   return $self;
 }

diff --git a/conf/checksetup_answers.txt b/conf/checksetup_answers.txt
@@ -50,3 +50,13 @@ $answer{'google_storage_host'}            = 'gcs';
 $answer{'google_storage_bucket'}          = 'attachments';
 $answer{'google_storage_service_account'} = 'test';
 $answer{'attachment_google_minsize'}      = '10';
+
+$answer{'mining_enabled'}                 = 1;
+$answer{'mining_google_host'}            = 'gcs';
+$answer{'mining_google_bucket'}          = 'mining';
+$answer{'mining_google_service_account'} = 'test';
+
+$answer{'sitemapindex_enabled'}                = 1;
+$answer{'sitemapindex_google_host'}            = 'gcs';
+$answer{'sitemapindex_google_bucket'}          = 'sitemapindex';
+$answer{'sitemapindex_google_service_account'} = 'test';
diff --git a/docker-compose.test.yml b/docker-compose.test.yml
@@ -64,3 +64,5 @@ services:
     command: "-scheme http"
     volumes:
       - ./docker/gcs/attachments:/data/attachments
+      - ./docker/gcs/sitemapindex:/data/sitemapindex
+      - ./docker/gcs/mining:/data/mining
diff --git a/docker-compose.yml b/docker-compose.yml
@@ -38,9 +38,11 @@ services:
     depends_on:
       - bmo.db
       - memcached
-      - proxy
       - s3
       - gcs
+      - externalapi.test
+    ports:
+      - 8000:8000
 
   bmo.jobqueue:
     build: *bmo_build
@@ -109,6 +111,10 @@ services:
       - MYSQL_PASSWORD=bugs
       - MYSQL_ALLOW_EMPTY_PASSWORD=1
 
+  externalapi.test:
+    image: bmo
+    entrypoint: perl /app/external_test_api.pl daemon -l http://*:8000
+
   memcached:
     image: memcached:latest
 
@@ -121,7 +127,9 @@ services:
     image: fsouza/fake-gcs-server
     command: "-scheme http"
     volumes:
-      - bmo-gcs-data:/data/attachments
+      - bmo-gcs-attachments:/data/attachments
+      - bmo-gcs-sitemapindex:/data/sitemapindex
+      - bmo-gcs-mining:/data/mining
 
   proxy:
     image: haproxytech/haproxy-alpine
@@ -135,4 +143,6 @@ volumes:
   bmo-mysql-db:
   bmo-data-dir:
   bmo-s3-data:
-  bmo-gcs-data:
+  bmo-gcs-attachments:
+  bmo-gcs-sitemapindex:
+  bmo-gcs-mining:
diff --git a/extensions/SiteMapIndex/Extension.pm b/extensions/SiteMapIndex/Extension.pm
@@ -54,9 +54,12 @@ sub before_robots_txt {
   return if !Bugzilla->params->{sitemapindex_enabled};
 
   my $sitemap_url
-      = 'https://s3.us-west-2.amazonaws.com/'
-      . Bugzilla->params->{sitemapindex_s3_bucket}
-    . '/sitemap_index.xml';
+    = 'https://'
+    . Bugzilla->params->{sitemapindex_google_host}
+    . '/storage/v1/b/'
+    . Bugzilla->params->{sitemapindex_google_bucket} . '/o/'
+    . 'sitemap_index.xml';
+
   $args->{vars}{SITEMAP_URL} = $sitemap_url;
 }
 

diff --git a/extensions/SiteMapIndex/lib/Config.pm b/extensions/SiteMapIndex/lib/Config.pm
@@ -19,11 +19,10 @@ sub get_param_list {
   my ($class) = @_;
 
   my @params = (
-    {name => 'sitemapindex_enabled',           type => 'b', default => 0,},
-    {name => 'sitemapindex_s3_bucket',         type => 't', default => '',},
-    {name => 'sitemapindex_aws_region',        type => 't', default => 'us-west-2'},
-    {name => 'sitemapindex_aws_client_id',     type => 't', default => '',},
-    {name => 'sitemapindex_aws_client_secret', type => 't', default => '',},
+    {name => 'sitemapindex_enabled',                type => 'b', default => 0,},
+    {name => 'sitemapindex_google_bucket',          type => 't', default => '',},
+    {name => 'sitemapindex_google_host',            type => 't', default => '',},
+    {name => 'sitemapindex_google_service_account', type => 't', default => '',},
   );
 
   return @params;

diff --git a/extensions/SiteMapIndex/lib/Util.pm b/extensions/SiteMapIndex/lib/Util.pm
@@ -34,7 +34,7 @@ our @EXPORT_OK = qw(
 );
 
 use Bugzilla;
-use Bugzilla::Net::S3;
+use Bugzilla::Net::Google;
 use Bugzilla::Util qw(datetime_from url_quote);
 
 use Bugzilla::Extension::SiteMapIndex::Constants;
@@ -125,9 +125,8 @@ END
 
   my $sitemap_url
     = 'https://'
-    . Bugzilla->params->{sitemapindex_s3_bucket} . '.s3-'
-    . Bugzilla->params->{sitemapindex_aws_region}
-    . '.amazonaws.com';
+    . Bugzilla->params->{sitemapindex_google_host} . '/storage/v1/b/'
+    . Bugzilla->params->{sitemapindex_google_bucket} . '/o/';
 
   foreach my $filename (@$filelist) {
     $index_xml .= "
@@ -142,8 +141,8 @@ END
 </sitemapindex>
 END
 
-  # Upload index file to s3
-  _upload_s3('sitemap_index.xml', $index_xml);
+  # Upload index file to net storage
+  _upload_net('sitemap_index.xml', $index_xml);
 
   return 1;
 }
@@ -183,28 +182,29 @@ END
 </urlset>
 END
 
-  # Write the compressed sitemap data to a variable and then upload to s3
+  # Write the compressed sitemap data to a variable and then upload to net storage
   my $gzipped_data;
   gzip \$sitemap_xml => \$gzipped_data || die "gzip failed: $GzipError\n";
 
   my $filename = "sitemap$filecount.xml.gz";
-  _upload_s3($filename, $gzipped_data);
+  _upload_net($filename, $gzipped_data);
 
   return $filename;
 }
 
-sub _upload_s3 {
+sub _upload_net {
   my ($filename, $data) = @_;
-  my $s3 = Bugzilla::Net::S3->new({
-    client_id  => Bugzilla->params->{sitemapindex_aws_client_id},
-    secret_key => Bugzilla->params->{sitemapindex_aws_client_secret},
-    bucket     => Bugzilla->params->{sitemapindex_s3_bucket},
-    host       => Bugzilla->params->{aws_host},
-    secure     => 1,
-    retry      => 1,
+  my $driver = Bugzilla::Net::Google->new({
+    bucket          => Bugzilla->params->{sitemapindex_google_bucket},
+    host            => Bugzilla->params->{sitemapindex_google_host},
+    service_account => Bugzilla->params->{sitemapindex_google_service_account},
+    secure          => 1,
+    retry           => 1,
   });
-  $s3->delete_key($filename) || die $s3->error_string;
-  $s3->add_key($filename, $data) || die $s3->error_string;
+  if ($driver->head_key($filename)) {
+    $driver->delete_key($filename) || die $driver->error_string;
+  }
+  $driver->add_key($filename, $data) || die $driver->error_string;
 }
 
 1;
diff --git a/extensions/SiteMapIndex/template/en/default/admin/params/sitemapindex.html.tmpl b/extensions/SiteMapIndex/template/en/default/admin/params/sitemapindex.html.tmpl
@@ -14,9 +14,8 @@
 [%
    param_descs = {
       sitemapindex_enabled => "Enable SiteMapIndex",
-      sitemapindex_s3_bucket => "S3 bucket name to access sitemapindex files",
-      sitemapindex_aws_region => "SiteMapIndex AWS Region (default: us-west-2)",
-      sitemapindex_aws_client_id => "SiteMapIndex AWS Client ID",
-      sitemapindex_aws_client_secret => "SiteMapIndex AWS Client Secret"
+      sitemapindex_google_bucket => "Google Cloud Storage bucket name for sitemapindex files",
+      sitemapindex_google_host => "Google Cloud Storage host for sitemapindex files",
+      sitemapindex_google_service_account => "Google Cloud Storage service account for sitemapindex files",
    }
 %]
diff --git a/template/en/default/admin/params/reports.html.tmpl b/template/en/default/admin/params/reports.html.tmpl
@@ -16,13 +16,13 @@
       "Comma delimited list of the email addresses that the security $terms.bugs report will be sent to.",
     report_secbugs_teams =>
       "A one team per line list of the teams the security $terms.bugs report will report on.",
-    s3_mining_enabled =>
-      "Use AWS S3 bucket to store mining data for graph reports. Otherwise store on local filesystem.",
-    s3_mining_access_key_id =>
-      "Access key ID for AWS S3 account used to store mining data for graph reports.",
-    s3_mining_secret_access_key =>
-      "Secret access key for AWS S3 account used to store mining data for graph reports.",
-    s3_mining_bucket =>
-      "Name of AWS S3 bucket to store mining data for graph reports."
+    mining_enabled =>
+      "Use Google Cloud Storage bucket to store mining data for graph reports. Otherwise store on local filesystem.",
+    mining_google_bucket =>
+      "Google Cloud Storage bucket used to store mining data for graph reports.",
+    mining_google_host =>
+      "Google Cloud Storage host used to store mining data for graph reports.",
+    mining_google_service_account =>
+      "Google Cloud Storage service account used to store mining data for graph reports."
   }
 %]