Merge pull request #16377 from CartoDB/feature/sc-189175/sql-api-oom-…

…errors-when-exporting-data [sc189175] Add export timeout to users
CartoDB · Dec 29, 2021 · b8215b1 · b8215b1
2 parents 6704b14 + 2fd1b21
commit b8215b1
Show file tree

Hide file tree

Showing 7 changed files with 22 additions and 4 deletions.
diff --git a/NEWS.md b/NEWS.md
@@ -28,6 +28,7 @@ Development
 - Disable email validation in DO Premium Subscriptions [#16309](https://github.com/CartoDB/cartodb/pull/16309)
 - Invalidate sessions on 'session_salt' issue [#16376](https://github.com/CartoDB/cartodb/pull/16376)
 - Hide sharing tab from viewer in on-premises [#16299](https://github.com/CartoDB/cartodb/pull/16299)
+- Add timeout for SQL API exports [#16377](https://github.com/CartoDB/cartodb/pull/16377)
 - Remove all references to Spatial Data Catalog and Kepler GL maps in on-premises [#16293](https://github.com/CartoDB/cartodb/pull/16293)
 - Increase hard-limit of MAX_TABLES_PER_IMPORT [#16374](https://github.com/CartoDB/cartodb/pull/16374)
 - Guard code for vizjson users [#16267](https://github.com/CartoDB/cartodb/pull/16267)

diff --git a/app/models/carto/helpers/user_commons.rb b/app/models/carto/helpers/user_commons.rb
@@ -42,7 +42,7 @@ module Carto::UserCommons
                           salesforce_datasource_enabled builder_enabled geocoder_provider isolines_provider
                           routing_provider engine_enabled mapzen_routing_quota
                           mapzen_routing_block_price soft_mapzen_routing_limit no_map_logo org_admin
-                          user_render_timeout database_render_timeout frontend_version asset_host
+                          user_render_timeout database_render_timeout export_timeout frontend_version asset_host
                           state rate_limit_id public_map_quota regular_api_key_quota
                           maintenance_mode private_map_quota public_dataset_quota]
 

diff --git a/app/models/concerns/cartodb_central_synchronizable.rb b/app/models/concerns/cartodb_central_synchronizable.rb
@@ -115,7 +115,7 @@ def allowed_attributes_from_central(action)
          salesforce_datasource_enabled viewer geocoder_provider
          isolines_provider routing_provider engine_enabled builder_enabled
          mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit no_map_logo
-         user_render_timeout database_render_timeout state industry company phone job_role
+         user_render_timeout database_render_timeout export_timeout state industry company phone job_role
          password_reset_token password_reset_sent_at maintenance_mode company_employees use_case private_map_quota
          session_salt public_dataset_quota dashboard_viewed_at email_verification_token email_verification_sent_at)
     end

diff --git a/app/models/user.rb b/app/models/user.rb
@@ -787,7 +787,8 @@ def save_metadata
                           'db',                        user_timeout,
                           'db_public',                 database_timeout,
                           'render',                    user_render_timeout,
-                          'render_public',             database_render_timeout
+                          'render_public',             database_render_timeout,
+                          'export',                    export_timeout
     save_rate_limits
   end
 

diff --git a/app/services/carto/user_metadata_export_service.rb b/app/services/carto/user_metadata_export_service.rb
@@ -50,7 +50,7 @@ module UserMetadataExportServiceConfiguration
       mobile_gis_extension mobile_max_open_users mobile_max_private_users
       viewer salesforce_datasource_enabled builder_enabled geocoder_provider isolines_provider routing_provider
       github_user_id engine_enabled mapzen_routing_quota mapzen_routing_block_price soft_mapzen_routing_limit
-      no_map_logo org_admin last_name user_render_timeout database_render_timeout frontend_version
+      no_map_logo org_admin last_name user_render_timeout database_render_timeout export_timeout frontend_version
       asset_host state company phone industry job_role password_reset_token password_reset_sent_at maintenance_mode
       company_employees use_case private_map_quota session_salt public_dataset_quota
       email_verification_token email_verification_sent_at

diff --git a/db/migrate/20211122155410_add_export_timeout_to_users.rb b/db/migrate/20211122155410_add_export_timeout_to_users.rb
@@ -0,0 +1,15 @@
+require 'carto/db/migration_helper'
+
+# rubocop:disable Style/MixinUsage
+include Carto::Db::MigrationHelper
+# rubocop:enable Style/MixinUsage
+
+migration(
+  proc do
+    # The 0 value means: "apply default export timeout" (defined by the SQL API)
+    add_column :users, :export_timeout, :integer, default: 0, null: false
+  end,
+  proc do
+    drop_column :users, :export_timeout
+  end
+)
diff --git a/spec/factories/user_metadata_export.rb b/spec/factories/user_metadata_export.rb
@@ -31,6 +31,7 @@ def self.full_export(params = {})
         user_timeout: 300_000,
         database_render_timeout: 0,
         user_render_timeout: 0,
+        export_timeout: 0,
         upgraded_at: nil,
         map_view_block_price: nil,
         geocoding_quota: 0,