Merge 884f578 into c463559

wikimedia · Oct 18, 2017 · 0d5eb77 · 0d5eb77
2 parents c463559 + 884f578
commit 0d5eb77
Show file tree

Hide file tree

Showing 39 changed files with 4,049 additions and 1,073 deletions.
diff --git a/config.example.wikimedia.yaml b/config.example.wikimedia.yaml
@@ -24,7 +24,7 @@ default_project: &default_project
           password: cassandra
           defaultConsistency: one # or 'localQuorum' for production
           storage_groups:
-            - name: test.group.local
+            - name: default.group.local
               domains: /./
         parsoid:
           host: http://parsoid-beta.wmflabs.org
@@ -57,6 +57,9 @@ default_project: &default_project
           secret: secret
         transform:
           cx_host: https://cxserver-beta.wmflabs.org
+        summary:
+          new_storage_enabled:
+            - en.wikipedia.org
         skip_updates: false
 
 # A separate project for en.wikipedia because it is more feature-rich

diff --git a/config.test.yaml b/config.test.yaml
@@ -32,6 +32,9 @@ default_project: &default_project
         parsoid:
           host: https://parsoid-beta.wmflabs.org
           grace_ttl: 2
+          backends:
+            both:
+              - '/.*/'
         action:
           apiUriTemplate: "{{'https://{domain}/w/api.php'}}"
           baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}"
@@ -58,6 +61,9 @@ default_project: &default_project
           secret: secret
         transform:
           cx_host: https://cxserver-beta.wmflabs.org
+        summary:
+          new_storage_enabled:
+            - en.wikipedia.org
         skip_updates: false
 
 # A separate project for en.wikipedia because it is more feature-rich

diff --git a/lib/base_feed.js b/lib/base_feed.js
@@ -124,7 +124,7 @@ class BaseFeed {
     }
 
     _getHistoricBucketName() {
-        return `${this.options.name}.historic`;
+        return this.options.name;
     }
 
     getModuleDeclaration() {

diff --git a/lib/mwUtil.js b/lib/mwUtil.js
@@ -149,7 +149,7 @@ mwUtil.getLimit = (hyper, req) => {
  * if access is restricted, no-op otherwise.
  *
  * Item might be either a revision or a restriction.
- * @param {Object} restrictions the restrictions object
+ * @param {Object} restrictions the restrictions object or a revision object
  * @param {number} [targetRev] the item revision.
  *                             Used in case item is a restriction to
  *                             indicate the content revision num.
@@ -409,7 +409,7 @@ mwUtil.hydrateResponse = (response, fetch) => {
             for (let i = node.length - 1; i >= 0; i--) {
                 _traverse(node[i], () => node.splice(i, 1));
             }
-        } else if (typeof node === 'object') {
+        } else if (node && typeof node === 'object') {
             if (Array.isArray(node.$merge)) {
                 node.$merge.forEach(requestResource);
             } else {
@@ -493,4 +493,22 @@ mwUtil.getDateSafe = (rp) => {
     }
 };
 
+/**
+ * From a list of regexes and strings, constructs a regex that
+ * matches any of list items
+ */
+mwUtil.constructRegex = (list) => {
+    let regex = (list || []).map((regexString) => {
+        regexString = regexString.trim();
+        if (/^\/.+\/$/.test(regexString)) {
+            return `(?:${regexString.substring(1, regexString.length - 1)})`;
+        }
+        // Compare strings, instead
+        return `(?:^${decodeURIComponent(regexString)
+        .replace(/[-[\]/{}()*+?.\\^$|]/g, "\\$&")}$)`;
+    }).join('|');
+    regex = regex && regex.length > 0 ? new RegExp(regex) : undefined;
+    return regex;
+};
+
 module.exports = mwUtil;
diff --git a/projects/dev.yaml b/projects/dev.yaml
@@ -53,9 +53,12 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table_ng}}'
-            /key_value:
+            /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
+            /key_value_old: &sys_key_value_old
+              x-modules:
+                - path: sys/key_value_old.js
             /key_rev_value:
               x-modules:
                 - path: sys/key_rev_value.js
@@ -81,10 +84,39 @@ paths:
                   options:
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    renew_expiring: true
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    table_name_prefix: parsoid_ng
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /mobileapps:
               x-modules:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
+            /mobile_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    table_name_prefix: mobile
+                    main_content_type:
+                      name: lead
+                      value_type: json
+                    dependent_content_types:
+                      - name: remaining
+                        value_type: json
         options: '{{options}}'
 
diff --git a/projects/example.yaml b/projects/example.yaml
@@ -31,9 +31,12 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table}}'
-            /key_value:
+            /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
+            /key_value_old: &sys_key_value_old
+              x-modules:
+                - path: sys/key_value_old.js
             /key_rev_value:
               x-modules:
                 - path: sys/key_rev_value.js

diff --git a/projects/wikimedia.org.yaml b/projects/wikimedia.org.yaml
@@ -88,9 +88,12 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table}}'
-            /key_value:
+            /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
+            /key_value_old: &sys_key_value_old
+              x-modules:
+                - path: sys/key_value_old.js
             /key_rev_value:
               x-modules:
                 - path: sys/key_rev_value.js

diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml
@@ -69,9 +69,8 @@ paths:
                                 options.mobileapps)}}'
                 - path: v1/graphoid.yaml
                   options: '{{options.graphoid}}'
-                - path: v1/summary.js
-                  options:
-                    response_cache_control: '{{options.purged_cache_control_client_cache}}'
+                - path: v1/summary_proxy.js
+                  options: '{{merge(options.summary, {"response_cache_control": options.purged_cache_control_client_cache})}}'
                 - path: v1/related.js
                   options: '{{options.related}}'
                 - path: v1/random.yaml
@@ -119,9 +118,28 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table_ng}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    renew_expiring: true
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    table_name_prefix: parsoid_ng
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
+            /key_value_old: &sys_key_value_old
+              x-modules:
+                - path: sys/key_value_old.js
             /key_rev_value:
               x-modules:
                 - path: sys/key_rev_value.js
@@ -142,9 +160,13 @@ paths:
               x-modules:
                 - path: sys/page_save.js
             /parsoid:
+              x-modules:
+                - path: sys/parsoid_proxy.js
+                  options: '{{options.parsoid.backends}}'
+            /parsoid_new:
               x-modules:
                 - path: sys/parsoid.js
-                  options:
+                  options: &parsoid_options
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
@@ -230,6 +252,10 @@ paths:
 #                        - '/^User:Oxyman\/London\//'
 #                        - 'User:Stunteltje/gallery'
 #                        - 'User:PIERRE_ANDRE_LECLERCQ/gallery'
+            /parsoid_old:
+              x-modules:
+                - path: sys/parsoid_old.js
+                  options: *parsoid_options
             /mobileapps:
               x-modules:
                 - path: sys/mobileapps.js
@@ -239,7 +265,6 @@ paths:
               x-modules:
                 - path: sys/multi_content_bucket.js
                   options:
-                    # TODO: Decide on the retention period
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
                     delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
                     table_name_prefix: mobile_ng
@@ -253,4 +278,18 @@ paths:
               x-modules:
                 - path: sys/events.js
                   options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}'
+            /summary_old:
+              x-modules:
+                - path: sys/summary.js
+                  options:
+                    backend: key_value_old
+                    table: summary
+                    response_cache_control: '{{options.purged_cache_control_client_cache}}'
+            /summary_new:
+              x-modules:
+                - path: sys/summary.js
+                  options:
+                    backend: key_value
+                    table: page_  summary
+                    response_cache_control: '{{options.purged_cache_control_client_cache}}'
         options: '{{options}}'
diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml
@@ -69,9 +69,8 @@ paths:
                                 options.mobileapps)}}'
                 - path: v1/graphoid.yaml
                   options: '{{options.graphoid}}'
-                - path: v1/summary.js
-                  options:
-                    response_cache_control: '{{options.purged_cache_control_client_cache}}'
+                - path: v1/summary_proxy.js
+                  options: '{{merge(options.summary, {"response_cache_control": options.purged_cache_control_client_cache})}}'
                 - path: v1/related.js
                   options: '{{options.related}}'
                 - path: v1/random.yaml
@@ -121,9 +120,28 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table_ng}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    table_name_prefix: parsoid_ng
+                    renew_expiring: true
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
+            /key_value_old: &sys_key_value_old
+              x-modules:
+                - path: sys/key_value_old.js
             /key_rev_value:
               x-modules:
                 - path: sys/key_rev_value.js
@@ -145,8 +163,13 @@ paths:
                 - path: sys/page_save.js
             /parsoid:
               x-modules:
-                - path: sys/parsoid.js
+                - path: sys/parsoid_proxy.js
                   options:
+                    backends: '{{options.parsoid.backends}}'
+            /parsoid_new:
+              x-modules:
+                - path: sys/parsoid.js
+                  options: &parsoid_options
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
@@ -186,6 +209,10 @@ paths:
                         - 'Wikipedia:WikiProject_Deletion_sorting/United_States_of_America'
                         - 'Wikipedia:Articles_for_creation/Redirects'
                         - 'Wikipedia:Administrators%27_noticeboard/Incidents'
+            /parsoid_old:
+              x-modules:
+                - path: sys/parsoid_old.js
+                  options: *parsoid_options
             /mobileapps:
               x-modules:
                 - path: sys/mobileapps.js
@@ -195,7 +222,6 @@ paths:
               x-modules:
                 - path: sys/multi_content_bucket.js
                   options:
-                    # TODO: Decide on the retention period
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
                     delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
                     table_name_prefix: mobile_ng
@@ -209,4 +235,18 @@ paths:
               x-modules:
                 - path: sys/events.js
                   options: '{{merge({"skip_updates": options.skip_updates}, options.events)}}'
+            /summary_old:
+              x-modules:
+                - path: sys/summary.js
+                  options:
+                    backend: key_value_old
+                    table: summary
+                    response_cache_control: '{{options.purged_cache_control_client_cache}}'
+            /summary_new:
+              x-modules:
+                - path: sys/summary.js
+                  options:
+                    backend: key_value
+                    table: page_summary
+                    response_cache_control: '{{options.purged_cache_control_client_cache}}'
         options: '{{options}}'