Merge 149da36 into c463559

wikimedia · Oct 3, 2017 · d7172c8 · d7172c8
2 parents c463559 + 149da36
commit d7172c8
Show file tree

Hide file tree

Showing 10 changed files with 1,346 additions and 92 deletions.
diff --git a/config.test.yaml b/config.test.yaml
@@ -32,6 +32,8 @@ default_project: &default_project
         parsoid:
           host: https://parsoid-beta.wmflabs.org
           grace_ttl: 2
+          backends:
+            default: both
         action:
           apiUriTemplate: "{{'https://{domain}/w/api.php'}}"
           baseUriTemplate: "{{'https://{domain}/api/rest_v1'}}"

diff --git a/projects/dev.yaml b/projects/dev.yaml
@@ -81,10 +81,39 @@ paths:
                   options:
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    renew_expiring: true
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    table_name_prefix: parsoid_ng
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /mobileapps:
               x-modules:
                 - path: sys/mobileapps.js
                   options: '{{merge({"response_cache_control": options.purged_cache_control},
                                 options.mobileapps)}}'
+            /mobile_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    table_name_prefix: mobile
+                    main_content_type:
+                      name: lead
+                      value_type: json
+                    dependent_content_types:
+                      - name: remaining
+                        value_type: json
         options: '{{options}}'
 
diff --git a/projects/wmf_default.yaml b/projects/wmf_default.yaml
@@ -119,6 +119,22 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table_ng}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    renew_expiring: true
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    table_name_prefix: parsoid_ng
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
@@ -142,9 +158,13 @@ paths:
               x-modules:
                 - path: sys/page_save.js
             /parsoid:
+              x-modules:
+                - path: sys/parsoid_proxy.js
+                  options: '{{options.parsoid.backends}}'
+            /parsoid_new:
               x-modules:
                 - path: sys/parsoid.js
-                  options:
+                  options: &parsoid_options
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
@@ -230,6 +250,10 @@ paths:
 #                        - '/^User:Oxyman\/London\//'
 #                        - 'User:Stunteltje/gallery'
 #                        - 'User:PIERRE_ANDRE_LECLERCQ/gallery'
+            /parsoid_old:
+              x-modules:
+                - path: sys/parsoid_old.js
+                  options: *parsoid_options
             /mobileapps:
               x-modules:
                 - path: sys/mobileapps.js
@@ -239,7 +263,6 @@ paths:
               x-modules:
                 - path: sys/multi_content_bucket.js
                   options:
-                    # TODO: Decide on the retention period
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
                     delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
                     table_name_prefix: mobile_ng

diff --git a/projects/wmf_enwiki.yaml b/projects/wmf_enwiki.yaml
@@ -121,6 +121,22 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table_ng}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    table_name_prefix: parsoid_ng
+                    renew_expiring: true
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
@@ -145,8 +161,13 @@ paths:
                 - path: sys/page_save.js
             /parsoid:
               x-modules:
-                - path: sys/parsoid.js
+                - path: sys/parsoid_proxy.js
                   options:
+                    backends: '{{options.parsoid.backends}}'
+            /parsoid_new:
+              x-modules:
+                - path: sys/parsoid.js
+                  options: &parsoid_options
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
@@ -186,6 +207,10 @@ paths:
                         - 'Wikipedia:WikiProject_Deletion_sorting/United_States_of_America'
                         - 'Wikipedia:Articles_for_creation/Redirects'
                         - 'Wikipedia:Administrators%27_noticeboard/Incidents'
+            /parsoid_old:
+              x-modules:
+                - path: sys/parsoid_old.js
+                  options: *parsoid_options
             /mobileapps:
               x-modules:
                 - path: sys/mobileapps.js
@@ -195,7 +220,6 @@ paths:
               x-modules:
                 - path: sys/multi_content_bucket.js
                   options:
-                    # TODO: Decide on the retention period
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
                     delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
                     table_name_prefix: mobile_ng

diff --git a/projects/wmf_wikidata.yaml b/projects/wmf_wikidata.yaml
@@ -94,6 +94,22 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table_ng}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    table_name_prefix: parsoid_ng
+                    renew_expiring: true
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
@@ -117,12 +133,20 @@ paths:
               x-modules:
                 - path: sys/page_save.js
             /parsoid:
+              x-modules:
+                - path: sys/parsoid_proxy.js
+                  options: '{{options.parsoid.backends}}'
+            /parsoid_new:
               x-modules:
                 - path: sys/parsoid.js
-                  options:
+                  options: &parsoid_options
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
+            /parsoid_old:
+              x-modules:
+                - path: sys/parsoid_old.js
+                  options: *parsoid_options
             /events:
               x-modules:
                 - path: sys/events.js

diff --git a/projects/wmf_wiktionary.yaml b/projects/wmf_wiktionary.yaml
@@ -108,6 +108,22 @@ paths:
                 - path: sys/table.js
                   options:
                     conf: '{{options.table_ng}}'
+            /parsoid_bucket:
+              x-modules:
+                - path: sys/multi_content_bucket.js
+                  options:
+                    table_name_prefix: parsoid_ng
+                    renew_expiring: true
+                    time_to_live: '{{default(options.parsoid.time_to_live, 84600)}}'
+                    delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
+                    main_content_type:
+                      name: html
+                      value_type: blob
+                    dependent_content_types:
+                      - name: data-parsoid
+                        value_type: json
+                      - name: section-offsets
+                        value_type: json
             /key_value: &sys_key_value
               x-modules:
                 - path: sys/key_value.js
@@ -131,12 +147,20 @@ paths:
               x-modules:
                 - path: sys/page_save.js
             /parsoid:
+              x-modules:
+                - path: sys/parsoid_proxy.js
+                  options: '{{options.parsoid.backends}}'
+            /parsoid_new:
               x-modules:
                 - path: sys/parsoid.js
-                  options:
+                  options: &parsoid_options
                     parsoidHost: '{{options.parsoid.host}}'
                     response_cache_control: '{{options.purged_cache_control}}'
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
+            /parsoid_old:
+              x-modules:
+                - path: sys/parsoid_old.js
+                  options: *parsoid_options
             /mobileapps:
               x-modules:
                 - path: sys/mobileapps.js
@@ -146,7 +170,6 @@ paths:
               x-modules:
                 - path: sys/multi_content_bucket.js
                   options:
-                    # TODO: Decide on the retention period
                     grace_ttl: '{{default(options.parsoid.grace_ttl, 86400)}}'
                     delete_probability: '{{default(options.parsoid.delete_probability, 1)}}'
                     table_name_prefix: mobile_ng