Merge develop into misc/suppres-2-kl-divergence-datasets-for-bigquery

assets/scripts/AlgoliaScripts/upload_s3_expectation_to_algolia.js

@@ -1,7 +1,7 @@
 // load .env file (used while development) for loading env variables
 require('dotenv').config()
 const fetch = require('node-fetch')
-const expecS3URL = 'https://superconductive-public.s3.us-east-2.amazonaws.com/static/gallery/expectation_library_v2.json'
+const expecS3URL = process.env.ALGOLIA_S3_EXPECTATIONS_URL
 const algoliasearch = require('algoliasearch')
 const client = algoliasearch(process.env.ALGOLIA_ACCOUNT, process.env.ALGOLIA_WRITE_KEY)
 const expecAlgoliaIndex = process.env.ALGOLIA_EXPECTATION_INDEX

assets/scripts/AlgoliaScripts/upload_s3_packages_to_algolia.js

@@ -2,7 +2,7 @@
 require('dotenv').config();
 const fetch = require('node-fetch');
 const removeMd = require('remove-markdown');
-const packageS3URL = "https://superconductive-public.s3.us-east-2.amazonaws.com/static/gallery/package_manifests.json";
+const packageS3URL = process.env.ALGOLIA_S3_PACKAGES_URL;
 const algoliasearch = require("algoliasearch");
 const client = algoliasearch(process.env.ALGOLIA_ACCOUNT, process.env.ALGOLIA_WRITE_KEY);
 const packageAlgoliaIndex = process.env.ALGOLIA_PACKAGE_INDEX;
@@ -21,7 +21,7 @@ function removeMarkdown(str) {
     return plainText;
 }
 
-// Replica expectation Index Names And Sorting Order Settings 
+// Replica expectation Index Names And Sorting Order Settings
 const expecReplicaIndexAndSettings = [
     {
         replica: `${process.env.ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX}`, ranking: ['asc(description.snake_name)']
@@ -37,7 +37,7 @@ const expecReplicaIndexAndSettings = [
     },
 ]
 
-// Main Packages' Expectations Index setSettings 
+// Main Packages' Expectations Index setSettings
 const expecAttributesForFaceting = ["searchable(library_metadata.tags)", "searchable(engineSupported)", "searchable(exp_type)", "searchable(package)"];
 const maxFacetHits = 100;
 const epxecSearchableAttributes = ["description.snake_name", "description.short_description"]
@@ -143,12 +143,12 @@ function mainExpecIndexSetting() {
         maxFacetHits: maxFacetHits,
         searchableAttributes: epxecSearchableAttributes,
         customRanking: mainExpecIndexRanking,
-        // Creating replica index 
+        // Creating replica index
         replicas: expecReplicaIndexAndSettings.map(replica => replica.replica)
     })
         .then(() => {
             console.log('facets and replicas created. Now configuring expectation replica indices');
-            // Creating replica index setsettings 
+            // Creating replica index setsettings
             setExpecIndexReplicaSettings();
         })
         .catch(err => console.log("Error creating setting for Package Expectation Index !!", err));;

assets/scripts/copy_s3_staging_to_prod.sh

@@ -3,8 +3,5 @@
 aws --version
 aws s3 ls s3://superconductive-public/static/gallery/ | grep staging
 
-aws s3 cp s3://superconductive-public/static/gallery/expectation_library_v2.json s3://superconductive-public/static/gallery/expectation_library_v2--old-prod.json
-aws s3 cp s3://superconductive-public/static/gallery/package_manifests.json s3://superconductive-public/static/gallery/package_manifests--old-prod.json
-
 aws s3 cp s3://superconductive-public/static/gallery/expectation_library_v2--staging.json s3://superconductive-public/static/gallery/expectation_library_v2.json
 aws s3 cp s3://superconductive-public/static/gallery/package_manifests--staging.json s3://superconductive-public/static/gallery/package_manifests.json

azure-pipelines-expectation-gallery.yml

@@ -257,6 +257,9 @@ stages:
               ALGOLIA_PACKAGE_EXPEC_INDEX: $(ALGOLIA_PACKAGE_EXPEC_INDEX)
               ALGOLIA_PACKAGE_INDEX: $(ALGOLIA_PACKAGE_INDEX)
               ALGOLIA_WRITE_KEY: $(ALGOLIA_WRITE_KEY)
+              # Build Gallery Paths
+              ALGOLIA_S3_PACKAGES_URL: $(ALGOLIA_S3_PACKAGES_URL)
+              ALGOLIA_S3_EXPECTATIONS_URL: $(ALGOLIA_S3_EXPECTATIONS_URL)
               # replica indices from expectations for sorting
               ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX)
               ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX)

azure-pipelines-manual-staging-json-to-prod.yml

@@ -52,64 +52,9 @@ stages:
               ALGOLIA_PACKAGE_EXPEC_INDEX: $(ALGOLIA_PACKAGE_EXPEC_INDEX)
               ALGOLIA_PACKAGE_INDEX: $(ALGOLIA_PACKAGE_INDEX)
               ALGOLIA_WRITE_KEY: $(ALGOLIA_WRITE_KEY)
-              # replica indices from expectations for sorting
-              ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX)
-              ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX)
-              ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
-              ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX)
-              ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX)
-              ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX)
-              ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX)
-              ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX)
-              # replica indices from package expectations for sorting
-              ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX)
-              ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX)
-              ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
-              ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX)
-
-  - stage: revert_gallery_prod
-    pool:
-      vmImage: 'ubuntu-22.04'
-
-    jobs:
-      - job: copy_old_prod_to_prod
-        condition: eq(variables.isManual, true)
-        timeoutInMinutes: 15
-        variables:
-          python.version: '3.8'
-
-        steps:
-          - task: UsePythonVersion@0
-            inputs:
-              versionSpec: '$(python.version)'
-            displayName: 'Use Python $(python.version)'
-
-          - bash: python -m pip install --upgrade pip==21.3.1 awscli
-            displayName: 'Update pip and install awscli'
-
-          - bash: bash ./copy_s3_old_prod_to_prod.sh
-            workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
-            displayName: 'Copy old prod JSON files in S3 to prod'
-            env:
-              # AWS credentials
-              AWS_ACCESS_KEY_ID: $(AWS_ACCESS_KEY_ID)
-              AWS_SECRET_ACCESS_KEY: $(AWS_SECRET_ACCESS_KEY)
-              AWS_DEFAULT_REGION: $(AWS_DEFAULT_REGION)
-
-          - task: NodeTool@0
-            inputs:
-              versionSpec: '16.16'
-
-          - bash: bash ./trigger_algolia.sh
-            workingDirectory: $(Build.SourcesDirectory)/assets/scripts/
-            displayName: 'Update Algolia indexes from S3'
-            env:
-              # algolia credentials
-              ALGOLIA_ACCOUNT: $(ALGOLIA_ACCOUNT)
-              ALGOLIA_EXPECTATION_INDEX: $(ALGOLIA_EXPECTATION_INDEX)
-              ALGOLIA_PACKAGE_EXPEC_INDEX: $(ALGOLIA_PACKAGE_EXPEC_INDEX)
-              ALGOLIA_PACKAGE_INDEX: $(ALGOLIA_PACKAGE_INDEX)
-              ALGOLIA_WRITE_KEY: $(ALGOLIA_WRITE_KEY)
+              # Build Gallery Paths
+              ALGOLIA_S3_PACKAGES_URL: $(ALGOLIA_S3_PACKAGES_URL)
+              ALGOLIA_S3_EXPECTATIONS_URL: $(ALGOLIA_S3_EXPECTATIONS_URL)
               # replica indices from expectations for sorting
               ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX)
               ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX)