Skip to content

Commit

Permalink
[MAINTENANCE] Update aloglia scripts for new fields and replica indic…
Browse files Browse the repository at this point in the history 
…es (#6049)
  • Loading branch information
@rpanwar-winfo
rpanwar-winfo committed Sep 27, 2022
1 parent 2448b0d commit 10c3a27
Show file tree
Hide file tree
Showing 7 changed files with 208 additions and 37 deletions.
3 changes: 3 additions & 0 deletions assets/scripts/AlgoliaScripts/.gitignore
View file
Original file line number Diff line number Diff line change
Expand Up @@ -114,3 +114,6 @@ dist
.yarn/build-state.yml
.yarn/install-state.gz
.pnp.*

#environment files
.env*
24 changes: 24 additions & 0 deletions assets/scripts/AlgoliaScripts/README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,3 +35,27 @@ To run this project, you will need to add the following environment variables to
`ALGOLIA_PACKAGE_INDEX` : prod_packages

`ALGOLIA_PACKAGE_EXPEC_INDEX` : prod_packages_expectations

`ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX` : prod_expectations_alpha_asc

`ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX` : prod_expectations_alpha_dsc

`ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX` : prod_expectations_coverage_asc

`ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX` : prod_expectations_coverage_dsc

`ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX` : prod_expectations_created_asc

`ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX` : prod_expectations_created_dsc

`ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX` : prod_expectations_updated_asc

`ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX` : prod_expectations_updated_dsc

`ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX` : prod_pack_expectations_alpha_asc

`ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX` : prod_pack_expectations_alpha_dsc

`ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX` : prod_pack_expectations_coverage_asc

`ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX` : prod_pack_expectations_coverage_dsc
14 changes: 14 additions & 0 deletions assets/scripts/AlgoliaScripts/package-lock.json
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions assets/scripts/AlgoliaScripts/package.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
"license": "ISC",
"dependencies": {
"algoliasearch": "^4.12.1",
"dotenv": "^16.0.2",
"node-fetch": "^2.6.7",
"remove-markdown": "^0.5.0"
}
Expand Down
107 changes: 85 additions & 22 deletions assets/scripts/AlgoliaScripts/upload_s3_expectation_to_algolia.js
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,48 @@
// load .env file (used while development) for loading env variables
require('dotenv').config();
const fetch = require('node-fetch');
const expecS3URL = "https://superconductive-public.s3.us-east-2.amazonaws.com/static/gallery/expectation_library_v2.json";
const algoliasearch = require("algoliasearch");
const client = algoliasearch(process.env.ALGOLIA_ACCOUNT, process.env.ALGOLIA_WRITE_KEY);
const expecAlgoliaIndex = process.env.ALGOLIA_EXPECTATION_INDEX;
const index = client.initIndex(expecAlgoliaIndex);


// Replica Index Names And Sorting Order Settings
const replicaIndexAndSettings = [
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX}`, ranking: ['asc(description.snake_name)']
},
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX}`, ranking: ['desc(description.snake_name)']
},
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX}`, ranking: ['asc(coverage_score)']
},
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX}`, ranking: ['desc(coverage_score)']
},
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX}`, ranking: ['asc(created_at)']
},
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX}`, ranking: ['desc(created_at)']
},
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX}`, ranking: ['asc(updated_at)']
},
{
replica: `${process.env.ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX}`, ranking: ['desc(updated_at)']
},
]

// Main Index setSettings
const attributesForFaceting = ["searchable(library_metadata.tags)", "searchable(engineSupported)", "searchable(exp_type)"];
const maxFacetHits=100;
const searchableAttributes=["description.snake_name", "description.short_description"]
const customRanking=['asc(description.snake_name)']

//load data from S3
loadFromS3(expecS3URL).then(response => {
console.log("Length of expectation loaded from S3", Object.keys(response).length);
if (Object.keys(response).length > 0) {
Expand All @@ -15,6 +53,7 @@ loadFromS3(expecS3URL).then(response => {
return;
}
console.log("Formatted expectation sample: ", algDataset[0]);
// return;
deleteIndex(algDataset);
}
}).catch((error) => {
Expand All @@ -37,6 +76,7 @@ async function loadFromS3(URL) {
return await response.json();
}

// Format expectations and prepare JSON which will be sent to algolia
function formatExpectation(ExpecData) {
const ExpectationKeys = Object.keys(ExpecData);
let dataset = [];
Expand All @@ -48,50 +88,73 @@ function formatExpectation(ExpecData) {
data.execution_engines = ExpecData[key].execution_engines;
data.maturity_checklist = ExpecData[key].maturity_checklist;
data.backend_test_result_counts = ExpecData[key].backend_test_result_counts;
data.engineSupported = ExpecData[key].backend_test_result_counts.map((db) => db.backend);
data.engineSupported=ExpecData[key].backend_test_result_counts.map((db)=>db.backend);
data.coverage_score=ExpecData[key].coverage_score;
data.created_at=ExpecData[key].created_at;
data.updated_at=ExpecData[key].updated_at;
data.exp_type=ExpecData[key].exp_type;
dataset.push(data);
})
return dataset;
}

// Upload data to algolia index
function uploadToAlgolia(dataset) {
index.saveObjects(dataset)
.then(() => {
console.log('Expectations data uploaded to algolia');
indexSetting(dataset);
mainIndexSetting(dataset);
})
.catch(err => console.log(err))
}

function indexSetting(dataset) {
function mainIndexSetting(dataset) {
index.setSettings({
attributesForFaceting: ["searchable(library_metadata.tags)", "searchable(engineSupported)"],
maxFacetHits: 100,
searchableAttributes: ["description.snake_name", "description.short_description"],
customRanking: [
'asc(description.snake_name)',
]
attributesForFaceting:attributesForFaceting ,
maxFacetHits: maxFacetHits,
searchableAttributes:searchableAttributes,
customRanking: customRanking,
// Creating replica index
replicas:replicaIndexAndSettings.map(replica=>replica.replica)
})
.then(() => {
console.log('facets created.');
fetchAllAtrributes(dataset[0]);
}).catch((error) => {
console.log("Error in index settings", error);
});
.then(() => {
console.log('facets created.');
fetchAllAtrributes(dataset[0]);
// Creating replica index setsettings
setReplicaSettings();
}).catch((error) => {
console.log("Error in index settings", error);
});
}

function fetchAllAtrributes(data) {
console.log("data is ", data);
console.log("data is ",data);
let existingId = [data.description.snake_name];
let attributes = Object.keys(data);
console.log("Attributes are", attributes);
index.getObjects(existingId, {
attributesToRetrieve: attributes
})
.then((results) => {
console.log('fetching all attributes ', results);
console.log("Successfully fetched sample record from algolia !");
}).catch((error) => {
console.log('getting error while fetching', error);
})
.then((results) => {
console.log('fetching all attributes ', results);
console.log("Successfully fetched sample record from algolia !");
}).catch((error) => {
console.log('getting error while fetching', error);
})
};

//Replica Index Settings
function setReplicaSettings() {
replicaIndexAndSettings.map((repli) => {
const { replica, ranking } = repli;
client.initIndex(replica).setSettings({
attributesForFaceting: attributesForFaceting,
maxFacetHits: maxFacetHits,
searchableAttributes: searchableAttributes,
customRanking: ranking
})
.then(() => {
console.log(`Replica: ${replica} configured`)
})
})
}
82 changes: 67 additions & 15 deletions assets/scripts/AlgoliaScripts/upload_s3_packages_to_algolia.js
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
// load .env file (used while development) for env variables
require('dotenv').config();
const fetch = require('node-fetch');
const removeMd = require('remove-markdown');
const packageS3URL = "https://superconductive-public.s3.us-east-2.amazonaws.com/static/gallery/package_manifests.json";
Expand All @@ -19,6 +21,28 @@ function removeMarkdown(str) {
return plainText;
}

// Replica expectation Index Names And Sorting Order Settings
const expecReplicaIndexAndSettings = [
{
replica: `${process.env.ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX}`, ranking: ['asc(description.snake_name)']
},
{
replica: `${process.env.ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX}`, ranking: ['desc(description.snake_name)']
},
{
replica: `${process.env.ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX}`, ranking: ['asc(coverage_score)']
},
{
replica: `${process.env.ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX}`, ranking: ['desc(coverage_score)']
},
]

// Main Packages' Expectations Index setSettings
const expecAttributesForFaceting = ["searchable(library_metadata.tags)", "searchable(engineSupported)", "searchable(exp_type)", "searchable(package)"];
const maxFacetHits = 100;
const epxecSearchableAttributes = ["description.snake_name", "description.short_description"]
const mainExpecIndexRanking = ['asc(description.snake_name)']

let packageDataset = [];
let expectationDataset = [];

Expand Down Expand Up @@ -60,7 +84,7 @@ function formatPackageData(data) {
pdata["description"] = removeMarkdown(package.description);
packageDataset.push(pdata);
const expecKeys = Object.keys(package.expectations);
expecKeys.forEach(key => {
expecKeys.forEach((key) => {
const expectation = package.expectations[key];
let expecData = {
"objectID": key,
Expand All @@ -71,28 +95,32 @@ function formatPackageData(data) {
"execution_engines": expectation.execution_engines,
"package": package.package_name,
"backend_test_result_counts": expectation.backend_test_result_counts,
"coverage_score": expectation.coverage_score,
"exp_type": expectation.exp_type,
};
expectationDataset.push(expecData);
});
});
}

//Upload package data to package index
function uploadPackageToAlgolia(dataset) {
packageIndex.saveObjects(dataset)
.then(() => {
console.log("data uploaded to package index", packageAlgoliaIndex);
packageIndexSetting();
})
.catch(err => console.log(err))
.catch(err => console.log("Error uploading data to Package Index !!",err))
}

//Upload package expectation data to package expectation index
function uploadPackageExpecToAlgolia(dataset) {
packageExpecIndex.saveObjects(dataset)
.then(() => {
console.log("data uploaded to package expectations index", packageExpecAlgoliaIndex);
expecIndexSetting();
mainExpecIndexSetting();
})
.catch(err => console.log(err))
.catch(err => console.log("Error uploading data to Package Expectatiions Index !!",err))
}

function packageIndexSetting() {
Expand All @@ -102,19 +130,43 @@ function packageIndexSetting() {
'asc(package_name)',
]
})
.then(() => { });
.then(() => {
console.log("Package index settings done.")
})
.catch(err => console.log("Error creating settings for Package Index !!",err));
}

function expecIndexSetting() {
console.log("Creating package expectations facets and search settings");
function mainExpecIndexSetting() {
console.log("Creating package expectations replicas, facets and search settings");
packageExpecIndex.setSettings({
attributesForFaceting: ["searchable(library_metadata.tags)",
"searchable(package)", "searchable(engineSupported)"],
maxFacetHits: 100,
searchableAttributes: ["description.snake_name", "description.short_description"],
customRanking: [
'asc(description.snake_name)',
]
attributesForFaceting: expecAttributesForFaceting,
maxFacetHits: maxFacetHits,
searchableAttributes: epxecSearchableAttributes,
customRanking: mainExpecIndexRanking,
// Creating replica index
replicas: expecReplicaIndexAndSettings.map(replica => replica.replica)
})
.then(() => {
console.log('facets and replicas created. Now configuring expectation replica indices');
// Creating replica index setsettings
setExpecIndexReplicaSettings();
})
.catch(err => console.log("Error creating setting for Package Expectation Index !!", err));;
}

//Replica Index for expectations Settings
function setExpecIndexReplicaSettings() {
expecReplicaIndexAndSettings.map((repli) => {
const { replica, ranking } = repli;
client.initIndex(replica).setSettings({
attributesForFaceting: expecAttributesForFaceting,
maxFacetHits: maxFacetHits,
searchableAttributes: epxecSearchableAttributes,
customRanking: ranking
})
.then(() => {
console.log(`Replica: ${replica} configured !!`)
})
.catch(err => console.log(`Error creating settings for Packages' Expectation replica Index !! ${replica}`,err));
})
.then(() => { });
}
14 changes: 14 additions & 0 deletions azure-pipelines-expectation-gallery.yml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,20 @@ stages:
ALGOLIA_PACKAGE_EXPEC_INDEX: $(ALGOLIA_PACKAGE_EXPEC_INDEX)
ALGOLIA_PACKAGE_INDEX: $(ALGOLIA_PACKAGE_INDEX)
ALGOLIA_WRITE_KEY: $(ALGOLIA_WRITE_KEY)
# replica indices from expectations for sorting
ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_ALPHA_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_COVERAGE_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_CREATED_DSC_INDEX)
ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_ASC_INDEX)
ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX: $(ALGOLIA_EXPEC_REPLICA_UPDATED_DSC_INDEX)
# replica indices from package expectations for sorting
ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_ASC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_ALPHA_DSC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_ASC_INDEX)
ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX: $(ALGOLIA_PACK_EXPEC_REPLICA_COVERAGE_DSC_INDEX)

# - bash: |
# echo "About to trigger webhook: $GALLERY_BUILD_HOOK"
Expand Down

0 comments on commit 10c3a27

Please sign in to comment.