From 5640f6d345b9551ab4342ba775701f5293c51fd6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Dornell?= Date: Wed, 21 Aug 2019 18:44:25 +0200 Subject: [PATCH 1/3] Added Launch Library support --- src/scripts/upcoming.js | 109 +++++++++++++++++++++++++++++++++------- 1 file changed, 90 insertions(+), 19 deletions(-) diff --git a/src/scripts/upcoming.js b/src/scripts/upcoming.js index e5de7ef7..b1cde7e7 100644 --- a/src/scripts/upcoming.js +++ b/src/scripts/upcoming.js @@ -31,6 +31,21 @@ const missionNames = []; const promises = []; const precision = []; const flightNumbers = []; +const lastWikiUpdates = []; + + +// Launch Library settings + +// lsp=121 means that SpaceX is the agency that launches +const launchLibraryURL = "https://launchlibrary.net/1.4/launch?lsp=121&name="; + +// How close (in days) the launch needs to be in order to be updated +// NOTE: Only applies to upcoming launches +const minimumProximity = 2; + +// Threshold (in seconds) for getting data from Launch Library instead of the wiki manifest +// Represents how many hours without a Reddit update to wait before using data from Launch Library +const thresholdSeconds = 3 * 60 * 60; // E.g. 3 hours // RegEx expressions for matching dates in the wiki manifest @@ -61,7 +76,6 @@ const monthTbd = /^[0-9]{4}\s([a-z]{3}|[a-z]{3,9})\sTBD$/i; // 2020 Early/Mid/Late Nov const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; - (async () => { try { client = await MongoClient.connect(process.env.MONGO_URL, { useNewUrlParser: true }); @@ -81,6 +95,7 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; launches.forEach((launch) => { missionNames.push(launch.mission_name); sites.push(launch.launch_site.site_id); + lastWikiUpdates.push(launch.last_wiki_update) }); // Grab subreddit wiki manifest @@ -115,7 +130,7 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; for await (const [payloadIndex, missionName] of missionNames.entries()) { for await (const [manifestIndex, manifestPayload] of manifestPayloads.entries()) { if (fuzz.partial_ratio(missionName, manifestPayload) === 100) { - // Check and see if dates match a certain patten depending on the length of the + // Check and see if dates match a certain pattern depending on the length of the // date given. This sets the amount of precision needed for the date. let mdate = manifestDates[manifestIndex]; // 2020 Q3 @@ -196,22 +211,6 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; console.log(date); console.log(`${missionName} : ${manifestPayload}`); - // Strip brackets from time given, and tack on UTC time offset at the end for date parser - const parsedDate = `${date.replace(/(early|mid|late)/i, '').replace('[', '').replace(']', '')} +0000`; - const time = moment(parsedDate, ['YYYY MMM D HH:mm Z', 'YYYY MMM D Z', 'YYYY MMM Z', 'YYYY Q Z', 'YYYY Z']); - - // Feed stripped time into all possible date formats in the wiki currently - const zone = moment.tz(time, 'UTC'); - - // Use launch site id's to properly set timezone for local time - if (location === 'ccafs_slc_40' || location === 'ksc_lc_39a' || location === 'ccafs_lc_13') { - localTime = time.tz('America/New_York').format(); - } else if (location === 'vafb_slc_4e' || location === 'vafb_slc_4w') { - localTime = time.tz('America/Los_Angeles').format(); - } else { - localTime = time.tz('America/Chicago').format(); - } - // Add flight numbers to array to check for duplicates flightNumbers.push(baseFlightNumber + manifestIndex); @@ -239,6 +238,77 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; siteNameLong = 'SpaceX South Texas Launch Site'; } + // Launch Library code + let isDateFromWiki, lastUpdate, launchDate; + let daysToLaunch = time.diff(moment(), 'days'); + if (daysToLaunch <= minimumProximity) { + let wikiDelay, resultLL, query; + // Get the launch from Launch Library + // and check if it has updated more recently than the wiki. + + // First we look for the mission name in LL + query = launchLibraryURL + missionName.replace(/ /g, "+"); + try { + resultLL = await request(query); + resultLL = JSON.parse(resultLL); + // November 4, 2019 00:00:00 UTC + launchDate = moment(resultLL.launches[0].net.replace("UTC", "Z"), "MMMM D, YYYY hh:mm:ss Z"); + let changed = moment(resultLL.launches[0].changed, "YYYY-MM-DD hh:mm:ss"); + if (zone.diff(launchDate) != 0) { + // If the date in the wiki and Launch Library aren't the same + // we calculate the delay of the wiki in respect to Launch Library + let lastWikiUpdate = moment(lastWikiUpdates[payloadIndex]); + wikiDelay = changed.diff(lastWikiUpdate, 'seconds'); // if negative, the server is ahead + } else { + wikiDelay = 0; + } + } catch (e) { + if (resultLL) { + console.log(e) + } + // Mission not found in LL, so we use wiki's data + wikiDelay = 0; + } + // Use the LL data if the delay of the wiki is bigger than 'thresholdSeconds' + if (wikiDelay > thresholdSeconds) { + isDateFromWiki = false; + } else { + isDateFromWiki = true; + } + } else { + isDateFromWiki = true; + } + + if (isDateFromWiki) { + // If it was updated from the wiki, we save the current time in last_wiki_update + lastUpdate = moment().toISOString(); + } else { + // If it was updated from LL, we don't update last_wiki_update + lastUpdate = lastWikiUpdates[payloadIndex]; + } + + // Strip brackets from time given, and tack on UTC time offset at the end for date parser + if (isDateFromWiki) { + const parsedDate = `${date.replace(/(early|mid|late)/i, '').replace('[', '').replace(']', '')} +0000`; + const time = moment(parsedDate, ['YYYY MMM D HH:mm Z', 'YYYY MMM D Z', 'YYYY MMM Z', 'YYYY Q Z', 'YYYY Z']); + } else { + // Use date from Launch Library instead + const time = launchDate; + } + + + // Feed stripped time into all possible date formats in the wiki currently + const zone = moment.tz(time, 'UTC'); + + // Use launch site id's to properly set timezone for local time + if (location === 'ccafs_slc_40' || location === 'ksc_lc_39a' || location === 'ccafs_lc_13') { + localTime = time.tz('America/New_York').format(); + } else if (location === 'vafb_slc_4e' || location === 'vafb_slc_4w') { + localTime = time.tz('America/Los_Angeles').format(); + } else { + localTime = time.tz('America/Chicago').format(); + } + // Build launch time objects to update calculatedTimes = { flight_number: (baseFlightNumber + manifestIndex), @@ -246,13 +316,14 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; launch_date_unix: zone.unix(), launch_date_utc: zone.toISOString(), launch_date_local: localTime, + last_wiki_update: lastUpdate, + is_date_from_wiki: isDateFromWiki, is_tentative: isTentative, tentative_max_precision: precision[manifestIndex], tbd, 'launch_site.site_id': siteId, 'launch_site.site_name': siteName, 'launch_site.site_name_long': siteNameLong, - }; console.log(calculatedTimes); console.log(''); From e7aa7d7ae3fef17722266e7603e4922b7657cbd7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Dornell?= Date: Wed, 21 Aug 2019 20:57:01 +0200 Subject: [PATCH 2/3] Fixed various error and pleased eslint --- src/scripts/upcoming.js | 39 ++++++++++++++++++++++++--------------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/src/scripts/upcoming.js b/src/scripts/upcoming.js index b1cde7e7..0d0034df 100644 --- a/src/scripts/upcoming.js +++ b/src/scripts/upcoming.js @@ -37,7 +37,7 @@ const lastWikiUpdates = []; // Launch Library settings // lsp=121 means that SpaceX is the agency that launches -const launchLibraryURL = "https://launchlibrary.net/1.4/launch?lsp=121&name="; +const launchLibraryURL = 'https://launchlibrary.net/1.4/launch?lsp=121&name='; // How close (in days) the launch needs to be in order to be updated // NOTE: Only applies to upcoming launches @@ -95,7 +95,7 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; launches.forEach((launch) => { missionNames.push(launch.mission_name); sites.push(launch.launch_site.site_id); - lastWikiUpdates.push(launch.last_wiki_update) + lastWikiUpdates.push(launch.last_wiki_update); }); // Grab subreddit wiki manifest @@ -239,32 +239,42 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; } // Launch Library code - let isDateFromWiki, lastUpdate, launchDate; - let daysToLaunch = time.diff(moment(), 'days'); + let isDateFromWiki; + let lastUpdate; + let launchDate; + let time; + let zone; + + const parsedDate = `${date.replace(/(early|mid|late)/i, '').replace('[', '').replace(']', '')} +0000`; + time = moment(parsedDate, ['YYYY MMM D HH:mm Z', 'YYYY MMM D Z', 'YYYY MMM Z', 'YYYY Q Z', 'YYYY Z']); + zone = moment.tz(time, 'UTC'); + + const daysToLaunch = time.diff(moment(), 'days'); if (daysToLaunch <= minimumProximity) { - let wikiDelay, resultLL, query; + let wikiDelay; + let resultLL; // Get the launch from Launch Library // and check if it has updated more recently than the wiki. // First we look for the mission name in LL - query = launchLibraryURL + missionName.replace(/ /g, "+"); + const query = launchLibraryURL + missionName.replace(/ /g, '+'); try { resultLL = await request(query); resultLL = JSON.parse(resultLL); // November 4, 2019 00:00:00 UTC - launchDate = moment(resultLL.launches[0].net.replace("UTC", "Z"), "MMMM D, YYYY hh:mm:ss Z"); - let changed = moment(resultLL.launches[0].changed, "YYYY-MM-DD hh:mm:ss"); - if (zone.diff(launchDate) != 0) { + launchDate = moment(resultLL.launches[0].net.replace('UTC', 'Z'), 'MMMM D, YYYY hh:mm:ss Z'); + const changed = moment(resultLL.launches[0].changed, 'YYYY-MM-DD hh:mm:ss'); + if (zone.diff(launchDate) !== 0) { // If the date in the wiki and Launch Library aren't the same // we calculate the delay of the wiki in respect to Launch Library - let lastWikiUpdate = moment(lastWikiUpdates[payloadIndex]); + const lastWikiUpdate = moment(lastWikiUpdates[payloadIndex]); wikiDelay = changed.diff(lastWikiUpdate, 'seconds'); // if negative, the server is ahead } else { wikiDelay = 0; } } catch (e) { if (resultLL) { - console.log(e) + console.log(e); } // Mission not found in LL, so we use wiki's data wikiDelay = 0; @@ -289,16 +299,15 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; // Strip brackets from time given, and tack on UTC time offset at the end for date parser if (isDateFromWiki) { - const parsedDate = `${date.replace(/(early|mid|late)/i, '').replace('[', '').replace(']', '')} +0000`; - const time = moment(parsedDate, ['YYYY MMM D HH:mm Z', 'YYYY MMM D Z', 'YYYY MMM Z', 'YYYY Q Z', 'YYYY Z']); + time = moment(parsedDate, ['YYYY MMM D HH:mm Z', 'YYYY MMM D Z', 'YYYY MMM Z', 'YYYY Q Z', 'YYYY Z']); } else { // Use date from Launch Library instead - const time = launchDate; + time = launchDate; } // Feed stripped time into all possible date formats in the wiki currently - const zone = moment.tz(time, 'UTC'); + zone = moment.tz(time, 'UTC'); // Use launch site id's to properly set timezone for local time if (location === 'ccafs_slc_40' || location === 'ksc_lc_39a' || location === 'ccafs_lc_13') { From ec7f2c532f9db5ee494948a3576b7ce7bd8229c5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Dornell?= Date: Thu, 22 Aug 2019 07:24:27 +0200 Subject: [PATCH 3/3] Added fuzzy match for Launch Library launches with custom ratio Reduced number of requests to LL API to 1 Added negative bound to daysToLaunch to deal with undefined dates --- src/scripts/upcoming.js | 64 +++++++++++++++++++++++++++++++---------- 1 file changed, 49 insertions(+), 15 deletions(-) diff --git a/src/scripts/upcoming.js b/src/scripts/upcoming.js index 0d0034df..a6ed56aa 100644 --- a/src/scripts/upcoming.js +++ b/src/scripts/upcoming.js @@ -36,8 +36,9 @@ const lastWikiUpdates = []; // Launch Library settings +// Request next 100 launches (for reference, current amount is around 40) // lsp=121 means that SpaceX is the agency that launches -const launchLibraryURL = 'https://launchlibrary.net/1.4/launch?lsp=121&name='; +const launchLibraryURL = 'https://launchlibrary.net/1.4/launch?next=100&lsp=121'; // How close (in days) the launch needs to be in order to be updated // NOTE: Only applies to upcoming launches @@ -47,6 +48,9 @@ const minimumProximity = 2; // Represents how many hours without a Reddit update to wait before using data from Launch Library const thresholdSeconds = 3 * 60 * 60; // E.g. 3 hours +// Minimum partial ratio from the fuzzy match, needed to accept a LL mission +const minimumPartialRatio = 90; + // RegEx expressions for matching dates in the wiki manifest // Allows for long months or short months ex. September vs Sep @@ -124,6 +128,10 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; baseFlightNumber = pastLaunches[0].flight_number + 1; } + // Store all the upcoming launches from Launch Library + // if at least one launch is within the threshold + let launchLibraryNextLaunches; + // Compare each mission name against entire list of manifest payloads, and fuzzy match the // mission name against the manifest payload name. The partial match must be 100%, to avoid // conflicts like SSO-A and SSO-B, where a really close match would produce wrong results. @@ -250,35 +258,61 @@ const monthVague = /^[0-9]{4}\s(early|mid|late)\s([a-z]{3}|[a-z]{3,9})$/i; zone = moment.tz(time, 'UTC'); const daysToLaunch = time.diff(moment(), 'days'); - if (daysToLaunch <= minimumProximity) { + if (daysToLaunch <= minimumProximity && daysToLaunch > -2) { let wikiDelay; let resultLL; // Get the launch from Launch Library // and check if it has updated more recently than the wiki. // First we look for the mission name in LL - const query = launchLibraryURL + missionName.replace(/ /g, '+'); - try { - resultLL = await request(query); - resultLL = JSON.parse(resultLL); - // November 4, 2019 00:00:00 UTC - launchDate = moment(resultLL.launches[0].net.replace('UTC', 'Z'), 'MMMM D, YYYY hh:mm:ss Z'); - const changed = moment(resultLL.launches[0].changed, 'YYYY-MM-DD hh:mm:ss'); + // If the launches haven't been loaded, + // we request the next 100 launches from Launch Library + // in order to do fuzzy match locally + if (!launchLibraryNextLaunches) { + try { + resultLL = await request(launchLibraryURL); + resultLL = JSON.parse(resultLL); + // Save only 'name', net' and 'changed' + const unwrap = ({ name, net, changed }) => ({ name, net, changed }); + launchLibraryNextLaunches = resultLL.launches.map(launch => unwrap(launch)); + // Format: November 4, 2019 00:00:00 UTC + } catch (e) { + if (resultLL) { + console.log(e); + } + // No delay because LL is unavailable + wikiDelay = 0; + } + } + + // Fuzzy match against the stored list to use + let bestMatch = [-1, 0]; + launchLibraryNextLaunches.forEach((launch, index) => { + // Fuzzy match between local name and LL name + const partialRatio = fuzz.partial_ratio(missionName, launch.name); + // Return best match + if (partialRatio > bestMatch[1]) { + bestMatch = [index, partialRatio]; + } + }); + // Check that partial ratio is above the minimum + if (bestMatch[0] !== -1 && bestMatch[1] >= minimumPartialRatio) { + const launch = launchLibraryNextLaunches[bestMatch[0]]; + launchDate = moment(launch.net.replace('UTC', 'Z'), 'MMMM D, YYYY hh:mm:ss Z'); + const changed = moment(launch.changed, 'YYYY-MM-DD hh:mm:ss'); if (zone.diff(launchDate) !== 0) { // If the date in the wiki and Launch Library aren't the same // we calculate the delay of the wiki in respect to Launch Library const lastWikiUpdate = moment(lastWikiUpdates[payloadIndex]); - wikiDelay = changed.diff(lastWikiUpdate, 'seconds'); // if negative, the server is ahead + // If negative, the server is ahead + wikiDelay = changed.diff(lastWikiUpdate, 'seconds'); } else { wikiDelay = 0; } - } catch (e) { - if (resultLL) { - console.log(e); - } - // Mission not found in LL, so we use wiki's data + } else { wikiDelay = 0; } + // Use the LL data if the delay of the wiki is bigger than 'thresholdSeconds' if (wikiDelay > thresholdSeconds) { isDateFromWiki = false;