From 5b0456a6667ffea8216f507c10d7f8d17934c674 Mon Sep 17 00:00:00 2001 From: Kshitiz Kumar Date: Tue, 29 Dec 2020 22:08:09 +0530 Subject: [PATCH] Refactor the LastFm Code --- .../static/js/src/LastFMImporter.tsx | 307 +++++++++++++++++- 1 file changed, 304 insertions(+), 3 deletions(-) diff --git a/listenbrainz/webserver/static/js/src/LastFMImporter.tsx b/listenbrainz/webserver/static/js/src/LastFMImporter.tsx index e78668d8eb..3609034137 100644 --- a/listenbrainz/webserver/static/js/src/LastFMImporter.tsx +++ b/listenbrainz/webserver/static/js/src/LastFMImporter.tsx @@ -1,6 +1,13 @@ import * as ReactDOM from "react-dom"; import * as React from "react"; import Importer from "./Importer"; +// Importer.tsx +import APIService from "./APIService"; +import { faSpinner, faCheck } from "@fortawesome/free-solid-svg-icons"; +import { FontAwesomeIcon } from "@fortawesome/react-fontawesome"; +import { IconProp } from "@fortawesome/fontawesome-svg-core"; +import Scrobble from "./Scrobble"; + import LastFMImporterModal from "./LastFMImporterModal"; @@ -26,8 +33,36 @@ export type ImporterState = { export default class LastFmImporter extends React.Component< ImporterProps, ImporterState -> { + > { importer: any; + // Importer.tsx + APIService: APIService; + + private lastfmURL: string; + private lastfmKey: string; + + private userName: string; + private userToken: string; + + private page = 1; + private totalPages = 0; + + private playCount = -1; // the number of scrobbles reported by Last.FM + private countReceived = 0; // number of scrobbles the Last.FM API sends us, this can be diff from playCount + + private latestImportTime = 0; // the latest timestamp that we've imported earlier + private maxTimestampForImport = 0; // the latest listen found in this import + private incrementalImport = false; + + private numCompleted = 0; // number of pages completed till now + + // Variables used to honor LB's rate limit + private rlRemain = -1; + private rlReset = -1; + private rlOrigin = -1; + + public msg?: React.ReactElement; // Message to be displayed in modal + public canClose = true; constructor(props: ImporterProps) { super(props); @@ -38,6 +73,14 @@ export default class LastFmImporter extends React.Component< lastfmUsername: "", msg: "", }; + + this.APIService = new APIService( + props.apiUrl || `${window.location.origin}/1` + ); // Used to access LB API + this.lastfmURL = this.props.lastfmApiUrl; + this.lastfmKey = this.props.lastfmApiKey; + this.userName = this.props.user.name; + this.userToken = this.props.user.auth_token || ""; } handleChange = (event: React.ChangeEvent) => { @@ -48,10 +91,10 @@ export default class LastFmImporter extends React.Component< const { lastfmUsername } = this.state; this.toggleModal(); event.preventDefault(); - this.importer = new Importer(lastfmUsername, this.props); + // this.importer = new Importer(lastfmUsername, this.props); setInterval(this.updateMessage, 100); setInterval(this.setClose, 100); - this.importer.startImport(); + this.startImport(); }; toggleModal = () => { @@ -68,6 +111,264 @@ export default class LastFmImporter extends React.Component< this.setState({ msg: this.importer.msg }); }; + // Importer.tsx + + async startImport() { + this.canClose = false; // Disable the close button + this.updateMessageImporter(

Your import from Last.fm is starting!

); + this.playCount = await this.getTotalNumberOfScrobbles(); + this.latestImportTime = await this.APIService.getLatestImport( + this.userName + ); + this.incrementalImport = this.latestImportTime > 0; + this.totalPages = await this.getNumberOfPages(); + this.page = this.totalPages; // Start from the last page so that oldest scrobbles are imported first + + while (this.page > 0) { + // Fixing no-await-in-loop will require significant changes to the code, ignoring for now + const payload = await this.getPage(this.page); // eslint-disable-line + if (payload) { + // Submit only if response is valid + this.submitPage(payload); + } + + this.page -= 1; + this.numCompleted += 1; + + // Update message + const msg = ( +

+ Sending page{" "} + {this.numCompleted} of {this.totalPages} to ListenBrainz
+ + {this.incrementalImport && ( + + Note: This import will stop at the starting point of your last + import. :) +
+ + )} + Please don't close this page while this is running + +

+ ); + this.updateMessageImporter(msg); + } + + // Update latest import time on LB server + try { + this.maxTimestampForImport = Math.max( + Number(this.maxTimestampForImport), + this.latestImportTime + ); + this.APIService.setLatestImport( + this.userToken, + this.maxTimestampForImport + ); + } catch { + // console.warn("Error setting latest import timestamp, retrying in 3s"); + setTimeout( + () => + this.APIService.setLatestImport( + this.userToken, + this.maxTimestampForImport + ), + 3000 + ); + } + const finalMsg = ( +

+ Import finished +
+ + Successfully submitted {this.countReceived} listens to ListenBrainz +
+ + {/* if the count received is different from the api count, show a message accordingly + * also don't show this message if it's an incremental import, because countReceived + * and playCount will be different by definition in incremental imports + */} + {!this.incrementalImport && + this.playCount !== -1 && + this.countReceived !== this.playCount && ( + + + The number submitted listens is different from the{" "} + {this.playCount} that Last.fm reports due to an inconsistency in + their API, sorry! +
+ + + )} + + Thank you for using ListenBrainz! + +
+
+ + + Close and go to your ListenBrainz profile + + +

+ ); + this.updateMessageImporter(finalMsg); + this.canClose = true; + } + + updateMessageImporter = (msg: React.ReactElement) => { + this.msg = msg; + }; + + async getTotalNumberOfScrobbles() { + /* + * Get the total play count reported by Last.FM for user + */ + + const url = `${this.lastfmURL}?method=user.getinfo&user=${this.state.lastfmUsername}&api_key=${this.lastfmKey}&format=json`; + try { + const response = await fetch(encodeURI(url)); + const data = await response.json(); + if ("playcount" in data.user) { + return Number(data.user.playcount); + } + return -1; + } catch { + this.updateMessageImporter(

An error occurred, please try again. :(

); + this.canClose = true; // Enable the close button + const error = new Error(); + error.message = "Something went wrong"; + throw error; + } + } + + async getNumberOfPages() { + /* + * Get the total pages of data from last import + */ + + const url = `${this.lastfmURL}?method=user.getrecenttracks&user=${this.state.lastfmUsername + }&api_key=${this.lastfmKey}&from=${this.latestImportTime + 1}&format=json`; + try { + const response = await fetch(encodeURI(url)); + const data = await response.json(); + if ("recenttracks" in data) { + return Number(data.recenttracks["@attr"].totalPages); + } + return 0; + } catch (error) { + this.updateMessageImporter(

An error occurred, please try again. :(

); + this.canClose = true; // Enable the close button + return -1; + } + } + + async getPage(page: number) { + /* + * Fetch page from Last.fm + */ + + const retry = (reason: string) => { + // console.warn(`${reason} while fetching last.fm page=${page}, retrying in 3s`); + setTimeout(() => this.getPage(page), 3000); + }; + + const url = `${this.lastfmURL}?method=user.getrecenttracks&user=${this.state.lastfmUsername + }&api_key=${this.lastfmKey}&from=${this.latestImportTime + 1 + }&page=${page}&format=json`; + try { + const response = await fetch(encodeURI(url)); + if (response.ok) { + const data = await response.json(); + // Set latest import time + if ("date" in data.recenttracks.track[0]) { + this.maxTimestampForImport = Math.max( + data.recenttracks.track[0].date.uts, + this.maxTimestampForImport + ); + } else { + this.maxTimestampForImport = Math.floor(Date.now() / 1000); + } + + // Encode the page so that it can be submitted + const payload = Importer.encodeScrobbles(data); + this.countReceived += payload.length; + return payload; + } + if (/^5/.test(response.status.toString())) { + retry(`Got ${response.status}`); + } else { + // ignore 40x + // console.warn(`Got ${response.status} while fetching page last.fm page=${page}, skipping`); + } + } catch { + // Retry if there is a network error + retry("Error"); + } + return null; + } + + async submitPage(payload: Array) { + const delay = this.getRateLimitDelay(); + // Halt execution for some time + await new Promise((resolve) => { + setTimeout(resolve, delay); + }); + + const response = await this.APIService.submitListens( + this.userToken, + "import", + payload + ); + this.updateRateLimitParameters(response); + } + + static encodeScrobbles(scrobbles: LastFmScrobblePage): any { + const rawScrobbles = scrobbles.recenttracks.track; + const parsedScrobbles = Importer.map((rawScrobble: any) => { + const scrobble = new Scrobble(rawScrobble); + return scrobble.asJSONSerializable(); + }, rawScrobbles); + return parsedScrobbles; + } + + static map(applicable: (collection: any) => Listen, collection: any) { + const newCollection = []; + for (let i = 0; i < collection.length; i += 1) { + const result = applicable(collection[i]); + if (result.listened_at > 0) { + // If the 'listened_at' attribute is -1 then either the listen is invalid or the + // listen is currently playing. In both cases we need to skip the submission. + newCollection.push(result); + } + } + return newCollection; + } + + + getRateLimitDelay() { + /* Get the amount of time we should wait according to LB rate limits before making a request to LB */ + let delay = 0; + const current = new Date().getTime() / 1000; + if (this.rlReset < 0 || current > this.rlOrigin + this.rlReset) { + delay = 0; + } else if (this.rlRemain > 0) { + delay = Math.max(0, Math.ceil((this.rlReset * 1000) / this.rlRemain)); + } else { + delay = Math.max(0, Math.ceil(this.rlReset * 1000)); + } + return delay; + } + + updateRateLimitParameters(response: Response) { + /* Update the variables we use to honor LB's rate limits */ + this.rlRemain = Number(response.headers.get("X-RateLimit-Remaining")); + this.rlReset = Number(response.headers.get("X-RateLimit-Reset-In")); + this.rlOrigin = new Date().getTime() / 1000; + } + + + render() { const { show, canClose, lastfmUsername, msg } = this.state;