Merge branch 'dev'

src/FMBot.Bot/Builders/AlbumBuilders.cs

@@ -683,7 +683,7 @@ public class AlbumBuilders
         if (context.ContextUser.PrivacyLevel != PrivacyLevel.Global)
         {
             footer += $"\nYou are currently not globally visible - use " +
-                      $"'{context.Prefix}privacy global' to enable.";
+                      $"'{context.Prefix}privacy' to enable.";
         }
         if (settings.HidePrivateUsers)
         {

src/FMBot.Bot/Builders/ArtistBuilders.cs

@@ -1054,7 +1054,7 @@ public SelectMenuBuilder GetFilterSelectMenu(string customId, Guild guild, IDict
         }
         if (context.ContextUser.PrivacyLevel != PrivacyLevel.Global)
         {
-            footer.AppendLine($"You are currently not globally visible - use '{context.Prefix}privacy global' to enable.");
+            footer.AppendLine($"You are currently not globally visible - use '{context.Prefix}privacy' to enable.");
         }
 
         if (settings.HidePrivateUsers)

src/FMBot.Bot/Builders/TrackBuilders.cs

@@ -494,7 +494,7 @@ public class TrackBuilders
         }
         if (context.ContextUser.PrivacyLevel != PrivacyLevel.Global)
         {
-            footer += $"\nYou are currently not globally visible - use '{context.Prefix}privacy global' to enable.";
+            footer += $"\nYou are currently not globally visible - use '{context.Prefix}privacy' to enable.";
         }
         if (settings.HidePrivateUsers)
         {

src/FMBot.Bot/Builders/UserBuilder.cs

@@ -252,6 +252,46 @@ public async Task<ResponseModel> BotScrobblingAsync(ContextModel context)
         return response;
     }
 
+    public static ResponseModel Privacy(ContextModel context)
+    {
+        var response = new ResponseModel
+        {
+            ResponseType = ResponseType.Embed,
+        };
+
+        var privacySetting = new SelectMenuBuilder()
+                .WithPlaceholder("Select global WhoKnows privacy")
+                .WithCustomId(InteractionConstants.FmPrivacySetting)
+                .WithMinValues(1)
+                .WithMaxValues(1);
+
+        foreach (var name in Enum.GetNames(typeof(PrivacyLevel)).OrderBy(o => o))
+        {
+            privacySetting.AddOption(new SelectMenuOptionBuilder(name, name));
+        }
+
+        var builder = new ComponentBuilder()
+            .WithSelectMenu(privacySetting);
+
+        response.Components = builder;
+
+        response.Embed.WithAuthor("Configuring your global WhoKnows privacy");
+        response.Embed.WithColor(DiscordConstants.InformationColorBlue);
+
+        var embedDescription = new StringBuilder();
+
+        embedDescription.AppendLine("Use this option to change your visibility to other .fmbot users.");
+        embedDescription.AppendLine();
+
+        response.Embed.AddField("Options:",
+            "**Global**: You are visible everywhere in global WhoKnows with your Last.fm username\n" +
+            "**Server**: You are not visible in global WhoKnows, but users in the same server will still see your name.");
+
+        response.Embed.WithDescription(embedDescription.ToString());
+
+        return response;
+    }
+
     public async Task<ResponseModel> FeaturedLogAsync(ContextModel context, UserSettingsModel userSettings, FeaturedView view)
     {
         var response = new ResponseModel
@@ -988,7 +1028,7 @@ public static ResponseModel ImportMode(ContextModel context, bool hasImported =
                 .WithMinValues(1)
                 .WithMaxValues(1);
 
-        if (!hasImported)
+        if (!hasImported && context.ContextUser.DataSource == DataSource.LastFm)
         {
             importSetting.IsDisabled = true;
         }

src/FMBot.Bot/Resources/InteractionConstants.cs

@@ -8,6 +8,8 @@ public static class InteractionConstants
     public const string FmSettingFooter = "fm-footer-menu";
     public const string FmSettingFooterSupporter = "fm-footer-menu-supporter";
 
+    public const string FmPrivacySetting = "fm-gwk-privacy";
+
     public const string FmGuildSettingType = "fm-guild-type-menu";
 
     public const string SetAllowedRoleMenu = "guild-allowed-roles-menu";

src/FMBot.Bot/Services/ImportService.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.IO.Compression;
 using System.Linq;
 using System.Net.Http;
 using System.Text.Json;
@@ -35,14 +36,34 @@ public async Task<(bool success, List<SpotifyEndSongImportModel> result)> Handle
         {
             var spotifyPlays = new List<SpotifyEndSongImportModel>();
 
-            foreach (var attachment in attachments.Where(w => w?.Url != null).GroupBy(g => g.Filename))
+            foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".json")).GroupBy(g => g.Filename))
             {
                 await using var stream = await this._httpClient.GetStreamAsync(attachment.First().Url);
 
                 var result = await JsonSerializer.DeserializeAsync<List<SpotifyEndSongImportModel>>(stream);
 
                 spotifyPlays.AddRange(result);
             }
+            foreach (var attachment in attachments.Where(w => w?.Url != null && w.Filename.Contains(".zip")).GroupBy(g => g.Filename))
+            {
+                await using var stream = await this._httpClient.GetStreamAsync(attachment.First().Url);
+
+                using var zip = new ZipArchive(stream, ZipArchiveMode.Read);
+                foreach (var entry in zip.Entries.Where(w => w.Name.Contains(".json")))
+                {
+                    try
+                    {
+                        await using var zipStream = entry.Open();
+                        var result = await JsonSerializer.DeserializeAsync<List<SpotifyEndSongImportModel>>(zipStream);
+
+                        spotifyPlays.AddRange(result);
+                    }
+                    catch (Exception e)
+                    {
+                        Log.Error("Error in import .zip file ({fileName})", entry.Name, e);
+                    }
+                }
+            }
 
             return (true, spotifyPlays);
         }
@@ -85,7 +106,7 @@ public async Task<List<UserPlay>> SpotifyImportToUserPlays(int userId, List<Spot
             }
         }
 
-        Log.Information("Importing: SpotifyImportToUserPlays found {validPlays} and {invalidPlays}", userPlays.Count, invalidPlays);
+        Log.Information("Importing: SpotifyImportToUserPlays found {validPlays} valid plays and {invalidPlays} invalid plays", userPlays.Count, invalidPlays);
 
         return userPlays;
     }
@@ -131,8 +152,8 @@ public async Task InsertImportPlays(IList<UserPlay> plays)
             await using var connection = new NpgsqlConnection(this._botSettings.Database.ConnectionString);
             await connection.OpenAsync();
 
-            await PlayRepository.InsertTimeSeriesPlays(plays, connection);
-            Log.Information("Importing: Inserted {importCount} plays for {userId}", plays.Count(), plays.First().UserId);
+            var inserted = await PlayRepository.InsertTimeSeriesPlays(plays, connection);
+            Log.Information("Importing: Inserted {insertCount} plays (Should be {importCount}) for {userId}", inserted, plays.Count(), plays.First().UserId);
         }
         else
         {

src/FMBot.Bot/Services/IndexService.cs

@@ -101,7 +101,7 @@ public async Task<IndexedUserStats> IndexUser(IndexUserQueueItem queueItem)
             }
         }
 
-        Log.Information($"Starting index for {user.UserNameLastFM}");
+        Log.Information($"Index: Starting for {user.UserNameLastFM}");
         var now = DateTime.UtcNow;
 
         await using var connection = new NpgsqlConnection(this._botSettings.Database.ConnectionString);
@@ -121,13 +121,18 @@ public async Task<IndexedUserStats> IndexUser(IndexUserQueueItem queueItem)
             await PlayRepository.ReplaceAllPlays(plays, user.UserId, connection);
 
             var artists = await GetArtistsForUserFromLastFm(user);
-            await ArtistRepository.AddOrReplaceUserArtistsInDatabase(artists, user.UserId, connection);
+            var artistsInserted = await ArtistRepository.AddOrReplaceUserArtistsInDatabase(artists, user.UserId, connection);
 
             var albums = await GetAlbumsForUserFromLastFm(user);
-            await AlbumRepository.AddOrReplaceUserAlbumsInDatabase(albums, user.UserId, connection);
+            var albumsInserted = await AlbumRepository.AddOrReplaceUserAlbumsInDatabase(albums, user.UserId, connection);
 
             var tracks = await GetTracksForUserFromLastFm(user);
-            await TrackRepository.AddOrReplaceUserTracksInDatabase(tracks, user.UserId, connection);
+            var tracksInserted = await TrackRepository.AddOrReplaceUserTracksInDatabase(tracks, user.UserId, connection);
+
+            Log.Information("Index complete for {userId}: Artists found {artistCount}, inserted {artistsInserted} - " +
+                            "Albums found {albumCount}, inserted {albumsInserted} - "+
+                            "Tracks found {trackCount}, inserted {tracksInserted}",
+                user.UserId, artists.Count, artistsInserted, albums.Count, albumsInserted, tracks.Count, tracksInserted);
 
             var latestScrobbleDate = await GetLatestScrobbleDate(user);
 
@@ -148,7 +153,7 @@ public async Task<IndexedUserStats> IndexUser(IndexUserQueueItem queueItem)
         }
         catch (Exception e)
         {
-            Console.WriteLine(e);
+            Log.Error("Index: Error happened!", e);
             throw;
         }
     }
@@ -670,7 +675,7 @@ public async Task<IReadOnlyList<User>> GetOutdatedUsers(DateTime timeLastIndexed
     {
         await using var db = await this._contextFactory.CreateDbContextAsync();
 
-        var recentlyUsed = DateTime.UtcNow.AddDays(-2);
+        var recentlyUsed = DateTime.UtcNow.AddDays(-7);
         return await db.Users
             .AsQueryable()
             .Where(f => f.LastIndexed != null &&

src/FMBot.Bot/Services/SupporterService.cs

@@ -662,7 +662,9 @@ public async Task UpdateDiscordSupporters()
             .ToListAsync();
 
         var discordUsersLeft = existingSupporters
+            .OrderByDescending(o => o.LastPayment)
             .Select(s => s.DiscordUserId.Value)
+            .Distinct()
             .ToHashSet();
 
         foreach (var discordSupporter in discordSupporters)

src/FMBot.Bot/Services/TimerService.cs

@@ -183,10 +183,10 @@ public async Task AddUsersToIndexQueue()
         }
 
         Log.Information("Getting users to index");
-        var timeToIndex = DateTime.UtcNow.AddDays(-this._botSettings.LastFm.UserIndexFrequencyInDays.Value);
+        var timeToIndex = DateTime.UtcNow.AddDays(-15);
 
         var usersToUpdate = (await this._indexService.GetOutdatedUsers(timeToIndex))
-            .Take(300)
+            .Take(2000)
             .ToList();
 
         Log.Information($"Found {usersToUpdate.Count} outdated users, adding them to index queue");

src/FMBot.Bot/Services/UserService.cs

@@ -781,19 +781,21 @@ public async Task<PrivacyLevel> SetPrivacy(User userToUpdate, string[] extraOpti
         return userToUpdate.PrivacyLevel;
     }
 
-    public async Task<PrivacyLevel> SetPrivacyLevel(User userToUpdate, PrivacyLevel privacyLevel)
+    public async Task<PrivacyLevel> SetPrivacyLevel(int userId, PrivacyLevel privacyLevel)
     {
         await using var db = await this._contextFactory.CreateDbContextAsync();
 
-        userToUpdate.PrivacyLevel = privacyLevel;
+        var user = await db.Users.FirstAsync(f => f.UserId == userId);
 
-        db.Update(userToUpdate);
+        user.PrivacyLevel = privacyLevel;
+
+        db.Update(user);
 
         await db.SaveChangesAsync();
 
-        this._cache.Remove(UserCacheKey(userToUpdate.DiscordUserId));
+        this._cache.Remove(UserCacheKey(user.DiscordUserId));
 
-        return userToUpdate.PrivacyLevel;
+        return user.PrivacyLevel;
     }
 
     public static User SetWkMode(User userSettings, string[] extraOptions)

src/FMBot.Bot/SlashCommands/ImportSlashCommands.cs

@@ -48,7 +48,7 @@ public class ImportSlashCommands : InteractionModuleBase
         this.Interactivity = interactivity;
     }
 
-    private const string SpotifyFileDescription = "Spotify endsong.json file";
+    private const string SpotifyFileDescription = "Spotify history package (.zip) or history files (.json) ";
 
     [SlashCommand("spotify", "Import your Spotify history (Beta)")]
     [UsernameSetRequired]
@@ -114,14 +114,14 @@ public class ImportSlashCommands : InteractionModuleBase
 
             description.AppendLine("### Importing your data into .fmbot");
             description.AppendLine("1. Download the file Spotify provided");
-            description.AppendLine("2. Extract the `.zip` file so you have multiple `endsong_x.json` files ready");
-            description.AppendLine("3. Use this command and add each file as an attachment through the options");
+            description.AppendLine("2. Use this command and add the `.zip` file as an attachment through the options");
+            description.AppendLine("3. Having issues? You can also attach each `.json` file separately");
 
             description.AppendLine("### Notes");
             description.AppendLine("- We filter out duplicates, so don't worry about submitting the same file twice");
             description.AppendLine("- Spotify files includes plays that you skipped quickly, we filter those out as well");
             description.AppendLine("- You can select what from your import you want to use with `/import manage`");
-            description.AppendLine("- Discord mobile currently has an issue where it corrupts any `.json` file you send through it. Please only use this command with Discord desktop.");
+            description.AppendLine("- Discord mobile currently has an issue where it corrupts any `.json` file you send through it. Attach the `.zip` instead, or try using Discord desktop");
 
             var years = await this.GetImportedYears(contextUser.UserId);
             if (years.Length > 0)
@@ -151,7 +151,7 @@ public class ImportSlashCommands : InteractionModuleBase
             if (!imports.success)
             {
                 embed.WithColor(DiscordConstants.WarningColorOrange);
-                await UpdateImportEmbed(message, embed, description, $"- ❌ Invalid Spotify import file. Make sure you select the right files, for example `endsong_1.json`." , true);
+                await UpdateImportEmbed(message, embed, description, $"- ❌ Invalid Spotify import file. Make sure you select the right files, for example `my_spotify_data.zip` or `Streaming_History_Audio_x.json`." , true);
                 this.Context.LogCommandUsed(CommandResponse.WrongInput);
                 return;
             }
@@ -162,14 +162,14 @@ public class ImportSlashCommands : InteractionModuleBase
                 {
                     embed.WithColor(DiscordConstants.WarningColorOrange);
                     await UpdateImportEmbed(message, embed, description, $"❌ Invalid Spotify import file. We can only process files that are from the ['Extended Streaming History'](https://www.spotify.com/us/account/privacy/) package.\n\n" +
-                                                                         $"The files should have the name `endsong_x.json`. Files that are named `StreamingHistory.json` are incomplete and can't be processed.\n\n" +
+                                                                         $"The files should have names like `my_spotify_data.zip` or `Streaming_History_Audio_x.json`.\n\n" +
                                                                          $"The right files can take some more time to get, but actually contain your full Spotify history. Sorry for the inconvenience.", true);
                     this.Context.LogCommandUsed(CommandResponse.WrongInput);
                     return;
                 }
 
                 embed.WithColor(DiscordConstants.WarningColorOrange);
-                await UpdateImportEmbed(message, embed, description, $"- ❌ Invalid Spotify import file (contains no plays). Make sure you select the right files, for example `endsong_1.json`.\n\n" +
+                await UpdateImportEmbed(message, embed, description, $"- ❌ Invalid Spotify import file (contains no plays). Make sure you select the right files, for example `my_spotify_data.zip` or `Streaming_History_Audio_x.json`.\n\n" +
                                                                      $"The Discord mobile app currently empties all `.json` files you send through it. We've reported this to them, try using Discord on desktop in the meantime.", true);
                 this.Context.LogCommandUsed(CommandResponse.WrongInput);
                 return;

src/FMBot.Bot/SlashCommands/UserSlashCommands.cs

@@ -177,32 +177,31 @@ public async Task LoginAsync()
 
     [SlashCommand("privacy", "Changes your visibility to other .fmbot users in Global WhoKnows")]
     [UsernameSetRequired]
-    public async Task PrivacyAsync([Summary("level", "Privacy level for your .fmbot account")] PrivacyLevel privacyLevel)
+    public async Task PrivacyAsync()
     {
-        var userSettings = await this._userService.GetUserSettingsAsync(this.Context.User);
-
-        var newPrivacyLevel = await this._userService.SetPrivacyLevel(userSettings, privacyLevel);
+        var contextUser = await this._userService.GetUserSettingsAsync(this.Context.User);
 
-        var reply = new StringBuilder();
-        reply.AppendLine($"Your privacy level has been set to **{newPrivacyLevel}**.");
-        reply.AppendLine();
+        var response = UserBuilder.Privacy(new ContextModel(this.Context, contextUser));
 
-        if (newPrivacyLevel == PrivacyLevel.Global)
-        {
-            reply.AppendLine("You will now be visible in the global WhoKnows with your Last.fm username.");
-        }
-        if (newPrivacyLevel == PrivacyLevel.Server)
-        {
-            reply.AppendLine("You will not be visible in the global WhoKnows with your Last.fm username, but users you share a server with will still see it.");
-        }
+        await this.Context.SendResponse(this.Interactivity, response, ephemeral: true);
+        this.Context.LogCommandUsed(response.CommandResponse);
+    }
 
+    [ComponentInteraction(InteractionConstants.FmPrivacySetting)]
+    [UsernameSetRequired]
+    public async Task SetPrivacy(string[] inputs)
+    {
         var embed = new EmbedBuilder();
-        embed.WithColor(DiscordConstants.InformationColorBlue);
-        embed.WithDescription(reply.ToString());
+        var userSettings = await this._userService.GetUserSettingsAsync(this.Context.User);
 
-        await RespondAsync(null, new[] { embed.Build() }, ephemeral: true);
+        if (Enum.TryParse(inputs.FirstOrDefault(), out PrivacyLevel privacyLevel))
+        {
+            var newPrivacyLevel = await this._userService.SetPrivacyLevel(userSettings.UserId, privacyLevel);
 
-        this.Context.LogCommandUsed();
+            embed.WithDescription($"Your privacy level has been set to **{newPrivacyLevel}**.");
+            embed.WithColor(DiscordConstants.InformationColorBlue);
+            await RespondAsync(embed: embed.Build(), ephemeral: true);
+        }
     }
 
     [SlashCommand("fmmode", "Changes your '/fm' layout")]