Merge #3786 Skip duplicate repo URLs during update

CHANGELOG.md

@@ -19,6 +19,7 @@ All notable changes to this project will be documented in this file.
 - [Core] Fix handling of empty builds.json file (#3733 by: HebaruSan; reviewed: DasSkelett)
 - [Core] Fix FIPS-mode exceptions on Windows for SHA256 (#3774 by: HebaruSan; reviewed: techman83)
 - [Core] Support cancellation of download checksums (#3778 by: HebaruSan; reviewed: techman83)
+- [Core] Skip duplicate repo URLs during update (#3786 by: HebaruSan; reviewed: techman83)
 
 ### Internal
 

Core/Extensions/EnumerableExtensions.cs

@@ -68,7 +68,6 @@ public static TimeSpan Sum(this IEnumerable<TimeSpan> source)
             => source.Aggregate(TimeSpan.Zero,
                                 (a, b) => a + b);
 
-
         /// <summary>
         /// Select : SelectMany :: Zip : ZipMany
         /// </summary>
@@ -79,6 +78,15 @@ public static TimeSpan Sum(this IEnumerable<TimeSpan> source)
         public static IEnumerable<V> ZipMany<T, U, V>(this IEnumerable<T> seq1, IEnumerable<U> seq2, Func<T, U, IEnumerable<V>> func)
             => seq1.Zip(seq2, func).SelectMany(seqs => seqs);
 
+        /// <summary>
+        /// Eliminate duplicate elements based on the value returned by a callback
+        /// </summary>
+        /// <param name="seq">Sequence of elements to check</param>
+        /// <param name="func">Function to return unique value per element</param>
+        /// <returns>Sequence where each element has a unique return value</returns>
+        public static IEnumerable<T> DistinctBy<T, U>(this IEnumerable<T> seq, Func<T, U> func)
+            => seq.GroupBy(func).Select(grp => grp.First());
+
     }
 
     /// <summary>

Core/Net/Repo.cs

@@ -37,7 +37,9 @@ public static class Repo
         /// </summary>
         public static RepoUpdateResult UpdateAllRepositories(RegistryManager registry_manager, GameInstance ksp, NetAsyncDownloader downloader, NetModuleCache cache, IUser user)
         {
-            var repos = registry_manager.registry.Repositories.Values.ToArray();
+            var repos = registry_manager.registry.Repositories.Values
+                .DistinctBy(r => r.uri)
+                .ToArray();
 
             // Get latest copy of the game versions data (remote build map)
             user.RaiseMessage(Properties.Resources.NetRepoUpdatingBuildMap);
@@ -139,7 +141,7 @@ private static List<CkanModule> ModulesFromTarGz(Repository repo, string path, r
             // Create a handle for the tar stream
             using (TarInputStream tarStream = new TarInputStream(gzipStream, Encoding.UTF8))
             {
-                user.RaiseMessage("Loading modules from {0} repository...", repo.name);
+                user.RaiseMessage(Properties.Resources.NetRepoLoadingModulesFromRepo, repo.name);
                 TarEntry entry;
                 int prevPercent = 0;
                 while ((entry = tarStream.GetNextEntry()) != null)
@@ -150,15 +152,17 @@ private static List<CkanModule> ModulesFromTarGz(Repository repo, string path, r
                     {
                         downloadCounts = JsonConvert.DeserializeObject<SortedDictionary<string, int>>(
                             tarStreamString(tarStream, entry));
-                        user.RaiseMessage("Loaded download counts from {0} repository", repo.name);
+                        user.RaiseMessage(Properties.Resources.NetRepoLoadedDownloadCounts, repo.name);
                     }
                     else if (filename.EndsWith(".ckan"))
                     {
                         log.DebugFormat("Reading CKAN data from {0}", filename);
                         var percent = (int)(100 * inputStream.Position / inputStream.Length);
                         if (percent > prevPercent)
                         {
-                            user.RaiseProgress($"Loading modules from {repo.name} repository", percent);
+                            user.RaiseProgress(
+                                string.Format(Properties.Resources.NetRepoLoadingModulesFromRepo, repo.name),
+                                percent);
                             prevPercent = percent;
                         }
 
@@ -227,7 +231,9 @@ private static List<CkanModule> ModulesFromZip(Repository repo, string path, IUs
                         var percent = (int)(100 * index / zipfile.Count);
                         if (percent > prevPercent)
                         {
-                            user.RaiseProgress($"Loading modules from {repo.name} repository", percent);
+                            user.RaiseProgress(
+                                string.Format(Properties.Resources.NetRepoLoadingModulesFromRepo, repo.name),
+                                percent);
                         }
 
                         // Read each file into a string.

Core/Properties/Resources.resx

@@ -155,6 +155,8 @@ You should reinstall them in order to preserve consistency with the repository.
 
 Do you wish to reinstall now?</value></data>
   <data name="NetRepoInconsistenciesHeader" xml:space="preserve"><value>The following inconsistencies were found:</value></data>
+  <data name="NetRepoLoadingModulesFromRepo" xml:space="preserve"><value>Loading modules from {0} repository...</value></data>
+  <data name="NetRepoLoadedDownloadCounts" xml:space="preserve"><value>Loaded download counts from {0} repository</value></data>
   <data name="JsonRelationshipConverterAnyOfCombined" xml:space="preserve"><value>`any_of` should not be combined with `{0}`</value></data>
   <data name="RegistryFileConflict" xml:space="preserve"><value>{0} wishes to install {1}, but this file is registered to {2}</value></data>
   <data name="RegistryFileNotRemoved" xml:space="preserve"><value>{0} is registered to {1} but has not been removed!</value></data>