NadekoBot Patronage system, Search commands improvements + fixes

src/NadekoBot/Modules/Searches/Feeds/FeedsService.cs

@@ -1,6 +1,8 @@
 #nullable disable
 using CodeHollow.FeedReader;
 using CodeHollow.FeedReader.Feeds;
+using LinqToDB;
+using LinqToDB.EntityFrameworkCore;
 using Microsoft.EntityFrameworkCore;
 using NadekoBot.Db;
 using NadekoBot.Services.Database.Models;
@@ -10,11 +12,12 @@ namespace NadekoBot.Modules.Searches.Services;
 public class FeedsService : INService
 {
     private readonly DbService _db;
-    private readonly ConcurrentDictionary<string, HashSet<FeedSub>> _subs;
+    private readonly ConcurrentDictionary<string, List<FeedSub>> _subs;
     private readonly DiscordSocketClient _client;
     private readonly IEmbedBuilderService _eb;
 
     private readonly ConcurrentDictionary<string, DateTime> _lastPosts = new();
+    private readonly Dictionary<string, uint> _errorCounters = new();
 
     public FeedsService(
         Bot bot,
@@ -33,7 +36,7 @@ public class FeedsService : INService
                        .ToList()
                        .SelectMany(x => x.FeedSubs)
                        .GroupBy(x => x.Url.ToLower())
-                       .ToDictionary(x => x.Key, x => x.ToHashSet())
+                       .ToDictionary(x => x.Key, x => x.ToList())
                        .ToConcurrent();
         }
 
@@ -43,6 +46,35 @@ public class FeedsService : INService
         _ = Task.Run(TrackFeeds);
     }
 
+    private void ClearErrors(string url)
+        => _errorCounters.Remove(url);
+
+    private async Task AddError(string url, List<int> ids)
+    {
+        try
+        {
+            var newValue = _errorCounters[url] = _errorCounters.GetValueOrDefault(url) + 1;
+
+            if (newValue >= 100)
+            {
+                // remove from db
+                await using var ctx = _db.GetDbContext();
+                await ctx.GetTable<FeedSub>()
+                         .DeleteAsync(x => ids.Contains(x.Id));
+                
+                // remove from the local cache
+                _subs.TryRemove(url, out _);
+
+                // reset the error counter
+                ClearErrors(url);
+            }
+        }
+        catch (Exception ex)
+        {
+            Log.Error(ex, "Error adding rss errors...");
+        }
+    }
+
     public async Task<EmbedBuilder> TrackFeeds()
     {
         while (true)
@@ -134,13 +166,17 @@ public class FeedsService : INService
                             embed.WithDescription(desc.TrimTo(2048));
 
                         //send the created embed to all subscribed channels
-                        var feedSendTasks = kvp.Value.Where(x => x.GuildConfig is not null)
+                        var feedSendTasks = kvp.Value
+                                               .Where(x => x.GuildConfig is not null)
                                                .Select(x => _client.GetGuild(x.GuildConfig.GuildId)
                                                                    ?.GetTextChannel(x.ChannelId))
                                                .Where(x => x is not null)
                                                .Select(x => x.EmbedAsync(embed));
 
                         allSendTasks.Add(feedSendTasks.WhenAll());
+
+                        // as data retrieval was sucessful, reset error counter
+                        ClearErrors(rssUrl);
                     }
                 }
                 catch (Exception ex)
@@ -149,6 +185,8 @@ public class FeedsService : INService
                                 + "\n {Message}",
                         rssUrl,
                         $"[{ex.GetType().Name}]: {ex.Message}");
+
+                    await AddError(rssUrl, kvp.Value.Select(x => x.Id).ToList());
                 }
             }
 
@@ -188,7 +226,7 @@ public class FeedsService : INService
         foreach (var feed in gc.FeedSubs)
         {
             _subs.AddOrUpdate(feed.Url.ToLower(),
-                new HashSet<FeedSub>
+                new List<FeedSub>
                 {
                     feed
                 },
@@ -216,7 +254,7 @@ public class FeedsService : INService
             return false;
         var toRemove = items[index];
         _subs.AddOrUpdate(toRemove.Url.ToLower(),
-            new HashSet<FeedSub>(),
+            new List<FeedSub>(),
             (_, old) =>
             {
                 old.Remove(toRemove);