Upped version to 4.2.15

added redis as optional to install in windows guide docs

See merge request Kwoth/nadekobot!258

CHANGELOG.md

@@ -2,6 +2,15 @@
 
 Experimental changelog. Mostly based on [keepachangelog](https://keepachangelog.com/en/1.0.0/) except date format. a-c-f-r-o
 
+## [4.2.15] - 12.07.2022
+
+### Fixed
+
+- Fixed `.nh*ntai` nsfw command
+- Xp Freezes may have been fixed
+- `data/images.yml` should once again support local file paths
+- Fixed multiword aliases
+
 ## [4.2.14] - 03.07.2022
 
 ### Added

docs/guides/windows-guide.md

@@ -31,7 +31,8 @@
  ![Create a new bot](https://i.imgur.com/JxtRk9e.png "Create a new bot")
 - Click on **`DOWNLOAD`** at the lower right
  ![Bot Setup](https://i.imgur.com/HqAl36p.png "Bot Setup")
-- Click on **`Install`** next to **`Redis`**.
+- Click on **`Install`** next to **`Redis`**. 
+- **(Note: Redis is optional unless you are are using the bot on 2000+ servers)**
 - Note: If Redis fails to install, install Redis manually here: [Redis Installer](https://github.com/MicrosoftArchive/redis/releases/tag/win-3.0.504) Download and run the **`.msi`** file.
 - If you will use the music module, click on **`Install`** next to **`FFMPEG`** and **`Youtube-DL`**.
 - If any dependencies fail to install, you can temporarily disable your Windows Defender/AV until you install them. If you don't want to, then read [the last section of this guide](#Manual-Prerequisite-Installation).

src/NadekoBot/Common/QueueRunner.cs

@@ -57,6 +57,6 @@ public sealed class QueueRunner
         }
     }
     
-    public ValueTask Enqueue(Func<Task> action)
+    public ValueTask EnqueueAsync(Func<Task> action)
         => _channel.Writer.WriteAsync(action);
 }

src/NadekoBot/Modules/Administration/Self/SelfCommands.cs

@@ -335,7 +335,8 @@ public partial class Administration
         {
             try
             {
-                await _client.SetStatusAsync(UserStatus.DoNotDisturb);
+                await _client.SetStatusAsync(UserStatus.Invisible);
+                _ =  _client.StopAsync();
                 await ReplyConfirmLocalizedAsync(strs.shutting_down);
             }
             catch

src/NadekoBot/Modules/Gambling/Gambling.cs

@@ -138,7 +138,7 @@ public partial class Gambling : GamblingModule<GamblingService>
             var tt = TimestampTag.FromDateTime(when, TimestampTagStyles.Relative);
 
             await _remind.AddReminderAsync(ctx.User.Id,
-                ctx.Channel.Id,
+                ctx.User.Id,
                 ctx.Guild.Id,
                 true,
                 when,

src/NadekoBot/Modules/Nsfw/Nhentai/INhentaiService.cs

@@ -0,0 +1,9 @@
+using NadekoBot.Modules.Searches.Common;
+
+namespace NadekoBot.Modules.Nsfw;
+
+public interface INhentaiService
+{
+    Task<Gallery?> GetAsync(uint id);
+    Task<IReadOnlyList<uint>> GetIdsBySearchAsync(string search);
+}

src/NadekoBot/Modules/Nsfw/Nhentai/NhentaiScraperService.cs

@@ -0,0 +1,115 @@
+using AngleSharp.Html.Dom;
+using AngleSharp.Html.Parser;
+using NadekoBot.Modules.Searches.Common;
+
+namespace NadekoBot.Modules.Nsfw;
+
+public sealed class NhentaiScraperService : INhentaiService, INService
+{
+    private readonly IHttpClientFactory _httpFactory;
+
+    private static readonly HtmlParser _htmlParser = new(new()
+    {
+        IsScripting = false,
+        IsEmbedded = false,
+        IsSupportingProcessingInstructions = false,
+        IsKeepingSourceReferences = false,
+        IsNotSupportingFrames = true
+    });
+
+    public NhentaiScraperService(IHttpClientFactory httpFactory)
+    {
+        _httpFactory = httpFactory;
+    }
+
+    private HttpClient GetHttpClient()
+    {
+        var http = _httpFactory.CreateClient();
+        http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.5060.114 Safari/537.36");
+        http.DefaultRequestHeaders.Add("Cookie", "cf_clearance=I5pR71P4wJkRBFTLFjBndI.GwfKwT.Gx06uS8XNmRJo-1657214595-0-150; csrftoken=WMWRLtsQtBVQYvYkbqXKJHI9T1JwWCdd3tNhoxHn7aHLUYHAqe60XFUKAoWsJtda");
+        return http;
+    }
+    
+    public async Task<Gallery?> GetAsync(uint id)
+    {
+        using var http = GetHttpClient();
+        try
+        {
+            var url = $"https://nhentai.net/g/{id}/";
+            var strRes = await http.GetStringAsync(url);
+            var doc = await _htmlParser.ParseDocumentAsync(strRes);
+
+            var title = doc.QuerySelector("#info .title")?.TextContent;
+            var fullTitle = doc.QuerySelector("meta[itemprop=\"name\"]")?.Attributes["content"]?.Value
+                            ?? title;
+            var thumb = (doc.QuerySelector("#cover a img") as IHtmlImageElement)?.Dataset["src"];
+
+            var tagsElem = doc.QuerySelector("#tags");
+            
+            var pageCount = tagsElem?.QuerySelector("a.tag[href^=\"/search/?q=pages\"] span")?.TextContent;
+            var likes = doc.QuerySelector(".buttons .btn-disabled.btn.tooltip span span")?.TextContent?.Trim('(', ')');
+            var uploadedAt = (tagsElem?.QuerySelector(".tag-container .tags time.nobold") as IHtmlTimeElement)?.DateTime;
+
+            var tags = tagsElem?.QuerySelectorAll(".tag-container .tags > a.tag[href^=\"/tag\"]")
+                .Cast<IHtmlAnchorElement>()
+                .Select(x => new Tag()
+                {
+                    Name = x.QuerySelector("span:first-child")?.TextContent,
+                    Url = $"https://nhentai.net{x.PathName}"
+                })
+                .ToArray();
+
+            if (string.IsNullOrWhiteSpace(fullTitle))
+                return null;
+
+            if (!int.TryParse(pageCount, out var pc))
+                return null;
+
+            if (!int.TryParse(likes, out var lc))
+                return null;
+
+            if (!DateTime.TryParse(uploadedAt, out var ua))
+                return null;
+
+            return new Gallery(id,
+                url,
+                fullTitle,
+                title,
+                thumb,
+                pc,
+                lc,
+                ua,
+                tags);
+        }
+        catch (HttpRequestException)
+        {
+            Log.Warning("Nhentai with id {NhentaiId} not found", id);
+            return null;
+        }
+    }
+
+    public async Task<IReadOnlyList<uint>> GetIdsBySearchAsync(string search)
+    {
+        using var http = GetHttpClient();
+        try
+        {
+            var url = $"https://nhentai.net/search/?q={Uri.EscapeDataString(search)}&sort=popular-today";
+            var strRes = await http.GetStringAsync(url);
+            var doc = await _htmlParser.ParseDocumentAsync(strRes);
+
+            var elems = doc.QuerySelectorAll(".container .gallery a")
+                .Cast<IHtmlAnchorElement>()
+                .Where(x => x.PathName.StartsWith("/g/"))
+                .Select(x => x.PathName[3..^1])
+                .Select(uint.Parse)
+                .ToArray();
+            
+            return elems;
+        }
+        catch (HttpRequestException)
+        {
+            Log.Warning("Nhentai search for {NhentaiSearch} failed", search);
+            return Array.Empty<uint>();
+        }
+    }
+}

src/NadekoBot/Modules/Nsfw/Nsfw.cs

@@ -404,15 +404,19 @@ public partial class NSFW : NadekoModule<ISearchImagesService>
                          .Join(" ");
 
         var embed = _eb.Create()
-                       .WithTitle(g.Title)
-                       .WithDescription(g.FullTitle)
-                       .WithImageUrl(g.Thumbnail)
-                       .WithUrl(g.Url)
-                       .AddField(GetText(strs.favorites), g.Likes, true)
-                       .AddField(GetText(strs.pages), g.PageCount, true)
-                       .AddField(GetText(strs.tags), tagString, true)
-                       .WithFooter(g.UploadedAt.ToString("f"))
-                       .WithOkColor();
+            .WithTitle(g.Title)
+            .WithDescription(g.FullTitle)
+            .WithImageUrl(g.Thumbnail)
+            .WithUrl(g.Url)
+            .AddField(GetText(strs.favorites), g.Likes, true)
+            .AddField(GetText(strs.pages), g.PageCount, true)
+            .AddField(GetText(strs.tags),
+                string.IsNullOrWhiteSpace(tagString)
+                    ? "?"
+                    : tagString,
+                true)
+            .WithFooter(g.UploadedAt.ToString("f"))
+            .WithOkColor();
 
         await ctx.Channel.EmbedAsync(embed);
     }

src/NadekoBot/Modules/Nsfw/NsfwService.cs

@@ -1,10 +0,0 @@
-#nullable disable
-namespace NadekoBot.Modules.Nsfw;
-
-public interface INsfwService
-{
-}
-
-public class NsfwService
-{
-}

src/NadekoBot/Modules/Nsfw/SearchImagesService.cs

@@ -1,21 +1,11 @@
-#nullable disable
+#nullable disable warnings
 using LinqToDB;
 using NadekoBot.Modules.Nsfw.Common;
 using NadekoBot.Modules.Searches.Common;
-using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 
 namespace NadekoBot.Modules.Nsfw;
 
-public record UrlReply
-{
-    public string Error { get; init; }
-    public string Url { get; init; }
-    public string Rating { get; init; }
-    public string Provider { get; init; }
-    public List<string> Tags { get; } = new();
-}
-
 public class SearchImagesService : ISearchImagesService, INService
 {
     private ConcurrentDictionary<ulong, HashSet<string>> BlacklistedTags { get; }
@@ -23,18 +13,22 @@ public class SearchImagesService : ISearchImagesService, INService
     public ConcurrentDictionary<ulong, Timer> AutoHentaiTimers { get; } = new();
     public ConcurrentDictionary<ulong, Timer> AutoBoobTimers { get; } = new();
     public ConcurrentDictionary<ulong, Timer> AutoButtTimers { get; } = new();
+    
     private readonly Random _rng;
     private readonly SearchImageCacher _cache;
     private readonly IHttpClientFactory _httpFactory;
     private readonly DbService _db;
+    private readonly INhentaiService _nh;
 
     private readonly object _taglock = new();
 
     public SearchImagesService(
         DbService db,
         SearchImageCacher cacher,
-        IHttpClientFactory httpFactory)
+        IHttpClientFactory httpFactory,
+        INhentaiService nh)
     {
+        _nh = nh;
         _db = db;
         _rng = new NadekoRandom();
         _cache = cacher;
@@ -284,85 +278,18 @@ public class SearchImagesService : ISearchImagesService, INService
 
     #region Nhentai
 
-    private string GetNhentaiExtensionInternal(string s)
-        => s switch
-        {
-            "j" => "jpg",
-            "p" => "png",
-            "g" => "gif",
-            _ => "jpg"
-        };
+    public Task<Gallery?> GetNhentaiByIdAsync(uint id)
+        => _nh.GetAsync(id);
 
-    private Gallery ModelToGallery(NhentaiApiModel.Gallery model)
+    public async Task<Gallery?> GetNhentaiBySearchAsync(string search)
     {
-        var thumbnail = $"https://t.nhentai.net/galleries/{model.MediaId}/thumb."
-                        + GetNhentaiExtensionInternal(model.Images.Thumbnail.T);
+        var ids = await _nh.GetIdsBySearchAsync(search);
 
-        var url = $"https://nhentai.net/g/{model.Id}";
-        return new(model.Id.ToString(),
-            url,
-            model.Title.English,
-            model.Title.Pretty,
-            thumbnail,
-            model.NumPages,
-            model.NumFavorites,
-            model.UploadDate.ToUnixTimestamp().UtcDateTime,
-            model.Tags.Map(x => new Tag
-            {
-                Name = x.Name,
-                Url = "https://nhentai.com/" + x.Url
-            }));
-    }
-
-    private async Task<NhentaiApiModel.Gallery> GetNhentaiByIdInternalAsync(uint id)
-    {
-        using var http = _httpFactory.CreateClient();
-        try
-        {
-            var res = await http.GetStringAsync("https://nhentai.net/api/gallery/" + id);
-            return JsonConvert.DeserializeObject<NhentaiApiModel.Gallery>(res);
-        }
-        catch (HttpRequestException)
-        {
-            Log.Warning("Nhentai with id {NhentaiId} not found", id);
+        if (ids.Count == 0)
             return null;
-        }
-    }
-
-    private async Task<NhentaiApiModel.Gallery[]> SearchNhentaiInternalAsync(string search)
-    {
-        using var http = _httpFactory.CreateClient();
-        try
-        {
-            var res = await http.GetStringAsync("https://nhentai.net/api/galleries/search?query=" + search);
-            return JsonConvert.DeserializeObject<NhentaiApiModel.SearchResult>(res).Result;
-        }
-        catch (HttpRequestException)
-        {
-            Log.Warning("Nhentai with search {NhentaiSearch} not found", search);
-            return null;
-        }
-    }
-
-    public async Task<Gallery> GetNhentaiByIdAsync(uint id)
-    {
-        var model = await GetNhentaiByIdInternalAsync(id);
-
-        return ModelToGallery(model);
-    }
-
-    private static readonly string[] _bannedTags = { "loli", "lolicon", "shota", "shotacon", "cub" };
-
-    public async Task<Gallery> GetNhentaiBySearchAsync(string search)
-    {
-        var models = await SearchNhentaiInternalAsync(search);
-
-        models = models.Where(x => !x.Tags.Any(t => _bannedTags.Contains(t.Name))).ToArray();
-
-        if (models.Length == 0)
-            return null;
-
-        return ModelToGallery(models[_rng.Next(0, models.Length)]);
+        
+        var id = ids[_rng.Next(0, ids.Count)];
+        return await _nh.GetAsync(id);
     }
 
     #endregion

src/NadekoBot/Modules/Nsfw/UrlReply.cs

@@ -0,0 +1,10 @@
+namespace NadekoBot.Modules.Nsfw;
+
+public record UrlReply
+{
+    public string Error { get; init; }
+    public string Url { get; init; }
+    public string Rating { get; init; }
+    public string Provider { get; init; }
+    public List<string> Tags { get; } = new();
+}

src/NadekoBot/Modules/Searches/_Common/Gallery.cs

@@ -9,7 +9,7 @@ public sealed class Tag
 
 public sealed class Gallery
 {
-    public string Id { get; }
+    public uint Id { get; }
     public string Url { get; }
     public string FullTitle { get; }
     public string Title { get; }
@@ -21,7 +21,7 @@ public sealed class Gallery
 
 
     public Gallery(
-        string id,
+        uint id,
         string url,
         string fullTitle,
         string title,

src/NadekoBot/Modules/Searches/_Common/NhentaiApiModel.cs

@@ -1,121 +0,0 @@
-#nullable disable
-using Newtonsoft.Json;
-
-namespace NadekoBot.Modules.Searches.Common;
-
-public static class NhentaiApiModel
-{
-    public class Title
-    {
-        [JsonProperty("english")]
-        public string English { get; set; }
-
-        [JsonProperty("japanese")]
-        public string Japanese { get; set; }
-
-        [JsonProperty("pretty")]
-        public string Pretty { get; set; }
-    }
-
-    public class Page
-    {
-        [JsonProperty("t")]
-        public string T { get; set; }
-
-        [JsonProperty("w")]
-        public int W { get; set; }
-
-        [JsonProperty("h")]
-        public int H { get; set; }
-    }
-
-    public class Cover
-    {
-        [JsonProperty("t")]
-        public string T { get; set; }
-
-        [JsonProperty("w")]
-        public int W { get; set; }
-
-        [JsonProperty("h")]
-        public int H { get; set; }
-    }
-
-    public class Thumbnail
-    {
-        [JsonProperty("t")]
-        public string T { get; set; }
-
-        [JsonProperty("w")]
-        public int W { get; set; }
-
-        [JsonProperty("h")]
-        public int H { get; set; }
-    }
-
-    public class Images
-    {
-        [JsonProperty("pages")]
-        public List<Page> Pages { get; set; }
-
-        [JsonProperty("cover")]
-        public Cover Cover { get; set; }
-
-        [JsonProperty("thumbnail")]
-        public Thumbnail Thumbnail { get; set; }
-    }
-
-    public class Tag
-    {
-        [JsonProperty("id")]
-        public int Id { get; set; }
-
-        [JsonProperty("type")]
-        public string Type { get; set; }
-
-        [JsonProperty("name")]
-        public string Name { get; set; }
-
-        [JsonProperty("url")]
-        public string Url { get; set; }
-
-        [JsonProperty("count")]
-        public int Count { get; set; }
-    }
-
-    public class Gallery
-    {
-        [JsonProperty("id")]
-        public int Id { get; set; }
-
-        [JsonProperty("media_id")]
-        public string MediaId { get; set; }
-
-        [JsonProperty("title")]
-        public Title Title { get; set; }
-
-        [JsonProperty("images")]
-        public Images Images { get; set; }
-
-        [JsonProperty("scanlator")]
-        public string Scanlator { get; set; }
-
-        [JsonProperty("upload_date")]
-        public double UploadDate { get; set; }
-
-        [JsonProperty("tags")]
-        public Tag[] Tags { get; set; }
-
-        [JsonProperty("num_pages")]
-        public int NumPages { get; set; }
-
-        [JsonProperty("num_favorites")]
-        public int NumFavorites { get; set; }
-    }
-
-    public class SearchResult
-    {
-        [JsonProperty("result")]
-        public Gallery[] Result { get; set; }
-    }
-}

src/NadekoBot/Modules/Utility/CommandMap/CommandMapService.cs

@@ -57,45 +57,41 @@ public class CommandMapService : IInputTransformer, INService
         
         if (AliasMaps.TryGetValue(guild.Id, out var maps))
         {
-            string word;
-            var index = input.IndexOf(' ', StringComparison.InvariantCulture);
-            if (index == -1)
+            string newInput = null;
+            foreach (var (k, v) in maps)
             {
-                word = input;
-            }
-            else
-            {
-                word = input[..index];
-            }
-
-            string newInput;
-            if (maps.TryGetValue(word, out var alias))
-            {
-                if (index == -1)
-                    newInput = alias;
-                else
-                    newInput = alias + ' ' + input[index..];
-            }
-            else
-            {
-                return null;
-            }
-            
-            try
-            {
-                var toDelete = await channel.SendConfirmAsync(_eb, $"{input} => {newInput}");
-                _ = Task.Run(async () =>
+                if (string.Equals(input, k, StringComparison.OrdinalIgnoreCase))
                 {
-                    await Task.Delay(1500);
-                    await toDelete.DeleteAsync(new()
-                    {
-                        RetryMode = RetryMode.AlwaysRetry
-                    });
-                });
-            }
-            catch { }
+                    newInput = v;
+                }
+                else if (input.StartsWith(k + ' ', StringComparison.OrdinalIgnoreCase))
+                {
+                    newInput = v + ' ' + input[k.Length..];
+                }
 
-            return newInput;
+                if (newInput is not null)
+                {
+                    try
+                    {
+                        var toDelete = await channel.SendConfirmAsync(_eb, $"{input} => {newInput}");
+                        _ = Task.Run(async () =>
+                        {
+                            await Task.Delay(1500);
+                            await toDelete.DeleteAsync(new()
+                            {
+                                RetryMode = RetryMode.AlwaysRetry
+                            });
+                        });
+                    }
+                    catch
+                    {
+                    }
+
+                    return newInput;
+                }
+            }
+
+            return null;
 
             // var keys = maps.Keys.OrderByDescending(x => x.Length);
             // foreach (var k in keys)

src/NadekoBot/Modules/Utility/Remind/RemindService.cs

@@ -180,7 +180,7 @@ public class RemindService : INService, IReadyExecutor
     }
 
     public async Task AddReminderAsync(ulong userId,
-        ulong channelId,
+        ulong targetId,
         ulong? guildId,
         bool isPrivate,
         DateTime time,
@@ -189,15 +189,16 @@ public class RemindService : INService, IReadyExecutor
         var rem = new Reminder
         {
             UserId = userId,
-            ChannelId = channelId,
+            ChannelId = targetId,
             ServerId = guildId ?? 0,
             IsPrivate = isPrivate,
             When = time,
-            Message = message,
+            Message = message, 
         };
 
         await using var ctx = _db.GetDbContext();
         await ctx.Reminders
                  .AddAsync(rem);
+        await ctx.SaveChangesAsync();
     }
 }

src/NadekoBot/Modules/Utility/StreamRole/StreamRoleService.cs

@@ -221,7 +221,7 @@ public class StreamRoleService : IReadyExecutor, INService
     }
 
     private async ValueTask RescanUser(IGuildUser user, StreamRoleSettings setting, IRole addRole = null)
-        => await _queueRunner.Enqueue(() => RescanUserInternal(user, setting, addRole));
+        => await _queueRunner.EnqueueAsync(() => RescanUserInternal(user, setting, addRole));
 
     private async Task RescanUserInternal(IGuildUser user, StreamRoleSettings setting, IRole addRole = null)
     {
@@ -239,7 +239,7 @@ public class StreamRoleService : IReadyExecutor, INService
             && setting.Blacklist.All(x => x.UserId != user.Id)
             && user.RoleIds.Contains(setting.FromRoleId))
         {
-            await _queueRunner.Enqueue(async () =>
+            await _queueRunner.EnqueueAsync(async () =>
             {
                 try
                 {
@@ -277,7 +277,7 @@ public class StreamRoleService : IReadyExecutor, INService
             //check if user is in the addrole
             if (user.RoleIds.Contains(setting.AddRoleId))
             {
-                await _queueRunner.Enqueue(async () =>
+                await _queueRunner.EnqueueAsync(async () =>
                 {
                     try
                     {

src/NadekoBot/Modules/Xp/XpService.cs

@@ -1,4 +1,5 @@
 #nullable disable
+using LinqToDB;
 using Microsoft.EntityFrameworkCore;
 using NadekoBot.Common.ModuleBehaviors;
 using NadekoBot.Db;
@@ -12,7 +13,7 @@ using SixLabors.ImageSharp.Drawing.Processing;
 using SixLabors.ImageSharp.Formats;
 using SixLabors.ImageSharp.PixelFormats;
 using SixLabors.ImageSharp.Processing;
-using StackExchange.Redis;
+using System.Threading.Channels;
 using Color = SixLabors.ImageSharp.Color;
 using Image = SixLabors.ImageSharp.Image;
 
@@ -37,13 +38,16 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
     private readonly ConcurrentDictionary<ulong, ConcurrentHashSet<ulong>> _excludedChannels;
     private readonly ConcurrentHashSet<ulong> _excludedServers;
 
-    private readonly System.Collections.Concurrent.ConcurrentQueue<UserCacheItem> _addMessageXp = new();
     private XpTemplate template;
     private readonly DiscordSocketClient _client;
 
     private readonly TypedKey<bool> _xpTemplateReloadKey;
     private readonly IPatronageService _ps;
     private readonly IBotCache _c;
+    
+    
+    private readonly QueueRunner _levelUpQueue = new QueueRunner(0, 50);
+    private readonly Channel<UserXpGainData> _xpGainQueue = Channel.CreateUnbounded<UserXpGainData>();
 
     public XpService(
         DiscordSocketClient client,
@@ -122,147 +126,131 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
 
     public async Task OnReadyAsync()
     {
+        _ = Task.Run(() => _levelUpQueue.RunAsync());
+        
         using var timer = new PeriodicTimer(5.Seconds());
         while (await timer.WaitForNextTickAsync())
         {
-            await UpdateLoop();
+            await UpdateXp();
         }
     }
 
-    private async Task UpdateLoop()
+    public sealed class MiniGuildXpStats
+    {
+        public long Xp { get; set; }
+        public XpNotificationLocation NotifyOnLevelUp { get; set; }
+        public ulong GuildId { get; set; }
+        public ulong UserId { get; set; }
+    }
+    private async Task UpdateXp()
     {
         try
         {
-            var toNotify =
-                new List<(IGuild Guild, IMessageChannel MessageChannel, IUser User, long Level,
-                    XpNotificationLocation NotifyType, NotifOf NotifOf)>();
-            var roleRewards = new Dictionary<ulong, List<XpRoleReward>>();
-            var curRewards = new Dictionary<ulong, List<XpCurrencyReward>>();
-
-            var toAddTo = new List<UserCacheItem>();
-            while (_addMessageXp.TryDequeue(out var usr))
-                toAddTo.Add(usr);
-
-            var group = toAddTo.GroupBy(x => (GuildId: x.Guild.Id, x.User));
-            if (toAddTo.Count == 0)
-                return;
-
-            await using (var uow = _db.GetDbContext())
+            var reader = _xpGainQueue.Reader;
+            
+            // sum up all gains into a single UserCacheItem
+            var globalToAdd = new Dictionary<ulong, UserXpGainData>();
+            var guildToAdd = new Dictionary<ulong, Dictionary<ulong, UserXpGainData>>();
+            while (reader.TryRead(out var item))
             {
-                foreach (var item in group)
+                // add global xp to these users
+                if (!globalToAdd.TryGetValue(item.User.Id, out var ci))
+                    globalToAdd[item.User.Id] = item.Clone();
+                else
+                    ci.XpAmount += item.XpAmount;
+
+                
+                // ad guild xp in these guilds to these users
+                if (!guildToAdd.TryGetValue(item.Guild.Id, out var users))
+                    users = guildToAdd[item.Guild.Id] = new();
+
+                if (!users.TryGetValue(item.User.Id, out ci))
+                    users[item.User.Id] = item.Clone();
+                else
+                    ci.XpAmount += item.XpAmount;
+            }
+            
+            await using var ctx = _db.GetDbContext();
+            await using var tran = await ctx.Database.BeginTransactionAsync();
+            
+            // update global user xp in batches
+            // group by xp amount and update the same amounts at the same time
+            var dus = new List<DiscordUser>(globalToAdd.Count);
+            foreach (var group in globalToAdd.GroupBy(x => x.Value.XpAmount, x => x.Key))
+            {
+                var items = await ctx.DiscordUser
+                    .Where(x => group.Contains(x.UserId))
+                    .UpdateWithOutputAsync(old => new()
+                        {
+                            TotalXp = old.TotalXp + group.Key
+                        },
+                        (_, n) => n);
+            
+                dus.AddRange(items);
+            }
+            
+            // update guild user xp in batches
+            var gxps = new List<UserXpStats>(globalToAdd.Count);
+            foreach (var (guildId, toAdd) in guildToAdd)
+            {
+                foreach (var group in toAdd.GroupBy(x => x.Value.XpAmount, x => x.Key))
                 {
-                    var xp = item.Sum(x => x.XpAmount);
-
-                    var usr = uow.GetOrCreateUserXpStats(item.Key.GuildId, item.Key.User.Id);
-                    var du = uow.GetOrCreateUser(item.Key.User);
-
-                    var globalXp = du.TotalXp;
-                    var oldGlobalLevelData = new LevelStats(globalXp);
-                    var newGlobalLevelData = new LevelStats(globalXp + xp);
-
-                    var oldGuildLevelData = new LevelStats(usr.Xp + usr.AwardedXp);
-                    usr.Xp += xp;
-                    du.TotalXp += xp;
-                    if (du.Club is not null)
-                        du.Club.Xp += xp;
-                    var newGuildLevelData = new LevelStats(usr.Xp + usr.AwardedXp);
-
-                    if (oldGlobalLevelData.Level < newGlobalLevelData.Level)
-                    {
-                        var first = item.First();
-                        if (du.NotifyOnLevelUp != XpNotificationLocation.None)
-                        {
-                            toNotify.Add((first.Guild, first.Channel, first.User, newGlobalLevelData.Level,
-                                du.NotifyOnLevelUp, NotifOf.Global));
-                        }
-                    }
-
-                    if (oldGuildLevelData.Level < newGuildLevelData.Level)
-                    {
-                        //send level up notification
-                        var first = item.First();
-                        if (usr.NotifyOnLevelUp != XpNotificationLocation.None)
-                        {
-                            toNotify.Add((first.Guild, first.Channel, first.User, newGuildLevelData.Level,
-                                usr.NotifyOnLevelUp, NotifOf.Server));
-                        }
-
-                        //give role
-                        if (!roleRewards.TryGetValue(usr.GuildId, out var rrews))
-                        {
-                            rrews = uow.XpSettingsFor(usr.GuildId).RoleRewards.ToList();
-                            roleRewards.Add(usr.GuildId, rrews);
-                        }
-
-                        if (!curRewards.TryGetValue(usr.GuildId, out var crews))
-                        {
-                            crews = uow.XpSettingsFor(usr.GuildId).CurrencyRewards.ToList();
-                            curRewards.Add(usr.GuildId, crews);
-                        }
-
-                        //loop through levels since last level up, so if a high amount of xp is gained, reward are still applied.
-                        for (var i = oldGuildLevelData.Level + 1; i <= newGuildLevelData.Level; i++)
-                        {
-                            var rrew = rrews.FirstOrDefault(x => x.Level == i);
-                            if (rrew is not null)
+                    var items = await ctx
+                        .UserXpStats
+                        .Where(x => x.GuildId == guildId)
+                        .Where(x => group.Contains(x.UserId))
+                        .UpdateWithOutputAsync(old => new()
                             {
-                                var role = first.User.Guild.GetRole(rrew.RoleId);
-                                if (role is not null)
-                                {
-                                    if (rrew.Remove)
-                                        _ = first.User.RemoveRoleAsync(role);
-                                    else
-                                        _ = first.User.AddRoleAsync(role);
-                                }
-                            }
+                                Xp = old.Xp + group.Key
+                            },
+                            (_, n) => n);
 
-                            //get currency reward for this level
-                            var crew = crews.FirstOrDefault(x => x.Level == i);
-                            if (crew is not null)
-                                //give the user the reward if it exists
-                                await _cs.AddAsync(item.Key.User.Id, crew.Amount, new("xp", "level-up"));
-                        }
-                    }
+                    gxps.AddRange(items);
+                }
+            }
+            
+            await tran.CommitAsync();
+            
+            foreach (var du in dus)
+            {
+                var oldLevel = new LevelStats(du.TotalXp - globalToAdd[du.UserId].XpAmount);
+                var newLevel = new LevelStats(du.TotalXp);
+            
+                if (oldLevel.Level != newLevel.Level)
+                {
+                    var item = globalToAdd[du.UserId];
+                    await _levelUpQueue.EnqueueAsync(
+                        NotifyUser(item.Guild.Id,
+                            item.Channel.Id,
+                            du.UserId,
+                            false,
+                            oldLevel.Level,
+                            newLevel.Level,
+                            du.NotifyOnLevelUp));
                 }
-
-                uow.SaveChanges();
             }
 
-            await toNotify.Select(async x =>
-                          {
-                              if (x.NotifOf == NotifOf.Server)
-                              {
-                                  if (x.NotifyType == XpNotificationLocation.Dm)
-                                  {
-                                      await x.User.SendConfirmAsync(_eb,
-                                          _strings.GetText(strs.level_up_dm(x.User.Mention,
-                                                  Format.Bold(x.Level.ToString()),
-                                                  Format.Bold(x.Guild.ToString() ?? "-")),
-                                              x.Guild.Id));
-                                  }
-                                  else if (x.MessageChannel is not null) // channel
-                                  {
-                                      await x.MessageChannel.SendConfirmAsync(_eb,
-                                          _strings.GetText(strs.level_up_channel(x.User.Mention,
-                                                  Format.Bold(x.Level.ToString())),
-                                              x.Guild.Id));
-                                  }
-                              }
-                              else
-                              {
-                                  IMessageChannel chan;
-                                  if (x.NotifyType == XpNotificationLocation.Dm)
-                                      chan = await x.User.CreateDMChannelAsync();
-                                  else // channel
-                                      chan = x.MessageChannel;
+            foreach (var du in gxps)
+            {
+                if (guildToAdd.TryGetValue(du.GuildId, out var users)
+                    && users.TryGetValue(du.UserId, out var xpGainData))
+                {
+                    var oldLevel = new LevelStats(du.Xp - xpGainData.XpAmount);
+                    var newLevel = new LevelStats(du.Xp);
 
-                                  await chan.SendConfirmAsync(_eb,
-                                      _strings.GetText(strs.level_up_global(x.User.Mention,
-                                              Format.Bold(x.Level.ToString())),
-                                          x.Guild.Id));
-                              }
-                          })
-                          .WhenAll();
+                    if (oldLevel.Level < newLevel.Level)
+                    {
+                        await _levelUpQueue.EnqueueAsync(
+                            NotifyUser(xpGainData.Guild.Id,
+                                xpGainData.Channel.Id,
+                                du.UserId,
+                                true,
+                                oldLevel.Level,
+                                newLevel.Level,
+                                du.NotifyOnLevelUp));
+                    }
+                }
+            }
         }
         catch (Exception ex)
         {
@@ -270,7 +258,115 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
         }
     }
 
+    private Func<Task> NotifyUser(
+        ulong guildId,
+        ulong channelId,
+        ulong userId,
+        bool isServer,
+        long oldLevel,
+        long newLevel,
+        XpNotificationLocation notifyLoc)
+        => async () =>
+        {
+            if (isServer)
+            {
+                await HandleRewardsInternalAsync(guildId, userId, oldLevel, newLevel);
+            }
+
+            await HandleNotifyInternalAsync(guildId, channelId, userId, isServer, newLevel, notifyLoc);
+        };
+
+    private async Task HandleRewardsInternalAsync(ulong guildId, ulong userId, long oldLevel, long newLevel)
+    {
+        await using var ctx = _db.GetDbContext();
+            var rrews = ctx.XpSettingsFor(guildId).RoleRewards.ToList();
+            var crews = ctx.XpSettingsFor(guildId).CurrencyRewards.ToList();
+        
+        //loop through levels since last level up, so if a high amount of xp is gained, reward are still applied.
+        for (var i = oldLevel + 1; i <= newLevel; i++)
+        {
+            var rrew = rrews.FirstOrDefault(x => x.Level == i);
+            if (rrew is not null)
+            {
+                var guild = _client.GetGuild(guildId);
+                var role = guild?.GetRole(rrew.RoleId);
+                var user = guild?.GetUser(userId);
+
+                if (role is not null && user is not null)
+                {
+                    if (rrew.Remove)
+                        _ = user.RemoveRoleAsync(role);
+                    else
+                        _ = user.AddRoleAsync(role);
+                }
+            }
+
+            //get currency reward for this level
+            var crew = crews.FirstOrDefault(x => x.Level == i);
+            if (crew is not null)
+            {
+                //give the user the reward if it exists
+                await _cs.AddAsync(userId, crew.Amount, new("xp", "level-up"));
+            }
+        }
+    }
+
+    private async Task HandleNotifyInternalAsync(ulong guildId, 
+        ulong channelId, 
+        ulong userId, 
+        bool isServer,
+        long newLevel,
+        XpNotificationLocation notifyLoc)
+    {
+        if (notifyLoc == XpNotificationLocation.None)
+            return;
+            
+        var guild = _client.GetGuild(guildId);
+        var user = guild?.GetUser(userId);
+        var ch = guild?.GetTextChannel(channelId);
+
+        if (guild is null || user is null)
+            return;
+
+        if (isServer)
+        {
+            if (notifyLoc == XpNotificationLocation.Dm)
+            {
+                await user.SendConfirmAsync(_eb,
+                    _strings.GetText(strs.level_up_dm(user.Mention,
+                            Format.Bold(newLevel.ToString()),
+                            Format.Bold(guild.ToString() ?? "-")),
+                        guild.Id));
+            }
+            else // channel
+            {
+                await ch.SendConfirmAsync(_eb,
+                    _strings.GetText(strs.level_up_channel(user.Mention,
+                            Format.Bold(newLevel.ToString())),
+                        guild.Id));
+            }
+        }
+        else // global level
+        {
+            var chan = notifyLoc switch
+            {
+                XpNotificationLocation.Dm => (IMessageChannel)await user.CreateDMChannelAsync(),
+                XpNotificationLocation.Channel => ch,
+                _ => null
+            };
+
+            if (chan is null)
+                return;
+
+            await chan.SendConfirmAsync(_eb,
+                _strings.GetText(strs.level_up_global(user.Mention,
+                        Format.Bold(newLevel.ToString())),
+                    guild.Id));
+        }
+    }
+
     private const string XP_TEMPLATE_PATH = "./data/xp_template.json";
+
     private void InternalReloadXpTemplate()
     {
         try
@@ -295,7 +391,8 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
             {
                 Log.Warning("Loaded default xp_template.json values as the old one was version 0. "
                             + "Old one was renamed to xp_template.json.old");
-                File.WriteAllText("./data/xp_template.json.old", JsonConvert.SerializeObject(template, Formatting.Indented));
+                File.WriteAllText("./data/xp_template.json.old",
+                    JsonConvert.SerializeObject(template, Formatting.Indented));
                 template = new();
                 template.Version = 1;
                 File.WriteAllText(XP_TEMPLATE_PATH, JsonConvert.SerializeObject(template, Formatting.Indented));
@@ -473,9 +570,11 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
             if (after.VoiceChannel is not null && after.VoiceChannel != before.VoiceChannel)
                 await ScanChannelForVoiceXp(after.VoiceChannel);
             else if (after.VoiceChannel is null)
+            {
                 // In this case, the user left the channel and the previous for loops didn't catch
                 // it because it wasn't in any new channel. So we need to get rid of it.
                 await UserLeftVoiceChannel(user, before.VoiceChannel);
+            }
         });
 
         return Task.CompletedTask;
@@ -546,7 +645,7 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
 
         if (actualXp > 0)
         {
-            _addMessageXp.Enqueue(new()
+            await _xpGainQueue.Writer.WriteAsync(new()
             {
                 Guild = channel.Guild,
                 User = user,
@@ -593,7 +692,7 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
             if (!await SetUserRewardedAsync(user.Id))
                 return;
 
-            _addMessageXp.Enqueue(new()
+            await _xpGainQueue.Writer.WriteAsync(new()
             {
                 Guild = user.Guild,
                 Channel = arg.Channel,
@@ -604,19 +703,19 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
         return Task.CompletedTask;
     }
 
-    public void AddXpDirectly(IGuildUser user, IMessageChannel channel, int amount)
-    {
-        if (amount <= 0)
-            throw new ArgumentOutOfRangeException(nameof(amount));
-
-        _addMessageXp.Enqueue(new()
-        {
-            Guild = user.Guild,
-            Channel = channel,
-            User = user,
-            XpAmount = amount
-        });
-    }
+    // public void AddXpDirectly(IGuildUser user, IMessageChannel channel, int amount)
+    // {
+    //     if (amount <= 0)
+    //         throw new ArgumentOutOfRangeException(nameof(amount));
+    //
+    //     _xpGainQueue.Writer.WriteAsync(new()
+    //     {
+    //         Guild = user.Guild,
+    //         Channel = channel,
+    //         User = user,
+    //         XpAmount = amount
+    //     });
+    // }
 
     public void AddXp(ulong userId, ulong guildId, int amount)
     {
@@ -649,7 +748,7 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
 
     private static TypedKey<bool> GetUserRewKey(ulong userId)
         => new($"xp:user_gain:{userId}");
-
+    
     private async Task<bool> SetUserRewardedAsync(ulong userId)
         => await _c.AddAsync(GetUserRewKey(userId),
             true,
@@ -829,8 +928,8 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
 
                 return font;
             }
-            
-            
+
+
             if (template.User.GlobalLevel.Show)
             {
                 // up to 83 width
@@ -841,7 +940,7 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
                     FontStyle.Bold,
                     stats.Global.Level.ToString(),
                     75);
-                
+
                 img.Mutate(x =>
                 {
                     x.DrawText(stats.Global.Level.ToString(),
@@ -859,7 +958,7 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
                     FontStyle.Bold,
                     stats.Guild.Level.ToString(),
                     75);
-                
+
                 img.Mutate(x =>
                 {
                     x.DrawText(stats.Guild.Level.ToString(),
@@ -936,14 +1035,14 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
             if (template.User.GlobalRank.Show)
             {
                 var globalRankStr = stats.GlobalRanking.ToString();
-                
+
                 var globalRankFont = GetTruncatedFont(
                     _fonts.UniSans,
                     template.User.GlobalRank.FontSize,
                     FontStyle.Bold,
                     globalRankStr,
                     68);
-                
+
                 img.Mutate(x => x.DrawText(globalRankStr,
                     globalRankFont,
                     template.User.GlobalRank.Color,
@@ -953,20 +1052,20 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
             if (template.User.GuildRank.Show)
             {
                 var guildRankStr = stats.GuildRanking.ToString();
-                
+
                 var guildRankFont = GetTruncatedFont(
                     _fonts.UniSans,
                     template.User.GuildRank.FontSize,
                     FontStyle.Bold,
                     guildRankStr,
                     43);
-                
+
                 img.Mutate(x => x.DrawText(guildRankStr,
                     guildRankFont,
                     template.User.GuildRank.Color,
                     new(template.User.GuildRank.Pos.X, template.User.GuildRank.Pos.Y)));
             }
-            
+
             //avatar
             if (stats.User.AvatarId is not null && template.User.Icon.Show)
             {
@@ -1018,13 +1117,13 @@ public class XpService : INService, IReadyExecutor, IExecNoCommand
 // #if GLOBAL_NADEKO
             await DrawFrame(img, stats.User.UserId);
 // #endif
-            
+
             var outputSize = template.OutputSize;
             if (outputSize.X != img.Width || outputSize.Y != img.Height)
                 img.Mutate(x => x.Resize(template.OutputSize.X, template.OutputSize.Y));
-            
+
             var output = ((Stream)await img.ToStreamAsync(imageFormat), imageFormat);
-            
+
             return output;
         });
 

src/NadekoBot/Modules/Xp/_Common/LevelStats.cs

@@ -3,7 +3,7 @@ using NadekoBot.Modules.Xp.Services;
 
 namespace NadekoBot.Modules.Xp;
 
-public class LevelStats
+public readonly struct LevelStats
 {
     public long Level { get; }
     public long LevelXp { get; }

src/NadekoBot/Modules/Xp/_Common/UserCacheItem.cs

@@ -1,16 +1,13 @@
 #nullable disable
+using Cloneable;
+
 namespace NadekoBot.Modules.Xp.Services;
 
-public class UserCacheItem
+[Cloneable]
+public sealed partial class UserXpGainData : ICloneable<UserXpGainData>
 {
     public IGuildUser User { get; set; }
     public IGuild Guild { get; set; }
     public IMessageChannel Channel { get; set; }
     public int XpAmount { get; set; }
-
-    public override int GetHashCode()
-        => User.GetHashCode();
-
-    public override bool Equals(object obj)
-        => obj is UserCacheItem uci && uci.User == User;
 }

src/NadekoBot/Services/Impl/RedisImagesCache.cs

@@ -59,6 +59,11 @@ public sealed class ImageCache : IImageCache, INService
             GetImageKey(url),
             async () =>
             {
+                if (url.IsFile)
+                {
+                    return await File.ReadAllBytesAsync(url.LocalPath);
+                }
+
                 using var http = _httpFactory.CreateClient();
                 var bytes = await http.GetByteArrayAsync(url);
                 return bytes;

src/NadekoBot/Services/Impl/StatsService.cs

@@ -7,7 +7,7 @@ namespace NadekoBot.Services;
 
 public sealed class StatsService : IStatsService, IReadyExecutor, INService
 {
-    public const string BOT_VERSION = "4.2.14";
+    public const string BOT_VERSION = "4.2.15";
 
     public string Author
         => "Kwoth#2452";

src/NadekoBot/data/bot.yml

@@ -67,8 +67,7 @@ helpText: |-
 # List of modules and commands completely blocked on the bot
 blocked:
   commands: []
-  modules:
-    - nsfw
+  modules: []
 # Which string will be used to recognize the commands
 prefix: .
 # Toggles whether your bot will group greet/bye messages into a single message every 5 seconds.