Fixed some crashes in response strings source generator, reorganized more submodules into their folders

src/NadekoBot/Modules/Nsfw/_Common/Booru.cs

@@ -0,0 +1,15 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public enum Booru
+{
+    Safebooru,
+    E621,
+    Derpibooru,
+    Rule34,
+    Gelbooru,
+    Konachan,
+    Yandere,
+    Danbooru,
+    Sankaku
+}

src/NadekoBot/Modules/Nsfw/_Common/DapiImageObject.cs

@@ -0,0 +1,21 @@
+#nullable disable
+using System.Text.Json.Serialization;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class DapiImageObject : IImageData
+{
+    [JsonPropertyName("File_Url")]
+    public string FileUrl { get; set; }
+
+    public string Tags { get; set; }
+
+    [JsonPropertyName("Tag_String")]
+    public string TagString { get; set; }
+
+    public int Score { get; set; }
+    public string Rating { get; set; }
+
+    public ImageData ToCachedImageData(Booru type)
+        => new(FileUrl, type, Tags?.Split(' ') ?? TagString?.Split(' '), Score.ToString() ?? Rating);
+}

src/NadekoBot/Modules/Nsfw/_Common/DapiTag.cs

@@ -0,0 +1,13 @@
+#nullable disable
+using System.Text.Json.Serialization;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public readonly struct DapiTag
+{
+    public string Name { get; }
+
+    [JsonConstructor]
+    public DapiTag(string name)
+        => Name = name;
+}

src/NadekoBot/Modules/Nsfw/_Common/DerpiContainer.cs

@@ -0,0 +1,21 @@
+#nullable disable
+using System.Text.Json.Serialization;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class DerpiContainer
+{
+    public DerpiImageObject[] Images { get; set; }
+}
+
+public class DerpiImageObject : IImageData
+{
+    [JsonPropertyName("view_url")]
+    public string ViewUrl { get; set; }
+
+    public string[] Tags { get; set; }
+    public int Score { get; set; }
+
+    public ImageData ToCachedImageData(Booru type)
+        => new(ViewUrl, type, Tags, Score.ToString("F1"));
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DanbooruImageDownloader.cs

@@ -0,0 +1,33 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public sealed class DanbooruImageDownloader : DapiImageDownloader
+{
+    // using them as concurrent hashsets, value doesn't matter
+    private static readonly ConcurrentDictionary<string, bool> _existentTags = new();
+    private static readonly ConcurrentDictionary<string, bool> _nonexistentTags = new();
+
+    public DanbooruImageDownloader(HttpClient http)
+        : base(Booru.Danbooru, http, "http://danbooru.donmai.us")
+    {
+    }
+
+    public override async Task<bool> IsTagValid(string tag, CancellationToken cancel = default)
+    {
+        if (_existentTags.ContainsKey(tag))
+            return true;
+
+        if (_nonexistentTags.ContainsKey(tag))
+            return false;
+
+        var tags = await _http.GetFromJsonAsync<DapiTag[]>(
+            _baseUrl + "/tags.json" + $"?search[name_or_alias_matches]={tag}",
+            _serializerOptions,
+            cancel);
+        if (tags is { Length: > 0 }) return _existentTags[tag] = true;
+
+        return _nonexistentTags[tag] = false;
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DapiImageDownloader.cs

@@ -0,0 +1,49 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public abstract class DapiImageDownloader : ImageDownloader<DapiImageObject>
+{
+    protected readonly string _baseUrl;
+
+    public DapiImageDownloader(Booru booru, HttpClient http, string baseUrl)
+        : base(booru, http)
+        => _baseUrl = baseUrl;
+
+    public abstract Task<bool> IsTagValid(string tag, CancellationToken cancel = default);
+
+    protected async Task<bool> AllTagsValid(string[] tags, CancellationToken cancel = default)
+    {
+        var results = await tags.Select(tag => IsTagValid(tag, cancel)).WhenAll();
+
+        // if any of the tags is not valid, the query is not valid
+        foreach (var result in results)
+            if (!result)
+                return false;
+
+        return true;
+    }
+
+    public override async Task<List<DapiImageObject>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        // up to 2 tags allowed on danbooru
+        if (tags.Length > 2)
+            return new();
+
+        if (!await AllTagsValid(tags, cancel))
+            return new();
+
+        var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
+
+        var uri = $"{_baseUrl}/posts.json?limit=200&tags={tagString}&page={page}";
+        var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
+        if (imageObjects is null)
+            return new();
+        return imageObjects.Where(x => x.FileUrl is not null).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DerpibooruImageDownloader.cs

@@ -0,0 +1,33 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class DerpibooruImageDownloader : ImageDownloader<DerpiImageObject>
+{
+    public DerpibooruImageDownloader(HttpClient http)
+        : base(Booru.Derpibooru, http)
+    {
+    }
+
+    public override async Task<List<DerpiImageObject>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
+        var uri =
+            $"https://www.derpibooru.org/api/v1/json/search/images?q={tagString.Replace('+', ',')}&per_page=49&page={page}";
+        using var req = new HttpRequestMessage(HttpMethod.Get, uri);
+        req.Headers.AddFakeHeaders();
+        using var res = await _http.SendAsync(req, cancel);
+        res.EnsureSuccessStatusCode();
+
+        var container = await res.Content.ReadFromJsonAsync<DerpiContainer>(_serializerOptions, cancel);
+        if (container?.Images is null)
+            return new();
+
+        return container.Images.Where(x => !string.IsNullOrWhiteSpace(x.ViewUrl)).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/E621ImageDownloader.cs

@@ -0,0 +1,32 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class E621ImageDownloader : ImageDownloader<E621Object>
+{
+    public E621ImageDownloader(HttpClient http)
+        : base(Booru.E621, http)
+    {
+    }
+
+    public override async Task<List<E621Object>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
+        var uri = $"https://e621.net/posts.json?limit=32&tags={tagString}&page={page}";
+        using var req = new HttpRequestMessage(HttpMethod.Get, uri);
+        req.Headers.AddFakeHeaders();
+        using var res = await _http.SendAsync(req, cancel);
+        res.EnsureSuccessStatusCode();
+
+        var data = await res.Content.ReadFromJsonAsync<E621Response>(_serializerOptions, cancel);
+        if (data?.Posts is null)
+            return new();
+
+        return data.Posts.Where(x => !string.IsNullOrWhiteSpace(x.File?.Url)).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/E621Response.cs

@@ -0,0 +1,7 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class E621Response
+{
+    public List<E621Object> Posts { get; set; }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/GelbooruImageDownloader.cs

@@ -0,0 +1,35 @@
+#nullable disable
+using System.Text.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class GelbooruImageDownloader : ImageDownloader<DapiImageObject>
+{
+    public GelbooruImageDownloader(HttpClient http)
+        : base(Booru.Gelbooru, http)
+    {
+    }
+
+    public override async Task<List<DapiImageObject>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
+        var uri = "http://gelbooru.com/index.php?page=dapi&s=post&json=1&q=index&limit=100"
+                  + $"&tags={tagString}&pid={page}";
+        using var req = new HttpRequestMessage(HttpMethod.Get, uri);
+        using var res = await _http.SendAsync(req, cancel);
+        res.EnsureSuccessStatusCode();
+        var resString = await res.Content.ReadAsStringAsync(cancel);
+        if (string.IsNullOrWhiteSpace(resString))
+            return new();
+
+        var images = JsonSerializer.Deserialize<List<DapiImageObject>>(resString, _serializerOptions);
+        if (images is null)
+            return new();
+
+        return images.Where(x => x.FileUrl is not null).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/IImageDownloader.cs

@@ -0,0 +1,11 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public interface IImageDownloader
+{
+    Task<List<ImageData>> DownloadImageDataAsync(
+        string[] tags,
+        int page = 0,
+        bool isExplicit = false,
+        CancellationToken cancel = default);
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/ImageDownloader.cs

@@ -0,0 +1,40 @@
+#nullable disable
+using System.Text.Json;
+using System.Text.Json.Serialization;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public abstract class ImageDownloader<T> : IImageDownloader
+    where T : IImageData
+{
+    public Booru Booru { get; }
+    protected readonly HttpClient _http;
+
+    protected JsonSerializerOptions _serializerOptions = new()
+    {
+        PropertyNameCaseInsensitive = true,
+        NumberHandling = JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString
+    };
+
+    public ImageDownloader(Booru booru, HttpClient http)
+    {
+        _http = http;
+        Booru = booru;
+    }
+
+    public abstract Task<List<T>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default);
+
+    public async Task<List<ImageData>> DownloadImageDataAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var images = await DownloadImagesAsync(tags, page, isExplicit, cancel);
+        return images.Select(x => x.ToCachedImageData(Booru)).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/ImageDownloaderHelper.cs

@@ -0,0 +1,13 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public static class ImageDownloaderHelper
+{
+    public static string GetTagString(IEnumerable<string> tags, bool isExplicit = false)
+    {
+        if (isExplicit)
+            tags = tags.Append("rating:explicit");
+
+        return string.Join('+', tags.Select(x => x.ToLowerInvariant()));
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/KonachanImageDownloader.cs

@@ -0,0 +1,27 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public sealed class KonachanImageDownloader : ImageDownloader<DapiImageObject>
+{
+    private readonly string _baseUrl;
+
+    public KonachanImageDownloader(HttpClient http)
+        : base(Booru.Konachan, http)
+        => _baseUrl = "https://konachan.com";
+
+    public override async Task<List<DapiImageObject>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
+        var uri = $"{_baseUrl}/post.json?s=post&q=index&limit=200&tags={tagString}&page={page}";
+        var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
+        if (imageObjects is null)
+            return new();
+        return imageObjects.Where(x => x.FileUrl is not null).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/Rule34ImageDownloader.cs

@@ -0,0 +1,29 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class Rule34ImageDownloader : ImageDownloader<Rule34Object>
+{
+    public Rule34ImageDownloader(HttpClient http)
+        : base(Booru.Rule34, http)
+    {
+    }
+
+    public override async Task<List<Rule34Object>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var tagString = ImageDownloaderHelper.GetTagString(tags);
+        var uri = "https://rule34.xxx/index.php?page=dapi&s=post&q=index&json=1&limit=100"
+                  + $"&tags={tagString}&pid={page}";
+        var images = await _http.GetFromJsonAsync<List<Rule34Object>>(uri, _serializerOptions, cancel);
+
+        if (images is null)
+            return new();
+
+        return images.Where(img => !string.IsNullOrWhiteSpace(img.Image)).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/SafebooruImageDownloader.cs

@@ -0,0 +1,28 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class SafebooruImageDownloader : ImageDownloader<SafebooruElement>
+{
+    public SafebooruImageDownloader(HttpClient http)
+        : base(Booru.Safebooru, http)
+    {
+    }
+
+    public override async Task<List<SafebooruElement>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var tagString = ImageDownloaderHelper.GetTagString(tags);
+        var uri =
+            $"https://safebooru.org/index.php?page=dapi&s=post&q=index&limit=200&tags={tagString}&json=1&pid={page}";
+        var images = await _http.GetFromJsonAsync<List<SafebooruElement>>(uri, _serializerOptions, cancel);
+        if (images is null)
+            return new();
+
+        return images;
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/SankakuImageDownloader.cs

@@ -0,0 +1,32 @@
+#nullable disable
+using System.Text.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public sealed class SankakuImageDownloader : ImageDownloader<SankakuImageObject>
+{
+    private readonly string _baseUrl;
+
+    public SankakuImageDownloader(HttpClient http)
+        : base(Booru.Sankaku, http)
+    {
+        _baseUrl = "https://capi-v2.sankakucomplex.com";
+        _http.AddFakeHeaders();
+    }
+
+    public override async Task<List<SankakuImageObject>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        // explicit probably not supported
+        var tagString = ImageDownloaderHelper.GetTagString(tags);
+
+        var uri = $"{_baseUrl}/posts?tags={tagString}&limit=50";
+        var data = await _http.GetStringAsync(uri);
+        return JsonSerializer.Deserialize<SankakuImageObject[]>(data, _serializerOptions)
+                             .Where(x => !string.IsNullOrWhiteSpace(x.FileUrl) && x.FileType.StartsWith("image"))
+                             .ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/YandereImageDownloader.cs

@@ -0,0 +1,28 @@
+#nullable disable
+using System.Net.Http.Json;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public sealed class YandereImageDownloader : ImageDownloader<DapiImageObject>
+{
+    private readonly string _baseUrl;
+
+    public YandereImageDownloader(HttpClient http)
+        : base(Booru.Yandere, http)
+        => _baseUrl = "https://yande.re";
+
+    public override async Task<List<DapiImageObject>> DownloadImagesAsync(
+        string[] tags,
+        int page,
+        bool isExplicit = false,
+        CancellationToken cancel = default)
+    {
+        var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
+
+        var uri = $"{_baseUrl}/post.json?limit=200&tags={tagString}&page={page}";
+        var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
+        if (imageObjects is null)
+            return new();
+        return imageObjects.Where(x => x.FileUrl is not null).ToList();
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/E621Object.cs

@@ -0,0 +1,27 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class E621Object : IImageData
+{
+    public FileData File { get; set; }
+    public TagData Tags { get; set; }
+    public ScoreData Score { get; set; }
+
+    public ImageData ToCachedImageData(Booru type)
+        => new(File.Url, Booru.E621, Tags.General, Score.Total.ToString());
+
+    public class FileData
+    {
+        public string Url { get; set; }
+    }
+
+    public class TagData
+    {
+        public string[] General { get; set; }
+    }
+
+    public class ScoreData
+    {
+        public int Total { get; set; }
+    }
+}

src/NadekoBot/Modules/Nsfw/_Common/IImageData.cs

@@ -0,0 +1,7 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public interface IImageData
+{
+    ImageData ToCachedImageData(Booru type);
+}

src/NadekoBot/Modules/Nsfw/_Common/ImageData.cs

@@ -0,0 +1,39 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class ImageData : IComparable<ImageData>
+{
+    public Booru SearchType { get; }
+    public string FileUrl { get; }
+    public HashSet<string> Tags { get; }
+    public string Rating { get; }
+
+    public ImageData(
+        string url,
+        Booru type,
+        string[] tags,
+        string rating)
+    {
+        if (type == Booru.Danbooru && !Uri.IsWellFormedUriString(url, UriKind.Absolute))
+            FileUrl = "https://danbooru.donmai.us" + url;
+        else
+            FileUrl = url.StartsWith("http", StringComparison.InvariantCulture) ? url : "https:" + url;
+
+        SearchType = type;
+        FileUrl = url;
+        Tags = tags.ToHashSet();
+        Rating = rating;
+    }
+
+    public override string ToString()
+        => FileUrl;
+
+    public override int GetHashCode()
+        => FileUrl.GetHashCode();
+
+    public override bool Equals(object obj)
+        => obj is ImageData ico && ico.FileUrl == FileUrl;
+
+    public int CompareTo(ImageData other)
+        => string.Compare(FileUrl, other.FileUrl, StringComparison.InvariantCulture);
+}

src/NadekoBot/Modules/Nsfw/_Common/Rule34Object.cs

@@ -0,0 +1,13 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class Rule34Object : IImageData
+{
+    public string Image { get; init; }
+    public int Directory { get; init; }
+    public string Tags { get; init; }
+    public int Score { get; init; }
+
+    public ImageData ToCachedImageData(Booru type)
+        => new($"https://img.rule34.xxx//images/{Directory}/{Image}", Booru.Rule34, Tags.Split(' '), Score.ToString());
+}

src/NadekoBot/Modules/Nsfw/_Common/SafebooruElement.cs

@@ -0,0 +1,18 @@
+#nullable disable
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class SafebooruElement : IImageData
+{
+    public string Directory { get; set; }
+    public string Image { get; set; }
+
+
+    public string FileUrl
+        => $"https://safebooru.org/images/{Directory}/{Image}";
+
+    public string Rating { get; set; }
+    public string Tags { get; set; }
+
+    public ImageData ToCachedImageData(Booru type)
+        => new(FileUrl, Booru.Safebooru, Tags.Split(' '), Rating);
+}

src/NadekoBot/Modules/Nsfw/_Common/SankakuImageObject.cs

@@ -0,0 +1,26 @@
+#nullable disable
+using System.Text.Json.Serialization;
+
+namespace NadekoBot.Modules.Nsfw.Common;
+
+public class SankakuImageObject : IImageData
+{
+    [JsonPropertyName("file_url")]
+    public string FileUrl { get; set; }
+
+    [JsonPropertyName("file_type")]
+    public string FileType { get; set; }
+
+    public Tag[] Tags { get; set; }
+
+    [JsonPropertyName("total_score")]
+    public int Score { get; set; }
+
+    public ImageData ToCachedImageData(Booru type)
+        => new(FileUrl, Booru.Sankaku, Tags.Select(x => x.Name).ToArray(), Score.ToString());
+
+    public class Tag
+    {
+        public string Name { get; set; }
+    }
+}