Fixed rule34, small refactor of the downloader classes

src/NadekoBot/Modules/Nsfw/SearchImageCacher.cs

@@ -264,18 +264,18 @@ public class SearchImageCacher : INService
         return new();
     }
 
-    private IImageDownloader GetImageDownloader(Booru booru, HttpClient http)
+    private IImageDownloader GetImageDownloader(Booru booru)
         => booru switch
         {
-            Booru.Danbooru => new DanbooruImageDownloader(http),
+            Booru.Danbooru => new DanbooruImageDownloader(_httpFactory),
-            Booru.Yandere => new YandereImageDownloader(http),
+            Booru.Yandere => new YandereImageDownloader(_httpFactory),
-            Booru.Konachan => new KonachanImageDownloader(http),
+            Booru.Konachan => new KonachanImageDownloader(_httpFactory),
-            Booru.Safebooru => new SafebooruImageDownloader(http),
+            Booru.Safebooru => new SafebooruImageDownloader(_httpFactory),
-            Booru.E621 => new E621ImageDownloader(http),
+            Booru.E621 => new E621ImageDownloader(_httpFactory),
-            Booru.Derpibooru => new DerpibooruImageDownloader(http),
+            Booru.Derpibooru => new DerpibooruImageDownloader(_httpFactory),
-            Booru.Gelbooru => new GelbooruImageDownloader(http),
+            Booru.Gelbooru => new GelbooruImageDownloader(_httpFactory),
-            Booru.Rule34 => new Rule34ImageDownloader(http),
+            Booru.Rule34 => new Rule34ImageDownloader(_httpFactory),
-            Booru.Sankaku => new SankakuImageDownloader(http),
+            Booru.Sankaku => new SankakuImageDownloader(_httpFactory),
             _ => throw new NotImplementedException($"{booru} downloader not implemented.")
         };
 
@@ -290,8 +290,7 @@ public class SearchImageCacher : INService
         {
             Log.Information("Downloading from {Type} (page {Page})...", type, page);
 
-            using var http = _httpFactory.CreateClient();
+            var downloader = GetImageDownloader(type);
-            var downloader = GetImageDownloader(type, http);
 
             var images = await downloader.DownloadImageDataAsync(tags, page, isExplicit, cancel);
             if (images.Count == 0)

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DanbooruImageDownloader.cs

@@ -9,7 +9,7 @@ public sealed class DanbooruImageDownloader : DapiImageDownloader
     private static readonly ConcurrentDictionary<string, bool> _existentTags = new();
     private static readonly ConcurrentDictionary<string, bool> _nonexistentTags = new();
 
-    public DanbooruImageDownloader(HttpClient http)
+    public DanbooruImageDownloader(IHttpClientFactory http)
         : base(Booru.Danbooru, http, "http://danbooru.donmai.us")
     {
     }
@@ -22,7 +22,8 @@ public sealed class DanbooruImageDownloader : DapiImageDownloader
         if (_nonexistentTags.ContainsKey(tag))
             return false;
 
-        var tags = await _http.GetFromJsonAsync<DapiTag[]>(
+        using var http = _http.CreateClient();
+        var tags = await http.GetFromJsonAsync<DapiTag[]>(
             _baseUrl + "/tags.json" + $"?search[name_or_alias_matches]={tag}",
             _serializerOptions,
             cancel);

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DapiImageDownloader.cs

@@ -7,7 +7,7 @@ public abstract class DapiImageDownloader : ImageDownloader<DapiImageObject>
 {
     protected readonly string _baseUrl;
 
-    public DapiImageDownloader(Booru booru, HttpClient http, string baseUrl)
+    public DapiImageDownloader(Booru booru, IHttpClientFactory http, string baseUrl)
         : base(booru, http)
         => _baseUrl = baseUrl;
 
@@ -43,7 +43,8 @@ public abstract class DapiImageDownloader : ImageDownloader<DapiImageObject>
         var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
 
         var uri = $"{_baseUrl}/posts.json?limit=200&tags={tagString}&page={page}";
-        var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
+        using var http = _http.CreateClient();
+        var imageObjects = await http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
         if (imageObjects is null)
             return new();
         return imageObjects.Where(x => x.FileUrl is not null).ToList();

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DerpibooruImageDownloader.cs

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
 
 public class DerpibooruImageDownloader : ImageDownloader<DerpiImageObject>
 {
-    public DerpibooruImageDownloader(HttpClient http)
+    public DerpibooruImageDownloader(IHttpClientFactory http)
         : base(Booru.Derpibooru, http)
     {
     }
@@ -21,7 +21,8 @@ public class DerpibooruImageDownloader : ImageDownloader<DerpiImageObject>
             $"https://www.derpibooru.org/api/v1/json/search/images?q={tagString.Replace('+', ',')}&per_page=49&page={page}";
         using var req = new HttpRequestMessage(HttpMethod.Get, uri);
         req.Headers.AddFakeHeaders();
-        using var res = await _http.SendAsync(req, cancel);
+        using var http = _http.CreateClient();
+        using var res = await http.SendAsync(req, cancel);
         res.EnsureSuccessStatusCode();
 
         var container = await res.Content.ReadFromJsonAsync<DerpiContainer>(_serializerOptions, cancel);

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/E621ImageDownloader.cs

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
 
 public class E621ImageDownloader : ImageDownloader<E621Object>
 {
-    public E621ImageDownloader(HttpClient http)
+    public E621ImageDownloader(IHttpClientFactory http)
         : base(Booru.E621, http)
     {
     }
@@ -20,7 +20,8 @@ public class E621ImageDownloader : ImageDownloader<E621Object>
         var uri = $"https://e621.net/posts.json?limit=32&tags={tagString}&page={page}";
         using var req = new HttpRequestMessage(HttpMethod.Get, uri);
         req.Headers.AddFakeHeaders();
-        using var res = await _http.SendAsync(req, cancel);
+        using var http = _http.CreateClient();
+        using var res = await http.SendAsync(req, cancel);
         res.EnsureSuccessStatusCode();
 
         var data = await res.Content.ReadFromJsonAsync<E621Response>(_serializerOptions, cancel);

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/GelbooruImageDownloader.cs

@@ -6,7 +6,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
 
 public class GelbooruImageDownloader : ImageDownloader<DapiImageObject>
 {
-    public GelbooruImageDownloader(HttpClient http)
+    public GelbooruImageDownloader(IHttpClientFactory http)
         : base(Booru.Gelbooru, http)
     {
     }
@@ -26,7 +26,8 @@ public class GelbooruImageDownloader : ImageDownloader<DapiImageObject>
                   + $"&tags={tagString}"
                   + $"&pid={page}";
         using var req = new HttpRequestMessage(HttpMethod.Get, uri);
-        using var res = await _http.SendAsync(req, cancel);
+        using var http = _http.CreateClient();
+        using var res = await http.SendAsync(req, cancel);
         res.EnsureSuccessStatusCode();
         var resString = await res.Content.ReadAsStringAsync(cancel);
         if (string.IsNullOrWhiteSpace(resString))

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/ImageDownloader.cs

@@ -8,7 +8,7 @@ public abstract class ImageDownloader<T> : IImageDownloader
     where T : IImageData
 {
     public Booru Booru { get; }
-    protected readonly HttpClient _http;
+    protected readonly IHttpClientFactory _http;
 
     protected readonly JsonSerializerOptions _serializerOptions = new()
     {
@@ -16,7 +16,7 @@ public abstract class ImageDownloader<T> : IImageDownloader
         NumberHandling = JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString
     };
 
-    public ImageDownloader(Booru booru, HttpClient http)
+    public ImageDownloader(Booru booru, IHttpClientFactory http)
     {
         _http = http;
         Booru = booru;

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/KonachanImageDownloader.cs

@@ -7,7 +7,7 @@ public sealed class KonachanImageDownloader : ImageDownloader<DapiImageObject>
 {
     private readonly string _baseUrl;
 
-    public KonachanImageDownloader(HttpClient http)
+    public KonachanImageDownloader(IHttpClientFactory http)
         : base(Booru.Konachan, http)
         => _baseUrl = "https://konachan.com";
 
@@ -19,7 +19,8 @@ public sealed class KonachanImageDownloader : ImageDownloader<DapiImageObject>
     {
         var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
         var uri = $"{_baseUrl}/post.json?s=post&q=index&limit=200&tags={tagString}&page={page}";
-        var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
+        using var http = _http.CreateClient();
+        var imageObjects = await http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
         if (imageObjects is null)
             return new();
         return imageObjects.Where(x => x.FileUrl is not null).ToList();

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/Rule34ImageDownloader.cs

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
 
 public class Rule34ImageDownloader : ImageDownloader<Rule34Object>
 {
-    public Rule34ImageDownloader(HttpClient http)
+    public Rule34ImageDownloader(IHttpClientFactory http)
         : base(Booru.Rule34, http)
     {
     }
@@ -17,9 +17,21 @@ public class Rule34ImageDownloader : ImageDownloader<Rule34Object>
         CancellationToken cancel = default)
     {
         var tagString = ImageDownloaderHelper.GetTagString(tags);
-        var uri = "https://rule34.xxx/index.php?page=dapi&s=post&q=index&json=1&limit=100"
+        var uri = $"https://api.rule34.xxx//index.php?page=dapi&s=post"
-                  + $"&tags={tagString}&pid={page}";
+                  + $"&q=index"
-        var images = await _http.GetFromJsonAsync<List<Rule34Object>>(uri, _serializerOptions, cancel);
+                  + $"&json=1"
+                  + $"&limit=100"
+                  + $"&tags={tagString}"
+                  + $"&pid={page}";
+        
+        using var http = _http.CreateClient();
+        http.DefaultRequestHeaders
+            .TryAddWithoutValidation("cookie", "cf_clearance=Gg3bVffg9fOL_.9fIdKmu5PJS86eTI.yTrhbR8z2tPc-1652310659-0-250");
+
+        http.DefaultRequestHeaders
+            .TryAddWithoutValidation("user-agent",
+                "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36");
+        var images = await http.GetFromJsonAsync<List<Rule34Object>>(uri, _serializerOptions, cancel);
 
         if (images is null)
             return new();

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/SafebooruImageDownloader.cs

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
 
 public class SafebooruImageDownloader : ImageDownloader<SafebooruElement>
 {
-    public SafebooruImageDownloader(HttpClient http)
+    public SafebooruImageDownloader(IHttpClientFactory http)
         : base(Booru.Safebooru, http)
     {
     }
@@ -19,7 +19,9 @@ public class SafebooruImageDownloader : ImageDownloader<SafebooruElement>
         var tagString = ImageDownloaderHelper.GetTagString(tags);
         var uri =
             $"https://safebooru.org/index.php?page=dapi&s=post&q=index&limit=200&tags={tagString}&json=1&pid={page}";
-        var images = await _http.GetFromJsonAsync<List<SafebooruElement>>(uri, _serializerOptions, cancel);
+        
+        using var http = _http.CreateClient();
+        var images = await http.GetFromJsonAsync<List<SafebooruElement>>(uri, _serializerOptions, cancel);
         if (images is null)
             return new();
 

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/SankakuImageDownloader.cs

@@ -7,11 +7,10 @@ public sealed class SankakuImageDownloader : ImageDownloader<SankakuImageObject>
 {
     private readonly string _baseUrl;
 
-    public SankakuImageDownloader(HttpClient http)
+    public SankakuImageDownloader(IHttpClientFactory http)
         : base(Booru.Sankaku, http)
     {
         _baseUrl = "https://capi-v2.sankakucomplex.com";
-        _http.AddFakeHeaders();
     }
 
     public override async Task<List<SankakuImageObject>> DownloadImagesAsync(
@@ -24,9 +23,12 @@ public sealed class SankakuImageDownloader : ImageDownloader<SankakuImageObject>
         var tagString = ImageDownloaderHelper.GetTagString(tags);
 
         var uri = $"{_baseUrl}/posts?tags={tagString}&limit=50";
-        var data = await _http.GetStringAsync(uri);
+        
+        using var http = _http.CreateClient();
+        http.AddFakeHeaders();
+        var data = await http.GetStringAsync(uri, cancel);
         return JsonSerializer.Deserialize<SankakuImageObject[]>(data, _serializerOptions)
-                             .Where(x => !string.IsNullOrWhiteSpace(x.FileUrl) && x.FileType.StartsWith("image"))
+                             ?.Where(x => !string.IsNullOrWhiteSpace(x.FileUrl) && x.FileType.StartsWith("image"))
                              .ToList();
     }
 }

src/NadekoBot/Modules/Nsfw/_Common/Downloaders/YandereImageDownloader.cs

@@ -7,7 +7,7 @@ public sealed class YandereImageDownloader : ImageDownloader<DapiImageObject>
 {
     private readonly string _baseUrl;
 
-    public YandereImageDownloader(HttpClient http)
+    public YandereImageDownloader(IHttpClientFactory http)
         : base(Booru.Yandere, http)
         => _baseUrl = "https://yande.re";
 
@@ -20,7 +20,9 @@ public sealed class YandereImageDownloader : ImageDownloader<DapiImageObject>
         var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
 
         var uri = $"{_baseUrl}/post.json?limit=200&tags={tagString}&page={page}";
-        var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
+        
+        using var http = _http.CreateClient();
+        var imageObjects = await http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
         if (imageObjects is null)
             return new();
         return imageObjects.Where(x => x.FileUrl is not null).ToList();

src/NadekoBot/Modules/Nsfw/_Common/Rule34Object.cs

@@ -1,4 +1,6 @@
 #nullable disable
+using System.Text.Json.Serialization;
+
 namespace NadekoBot.Modules.Nsfw.Common;
 
 public class Rule34Object : IImageData
@@ -7,7 +9,9 @@ public class Rule34Object : IImageData
     public int Directory { get; init; }
     public string Tags { get; init; }
     public int Score { get; init; }
+    [JsonPropertyName("file_url")]
+    public string FileUrl { get; init; }
 
     public ImageData ToCachedImageData(Booru type)
-        => new($"https://img.rule34.xxx//images/{Directory}/{Image}", Booru.Rule34, Tags.Split(' '), Score.ToString());
+        => new(FileUrl, Booru.Rule34, Tags.Split(' '), Score.ToString());
 }

src/NadekoBot/Modules/Searches/Crypto/_Common/QuoteResponse.cs

@@ -1,4 +1,5 @@
-using System.Text.Json.Serialization;
+#nullable disable
+using System.Text.Json.Serialization;
 
 namespace NadekoBot.Modules.Searches;
 

src/NadekoBot/Modules/Searches/Crypto/_Common/YahooQueryModel.cs

@@ -5,5 +5,5 @@ namespace NadekoBot.Modules.Searches;
 public class YahooQueryModel
 {
     [JsonPropertyName("quoteResponse")]
-    public QuoteResponse QuoteResponse { get; set; }
+    public QuoteResponse QuoteResponse { get; set; } = null!;
 }