minster586/nadekobot
1
0
Fork 0
mirror of https://gitlab.com/Kwoth/nadekobot.git synced 2025-11-04 00:34:26 -05:00
Code Issues Packages Projects Releases Wiki Activity

Fixed rule34, small refactor of the downloader classes

Browse Source
This commit is contained in:
Kwoth
2022-05-12 10:23:01 +02:00
parent b02768a08e
commit 1db97decd1
15 changed files with 69 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
src/NadekoBot/Modules/Nsfw/SearchImageCacher.cs
View File

@@ -264,18 +264,18 @@ public class SearchImageCacher : INService
return new();
}
private IImageDownloader GetImageDownloader(Booru booru, HttpClient http)
private IImageDownloader GetImageDownloader(Booru booru)
=> booru switch
{
Booru.Danbooru => new DanbooruImageDownloader(http),
Booru.Yandere => new YandereImageDownloader(http),
Booru.Konachan => new KonachanImageDownloader(http),
Booru.Safebooru => new SafebooruImageDownloader(http),
Booru.E621 => new E621ImageDownloader(http),
Booru.Derpibooru => new DerpibooruImageDownloader(http),
Booru.Gelbooru => new GelbooruImageDownloader(http),
Booru.Rule34 => new Rule34ImageDownloader(http),
Booru.Sankaku => new SankakuImageDownloader(http),
Booru.Danbooru => new DanbooruImageDownloader(_httpFactory),
Booru.Yandere => new YandereImageDownloader(_httpFactory),
Booru.Konachan => new KonachanImageDownloader(_httpFactory),
Booru.Safebooru => new SafebooruImageDownloader(_httpFactory),
Booru.E621 => new E621ImageDownloader(_httpFactory),
Booru.Derpibooru => new DerpibooruImageDownloader(_httpFactory),
Booru.Gelbooru => new GelbooruImageDownloader(_httpFactory),
Booru.Rule34 => new Rule34ImageDownloader(_httpFactory),
Booru.Sankaku => new SankakuImageDownloader(_httpFactory),
_ => throw new NotImplementedException($"{booru} downloader not implemented.")
};
@@ -290,8 +290,7 @@ public class SearchImageCacher : INService
{
Log.Information("Downloading from {Type} (page {Page})...", type, page);
using var http = _httpFactory.CreateClient();
var downloader = GetImageDownloader(type, http);
var downloader = GetImageDownloader(type);
var images = await downloader.DownloadImageDataAsync(tags, page, isExplicit, cancel);
if (images.Count == 0)

5
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DanbooruImageDownloader.cs
View File

@@ -9,7 +9,7 @@ public sealed class DanbooruImageDownloader : DapiImageDownloader
private static readonly ConcurrentDictionary<string, bool> _existentTags = new();
private static readonly ConcurrentDictionary<string, bool> _nonexistentTags = new();
public DanbooruImageDownloader(HttpClient http)
public DanbooruImageDownloader(IHttpClientFactory http)
: base(Booru.Danbooru, http, "http://danbooru.donmai.us")
{
}
@@ -22,7 +22,8 @@ public sealed class DanbooruImageDownloader : DapiImageDownloader
if (_nonexistentTags.ContainsKey(tag))
return false;
var tags = await _http.GetFromJsonAsync<DapiTag[]>(
using var http = _http.CreateClient();
var tags = await http.GetFromJsonAsync<DapiTag[]>(
_baseUrl + "/tags.json" + $"?search[name_or_alias_matches]={tag}",
_serializerOptions,
cancel);

5
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DapiImageDownloader.cs
View File

@@ -7,7 +7,7 @@ public abstract class DapiImageDownloader : ImageDownloader<DapiImageObject>
{
protected readonly string _baseUrl;
public DapiImageDownloader(Booru booru, HttpClient http, string baseUrl)
public DapiImageDownloader(Booru booru, IHttpClientFactory http, string baseUrl)
: base(booru, http)
=> _baseUrl = baseUrl;
@@ -43,7 +43,8 @@ public abstract class DapiImageDownloader : ImageDownloader<DapiImageObject>
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"{_baseUrl}/posts.json?limit=200&tags={tagString}&page={page}";
var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
using var http = _http.CreateClient();
var imageObjects = await http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
if (imageObjects is null)
return new();
return imageObjects.Where(x => x.FileUrl is not null).ToList();

5
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/DerpibooruImageDownloader.cs
View File

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
public class DerpibooruImageDownloader : ImageDownloader<DerpiImageObject>
{
public DerpibooruImageDownloader(HttpClient http)
public DerpibooruImageDownloader(IHttpClientFactory http)
: base(Booru.Derpibooru, http)
{
}
@@ -21,7 +21,8 @@ public class DerpibooruImageDownloader : ImageDownloader<DerpiImageObject>
$"https://www.derpibooru.org/api/v1/json/search/images?q={tagString.Replace('+', ',')}&per_page=49&page={page}";
using var req = new HttpRequestMessage(HttpMethod.Get, uri);
req.Headers.AddFakeHeaders();
using var res = await _http.SendAsync(req, cancel);
using var http = _http.CreateClient();
using var res = await http.SendAsync(req, cancel);
res.EnsureSuccessStatusCode();
var container = await res.Content.ReadFromJsonAsync<DerpiContainer>(_serializerOptions, cancel);

5
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/E621ImageDownloader.cs
View File

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
public class E621ImageDownloader : ImageDownloader<E621Object>
{
public E621ImageDownloader(HttpClient http)
public E621ImageDownloader(IHttpClientFactory http)
: base(Booru.E621, http)
{
}
@@ -20,7 +20,8 @@ public class E621ImageDownloader : ImageDownloader<E621Object>
var uri = $"https://e621.net/posts.json?limit=32&tags={tagString}&page={page}";
using var req = new HttpRequestMessage(HttpMethod.Get, uri);
req.Headers.AddFakeHeaders();
using var res = await _http.SendAsync(req, cancel);
using var http = _http.CreateClient();
using var res = await http.SendAsync(req, cancel);
res.EnsureSuccessStatusCode();
var data = await res.Content.ReadFromJsonAsync<E621Response>(_serializerOptions, cancel);

5
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/GelbooruImageDownloader.cs
View File

@@ -6,7 +6,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
public class GelbooruImageDownloader : ImageDownloader<DapiImageObject>
{
public GelbooruImageDownloader(HttpClient http)
public GelbooruImageDownloader(IHttpClientFactory http)
: base(Booru.Gelbooru, http)
{
}
@@ -26,7 +26,8 @@ public class GelbooruImageDownloader : ImageDownloader<DapiImageObject>
+ $"&tags={tagString}"
+ $"&pid={page}";
using var req = new HttpRequestMessage(HttpMethod.Get, uri);
using var res = await _http.SendAsync(req, cancel);
using var http = _http.CreateClient();
using var res = await http.SendAsync(req, cancel);
res.EnsureSuccessStatusCode();
var resString = await res.Content.ReadAsStringAsync(cancel);
if (string.IsNullOrWhiteSpace(resString))

4
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/ImageDownloader.cs
View File

@@ -8,7 +8,7 @@ public abstract class ImageDownloader<T> : IImageDownloader
where T : IImageData
{
public Booru Booru { get; }
protected readonly HttpClient _http;
protected readonly IHttpClientFactory _http;
protected readonly JsonSerializerOptions _serializerOptions = new()
{
@@ -16,7 +16,7 @@ public abstract class ImageDownloader<T> : IImageDownloader
NumberHandling = JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString
};
public ImageDownloader(Booru booru, HttpClient http)
public ImageDownloader(Booru booru, IHttpClientFactory http)
{
_http = http;
Booru = booru;

5
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/KonachanImageDownloader.cs
View File

@@ -7,7 +7,7 @@ public sealed class KonachanImageDownloader : ImageDownloader<DapiImageObject>
{
private readonly string _baseUrl;
public KonachanImageDownloader(HttpClient http)
public KonachanImageDownloader(IHttpClientFactory http)
: base(Booru.Konachan, http)
=> _baseUrl = "https://konachan.com";
@@ -19,7 +19,8 @@ public sealed class KonachanImageDownloader : ImageDownloader<DapiImageObject>
{
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"{_baseUrl}/post.json?s=post&q=index&limit=200&tags={tagString}&page={page}";
var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
using var http = _http.CreateClient();
var imageObjects = await http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
if (imageObjects is null)
return new();
return imageObjects.Where(x => x.FileUrl is not null).ToList();

20
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/Rule34ImageDownloader.cs
View File

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
public class Rule34ImageDownloader : ImageDownloader<Rule34Object>
{
public Rule34ImageDownloader(HttpClient http)
public Rule34ImageDownloader(IHttpClientFactory http)
: base(Booru.Rule34, http)
{
}
@@ -17,9 +17,21 @@ public class Rule34ImageDownloader : ImageDownloader<Rule34Object>
CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags);
var uri = "https://rule34.xxx/index.php?page=dapi&s=post&q=index&json=1&limit=100"
+ $"&tags={tagString}&pid={page}";
var images = await _http.GetFromJsonAsync<List<Rule34Object>>(uri, _serializerOptions, cancel);
var uri = $"https://api.rule34.xxx//index.php?page=dapi&s=post"
+ $"&q=index"
+ $"&json=1"
+ $"&limit=100"
+ $"&tags={tagString}"
+ $"&pid={page}";
using var http = _http.CreateClient();
http.DefaultRequestHeaders
.TryAddWithoutValidation("cookie", "cf_clearance=Gg3bVffg9fOL_.9fIdKmu5PJS86eTI.yTrhbR8z2tPc-1652310659-0-250");
http.DefaultRequestHeaders
.TryAddWithoutValidation("user-agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/100.0.4896.60 Safari/537.36");
var images = await http.GetFromJsonAsync<List<Rule34Object>>(uri, _serializerOptions, cancel);
if (images is null)
return new();

6
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/SafebooruImageDownloader.cs
View File

@@ -5,7 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
public class SafebooruImageDownloader : ImageDownloader<SafebooruElement>
{
public SafebooruImageDownloader(HttpClient http)
public SafebooruImageDownloader(IHttpClientFactory http)
: base(Booru.Safebooru, http)
{
}
@@ -19,7 +19,9 @@ public class SafebooruImageDownloader : ImageDownloader<SafebooruElement>
var tagString = ImageDownloaderHelper.GetTagString(tags);
var uri =
$"https://safebooru.org/index.php?page=dapi&s=post&q=index&limit=200&tags={tagString}&json=1&pid={page}";
var images = await _http.GetFromJsonAsync<List<SafebooruElement>>(uri, _serializerOptions, cancel);
using var http = _http.CreateClient();
var images = await http.GetFromJsonAsync<List<SafebooruElement>>(uri, _serializerOptions, cancel);
if (images is null)
return new();

10
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/SankakuImageDownloader.cs
View File

@@ -7,11 +7,10 @@ public sealed class SankakuImageDownloader : ImageDownloader<SankakuImageObject>
{
private readonly string _baseUrl;
public SankakuImageDownloader(HttpClient http)
public SankakuImageDownloader(IHttpClientFactory http)
: base(Booru.Sankaku, http)
{
_baseUrl = "https://capi-v2.sankakucomplex.com";
_http.AddFakeHeaders();
}
public override async Task<List<SankakuImageObject>> DownloadImagesAsync(
@@ -24,9 +23,12 @@ public sealed class SankakuImageDownloader : ImageDownloader<SankakuImageObject>
var tagString = ImageDownloaderHelper.GetTagString(tags);
var uri = $"{_baseUrl}/posts?tags={tagString}&limit=50";
var data = await _http.GetStringAsync(uri);
using var http = _http.CreateClient();
http.AddFakeHeaders();
var data = await http.GetStringAsync(uri, cancel);
return JsonSerializer.Deserialize<SankakuImageObject[]>(data, _serializerOptions)
.Where(x => !string.IsNullOrWhiteSpace(x.FileUrl) && x.FileType.StartsWith("image"))
?.Where(x => !string.IsNullOrWhiteSpace(x.FileUrl) && x.FileType.StartsWith("image"))
.ToList();
}
}

6
src/NadekoBot/Modules/Nsfw/_Common/Downloaders/YandereImageDownloader.cs
View File

@@ -7,7 +7,7 @@ public sealed class YandereImageDownloader : ImageDownloader<DapiImageObject>
{
private readonly string _baseUrl;
public YandereImageDownloader(HttpClient http)
public YandereImageDownloader(IHttpClientFactory http)
: base(Booru.Yandere, http)
=> _baseUrl = "https://yande.re";
@@ -20,7 +20,9 @@ public sealed class YandereImageDownloader : ImageDownloader<DapiImageObject>
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"{_baseUrl}/post.json?limit=200&tags={tagString}&page={page}";
var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
using var http = _http.CreateClient();
var imageObjects = await http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel);
if (imageObjects is null)
return new();
return imageObjects.Where(x => x.FileUrl is not null).ToList();

6
src/NadekoBot/Modules/Nsfw/_Common/Rule34Object.cs
View File

@@ -1,4 +1,6 @@
#nullable disable
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Nsfw.Common;
public class Rule34Object : IImageData
@@ -7,7 +9,9 @@ public class Rule34Object : IImageData
public int Directory { get; init; }
public string Tags { get; init; }
public int Score { get; init; }
[JsonPropertyName("file_url")]
public string FileUrl { get; init; }
public ImageData ToCachedImageData(Booru type)
=> new($"https://img.rule34.xxx//images/{Directory}/{Image}", Booru.Rule34, Tags.Split(' '), Score.ToString());
=> new(FileUrl, Booru.Rule34, Tags.Split(' '), Score.ToString());
}

3
src/NadekoBot/Modules/Searches/Crypto/_Common/QuoteResponse.cs
View File

@@ -1,4 +1,5 @@
using System.Text.Json.Serialization;
#nullable disable
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;

2
src/NadekoBot/Modules/Searches/Crypto/_Common/YahooQueryModel.cs
View File

@@ -5,5 +5,5 @@ namespace NadekoBot.Modules.Searches;
public class YahooQueryModel
{
[JsonPropertyName("quoteResponse")]
public QuoteResponse QuoteResponse { get; set; }
public QuoteResponse QuoteResponse { get; set; } = null!;
}