minster586/nadekobot
1
0
Fork 0
mirror of https://gitlab.com/Kwoth/nadekobot.git synced 2025-09-11 17:58:26 -04:00
Code Issues Packages Projects Releases Wiki Activity

backport of public nsfw module

Browse Source
This commit is contained in:
Kwoth
2021-10-21 23:35:58 +00:00
parent 24a4745193
commit 1141791ce5
44 changed files with 4284 additions and 713 deletions
Download Patch File Download Diff File Expand all files Collapse all files

41
src/NadekoBot/Modules/Nsfw/Common/Downloaders/DanbooruImageDownloader.cs Normal file
View File

@@ -0,0 +1,41 @@
using System.Collections.Concurrent;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public sealed class DanbooruImageDownloader : DapiImageDownloader
{
// using them as concurrent hashsets, value doesn't matter
private static readonly ConcurrentDictionary<string, bool> _existentTags = new();
private static readonly ConcurrentDictionary<string, bool> _nonexistentTags = new();
public override async Task<bool> IsTagValid(string tag, CancellationToken cancel = default)
{
if (_existentTags.ContainsKey(tag))
return true;
if (_nonexistentTags.ContainsKey(tag))
return false;
var tags = await _http.GetFromJsonAsync<DapiTag[]>(_baseUrl +
"/tags.json" +
$"?search[name_or_alias_matches]={tag}",
options: this._serializerOptions,
cancellationToken: cancel);
if (tags is {Length: > 0})
{
return _existentTags[tag] = true;
}
return _nonexistentTags[tag] = false;
}
public DanbooruImageDownloader(HttpClient http)
: base(Booru.Danbooru, http, "http://danbooru.donmai.us")
{
}
}
}

56
src/NadekoBot/Modules/Nsfw/Common/Downloaders/DapiImageDownloader.cs Normal file
View File

@@ -0,0 +1,56 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public abstract class DapiImageDownloader : ImageDownloader<DapiImageObject>
{
protected readonly string _baseUrl;
public DapiImageDownloader(Booru booru, HttpClient http, string baseUrl) : base(booru, http)
{
_baseUrl = baseUrl;
}
public abstract Task<bool> IsTagValid(string tag, CancellationToken cancel = default);
protected async Task<bool> AllTagsValid(string[] tags, CancellationToken cancel = default)
{
var results = await Task.WhenAll(tags.Select(tag => IsTagValid(tag, cancel)));
// if any of the tags is not valid, the query is not valid
foreach (var result in results)
{
if (!result)
return false;
}
return true;
}
public override async Task<List<DapiImageObject>> DownloadImagesAsync(string[] tags, int page,
bool isExplicit = false, CancellationToken cancel = default)
{
// up to 2 tags allowed on danbooru
if (tags.Length > 2)
return new();
if (!await AllTagsValid(tags, cancel).ConfigureAwait(false))
return new();
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"{_baseUrl}/posts.json?limit=200&tags={tagString}&page={page}";
var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel)
.ConfigureAwait(false);
if (imageObjects is null)
return new();
return imageObjects
.Where(x => x.FileUrl is not null)
.ToList();
}
}
}

35
src/NadekoBot/Modules/Nsfw/Common/Downloaders/DerpibooruImageDownloader.cs Normal file
View File

@@ -0,0 +1,35 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using NadekoBot.Extensions;
namespace NadekoBot.Modules.Nsfw.Common
{
public class DerpibooruImageDownloader : ImageDownloader<DerpiImageObject>
{
public DerpibooruImageDownloader(HttpClient http) : base(Booru.Derpibooru, http)
{
}
public override async Task<List<DerpiImageObject>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"https://www.derpibooru.org/api/v1/json/search/images?q={tagString.Replace('+', ',')}&per_page=49&page={page}";
using var req = new HttpRequestMessage(HttpMethod.Get, uri);
req.Headers.AddFakeHeaders();
using var res = await _http.SendAsync(req, cancel).ConfigureAwait(false);
res.EnsureSuccessStatusCode();
var container = await res.Content.ReadFromJsonAsync<DerpiContainer>(_serializerOptions, cancel).ConfigureAwait(false);
if (container?.Images is null)
return new();
return container.Images
.Where(x => !string.IsNullOrWhiteSpace(x.ViewUrl))
.ToList();
}
}
}

35
src/NadekoBot/Modules/Nsfw/Common/Downloaders/E621ImageDownloader.cs Normal file
View File

@@ -0,0 +1,35 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using NadekoBot.Extensions;
namespace NadekoBot.Modules.Nsfw.Common
{
public class E621ImageDownloader : ImageDownloader<E621Object>
{
public E621ImageDownloader(HttpClient http) : base(Booru.E621, http)
{
}
public override async Task<List<E621Object>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit: isExplicit);
var uri = $"https://e621.net/posts.json?limit=32&tags={tagString}&page={page}";
using var req = new HttpRequestMessage(HttpMethod.Get, uri);
req.Headers.AddFakeHeaders();
using var res = await _http.SendAsync(req, cancel).ConfigureAwait(false);
res.EnsureSuccessStatusCode();
var data = await res.Content.ReadFromJsonAsync<E621Response>(_serializerOptions, cancel).ConfigureAwait(false);
if (data?.Posts is null)
return new();
return data.Posts
.Where(x => !string.IsNullOrWhiteSpace(x.File?.Url))
.ToList();
}
}
}

9
src/NadekoBot/Modules/Nsfw/Common/Downloaders/E621Response.cs Normal file
View File

@@ -0,0 +1,9 @@
using System.Collections.Generic;
namespace NadekoBot.Modules.Nsfw.Common
{
public class E621Response
{
public List<E621Object> Posts { get; set; }
}
}

35
src/NadekoBot/Modules/Nsfw/Common/Downloaders/GelbooruImageDownloader.cs Normal file
View File

@@ -0,0 +1,35 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public class GelbooruImageDownloader : ImageDownloader<DapiImageObject>
{
public GelbooruImageDownloader(HttpClient http) : base(Booru.Gelbooru, http)
{
}
public override async Task<List<DapiImageObject>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"http://gelbooru.com/index.php?page=dapi&s=post&json=1&q=index&limit=100" +
$"&tags={tagString}&pid={page}";
using var req = new HttpRequestMessage(HttpMethod.Get, uri);
using var res = await _http.SendAsync(req, cancel).ConfigureAwait(false);
res.EnsureSuccessStatusCode();
var resString = await res.Content.ReadAsStringAsync(cancel);
if (string.IsNullOrWhiteSpace(resString))
return new();
var images = JsonSerializer.Deserialize<List<DapiImageObject>>(resString, _serializerOptions);
if (images is null)
return new();
return images.Where(x => x.FileUrl is not null).ToList();
}
}
}

12
src/NadekoBot/Modules/Nsfw/Common/Downloaders/IImageDownloader.cs Normal file
View File

@@ -0,0 +1,12 @@
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public interface IImageDownloader
{
Task<List<ImageData>> DownloadImageDataAsync(string[] tags, int page = 0,
bool isExplicit = false, CancellationToken cancel = default);
}
}

40
src/NadekoBot/Modules/Nsfw/Common/Downloaders/ImageDownloader.cs Normal file
View File

@@ -0,0 +1,40 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public abstract class ImageDownloader<T> : IImageDownloader
where T : IImageData
{
protected readonly HttpClient _http;
protected JsonSerializerOptions _serializerOptions = new JsonSerializerOptions()
{
PropertyNameCaseInsensitive = true,
NumberHandling = JsonNumberHandling.WriteAsString | JsonNumberHandling.AllowReadingFromString,
};
public Booru Booru { get; }
public ImageDownloader(Booru booru, HttpClient http)
{
_http = http;
this.Booru = booru;
}
public abstract Task<List<T>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default);
public async Task<List<ImageData>> DownloadImageDataAsync(string[] tags, int page, bool isExplicit = false,
CancellationToken cancel = default)
{
var images = await DownloadImagesAsync(tags, page, isExplicit, cancel).ConfigureAwait(false);
return images.Select(x => x.ToCachedImageData(Booru)).ToList();
}
}
}

17
src/NadekoBot/Modules/Nsfw/Common/Downloaders/ImageDownloaderHelper.cs Normal file
View File

@@ -0,0 +1,17 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace NadekoBot.Modules.Nsfw.Common
{
public static class ImageDownloaderHelper
{
public static string GetTagString(IEnumerable<string> tags, bool isExplicit = false)
{
if (isExplicit)
tags = tags.Append("rating:explicit");
return string.Join('+', tags.Select(x => x.ToLowerInvariant()));
}
}
}

33
src/NadekoBot/Modules/Nsfw/Common/Downloaders/KonachanImageDownloader.cs Normal file
View File

@@ -0,0 +1,33 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public sealed class KonachanImageDownloader : ImageDownloader<DapiImageObject>
{
private readonly string _baseUrl;
public KonachanImageDownloader(HttpClient http)
: base(Booru.Konachan, http)
{
_baseUrl = "https://konachan.com";
}
public override async Task<List<DapiImageObject>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"{_baseUrl}/post.json?s=post&q=index&limit=200&tags={tagString}&page={page}";
var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel)
.ConfigureAwait(false);
if (imageObjects is null)
return new();
return imageObjects
.Where(x => x.FileUrl is not null)
.ToList();
}
}
}

31
src/NadekoBot/Modules/Nsfw/Common/Downloaders/Rule34ImageDownloader.cs Normal file
View File

@@ -0,0 +1,31 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public class Rule34ImageDownloader : ImageDownloader<Rule34Object>
{
public Rule34ImageDownloader(HttpClient http) : base(Booru.Rule34, http)
{
}
public override async Task<List<Rule34Object>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags);
var uri = $"https://rule34.xxx/index.php?page=dapi&s=post&q=index&json=1&limit=100" +
$"&tags={tagString}&pid={page}";
var images = await _http.GetFromJsonAsync<List<Rule34Object>>(uri, _serializerOptions, cancel).ConfigureAwait(false);
if (images is null)
return new();
return images
.Where(img => !string.IsNullOrWhiteSpace(img.Directory) && !string.IsNullOrWhiteSpace(img.Image))
.ToList();
}
}
}

26
src/NadekoBot/Modules/Nsfw/Common/Downloaders/SafebooruImageDownloader.cs Normal file
View File

@@ -0,0 +1,26 @@
using System.Collections.Generic;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
namespace NadekoBot.Modules.Nsfw.Common
{
public class SafebooruImageDownloader : ImageDownloader<SafebooruElement>
{
public SafebooruImageDownloader(HttpClient http) : base(Booru.Safebooru, http)
{
}
public override async Task<List<SafebooruElement>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit: false);
var uri = $"https://safebooru.org/index.php?page=dapi&s=post&q=index&limit=200&tags={tagString}&json=1&pid={page}";
var images = await _http.GetFromJsonAsync<List<SafebooruElement>>(uri, _serializerOptions, cancellationToken: cancel);
if (images is null)
return new();
return images;
}
}
}

34
src/NadekoBot/Modules/Nsfw/Common/Downloaders/SankakuImageDownloader.cs Normal file
View File

@@ -0,0 +1,34 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using NadekoBot.Extensions;
namespace NadekoBot.Modules.Nsfw.Common
{
public sealed class SankakuImageDownloader : ImageDownloader<SankakuImageObject>
{
private readonly string _baseUrl;
public SankakuImageDownloader(HttpClient http)
: base(Booru.Sankaku, http)
{
_baseUrl = "https://capi-v2.sankakucomplex.com";
_http.AddFakeHeaders();
}
public override async Task<List<SankakuImageObject>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
// explicit probably not supported
var tagString = ImageDownloaderHelper.GetTagString(tags, false);
var uri = $"{_baseUrl}/posts?tags={tagString}&limit=50";
var data = await _http.GetStringAsync(uri).ConfigureAwait(false);
return JsonSerializer.Deserialize<SankakuImageObject[]>(data, _serializerOptions)
.Where(x => !string.IsNullOrWhiteSpace(x.FileUrl) && x.FileType.StartsWith("image"))
.ToList();
}
}
}

34
src/NadekoBot/Modules/Nsfw/Common/Downloaders/YandereImageDownloader.cs Normal file
View File

@@ -0,0 +1,34 @@
using System.Collections.Generic;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Json;
using System.Threading;
using System.Threading.Tasks;
using NadekoBot.Extensions;
namespace NadekoBot.Modules.Nsfw.Common
{
public sealed class YandereImageDownloader : ImageDownloader<DapiImageObject>
{
private readonly string _baseUrl;
public YandereImageDownloader(HttpClient http)
: base(Booru.Yandere, http)
{
_baseUrl = "https://yande.re";
}
public override async Task<List<DapiImageObject>> DownloadImagesAsync(string[] tags, int page, bool isExplicit = false, CancellationToken cancel = default)
{
var tagString = ImageDownloaderHelper.GetTagString(tags, isExplicit);
var uri = $"{_baseUrl}/post.json?limit=200&tags={tagString}&page={page}";
var imageObjects = await _http.GetFromJsonAsync<DapiImageObject[]>(uri, _serializerOptions, cancel)
.ConfigureAwait(false);
if (imageObjects is null)
return new();
return imageObjects
.Where(x => x.FileUrl is not null)
.ToList();
}
}
}