backport of public nsfw module

src/NadekoBot/Modules/Searches/Common/ImageCacherObject.cs

@@ -1,18 +1,19 @@
 using System;
 using System.Collections.Generic;
+using NadekoBot.Modules.Nsfw.Common;
 
 namespace NadekoBot.Modules.Searches.Common
 {
     public class ImageCacherObject : IComparable<ImageCacherObject>
     {
-        public DapiSearchType SearchType { get; }
+        public Booru SearchType { get; }
         public string FileUrl { get; }
         public HashSet<string> Tags { get; }
         public string Rating { get; }
 
-        public ImageCacherObject(DapiImageObject obj, DapiSearchType type)
+        public ImageCacherObject(DapiImageObject obj, Booru type)
         {
-            if (type == DapiSearchType.Danbooru && !Uri.IsWellFormedUriString(obj.FileUrl, UriKind.Absolute))
+            if (type == Booru.Danbooru && !Uri.IsWellFormedUriString(obj.FileUrl, UriKind.Absolute))
             {
                 this.FileUrl = "https://danbooru.donmai.us" + obj.FileUrl;
             }
@@ -25,7 +26,7 @@ namespace NadekoBot.Modules.Searches.Common
             this.Tags = new HashSet<string>((obj.Tags ?? obj.TagString).Split(' '));
         }
 
-        public ImageCacherObject(string url, DapiSearchType type, string tags, string rating)
+        public ImageCacherObject(string url, Booru type, string tags, string rating)
         {
             this.SearchType = type;
             this.FileUrl = url;

src/NadekoBot/Modules/Searches/Common/SearchImageCacher.cs

@@ -1,312 +0,0 @@
-using NadekoBot.Extensions;
-using Newtonsoft.Json;
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Net.Http;
-using System.Threading;
-using System.Threading.Tasks;
-using System.Xml;
-using Serilog;
-
-namespace NadekoBot.Modules.Searches.Common
-{
-    // note: this is not the code that public nadeko is using
-    public class SearchImageCacher
-    {
-        private readonly SemaphoreSlim _lock = new SemaphoreSlim(1, 1);
-        private readonly IHttpClientFactory _httpFactory;
-        private readonly Random _rng;
-        private readonly SortedSet<ImageCacherObject> _cache;
-        private static readonly List<string> defaultTagBlacklist = new List<string>() {
-            "loli",
-            "lolicon",
-            "shota"
-        };
-
-        public SearchImageCacher(IHttpClientFactory http)
-        {
-            _httpFactory = http;
-            _rng = new Random();
-            _cache = new SortedSet<ImageCacherObject>();
-        }
-
-        public async Task<ImageCacherObject> GetImage(string[] tags, bool forceExplicit, DapiSearchType type,
-            HashSet<string> blacklistedTags = null)
-        {
-            tags = tags.Select(tag => tag?.ToLowerInvariant()).ToArray();
-
-            blacklistedTags = blacklistedTags ?? new HashSet<string>();
-
-            foreach (var item in defaultTagBlacklist)
-            {
-                blacklistedTags.Add(item);
-            }
-
-            blacklistedTags = blacklistedTags.Select(t => t.ToLowerInvariant()).ToHashSet();
-
-            if (tags.Any(x => blacklistedTags.Contains(x)))
-            {
-                throw new Exception("One of the specified tags is blacklisted");
-            }
-
-            if (type == DapiSearchType.E621)
-                tags = tags.Select(tag => tag?.Replace("yuri", "female/female", StringComparison.InvariantCulture))
-                    .ToArray();
-
-            await _lock.WaitAsync().ConfigureAwait(false);
-            try
-            {
-                ImageCacherObject[] imgs;
-                if (tags.Any())
-                {
-                    imgs = _cache.Where(x => x.Tags.IsSupersetOf(tags) && x.SearchType == type && (!forceExplicit || x.Rating == "e")).ToArray();
-                }
-                else
-                {
-                    imgs = _cache.Where(x => x.SearchType == type).ToArray();
-                }
-                imgs = imgs.Where(x => x.Tags.All(t => !blacklistedTags.Contains(t.ToLowerInvariant()))).ToArray();
-                ImageCacherObject img;
-                if (imgs.Length == 0)
-                    img = null;
-                else
-                    img = imgs[_rng.Next(imgs.Length)];
-
-                if (img != null)
-                {
-                    _cache.Remove(img);
-                    return img;
-                }
-                else
-                {
-                    var images = await DownloadImagesAsync(tags, forceExplicit, type).ConfigureAwait(false);
-                    images = images
-                        .Where(x => x.Tags.All(t => !blacklistedTags.Contains(t.ToLowerInvariant())))
-                        .ToArray();
-                    if (images.Length == 0)
-                        return null;
-                    var toReturn = images[_rng.Next(images.Length)];
-                    foreach (var dledImg in images)
-                    {
-                        if (dledImg != toReturn)
-                            _cache.Add(dledImg);
-                    }
-                    return toReturn;
-                }
-            }
-            finally
-            {
-                _lock.Release();
-            }
-        }
-
-        public async Task<ImageCacherObject[]> DownloadImagesAsync(string[] tags, bool isExplicit, DapiSearchType type)
-        {
-            isExplicit = type == DapiSearchType.Safebooru
-                ? false
-                : isExplicit;
-            var tag = "";
-            tag += string.Join('+', tags.Select(x => x.Replace(" ", "_", StringComparison.InvariantCulture).ToLowerInvariant()));
-            if (isExplicit)
-                tag = "rating%3Aexplicit+" + tag;
-            var website = "";
-            switch (type)
-            {
-                case DapiSearchType.Safebooru:
-                    website = $"https://safebooru.org/index.php?page=dapi&s=post&q=index&limit=1000&tags={tag}&json=1";
-                    break;
-                case DapiSearchType.E621:
-                    website = $"https://e621.net/posts.json?limit=200&tags={tag}";
-                    break;
-                case DapiSearchType.Danbooru:
-                    website = $"http://danbooru.donmai.us/posts.json?limit=100&tags={tag}";
-                    break;
-                case DapiSearchType.Gelbooru:
-                    website = $"http://gelbooru.com/index.php?page=dapi&s=post&q=index&limit=100&tags={tag}";
-                    break;
-                case DapiSearchType.Rule34:
-                    website = $"https://rule34.xxx/index.php?page=dapi&s=post&q=index&limit=100&tags={tag}";
-                    break;
-                case DapiSearchType.Konachan:
-                    website = $"https://konachan.com/post.json?s=post&q=index&limit=100&tags={tag}";
-                    break;
-                case DapiSearchType.Yandere:
-                    website = $"https://yande.re/post.json?limit=100&tags={tag}";
-                    break;
-                case DapiSearchType.Derpibooru:
-                    tag = string.IsNullOrWhiteSpace(tag) ? "safe" : tag;
-                    website = $"https://www.derpibooru.org/api/v1/json/search/images?q={tag?.Replace('+', ',')}&per_page=49";
-                    break;
-                case DapiSearchType.Sankaku:
-                    website = $"https://capi-v2.sankakucomplex.com/posts?tags={tag}&limit=50";
-                    break;
-            }
-
-            try
-            {
-                using (var _http = _httpFactory.CreateClient())
-                {
-                    _http.AddFakeHeaders();
-                    if (type == DapiSearchType.Konachan || type == DapiSearchType.Yandere || type == DapiSearchType.Danbooru)
-                    {
-                        var data = await _http.GetStringAsync(website).ConfigureAwait(false);
-                        return JsonConvert.DeserializeObject<DapiImageObject[]>(data)
-                            .Where(x => x.FileUrl != null)
-                            .Select(x => new ImageCacherObject(x, type))
-                            .ToArray();
-                    }
-
-                    if (type == DapiSearchType.Sankaku)
-                    {
-                        var data = await _http.GetStringAsync(website).ConfigureAwait(false);
-                        return JsonConvert.DeserializeObject<SankakuImageObject[]>(data)
-                            .Where(x => !string.IsNullOrWhiteSpace(x.FileUrl) && x.FileType.StartsWith("image"))
-                            .Select(x => new ImageCacherObject(
-                                x.FileUrl,
-                                DapiSearchType.Sankaku,
-                                x.Tags.Select(x => x.Name).JoinWith(','),
-                                x.Score))
-                            .ToArray();
-                    }
-
-                    if (type == DapiSearchType.E621)
-                    {
-                        var data = await _http.GetStringAsync(website).ConfigureAwait(false);
-                        return JsonConvert.DeserializeAnonymousType(data, new { posts = new List<E621Object>() })
-                            .posts
-                            .Where(x => !string.IsNullOrWhiteSpace(x.File?.Url))
-                            .Select(x => new ImageCacherObject(x.File.Url,
-                                type, string.Join(' ', x.Tags.General), x.Score.Total))
-                            .ToArray();
-                    }
-
-                    if (type == DapiSearchType.Derpibooru)
-                    {
-                        var data = await _http.GetStringAsync(website).ConfigureAwait(false);
-                        return JsonConvert.DeserializeObject<DerpiContainer>(data)
-                            .Images
-                            .Where(x => !string.IsNullOrWhiteSpace(x.ViewUrl))
-                            .Select(x => new ImageCacherObject(x.ViewUrl,
-                                type, string.Join("\n", x.Tags), x.Score))
-                            .ToArray();
-                    }
-
-                    if (type == DapiSearchType.Safebooru)
-                    {
-                        var data = await _http.GetStringAsync(website).ConfigureAwait(false);
-                        return JsonConvert.DeserializeObject<SafebooruElement[]>(data)
-                            .Select(x => new ImageCacherObject(x.FileUrl, type, x.Tags, x.Rating))
-                            .ToArray();
-                    }
-
-                    return (await LoadXmlAsync(website, type).ConfigureAwait(false)).ToArray();
-                }
-            }
-            catch (Exception ex)
-            {
-                Log.Warning(ex, "Error downloading an image: {Message}", ex.Message);
-                return Array.Empty<ImageCacherObject>();
-            }
-        }
-
-        private async Task<ImageCacherObject[]> LoadXmlAsync(string website, DapiSearchType type)
-        {
-            var list = new List<ImageCacherObject>();
-            using (var http = _httpFactory.CreateClient())
-            using (var stream = await http.GetStreamAsync(website).ConfigureAwait(false))
-            using (var reader = XmlReader.Create(stream, new XmlReaderSettings()
-            {
-                Async = true,
-            }))
-            {
-                while (await reader.ReadAsync().ConfigureAwait(false))
-                {
-                    if (reader.NodeType == XmlNodeType.Element &&
-                        reader.Name == "post")
-                    {
-                        list.Add(new ImageCacherObject(new DapiImageObject()
-                        {
-                            FileUrl = reader["file_url"],
-                            Tags = reader["tags"],
-                            Rating = reader["rating"] ?? "e"
-
-                        }, type));
-                    }
-                }
-            }
-            return list.ToArray();
-        }
-
-        public void Clear()
-        {
-            _cache.Clear();
-        }
-    }
-
-
-    public class DapiImageObject
-    {
-        [JsonProperty("File_Url")]
-        public string FileUrl { get; set; }
-        public string Tags { get; set; }
-        [JsonProperty("Tag_String")]
-        public string TagString { get; set; }
-        public string Rating { get; set; }
-    }
-
-    public class DerpiContainer
-    {
-        public DerpiImageObject[] Images { get; set; }
-    }
-
-    public class DerpiImageObject
-    {
-        [JsonProperty("view_url")]
-        public string ViewUrl { get; set; }
-        public string[] Tags { get; set; }
-        public string Score { get; set; }
-    }
-
-    public class SankakuImageObject
-    {
-        public class Tag
-        {
-            public string Name { get; set; }
-        }
-        
-        [JsonProperty("file_url")]
-        public string FileUrl { get; set; }
-        
-        [JsonProperty("file_type")]
-        public string FileType { get; set; }
-        
-        public Tag[] Tags { get; set; }
-        
-        [JsonProperty("total_score")]
-        public string Score { get; set; }
-    }
-
-    public enum DapiSearchType
-    {
-        Safebooru,
-        E621,
-        Derpibooru,
-        Gelbooru,
-        Konachan,
-        Rule34,
-        Yandere,
-        Danbooru,
-        Sankaku,
-    }
-    public class SafebooruElement
-    {
-        public string Directory { get; set; }
-        public string Image { get; set; }
-
-
-        public string FileUrl => $"https://safebooru.org/images/{Directory}/{Image}";
-        public string Rating { get; set; }
-        public string Tags { get; set; }
-    }
-}

src/NadekoBot/Modules/Searches/Searches.cs

@@ -22,6 +22,7 @@ using System.Net;
 using System.Net.Http;
 using System.Threading.Tasks;
 using NadekoBot.Modules.Administration.Services;
+using NadekoBot.Modules.Nsfw.Common;
 using Serilog;
 using Configuration = AngleSharp.Configuration;
 
@@ -589,10 +590,6 @@ namespace NadekoBot.Modules.Searches
             await SendConfirmAsync($"https://images.google.com/searchbyimage?image_url={imageLink}").ConfigureAwait(false);
         }
 
-        [NadekoCommand, Aliases]
-        public Task Safebooru([Leftover] string tag = null)
-            => InternalDapiCommand(tag, DapiSearchType.Safebooru);
-
         [NadekoCommand, Aliases]
         public async Task Wiki([Leftover] string query = null)
         {
@@ -760,21 +757,6 @@ namespace NadekoBot.Modules.Searches
             await ctx.Channel.SendMessageAsync($"https://store.steampowered.com/app/{appId}").ConfigureAwait(false);
         }
 
-        public async Task InternalDapiCommand(string tag, DapiSearchType type)
-        {
-            tag = tag?.Trim() ?? "";
-
-            var imgObj = await _service.DapiSearch(tag, type, ctx.Guild?.Id).ConfigureAwait(false);
-
-            if (imgObj is null)
-                await SendErrorAsync(ctx.User.Mention + " " + GetText(strs.no_results)).ConfigureAwait(false);
-            else
-                await ctx.Channel.EmbedAsync(_eb.Create().WithOkColor()
-                    .WithDescription($"{ctx.User.Mention} [{tag ?? "url"}]({imgObj.FileUrl})")
-                    .WithImageUrl(imgObj.FileUrl)
-                    .WithFooter(type.ToString())).ConfigureAwait(false);
-        }
-
         public async Task<bool> ValidateQuery(IMessageChannel ch, string query)
         {
             if (!string.IsNullOrWhiteSpace(query))

src/NadekoBot/Modules/Searches/Services/SearchesService.cs

@@ -50,14 +50,6 @@ namespace NadekoBot.Modules.Searches.Services
 
         public List<WoWJoke> WowJokes { get; } = new List<WoWJoke>();
         public List<MagicItem> MagicItems { get; } = new List<MagicItem>();
-
-        private readonly ConcurrentDictionary<ulong, SearchImageCacher> _imageCacher = new ConcurrentDictionary<ulong, SearchImageCacher>();
-
-        public ConcurrentDictionary<ulong, Timer> AutoHentaiTimers { get; } = new ConcurrentDictionary<ulong, Timer>();
-        public ConcurrentDictionary<ulong, Timer> AutoBoobTimers { get; } = new ConcurrentDictionary<ulong, Timer>();
-        public ConcurrentDictionary<ulong, Timer> AutoButtTimers { get; } = new ConcurrentDictionary<ulong, Timer>();
-
-        private readonly ConcurrentDictionary<ulong, HashSet<string>> _blacklistedTags = new ConcurrentDictionary<ulong, HashSet<string>>();
         private readonly List<string> _yomamaJokes;
 
         public SearchesService(DiscordSocketClient client, IGoogleApiService google,
@@ -75,11 +67,6 @@ namespace NadekoBot.Modules.Searches.Services
             _eb = eb;
             _rng = new NadekoRandom();
 
-            _blacklistedTags = new ConcurrentDictionary<ulong, HashSet<string>>(
-                bot.AllGuildConfigs.ToDictionary(
-                    x => x.GuildId,
-                    x => new HashSet<string>(x.NsfwBlacklistedTags.Select(y => y.Tag))));
-
             //translate commands
             _client.MessageReceived += (msg) =>
             {
@@ -366,80 +353,6 @@ namespace NadekoBot.Modules.Searches.Services
             return (await _google.Translate(text, from, to).ConfigureAwait(false)).SanitizeMentions(true);
         }
 
-        public Task<ImageCacherObject> DapiSearch(string tag, DapiSearchType type, ulong? guild, bool isExplicit = false)
-        {
-            tag = tag ?? "";
-            if (string.IsNullOrWhiteSpace(tag)
-                && (tag.Contains("loli") || tag.Contains("shota")))
-            {
-                return null;
-            }
-
-            var tags = tag
-                .Split('+')
-                .Select(x => x.ToLowerInvariant().Replace(' ', '_'))
-                .ToArray();
-
-            if (guild.HasValue)
-            {
-                var blacklistedTags = GetBlacklistedTags(guild.Value);
-
-                var cacher = _imageCacher.GetOrAdd(guild.Value, (key) => new SearchImageCacher(_httpFactory));
-
-                return cacher.GetImage(tags, isExplicit, type, blacklistedTags);
-            }
-            else
-            {
-                var cacher = _imageCacher.GetOrAdd(guild ?? 0, (key) => new SearchImageCacher(_httpFactory));
-
-                return cacher.GetImage(tags, isExplicit, type);
-            }
-        }
-
-        public HashSet<string> GetBlacklistedTags(ulong guildId)
-        {
-            if (_blacklistedTags.TryGetValue(guildId, out var tags))
-                return tags;
-            return new HashSet<string>();
-        }
-
-        public bool ToggleBlacklistedTag(ulong guildId, string tag)
-        {
-            var tagObj = new NsfwBlacklitedTag
-            {
-                Tag = tag
-            };
-
-            bool added;
-            using (var uow = _db.GetDbContext())
-            {
-                var gc = uow.GuildConfigsForId(guildId, set => set.Include(y => y.NsfwBlacklistedTags));
-                if (gc.NsfwBlacklistedTags.Add(tagObj))
-                    added = true;
-                else
-                {
-                    gc.NsfwBlacklistedTags.Remove(tagObj);
-                    var toRemove = gc.NsfwBlacklistedTags.FirstOrDefault(x => x.Equals(tagObj));
-                    if (toRemove != null)
-                        uow.Remove(toRemove);
-                    added = false;
-                }
-                var newTags = new HashSet<string>(gc.NsfwBlacklistedTags.Select(x => x.Tag));
-                _blacklistedTags.AddOrUpdate(guildId, newTags, delegate { return newTags; });
-
-                uow.SaveChanges();
-            }
-            return added;
-        }
-
-        public void ClearCache()
-        {
-            foreach (var c in _imageCacher)
-            {
-                c.Value?.Clear();
-            }
-        }
-
         private readonly object yomamaLock = new object();
         private int yomamaJokeIndex = 0;
         public Task<string> GetYomamaJoke()
@@ -838,95 +751,5 @@ namespace NadekoBot.Modules.Searches.Services
                 fullQueryLink,
                 "0");
         }
-        #region Nhentai
-        private string GetNhentaiExtensionInternal(string s)
-            => s switch
-            {
-                "j" => "jpg",
-                "p" => "png",
-                "g" => "gif",
-                _ => "jpg"
-            };
-        
-        private Gallery ModelToGallery(NhentaiApiModel.Gallery model)
-        {
-            var thumbnail = $"https://t.nhentai.net/galleries/{model.MediaId}/thumb."
-                            + GetNhentaiExtensionInternal(model.Images.Thumbnail.T);
-
-            var url = $"https://nhentai.net/g/{model.Id}";
-            return new Gallery(
-                model.Id.ToString(),
-                url,
-                model.Title.English,
-                model.Title.Pretty,
-                thumbnail,
-                model.NumPages,
-                model.NumFavorites,
-                model.UploadDate.ToUnixTimestamp().UtcDateTime,
-                model.Tags.Map(x => new Tag()
-                {
-                    Name = x.Name,
-                    Url = "https://nhentai.com/" + x.Url
-                }));
-        }
-        
-        public async Task<NhentaiApiModel.Gallery> GetNhentaiByIdInternalAsync(uint id)
-        {
-            using var http = _httpFactory.CreateClient();
-            try
-            {
-                var res = await http.GetStringAsync("https://nhentai.net/api/gallery/" + id);
-                return JsonConvert.DeserializeObject<NhentaiApiModel.Gallery>(res);
-            }
-            catch (HttpRequestException)
-            {
-                Log.Warning("Nhentai with id {NhentaiId} not found", id);
-                return null;
-            }
-        }
-        
-        private async Task<NhentaiApiModel.Gallery[]> SearchNhentaiInternalAsync(string search)
-        {
-            using var http = _httpFactory.CreateClient();
-            try
-            {
-                var res = await http.GetStringAsync("https://nhentai.net/api/galleries/search?query=" + search);
-                return JsonConvert.DeserializeObject<NhentaiApiModel.SearchResult>(res).Result;
-            }
-            catch (HttpRequestException)
-            {
-                Log.Warning("Nhentai with search {NhentaiSearch} not found", search);
-                return null;
-            }
-        }
-        
-        public async Task<Gallery> GetNhentaiByIdAsync(uint id)
-        {
-            var model = await GetNhentaiByIdInternalAsync(id);
-
-            return ModelToGallery(model);
-        }
-
-        private static readonly string[] _bannedTags =
-        {
-            "loli",
-            "lolicon",
-            "shota",
-            "shotacon",
-            "cub"
-        };
-        
-        public async Task<Gallery> GetNhentaiBySearchAsync(string search)
-        {
-            var models = await SearchNhentaiInternalAsync(search);
-
-            models = models.Where(x => !x.Tags.Any(t => _bannedTags.Contains(t.Name))).ToArray();
-            
-            if (models.Length == 0)
-                return null;
-            
-            return ModelToGallery(models[_rng.Next(0, models.Length)]);
-        }
-        #endregion
     }
 }