minster586/nadekobot
1
0
Fork 0
mirror of https://gitlab.com/Kwoth/nadekobot.git synced 2025-11-04 16:44:28 -05:00
Code Issues Packages Projects Releases Wiki Activity

Applied codestyle to all .cs files

Browse Source
This commit is contained in:
Kwoth
2021-12-29 06:07:16 +01:00
parent 723447c7d4
commit 82000c97a4
543 changed files with 13221 additions and 14059 deletions
Download Patch File Download Diff File Expand all files Collapse all files

113
src/NadekoBot/Modules/Nsfw/SearchImageCacher.cs
View File

@@ -5,10 +5,7 @@ namespace NadekoBot.Modules.Nsfw.Common;
public class SearchImageCacher : INService
{
private readonly IHttpClientFactory _httpFactory;
private readonly Random _rng;
private static readonly ISet<string> defaultTagBlacklist = new HashSet<string>()
private static readonly ISet<string> defaultTagBlacklist = new HashSet<string>
{
"loli",
"lolicon",
@@ -17,10 +14,15 @@ public class SearchImageCacher : INService
"cub"
};
private readonly IHttpClientFactory _httpFactory;
private readonly Random _rng;
private readonly Dictionary<Booru, object> _typeLocks = new();
private readonly Dictionary<Booru, HashSet<string>> _usedTags = new();
private readonly IMemoryCache _cache;
private readonly ConcurrentDictionary<(Booru, string), int> maxPages = new();
public SearchImageCacher(IHttpClientFactory httpFactory, IMemoryCache cache)
{
_httpFactory = httpFactory;
@@ -39,21 +41,23 @@ public class SearchImageCacher : INService
=> $"booru:{boory}__tag:{tag}";
/// <summary>
/// Download images of the specified type, and cache them.
/// Download images of the specified type, and cache them.
/// </summary>
/// <param name="tags">Required tags</param>
/// <param name="forceExplicit">Whether images will be forced to be explicit</param>
/// <param name="type">Provider type</param>
/// <param name="cancel">Cancellation token</param>
/// <returns>Whether any image is found.</returns>
private async Task<bool> UpdateImagesInternalAsync(string[] tags, bool forceExplicit, Booru type, CancellationToken cancel)
private async Task<bool> UpdateImagesInternalAsync(
string[] tags,
bool forceExplicit,
Booru type,
CancellationToken cancel)
{
var images = await DownloadImagesAsync(tags, forceExplicit, type, cancel);
if (images is null || images.Count == 0)
{
// Log.Warning("Got no images for {0}, tags: {1}", type, string.Join(", ", tags));
return false;
}
Log.Information("Updating {0}...", type);
lock (_typeLocks[type])
@@ -65,12 +69,7 @@ public class SearchImageCacher : INService
// if user uses no tags for the hentai command and there are no used
// tags atm, just select 50 random tags from downloaded images to seed
if (typeUsedTags.Count == 0)
images.SelectMany(x => x.Tags)
.Distinct()
.Shuffle()
.Take(50)
.ToList()
.ForEach(x => typeUsedTags.Add(x));
images.SelectMany(x => x.Tags).Distinct().Shuffle().Take(50).ToList().ForEach(x => typeUsedTags.Add(x));
foreach (var img in images)
{
@@ -78,7 +77,7 @@ public class SearchImageCacher : INService
// do not put that image in the cache
if (defaultTagBlacklist.Overlaps(img.Tags))
continue;
// if image doesn't have a proper absolute uri, skip it
if (!Uri.IsWellFormedUriString(img.FileUrl, UriKind.Absolute))
continue;
@@ -88,26 +87,29 @@ public class SearchImageCacher : INService
// both 'kiss' (real tag returned by the image) and 'kissing' will be populated with
// retreived images
foreach (var tag in img.Tags.Concat(tags).Distinct())
{
if (typeUsedTags.Contains(tag))
{
var set = _cache.GetOrCreate<HashSet<ImageData>>(Key(type, tag), e =>
{
e.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(30);
return new();
});
if(set.Count < 100)
var set = _cache.GetOrCreate<HashSet<ImageData>>(Key(type, tag),
e =>
{
e.AbsoluteExpirationRelativeToNow = TimeSpan.FromMinutes(30);
return new();
});
if (set.Count < 100)
set.Add(img);
}
}
}
}
return true;
}
private ImageData QueryLocal(string[] tags, bool forceExplicit, Booru type, HashSet<string> blacklistedTags)
private ImageData QueryLocal(
string[] tags,
bool forceExplicit,
Booru type,
HashSet<string> blacklistedTags)
{
var setList = new List<HashSet<ImageData>>();
@@ -118,25 +120,18 @@ public class SearchImageCacher : INService
if (tags.Length == 0)
{
// get all tags in the cache
if (_usedTags.TryGetValue(type, out var allTags)
&& allTags.Count > 0)
{
tags = new[] {allTags.ToList()[_rng.Next(0, allTags.Count)]};
}
if (_usedTags.TryGetValue(type, out var allTags) && allTags.Count > 0)
tags = new[] { allTags.ToList()[_rng.Next(0, allTags.Count)] };
else
{
return null;
}
}
foreach (var tag in tags)
{
// if any tag is missing from cache, that means there is no result
if (_cache.TryGetValue<HashSet<ImageData>>(Key(type, tag), out var set))
setList.Add(set);
else
return null;
}
if (setList.Count == 0)
return null;
@@ -152,14 +147,11 @@ public class SearchImageCacher : INService
// go through all other sets, and
for (var i = 1; i < setList.Count; ++i)
{
// if any of the elements in result set are not present in the current set
// remove it from the result set
resultSet.IntersectWith(setList[i]);
}
resultList = resultSet.ToList();
}
else
{
@@ -173,30 +165,26 @@ public class SearchImageCacher : INService
// if no items in the set -> not found
if (resultList.Count == 0)
return null;
var toReturn = resultList[_rng.Next(0, resultList.Count)];
// remove from cache
foreach (var tag in tags)
{
if (_cache.TryGetValue<HashSet<ImageData>>(Key(type, tag), out var items))
{
items.Remove(toReturn);
}
}
return toReturn;
}
}
public async Task<ImageData> GetImageNew(string[] tags, bool forceExplicit, Booru type,
HashSet<string> blacklistedTags, CancellationToken cancel)
public async Task<ImageData> GetImageNew(
string[] tags,
bool forceExplicit,
Booru type,
HashSet<string> blacklistedTags,
CancellationToken cancel)
{
// make sure tags are proper
tags = tags
.Where(x => x is not null)
.Select(tag => tag.ToLowerInvariant().Trim())
.Distinct()
.ToArray();
tags = tags.Where(x => x is not null).Select(tag => tag.ToLowerInvariant().Trim()).Distinct().ToArray();
if (tags.Length > 2 && type == Booru.Danbooru)
tags = tags[..2];
@@ -222,15 +210,17 @@ public class SearchImageCacher : INService
if (!success)
return default;
image = QueryLocal(tags, forceExplicit, type, blacklistedTags);
return image;
}
private readonly ConcurrentDictionary<(Booru, string), int> maxPages = new();
public async Task<List<ImageData>> DownloadImagesAsync(string[] tags, bool isExplicit, Booru type, CancellationToken cancel)
public async Task<List<ImageData>> DownloadImagesAsync(
string[] tags,
bool isExplicit,
Booru type,
CancellationToken cancel)
{
var tagStr = string.Join(' ', tags.OrderByDescending(x => x));
var page = 0;
@@ -257,7 +247,10 @@ public class SearchImageCacher : INService
if (result is null or { Count: 0 })
{
Log.Information("Tag {0}, page {1} has no result on {2}.", string.Join(", ", tags), page, type.ToString());
Log.Information("Tag {0}, page {1} has no result on {2}.",
string.Join(", ", tags),
page,
type.ToString());
continue;
}
@@ -282,7 +275,12 @@ public class SearchImageCacher : INService
_ => throw new NotImplementedException($"{booru} downloader not implemented.")
};
private async Task<List<ImageData>> DownloadImagesAsync(string[] tags, bool isExplicit, Booru type, int page, CancellationToken cancel)
private async Task<List<ImageData>> DownloadImagesAsync(
string[] tags,
bool isExplicit,
Booru type,
int page,
CancellationToken cancel)
{
try
{
@@ -306,7 +304,8 @@ public class SearchImageCacher : INService
}
catch (Exception ex)
{
Log.Error(ex, "Error downloading an image:\nTags: {0}\nType: {1}\nPage: {2}\nMessage: {3}",
Log.Error(ex,
"Error downloading an image:\nTags: {0}\nType: {1}\nPage: {2}\nMessage: {3}",
string.Join(", ", tags),
type,
page,
@@ -314,4 +313,4 @@ public class SearchImageCacher : INService
return new();
}
}
}
}