minster586/nadekobot
1
0
Fork 0
mirror of https://gitlab.com/Kwoth/nadekobot.git synced 2025-09-11 17:58:26 -04:00
Code Issues Packages Projects Releases Wiki Activity

Abstract away cache. 2 implementations: redis and memory

Browse Source
This commit is contained in:
Kwoth
2022-06-23 13:07:45 +00:00
parent 1716c69132
commit 210da263ad
75 changed files with 11525 additions and 1547 deletions
Download Patch File Download Diff File Expand all files Collapse all files

26
src/NadekoBot/Modules/Searches/Anime/AnimeResult.cs
View File

@@ -1,34 +1,38 @@
#nullable disable
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches.Common;
public class AnimeResult
{
[JsonPropertyName("id")]
public int Id { get; set; }
public string AiringStatus
=> AiringStatusParsed.ToTitleCase();
[JsonProperty("airing_status")]
[JsonPropertyName("airing_status")]
public string AiringStatusParsed { get; set; }
[JsonProperty("title_english")]
[JsonPropertyName("title_english")]
public string TitleEnglish { get; set; }
[JsonProperty("total_episodes")]
[JsonPropertyName("total_episodes")]
public int TotalEpisodes { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; }
[JsonProperty("image_url_lge")]
[JsonPropertyName("image_url_lge")]
public string ImageUrlLarge { get; set; }
[JsonPropertyName("genres")]
public string[] Genres { get; set; }
[JsonProperty("average_score")]
public string AverageScore { get; set; }
[JsonPropertyName("average_score")]
public float AverageScore { get; set; }
public string AiringStatus
=> AiringStatusParsed.ToTitleCase();
public string Link
=> "http://anilist.co/anime/" + Id;

117
src/NadekoBot/Modules/Searches/Anime/AnimeSearchService.cs
View File

@@ -1,17 +1,15 @@
#nullable disable
using AngleSharp;
using AngleSharp.Html.Dom;
using NadekoBot.Modules.Searches.Common;
using Newtonsoft.Json;
using System.Net.Http.Json;
namespace NadekoBot.Modules.Searches.Services;
public class AnimeSearchService : INService
{
private readonly IDataCache _cache;
private readonly IBotCache _cache;
private readonly IHttpClientFactory _httpFactory;
public AnimeSearchService(IDataCache cache, IHttpClientFactory httpFactory)
public AnimeSearchService(IBotCache cache, IHttpClientFactory httpFactory)
{
_cache = cache;
_httpFactory = httpFactory;
@@ -21,24 +19,25 @@ public class AnimeSearchService : INService
{
if (string.IsNullOrWhiteSpace(query))
throw new ArgumentNullException(nameof(query));
TypedKey<AnimeResult> GetKey(string link)
=> new TypedKey<AnimeResult>($"anime2:{link}");
try
{
var link = "https://aniapi.nadeko.bot/anime/"
+ Uri.EscapeDataString(query.Replace("/", " ", StringComparison.InvariantCulture));
var suffix = Uri.EscapeDataString(query.Replace("/", " ", StringComparison.InvariantCulture));
var link = $"https://aniapi.nadeko.bot/anime/{suffix}";
link = link.ToLowerInvariant();
var (ok, data) = await _cache.TryGetAnimeDataAsync(link);
if (!ok)
var result = await _cache.GetAsync(GetKey(link));
if (!result.TryPickT0(out var data, out _))
{
using (var http = _httpFactory.CreateClient())
{
data = await http.GetStringAsync(link);
}
using var http = _httpFactory.CreateClient();
data = await http.GetFromJsonAsync<AnimeResult>(link);
await _cache.SetAnimeDataAsync(link, data);
await _cache.AddAsync(GetKey(link), data, expiry: TimeSpan.FromHours(12));
}
return JsonConvert.DeserializeObject<AnimeResult>(data);
return data;
}
catch
{
@@ -46,95 +45,31 @@ public class AnimeSearchService : INService
}
}
public async Task<NovelResult> GetNovelData(string query)
{
if (string.IsNullOrWhiteSpace(query))
throw new ArgumentNullException(nameof(query));
query = query.Replace(" ", "-", StringComparison.InvariantCulture);
try
{
var link = "https://www.novelupdates.com/series/"
+ Uri.EscapeDataString(query.Replace(" ", "-").Replace("/", " "));
link = link.ToLowerInvariant();
var (ok, data) = await _cache.TryGetNovelDataAsync(link);
if (!ok)
{
var config = Configuration.Default.WithDefaultLoader();
using var document = await BrowsingContext.New(config).OpenAsync(link);
var imageElem = document.QuerySelector("div.seriesimg > img");
if (imageElem is null)
return null;
var imageUrl = ((IHtmlImageElement)imageElem).Source;
var descElem = document.QuerySelector("div#editdescription > p");
var desc = descElem.InnerHtml;
var genres = document.QuerySelector("div#seriesgenre")
.Children.Select(x => x as IHtmlAnchorElement)
.Where(x => x is not null)
.Select(x => $"[{x.InnerHtml}]({x.Href})")
.ToArray();
var authors = document.QuerySelector("div#showauthors")
.Children.Select(x => x as IHtmlAnchorElement)
.Where(x => x is not null)
.Select(x => $"[{x.InnerHtml}]({x.Href})")
.ToArray();
var score = ((IHtmlSpanElement)document.QuerySelector("h5.seriesother > span.uvotes")).InnerHtml;
var status = document.QuerySelector("div#editstatus").InnerHtml;
var title = document.QuerySelector("div.w-blog-content > div.seriestitlenu").InnerHtml;
var obj = new NovelResult
{
Description = desc,
Authors = authors,
Genres = genres,
ImageUrl = imageUrl,
Link = link,
Score = score,
Status = status,
Title = title
};
await _cache.SetNovelDataAsync(link, JsonConvert.SerializeObject(obj));
return obj;
}
return JsonConvert.DeserializeObject<NovelResult>(data);
}
catch (Exception ex)
{
Log.Error(ex, "Error getting novel data");
return null;
}
}
public async Task<MangaResult> GetMangaData(string query)
{
if (string.IsNullOrWhiteSpace(query))
throw new ArgumentNullException(nameof(query));
TypedKey<MangaResult> GetKey(string link)
=> new TypedKey<MangaResult>($"manga2:{link}");
try
{
var link = "https://aniapi.nadeko.bot/manga/"
+ Uri.EscapeDataString(query.Replace("/", " ", StringComparison.InvariantCulture));
link = link.ToLowerInvariant();
var (ok, data) = await _cache.TryGetAnimeDataAsync(link);
if (!ok)
var result = await _cache.GetAsync(GetKey(link));
if (!result.TryPickT0(out var data, out _))
{
using (var http = _httpFactory.CreateClient())
{
data = await http.GetStringAsync(link);
}
using var http = _httpFactory.CreateClient();
data = await http.GetFromJsonAsync<MangaResult>(link);
await _cache.SetAnimeDataAsync(link, data);
await _cache.AddAsync(GetKey(link), data, expiry: TimeSpan.FromHours(3));
}
return JsonConvert.DeserializeObject<MangaResult>(data);
return data;
}
catch
{

20
src/NadekoBot/Modules/Searches/Anime/MangaResult.cs
View File

@@ -1,32 +1,36 @@
#nullable disable
using Newtonsoft.Json;
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches.Common;
public class MangaResult
{
[JsonPropertyName("id")]
public int Id { get; set; }
[JsonProperty("publishing_status")]
[JsonPropertyName("publishing_status")]
public string PublishingStatus { get; set; }
[JsonProperty("image_url_lge")]
[JsonPropertyName("image_url_lge")]
public string ImageUrlLge { get; set; }
[JsonProperty("title_english")]
[JsonPropertyName("title_english")]
public string TitleEnglish { get; set; }
[JsonProperty("total_chapters")]
[JsonPropertyName("total_chapters")]
public int TotalChapters { get; set; }
[JsonProperty("total_volumes")]
[JsonPropertyName("total_volumes")]
public int TotalVolumes { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; }
[JsonPropertyName("genres")]
public string[] Genres { get; set; }
[JsonProperty("average_score")]
public string AverageScore { get; set; }
[JsonPropertyName("average_score")]
public float AverageScore { get; set; }
public string Link
=> "http://anilist.co/manga/" + Id;

100
src/NadekoBot/Modules/Searches/Crypto/CryptoService.cs
View File

@@ -15,13 +15,13 @@ namespace NadekoBot.Modules.Searches.Services;
public class CryptoService : INService
{
private readonly IDataCache _cache;
private readonly IBotCache _cache;
private readonly IHttpClientFactory _httpFactory;
private readonly IBotCredentials _creds;
private readonly SemaphoreSlim _getCryptoLock = new(1, 1);
public CryptoService(IDataCache cache, IHttpClientFactory httpFactory, IBotCredentials creds)
public CryptoService(IBotCache cache, IHttpClientFactory httpFactory, IBotCredentials creds)
{
_cache = cache;
_httpFactory = httpFactory;
@@ -40,7 +40,8 @@ public class CryptoService : INService
Span<PointF> points = new PointF[gElement.ChildNodes.Count];
var cnt = 0;
bool GetValuesFromAttributes(XmlAttributeCollection attrs,
bool GetValuesFromAttributes(
XmlAttributeCollection attrs,
out float x1,
out float y1,
out float x2,
@@ -56,7 +57,7 @@ public class CryptoService : INService
&& attrs["y2"]?.Value is string y2Str
&& float.TryParse(y2Str, NumberStyles.Any, CultureInfo.InvariantCulture, out y2);
}
foreach (XmlElement x in gElement.ChildNodes)
{
if (x.Name != "line")
@@ -67,22 +68,22 @@ public class CryptoService : INService
points[cnt++] = new(x1, y1);
// this point will be set twice to the same value
// on all points except the last one
if(cnt + 1 < points.Length)
if (cnt + 1 < points.Length)
points[cnt + 1] = new(x2, y2);
}
}
if (cnt == 0)
return Array.Empty<PointF>();
return points.Slice(0, cnt).ToArray();
}
private SixLabors.ImageSharp.Image<Rgba32> GenerateSparklineChart(PointF[] points, bool up)
{
const int width = 164;
const int height = 48;
var img = new Image<Rgba32>(width, height, Color.Transparent);
var color = up
? Color.Green
@@ -92,10 +93,10 @@ public class CryptoService : INService
{
x.DrawLines(color, 2, points);
});
return img;
}
public async Task<(CmcResponseData? Data, CmcResponseData? Nearest)> GetCryptoData(string name)
{
if (string.IsNullOrWhiteSpace(name))
@@ -130,20 +131,20 @@ public class CryptoService : INService
await _getCryptoLock.WaitAsync();
try
{
var fullStrData = await _cache.GetOrAddCachedDataAsync("nadeko:crypto_data",
async _ =>
var data = await _cache.GetOrAddAsync(new("nadeko:crypto_data"),
async () =>
{
try
{
using var http = _httpFactory.CreateClient();
var strData = await http.GetFromJsonAsync<CryptoResponse>(
var data = await http.GetFromJsonAsync<CryptoResponse>(
"https://pro-api.coinmarketcap.com/v1/cryptocurrency/listings/latest?"
+ $"CMC_PRO_API_KEY={_creds.CoinmarketcapApiKey}"
+ "&start=1"
+ "&limit=5000"
+ "&convert=USD");
return JsonSerializer.Serialize(strData);
return data;
}
catch (Exception ex)
{
@@ -151,13 +152,12 @@ public class CryptoService : INService
return default;
}
},
"",
TimeSpan.FromHours(2));
if (fullStrData is null)
if (data is null)
return default;
return JsonSerializer.Deserialize<CryptoResponse>(fullStrData)?.Data ?? new();
return data.Data;
}
catch (Exception ex)
{
@@ -170,44 +170,33 @@ public class CryptoService : INService
}
}
private TypedKey<byte[]> GetSparklineKey(int id)
=> new($"crypto:sparkline:{id}");
public async Task<Stream?> GetSparklineAsync(int id, bool up)
{
var key = $"crypto:sparkline:{id}";
// attempt to get from cache
var db = _cache.Redis.GetDatabase();
byte[] bytes = await db.StringGetAsync(key);
// if it succeeds, return it
if (bytes is { Length: > 0 })
{
return bytes.ToStream();
}
// if it fails, generate a new one
var points = await DownloadSparklinePointsAsync(id);
if (points is null)
return default;
var sparkline = GenerateSparklineChart(points, up);
// add to cache for 1h and return it
var stream = sparkline.ToStream();
await db.StringSetAsync(key, stream.ToArray(), expiry: TimeSpan.FromHours(1));
return stream;
}
private async Task<PointF[]?> DownloadSparklinePointsAsync(int id)
{
try
{
using var http = _httpFactory.CreateClient();
var str = await http.GetStringAsync(
$"https://s3.coinmarketcap.com/generated/sparklines/web/7d/usd/{id}.svg");
var points = GetSparklinePointsFromSvgText(str);
return points;
var bytes = await _cache.GetOrAddAsync(GetSparklineKey(id),
async () =>
{
// if it fails, generate a new one
var points = await DownloadSparklinePointsAsync(id);
var sparkline = GenerateSparklineChart(points, up);
using var stream = await sparkline.ToStreamAsync();
return stream.ToArray();
},
TimeSpan.FromHours(1));
if (bytes is { Length: > 0 })
{
return bytes.ToStream();
}
return default;
}
catch(Exception ex)
catch (Exception ex)
{
Log.Warning(ex,
"Exception occurred while downloading sparkline points: {ErrorMessage}",
@@ -215,4 +204,13 @@ public class CryptoService : INService
return default;
}
}
private async Task<PointF[]> DownloadSparklinePointsAsync(int id)
{
using var http = _httpFactory.CreateClient();
var str = await http.GetStringAsync(
$"https://s3.coinmarketcap.com/generated/sparklines/web/7d/usd/{id}.svg");
var points = GetSparklinePointsFromSvgText(str);
return points;
}
}

14
src/NadekoBot/Modules/Searches/PokemonSearchCommands.cs
View File

@@ -9,15 +9,9 @@ public partial class Searches
[Group]
public partial class PokemonSearchCommands : NadekoModule<SearchesService>
{
public IReadOnlyDictionary<string, SearchPokemon> Pokemons
=> _cache.LocalData.Pokemons;
private readonly ILocalDataCache _cache;
public IReadOnlyDictionary<string, SearchPokemonAbility> PokemonAbilities
=> _cache.LocalData.PokemonAbilities;
private readonly IDataCache _cache;
public PokemonSearchCommands(IDataCache cache)
public PokemonSearchCommands(ILocalDataCache cache)
=> _cache = cache;
[Cmd]
@@ -27,7 +21,7 @@ public partial class Searches
if (string.IsNullOrWhiteSpace(pokemon))
return;
foreach (var kvp in Pokemons)
foreach (var kvp in await _cache.GetPokemonsAsync())
{
if (kvp.Key.ToUpperInvariant() == pokemon.ToUpperInvariant())
{
@@ -58,7 +52,7 @@ public partial class Searches
ability = ability?.Trim().ToUpperInvariant().Replace(" ", "", StringComparison.InvariantCulture);
if (string.IsNullOrWhiteSpace(ability))
return;
foreach (var kvp in PokemonAbilities)
foreach (var kvp in await _cache.GetPokemonAbilitiesAsync())
{
if (kvp.Key.ToUpperInvariant() == ability)
{

19
src/NadekoBot/Modules/Searches/Search/SearchCommands.cs
View File

@@ -10,14 +10,14 @@ public partial class Searches
public partial class SearchCommands : NadekoModule
{
private readonly ISearchServiceFactory _searchFactory;
private readonly ConnectionMultiplexer _redis;
private readonly IBotCache _cache;
public SearchCommands(
ISearchServiceFactory searchFactory,
ConnectionMultiplexer redis)
IBotCache cache)
{
_searchFactory = searchFactory;
_redis = redis;
_cache = cache;
}
[Cmd]
@@ -127,18 +127,17 @@ public partial class Searches
await ctx.Channel.EmbedAsync(null, embeds: embeds);
}
private TypedKey<string> GetYtCacheKey(string query)
=> new($"search:youtube:{query}");
private async Task AddYoutubeUrlToCacheAsync(string query, string url)
{
var db = _redis.GetDatabase();
await db.StringSetAsync($"search:youtube:{query}", url, expiry: 1.Hours());
}
=> await _cache.AddAsync(GetYtCacheKey(query), url, expiry: 1.Hours());
private async Task<VideoInfo?> GetYoutubeUrlFromCacheAsync(string query)
{
var db = _redis.GetDatabase();
var url = await db.StringGetAsync($"search:youtube:{query}");
var result = await _cache.GetAsync(GetYtCacheKey(query));
if (string.IsNullOrWhiteSpace(url))
if (!result.TryGetValue(out var url) || string.IsNullOrWhiteSpace(url))
return null;
return new VideoInfo()

105
src/NadekoBot/Modules/Searches/SearchesService.cs
View File

@@ -1,6 +1,4 @@
#nullable disable
using AngleSharp.Html.Dom;
using AngleSharp.Html.Parser;
using Html2Markdown;
using NadekoBot.Modules.Searches.Common;
using Newtonsoft.Json;
@@ -10,7 +8,6 @@ using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Drawing.Processing;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using System.Net;
using Color = SixLabors.ImageSharp.Color;
using Image = SixLabors.ImageSharp.Image;
@@ -31,9 +28,9 @@ public class SearchesService : INService
private readonly IHttpClientFactory _httpFactory;
private readonly IGoogleApiService _google;
private readonly IImageCache _imgs;
private readonly IDataCache _cache;
private readonly IBotCache _c;
private readonly FontProvider _fonts;
private readonly IBotCredentials _creds;
private readonly IBotCredsProvider _creds;
private readonly NadekoRandom _rng;
private readonly List<string> _yomamaJokes;
@@ -42,15 +39,16 @@ public class SearchesService : INService
public SearchesService(
IGoogleApiService google,
IDataCache cache,
IImageCache images,
IBotCache c,
IHttpClientFactory factory,
FontProvider fonts,
IBotCredentials creds)
IBotCredsProvider creds)
{
_httpFactory = factory;
_google = google;
_imgs = cache.LocalImages;
_cache = cache;
_imgs = images;
_c = c;
_fonts = fonts;
_creds = creds;
_rng = new();
@@ -76,36 +74,28 @@ public class SearchesService : INService
}
public async Task<Stream> GetRipPictureAsync(string text, Uri imgUrl)
{
var data = await _cache.GetOrAddCachedDataAsync($"nadeko_rip_{text}_{imgUrl}",
GetRipPictureFactory,
(text, imgUrl),
TimeSpan.FromDays(1));
return data.ToStream();
}
=> (await GetRipPictureFactory(text, imgUrl)).ToStream();
private void DrawAvatar(Image bg, Image avatarImage)
=> bg.Mutate(x => x.Grayscale().DrawImage(avatarImage, new(83, 139), new GraphicsOptions()));
public async Task<byte[]> GetRipPictureFactory((string text, Uri avatarUrl) arg)
public async Task<byte[]> GetRipPictureFactory(string text, Uri avatarUrl)
{
var (text, avatarUrl) = arg;
using var bg = Image.Load<Rgba32>(_imgs.Rip.ToArray());
var (succ, data) = (false, (byte[])null); //await _cache.TryGetImageDataAsync(avatarUrl);
if (!succ)
using var bg = Image.Load<Rgba32>(await _imgs.GetRipBgAsync());
var result = await _c.GetImageDataAsync(avatarUrl);
if (!result.TryPickT0(out var data, out _))
{
using var http = _httpFactory.CreateClient();
data = await http.GetByteArrayAsync(avatarUrl);
using (var avatarImg = Image.Load<Rgba32>(data))
{
avatarImg.Mutate(x => x.Resize(85, 85).ApplyRoundedCorners(42));
await using var avStream = avatarImg.ToStream();
await using var avStream = await avatarImg.ToStreamAsync();
data = avStream.ToArray();
DrawAvatar(bg, avatarImg);
}
await _cache.SetImageDataAsync(avatarUrl, data);
await _c.SetImageDataAsync(avatarUrl, data);
}
else
{
@@ -128,7 +118,7 @@ public class SearchesService : INService
new(25, 225)));
//flowa
using (var flowers = Image.Load(_imgs.RipOverlay.ToArray()))
using (var flowers = Image.Load(await _imgs.GetRipOverlayAsync()))
{
bg.Mutate(x => x.DrawImage(flowers, new(0, 0), new GraphicsOptions()));
}
@@ -137,13 +127,12 @@ public class SearchesService : INService
return stream.ToArray();
}
public Task<WeatherData> GetWeatherDataAsync(string query)
public async Task<WeatherData> GetWeatherDataAsync(string query)
{
query = query.Trim().ToLowerInvariant();
return _cache.GetOrAddCachedDataAsync($"nadeko_weather_{query}",
GetWeatherDataFactory,
query,
return await _c.GetOrAddAsync(new($"nadeko_weather_{query}"),
async () => await GetWeatherDataFactory(query),
TimeSpan.FromHours(3));
}
@@ -184,26 +173,28 @@ public class SearchesService : INService
if (string.IsNullOrEmpty(query))
return (default, TimeErrors.InvalidInput);
if (string.IsNullOrWhiteSpace(_creds.LocationIqApiKey) || string.IsNullOrWhiteSpace(_creds.TimezoneDbApiKey))
var locIqKey = _creds.GetCreds().LocationIqApiKey;
var tzDbKey = _creds.GetCreds().TimezoneDbApiKey;
if (string.IsNullOrWhiteSpace(locIqKey) || string.IsNullOrWhiteSpace(tzDbKey))
return (default, TimeErrors.ApiKeyMissing);
try
{
using var http = _httpFactory.CreateClient();
var res = await _cache.GetOrAddCachedDataAsync($"geo_{query}",
_ =>
var res = await _c.GetOrAddAsync(new($"searches:geo:{query}"),
async () =>
{
var url = "https://eu1.locationiq.com/v1/search.php?"
+ (string.IsNullOrWhiteSpace(_creds.LocationIqApiKey)
+ (string.IsNullOrWhiteSpace(locIqKey)
? "key="
: $"key={_creds.LocationIqApiKey}&")
: $"key={locIqKey}&")
+ $"q={Uri.EscapeDataString(query)}&"
+ "format=json";
var res = http.GetStringAsync(url);
var res = await http.GetStringAsync(url);
return res;
},
"",
TimeSpan.FromHours(1));
var responses = JsonConvert.DeserializeObject<LocationIqResponse[]>(res);
@@ -217,7 +208,7 @@ public class SearchesService : INService
using var req = new HttpRequestMessage(HttpMethod.Get,
"http://api.timezonedb.com/v2.1/get-time-zone?"
+ $"key={_creds.TimezoneDbApiKey}"
+ $"key={tzDbKey}"
+ $"&format=json"
+ $"&by=position"
+ $"&lat={geoData.Lat}"
@@ -315,9 +306,8 @@ public class SearchesService : INService
public async Task<MtgData> GetMtgCardAsync(string search)
{
search = search.Trim().ToLowerInvariant();
var data = await _cache.GetOrAddCachedDataAsync($"nadeko_mtg_{search}",
GetMtgCardFactory,
search,
var data = await _c.GetOrAddAsync(new($"mtg:{search}"),
async () => await GetMtgCardFactory(search),
TimeSpan.FromDays(1));
if (data is null || data.Length == 0)
@@ -368,12 +358,11 @@ public class SearchesService : INService
return await cards.Select(GetMtgDataAsync).WhenAll();
}
public Task<HearthstoneCardData> GetHearthstoneCardDataAsync(string name)
public async Task<HearthstoneCardData> GetHearthstoneCardDataAsync(string name)
{
name = name.ToLowerInvariant();
return _cache.GetOrAddCachedDataAsync($"nadeko_hearthstone_{name}",
HearthstoneCardDataFactory,
name,
return await _c.GetOrAddAsync($"hearthstone:{name}",
() => HearthstoneCardDataFactory(name),
TimeSpan.FromDays(1));
}
@@ -381,7 +370,7 @@ public class SearchesService : INService
{
using var http = _httpFactory.CreateClient();
http.DefaultRequestHeaders.Clear();
http.DefaultRequestHeaders.Add("x-rapidapi-key", _creds.RapidApiKey);
http.DefaultRequestHeaders.Add("x-rapidapi-key", _creds.GetCreds().RapidApiKey);
try
{
var response = await http.GetStringAsync("https://omgvamp-hearthstone-v1.p.rapidapi.com/"
@@ -410,16 +399,22 @@ public class SearchesService : INService
}
}
public Task<OmdbMovie> GetMovieDataAsync(string name)
public async Task<OmdbMovie> GetMovieDataAsync(string name)
{
name = name.Trim().ToLowerInvariant();
return _cache.GetOrAddCachedDataAsync($"nadeko_movie_{name}", GetMovieDataFactory, name, TimeSpan.FromDays(1));
return await _c.GetOrAddAsync(new($"movie:{name}"),
() => GetMovieDataFactory(name),
TimeSpan.FromDays(1));
}
private async Task<OmdbMovie> GetMovieDataFactory(string name)
{
using var http = _httpFactory.CreateClient();
var res = await http.GetStringAsync(string.Format("https://omdbapi.nadeko.bot/?t={0}&y=&plot=full&r=json",
var res = await http.GetStringAsync(string.Format("https://omdbapi.nadeko.bot/"
+ "?t={0}"
+ "&y="
+ "&plot=full"
+ "&r=json",
name.Trim().Replace(' ', '+')));
var movie = JsonConvert.DeserializeObject<OmdbMovie>(res);
if (movie?.Title is null)
@@ -432,10 +427,11 @@ public class SearchesService : INService
{
const string steamGameIdsKey = "steam_names_to_appid";
var gamesMap = await _cache.GetOrAddCachedDataAsync(steamGameIdsKey,
async _ =>
var gamesMap = await _c.GetOrAddAsync(new(steamGameIdsKey),
async () =>
{
using var http = _httpFactory.CreateClient();
// https://api.steampowered.com/ISteamApps/GetAppList/v2/
var gamesStr = await http.GetStringAsync("https://api.steampowered.com/ISteamApps/GetAppList/v2/");
var apps = JsonConvert
@@ -446,23 +442,18 @@ public class SearchesService : INService
{
apps = new List<SteamGameId>()
}
})
})!
.applist.apps;
return apps.OrderBy(x => x.Name, StringComparer.OrdinalIgnoreCase)
.GroupBy(x => x.Name)
.ToDictionary(x => x.Key, x => x.First().AppId);
//await db.HashSetAsync("steam_game_ids", apps.Select(app => new HashEntry(app.Name.Trim().ToLowerInvariant(), app.AppId)).ToArray());
//await db.StringSetAsync("steam_game_ids", gamesStr, TimeSpan.FromHours(24));
//await db.KeyExpireAsync("steam_game_ids", TimeSpan.FromHours(24), CommandFlags.FireAndForget);
},
default(string),
TimeSpan.FromHours(24));
if (gamesMap is null)
return -1;
query = query.Trim();
var keyList = gamesMap.Keys.ToList();

174
src/NadekoBot/Modules/Searches/StreamNotification/StreamNotificationService.cs
View File

@@ -1,4 +1,6 @@
#nullable disable
using LinqToDB;
using LinqToDB.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Common.ModuleBehaviors;
using NadekoBot.Db;
@@ -6,10 +8,98 @@ using NadekoBot.Db.Models;
using NadekoBot.Modules.Searches.Common;
using NadekoBot.Modules.Searches.Common.StreamNotifications;
using NadekoBot.Services.Database.Models;
using StackExchange.Redis;
namespace NadekoBot.Modules.Searches.Services;
public sealed class StreamOnlineMessageDeleterService : INService, IReadyExecutor
{
private readonly StreamNotificationService _notifService;
private readonly DbService _db;
private readonly DiscordSocketClient _client;
private readonly IPubSub _pubSub;
public StreamOnlineMessageDeleterService(
StreamNotificationService notifService,
DbService db,
IPubSub pubSub,
DiscordSocketClient client)
{
_notifService = notifService;
_db = db;
_client = client;
_pubSub = pubSub;
}
public async Task OnReadyAsync()
{
_notifService.OnlineMessagesSent += OnOnlineMessagesSent;
if(_client.ShardId == 0)
await _pubSub.Sub(_notifService.StreamsOfflineKey, OnStreamsOffline);
}
private async Task OnOnlineMessagesSent(FollowedStream.FType type, string name, IReadOnlyCollection<(ulong, ulong)> pairs)
{
await using var ctx = _db.GetDbContext();
foreach (var (channelId, messageId) in pairs)
{
await ctx.GetTable<StreamOnlineMessage>()
.InsertAsync(() => new()
{
Name = name,
Type = type,
MessageId = messageId,
ChannelId = channelId,
DateAdded = DateTime.UtcNow,
});
}
}
private async ValueTask OnStreamsOffline(List<StreamData> streamDatas)
{
if (_client.ShardId != 0)
return;
var pairs = await GetMessagesToDelete(streamDatas);
foreach (var (channelId, messageId) in pairs)
{
try
{
var textChannel = await _client.GetChannelAsync(channelId) as ITextChannel;
if (textChannel is null)
continue;
await textChannel.DeleteMessageAsync(messageId);
}
catch
{
continue;
}
}
}
private async Task<IEnumerable<(ulong, ulong)>> GetMessagesToDelete(List<StreamData> streamDatas)
{
await using var ctx = _db.GetDbContext();
var toReturn = new List<(ulong, ulong)>();
foreach (var sd in streamDatas)
{
var key = sd.CreateKey();
var toDelete = await ctx.GetTable<StreamOnlineMessage>()
.Where(x => (x.Type == key.Type && x.Name == key.Name)
|| Sql.DateDiff(Sql.DateParts.Day, x.DateAdded, DateTime.UtcNow) > 1)
.DeleteWithOutputAsync();
toReturn.AddRange(toDelete.Select(x => (x.ChannelId, x.MessageId)));
}
return toReturn;
}
}
public sealed class StreamNotificationService : INService, IReadyExecutor
{
private readonly DbService _db;
@@ -29,18 +119,22 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
private readonly IPubSub _pubSub;
private readonly IEmbedBuilderService _eb;
private readonly TypedKey<List<StreamData>> _streamsOnlineKey;
private readonly TypedKey<List<StreamData>> _streamsOfflineKey;
public TypedKey<List<StreamData>> StreamsOnlineKey { get; }
public TypedKey<List<StreamData>> StreamsOfflineKey { get; }
private readonly TypedKey<FollowStreamPubData> _streamFollowKey;
private readonly TypedKey<FollowStreamPubData> _streamUnfollowKey;
private readonly ConnectionMultiplexer _redis;
public event Func<
FollowedStream.FType,
string,
IReadOnlyCollection<(ulong, ulong)>,
Task> OnlineMessagesSent = static delegate { return Task.CompletedTask; };
public StreamNotificationService(
DbService db,
DiscordSocketClient client,
IBotStrings strings,
ConnectionMultiplexer redis,
IBotCredsProvider creds,
IHttpClientFactory httpFactory,
Bot bot,
@@ -52,11 +146,11 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
_strings = strings;
_pubSub = pubSub;
_eb = eb;
_redis = redis;
_streamTracker = new(httpFactory, creds, redis, creds.GetCreds().RedisKey(), client.ShardId == 0);
_streamTracker = new(httpFactory, creds);
_streamsOnlineKey = new("streams.online");
_streamsOfflineKey = new("streams.offline");
StreamsOnlineKey = new("streams.online");
StreamsOfflineKey = new("streams.offline");
_streamFollowKey = new("stream.follow");
_streamUnfollowKey = new("stream.unfollow");
@@ -100,7 +194,7 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
var allFollowedStreams = uow.Set<FollowedStream>().AsQueryable().ToList();
foreach (var fs in allFollowedStreams)
_streamTracker.CacheAddData(fs.CreateKey(), null, false);
_streamTracker.AddLastData(fs.CreateKey(), null, false);
_trackCounter = allFollowedStreams.GroupBy(x => new
{
@@ -112,8 +206,8 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
}
}
_pubSub.Sub(_streamsOfflineKey, HandleStreamsOffline);
_pubSub.Sub(_streamsOnlineKey, HandleStreamsOnline);
_pubSub.Sub(StreamsOfflineKey, HandleStreamsOffline);
_pubSub.Sub(StreamsOnlineKey, HandleStreamsOnline);
if (client.ShardId == 0)
{
@@ -186,7 +280,7 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
/// </summary>
private ValueTask HandleFollowStream(FollowStreamPubData info)
{
_streamTracker.CacheAddData(info.Key, null, false);
_streamTracker.AddLastData(info.Key, null, false);
lock (_shardLock)
{
var key = info.Key;
@@ -251,45 +345,8 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
.WhenAll();
}
}
if (_client.ShardId == 0)
{
foreach (var stream in offlineStreams)
{
await DeleteOnlineMessages(stream);
}
}
}
private async Task DeleteOnlineMessages(StreamData stream)
{
var db = _redis.GetDatabase();
var data = await db.ListRangeAsync($"streams_online_del:{stream.CreateKey()}");
await db.KeyDeleteAsync($"streams_online_del:{stream.CreateKey()}");
foreach (string pair in data)
{
var pairArr = pair.Split(',');
if (pairArr.Length != 2)
continue;
if (!ulong.TryParse(pairArr[0], out var chId) || !ulong.TryParse(pairArr[1], out var msgId))
continue;
try
{
var textChannel = await _client.GetChannelAsync(chId) as ITextChannel;
if (textChannel is null)
continue;
await textChannel.DeleteMessageAsync(msgId);
}
catch
{
continue;
}
}
}
private async ValueTask HandleStreamsOnline(List<StreamData> onlineStreams)
{
@@ -331,14 +388,11 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
{
var pairs = messages
.Where(x => x != default)
.Select(x => (RedisValue)$"{x.Item1},{x.Item2}")
.ToArray();
.Select(x => (x.Item1, x.Item2))
.ToList();
if (pairs.Length > 0)
{
var db = _redis.GetDatabase();
await db.ListRightPushAsync($"streams_online_del:{key}", pairs);
}
if (pairs.Count > 0)
await OnlineMessagesSent(key.Type, key.Name, pairs);
}
catch
{
@@ -349,10 +403,10 @@ public sealed class StreamNotificationService : INService, IReadyExecutor
}
private Task OnStreamsOnline(List<StreamData> data)
=> _pubSub.Pub(_streamsOnlineKey, data);
=> _pubSub.Pub(StreamsOnlineKey, data);
private Task OnStreamsOffline(List<StreamData> data)
=> _pubSub.Pub(_streamsOfflineKey, data);
=> _pubSub.Pub(StreamsOfflineKey, data);
private Task ClientOnJoinedGuild(GuildConfig guildConfig)
{

5
src/NadekoBot/Modules/Searches/_Common/StreamNotifications/Models/StreamDataKey.cs
View File

@@ -6,10 +6,9 @@ namespace NadekoBot.Modules.Searches.Common;
public readonly struct StreamDataKey
{
public FollowedStream.FType Type { get; }
public string Name { get; }
public FollowedStream.FType Type { get; init; }
public string Name { get; init; }
[JsonConstructor]
public StreamDataKey(FollowedStream.FType type, string name)
{
Type = type;

69
src/NadekoBot/Modules/Searches/_Common/StreamNotifications/NotifChecker.cs
View File

@@ -1,7 +1,5 @@
using NadekoBot.Db.Models;
using NadekoBot.Modules.Searches.Common.StreamNotifications.Providers;
using Newtonsoft.Json;
using StackExchange.Redis;
namespace NadekoBot.Modules.Searches.Common.StreamNotifications;
@@ -9,30 +7,22 @@ public class NotifChecker
{
public event Func<List<StreamData>, Task> OnStreamsOffline = _ => Task.CompletedTask;
public event Func<List<StreamData>, Task> OnStreamsOnline = _ => Task.CompletedTask;
private readonly ConnectionMultiplexer _multi;
private readonly string _key;
private readonly Dictionary<FollowedStream.FType, Provider> _streamProviders;
private readonly IReadOnlyDictionary<FollowedStream.FType, Provider> _streamProviders;
private readonly HashSet<(FollowedStream.FType, string)> _offlineBuffer;
private readonly ConcurrentDictionary<StreamDataKey, StreamData?> _cache = new();
public NotifChecker(
IHttpClientFactory httpClientFactory,
IBotCredsProvider credsProvider,
ConnectionMultiplexer multi,
string uniqueCacheKey,
bool isMaster)
IBotCredsProvider credsProvider)
{
_multi = multi;
_key = $"{uniqueCacheKey}_followed_streams_data";
_streamProviders = new()
_streamProviders = new Dictionary<FollowedStream.FType, Provider>()
{
{ FollowedStream.FType.Twitch, new TwitchHelixProvider(httpClientFactory, credsProvider) },
{ FollowedStream.FType.Picarto, new PicartoProvider(httpClientFactory) },
{ FollowedStream.FType.Trovo, new TrovoProvider(httpClientFactory, credsProvider) }
};
_offlineBuffer = new();
if (isMaster)
CacheClearAllData();
}
// gets all streams which have been failing for more than the provided timespan
@@ -61,7 +51,7 @@ public class NotifChecker
{
try
{
var allStreamData = CacheGetAllData();
var allStreamData = GetAllData();
var oldStreamDataDict = allStreamData
// group by type
@@ -101,7 +91,7 @@ public class NotifChecker
|| !typeDict.TryGetValue(key.Name, out var oldData)
|| oldData is null)
{
CacheAddData(key, newData, true);
AddLastData(key, newData, true);
continue;
}
@@ -109,7 +99,7 @@ public class NotifChecker
if (string.IsNullOrWhiteSpace(newData.Game))
newData.Game = oldData.Game;
CacheAddData(key, newData, true);
AddLastData(key, newData, true);
// if the stream is offline, we need to check if it was
// marked as offline once previously
@@ -158,39 +148,22 @@ public class NotifChecker
}
});
public bool CacheAddData(StreamDataKey key, StreamData? data, bool replace)
public bool AddLastData(StreamDataKey key, StreamData? data, bool replace)
{
var db = _multi.GetDatabase();
return db.HashSet(_key,
JsonConvert.SerializeObject(key),
JsonConvert.SerializeObject(data),
replace ? When.Always : When.NotExists);
if (replace)
{
_cache[key] = data;
return true;
}
return _cache.TryAdd(key, data);
}
public void CacheDeleteData(StreamDataKey key)
{
var db = _multi.GetDatabase();
db.HashDelete(_key, JsonConvert.SerializeObject(key));
}
public void DeleteLastData(StreamDataKey key)
=> _cache.TryRemove(key, out _);
public void CacheClearAllData()
{
var db = _multi.GetDatabase();
db.KeyDelete(_key);
}
public Dictionary<StreamDataKey, StreamData?> CacheGetAllData()
{
var db = _multi.GetDatabase();
if (!db.KeyExists(_key))
return new();
return db.HashGetAll(_key)
.ToDictionary(entry => JsonConvert.DeserializeObject<StreamDataKey>(entry.Name),
entry => entry.Value.IsNullOrEmpty
? default
: JsonConvert.DeserializeObject<StreamData>(entry.Value));
}
public Dictionary<StreamDataKey, StreamData?> GetAllData()
=> _cache.ToDictionary(x => x.Key, x => x.Value);
public async Task<StreamData?> GetStreamDataByUrlAsync(string url)
{
@@ -234,9 +207,9 @@ public class NotifChecker
// if stream is found, add it to the cache for tracking only if it doesn't already exist
// because stream will be checked and events will fire in a loop. We don't want to override old state
return CacheAddData(data.CreateKey(), data, false);
return AddLastData(data.CreateKey(), data, false);
}
public void UntrackStreamByKey(in StreamDataKey key)
=> CacheDeleteData(key);
=> DeleteLastData(key);
}