minster586/nadekobot
1
0
Fork 0
mirror of https://gitlab.com/Kwoth/nadekobot.git synced 2025-09-11 17:58:26 -04:00
Code Issues Packages Projects Releases Wiki Activity

NadekoBot Patronage system, Search commands improvements + fixes

Browse Source
This commit is contained in:
Kwoth
2022-06-14 07:24:33 +00:00
parent 18b10b8c6f
commit 7b5145f116
165 changed files with 14920 additions and 1457 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/NadekoBot/Modules/Searches/Feeds/FeedsService.cs
View File

@@ -1,6 +1,8 @@
#nullable disable
using CodeHollow.FeedReader;
using CodeHollow.FeedReader.Feeds;
using LinqToDB;
using LinqToDB.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Db;
using NadekoBot.Services.Database.Models;
@@ -10,11 +12,12 @@ namespace NadekoBot.Modules.Searches.Services;
public class FeedsService : INService
{
private readonly DbService _db;
private readonly ConcurrentDictionary<string, HashSet<FeedSub>> _subs;
private readonly ConcurrentDictionary<string, List<FeedSub>> _subs;
private readonly DiscordSocketClient _client;
private readonly IEmbedBuilderService _eb;
private readonly ConcurrentDictionary<string, DateTime> _lastPosts = new();
private readonly Dictionary<string, uint> _errorCounters = new();
public FeedsService(
Bot bot,
@@ -33,7 +36,7 @@ public class FeedsService : INService
.ToList()
.SelectMany(x => x.FeedSubs)
.GroupBy(x => x.Url.ToLower())
.ToDictionary(x => x.Key, x => x.ToHashSet())
.ToDictionary(x => x.Key, x => x.ToList())
.ToConcurrent();
}
@@ -43,6 +46,35 @@ public class FeedsService : INService
_ = Task.Run(TrackFeeds);
}
private void ClearErrors(string url)
=> _errorCounters.Remove(url);
private async Task AddError(string url, List<int> ids)
{
try
{
var newValue = _errorCounters[url] = _errorCounters.GetValueOrDefault(url) + 1;
if (newValue >= 100)
{
// remove from db
await using var ctx = _db.GetDbContext();
await ctx.GetTable<FeedSub>()
.DeleteAsync(x => ids.Contains(x.Id));
// remove from the local cache
_subs.TryRemove(url, out _);
// reset the error counter
ClearErrors(url);
}
}
catch (Exception ex)
{
Log.Error(ex, "Error adding rss errors...");
}
}
public async Task<EmbedBuilder> TrackFeeds()
{
while (true)
@@ -134,13 +166,17 @@ public class FeedsService : INService
embed.WithDescription(desc.TrimTo(2048));
//send the created embed to all subscribed channels
var feedSendTasks = kvp.Value.Where(x => x.GuildConfig is not null)
var feedSendTasks = kvp.Value
.Where(x => x.GuildConfig is not null)
.Select(x => _client.GetGuild(x.GuildConfig.GuildId)
?.GetTextChannel(x.ChannelId))
.Where(x => x is not null)
.Select(x => x.EmbedAsync(embed));
allSendTasks.Add(feedSendTasks.WhenAll());
// as data retrieval was sucessful, reset error counter
ClearErrors(rssUrl);
}
}
catch (Exception ex)
@@ -149,6 +185,8 @@ public class FeedsService : INService
+ "\n {Message}",
rssUrl,
$"[{ex.GetType().Name}]: {ex.Message}");
await AddError(rssUrl, kvp.Value.Select(x => x.Id).ToList());
}
}
@@ -188,7 +226,7 @@ public class FeedsService : INService
foreach (var feed in gc.FeedSubs)
{
_subs.AddOrUpdate(feed.Url.ToLower(),
new HashSet<FeedSub>
new List<FeedSub>
{
feed
},
@@ -216,7 +254,7 @@ public class FeedsService : INService
return false;
var toRemove = items[index];
_subs.AddOrUpdate(toRemove.Url.ToLower(),
new HashSet<FeedSub>(),
new List<FeedSub>(),
(_, old) =>
{
old.Remove(toRemove);

60
src/NadekoBot/Modules/Searches/Search/DefaultSearchServiceFactory.cs Normal file
View File

@@ -0,0 +1,60 @@
using NadekoBot.Modules.Searches.Youtube;
namespace NadekoBot.Modules.Searches;
public sealed class DefaultSearchServiceFactory : ISearchServiceFactory, INService
{
private readonly SearchesConfigService _scs;
private readonly SearxSearchService _sss;
private readonly GoogleSearchService _gss;
private readonly YtdlpYoutubeSearchService _ytdlp;
private readonly YtdlYoutubeSearchService _ytdl;
private readonly YoutubeDataApiSearchService _ytdata;
private readonly InvidiousYtSearchService _iYtSs;
public DefaultSearchServiceFactory(
SearchesConfigService scs,
GoogleSearchService gss,
SearxSearchService sss,
YtdlpYoutubeSearchService ytdlp,
YtdlYoutubeSearchService ytdl,
YoutubeDataApiSearchService ytdata,
InvidiousYtSearchService iYtSs)
{
_scs = scs;
_sss = sss;
_gss = gss;
_iYtSs = iYtSs;
_ytdlp = ytdlp;
_ytdl = ytdl;
_ytdata = ytdata;
}
public ISearchService GetSearchService(string? hint = null)
=> _scs.Data.WebSearchEngine switch
{
WebSearchEngine.Google => _gss,
WebSearchEngine.Searx => _sss,
_ => _gss
};
public ISearchService GetImageSearchService(string? hint = null)
=> _scs.Data.ImgSearchEngine switch
{
ImgSearchEngine.Google => _gss,
ImgSearchEngine.Searx => _sss,
_ => _gss
};
public IYoutubeSearchService GetYoutubeSearchService(string? hint = null)
=> _scs.Data.YtProvider switch
{
YoutubeSearcher.YtDataApiv3 => _ytdata,
YoutubeSearcher.Ytdlp => _ytdlp,
YoutubeSearcher.Ytdl => _ytdl,
YoutubeSearcher.Invidious => _iYtSs,
_ => _ytdl
};
}

65
src/NadekoBot/Modules/Searches/Search/DuckDuckGoScrape/DuckDuckGoSeachService.cs Normal file
View File

@@ -0,0 +1,65 @@
// using AngleSharp.Html.Dom;
// using MorseCode.ITask;
// using NadekoBot.Modules.Searches.Common;
// using System.Net;
//
// namespace NadekoBot.Modules.Searches.DuckDuckGo;
//
// public sealed class DuckDuckGoSeachService : SearchServiceBase
// {
// private static readonly HtmlParser _googleParser = new(new()
// {
// IsScripting = false,
// IsEmbedded = false,
// IsSupportingProcessingInstructions = false,
// IsKeepingSourceReferences = false,
// IsNotSupportingFrames = true
// });
//
// public override async ITask<SearchResultData> SearchAsync(string query)
// {
// query = WebUtility.UrlEncode(query)?.Replace(' ', '+');
//
// var fullQueryLink = "https://html.duckduckgo.com/html";
//
// using var http = _httpFactory.CreateClient();
// http.DefaultRequestHeaders.Clear();
// http.DefaultRequestHeaders.Add("User-Agent",
// "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
//
// using var formData = new MultipartFormDataContent();
// formData.Add(new StringContent(query), "q");
// using var response = await http.PostAsync(fullQueryLink, formData);
// var content = await response.Content.ReadAsStringAsync();
//
// using var document = await _googleParser.ParseDocumentAsync(content);
// var searchResults = document.QuerySelector(".results");
// var elems = searchResults.QuerySelectorAll(".result");
//
// if (!elems.Any())
// return default;
//
// var results = elems.Select(elem =>
// {
// if (elem.QuerySelector(".result__a") is not IHtmlAnchorElement anchor)
// return null;
//
// var href = anchor.Href;
// var name = anchor.TextContent;
//
// if (string.IsNullOrWhiteSpace(href) || string.IsNullOrWhiteSpace(name))
// return null;
//
// var txt = elem.QuerySelector(".result__snippet")?.TextContent;
//
// if (string.IsNullOrWhiteSpace(txt))
// return null;
//
// return new GoogleSearchResult(name, href, txt);
// })
// .Where(x => x is not null)
// .ToList();
//
// return new(results.AsReadOnly(), fullQueryLink, "0");
// }
// }

22
src/NadekoBot/Modules/Searches/Search/Google/GoogleCustomSearchResult.cs Normal file
View File

@@ -0,0 +1,22 @@
using NadekoBot.Modules.Searches;
using System.Text.Json.Serialization;
namespace NadekoBot.Services;
public sealed class GoogleCustomSearchResult : ISearchResult
{
ISearchResultInformation ISearchResult.Info
=> Info;
public string? Answer
=> null;
IReadOnlyCollection<ISearchResultEntry> ISearchResult.Entries
=> Entries ?? Array.Empty<OfficialGoogleSearchResultEntry>();
[JsonPropertyName("searchInformation")]
public GoogleSearchResultInformation Info { get; init; } = null!;
[JsonPropertyName("items")]
public IReadOnlyCollection<OfficialGoogleSearchResultEntry>? Entries { get; init; }
}

12
src/NadekoBot/Modules/Searches/Search/Google/GoogleImageData.cs Normal file
View File

@@ -0,0 +1,12 @@
using System.Text.Json.Serialization;
namespace NadekoBot.Services;
public sealed class GoogleImageData
{
[JsonPropertyName("contextLink")]
public string ContextLink { get; init; } = null!;
[JsonPropertyName("thumbnailLink")]
public string ThumbnailLink { get; init; } = null!;
}

19
src/NadekoBot/Modules/Searches/Search/Google/GoogleImageResult.cs Normal file
View File

@@ -0,0 +1,19 @@
using NadekoBot.Modules.Searches;
using System.Text.Json.Serialization;
namespace NadekoBot.Services;
public sealed class GoogleImageResult : IImageSearchResult
{
ISearchResultInformation IImageSearchResult.Info
=> Info;
IReadOnlyCollection<IImageSearchResultEntry> IImageSearchResult.Entries
=> Entries ?? Array.Empty<GoogleImageResultEntry>();
[JsonPropertyName("searchInformation")]
public GoogleSearchResultInformation Info { get; init; } = null!;
[JsonPropertyName("items")]
public IReadOnlyCollection<GoogleImageResultEntry>? Entries { get; init; }
}

13
src/NadekoBot/Modules/Searches/Search/Google/GoogleImageResultEntry.cs Normal file
View File

@@ -0,0 +1,13 @@
using NadekoBot.Modules.Searches;
using System.Text.Json.Serialization;
namespace NadekoBot.Services;
public sealed class GoogleImageResultEntry : IImageSearchResultEntry
{
[JsonPropertyName("link")]
public string Link { get; init; } = null!;
[JsonPropertyName("image")]
public GoogleImageData Image { get; init; } = null!;
}

13
src/NadekoBot/Modules/Searches/Search/Google/GoogleSearchResultInformation.cs Normal file
View File

@@ -0,0 +1,13 @@
using NadekoBot.Modules.Searches;
using System.Text.Json.Serialization;
namespace NadekoBot.Services;
public sealed class GoogleSearchResultInformation : ISearchResultInformation
{
[JsonPropertyName("formattedTotalResults")]
public string TotalResults { get; init; } = null!;
[JsonPropertyName("formattedSearchTime")]
public string SearchTime { get; init; } = null!;
}

66
src/NadekoBot/Modules/Searches/Search/Google/GoogleSearchService.cs Normal file
View File

@@ -0,0 +1,66 @@
using MorseCode.ITask;
namespace NadekoBot.Modules.Searches;
public sealed class GoogleSearchService : SearchServiceBase, INService
{
private readonly IBotCredsProvider _creds;
private readonly IHttpClientFactory _httpFactory;
public GoogleSearchService(IBotCredsProvider creds, IHttpClientFactory httpFactory)
{
_creds = creds;
_httpFactory = httpFactory;
}
public override async ITask<GoogleImageResult?> SearchImagesAsync(string query)
{
ArgumentNullException.ThrowIfNull(query);
var creds = _creds.GetCreds();
var key = creds.Google.ImageSearchId;
var cx = string.IsNullOrWhiteSpace(key)
? "c3f56de3be2034c07"
: key;
using var http = _httpFactory.CreateClient("google:search");
http.DefaultRequestHeaders.Add("Accept-Encoding", "gzip");
await using var stream = await http.GetStreamAsync(
$"https://customsearch.googleapis.com/customsearch/v1"
+ $"?cx={cx}"
+ $"&q={Uri.EscapeDataString(query)}"
+ $"&fields=items(image(contextLink%2CthumbnailLink)%2Clink)%2CsearchInformation"
+ $"&key={creds.GoogleApiKey}"
+ $"&searchType=image"
+ $"&safe=active");
var result = await System.Text.Json.JsonSerializer.DeserializeAsync<GoogleImageResult>(stream);
return result;
}
public override async ITask<GoogleCustomSearchResult?> SearchAsync(string query)
{
ArgumentNullException.ThrowIfNull(query);
var creds = _creds.GetCreds();
var key = creds.Google.SearchId;
var cx = string.IsNullOrWhiteSpace(key)
? "c7f1dac95987d4571"
: key;
using var http = _httpFactory.CreateClient("google:search");
http.DefaultRequestHeaders.Add("Accept-Encoding", "gzip");
await using var stream = await http.GetStreamAsync(
$"https://customsearch.googleapis.com/customsearch/v1"
+ $"?cx={cx}"
+ $"&q={Uri.EscapeDataString(query)}"
+ $"&fields=items(title%2Clink%2CdisplayLink%2Csnippet)%2CsearchInformation"
+ $"&key={creds.GoogleApiKey}"
+ $"&safe=active");
var result = await System.Text.Json.JsonSerializer.DeserializeAsync<GoogleCustomSearchResult>(stream);
return result;
}
}

19
src/NadekoBot/Modules/Searches/Search/Google/OfficialGoogleSearchResultEntry.cs Normal file
View File

@@ -0,0 +1,19 @@
using NadekoBot.Modules.Searches;
using System.Text.Json.Serialization;
namespace NadekoBot.Services;
public sealed class OfficialGoogleSearchResultEntry : ISearchResultEntry
{
[JsonPropertyName("title")]
public string Title { get; init; } = null!;
[JsonPropertyName("link")]
public string Url { get; init; } = null!;
[JsonPropertyName("displayLink")]
public string DisplayUrl { get; init; } = null!;
[JsonPropertyName("snippet")]
public string Description { get; init; } = null!;
}

62
src/NadekoBot/Modules/Searches/Search/GoogleScrape/GoogleScrapeService.cs Normal file
View File

@@ -0,0 +1,62 @@
// using AngleSharp.Html.Dom;
// using MorseCode.ITask;
// using NadekoBot.Modules.Searches.Common;
//
// namespace NadekoBot.Modules.Searches.GoogleScrape;
//
// public sealed class GoogleScrapeService : SearchServiceBase
// {
// public override async ITask<GoogleSearchResultData> SearchAsync(string query)
// {
// ArgumentNullException.ThrowIfNull(query);
//
// query = Uri.EscapeDataString(query)?.Replace(' ', '+');
//
// var fullQueryLink = $"https://www.google.ca/search?q={query}&safe=on&lr=lang_eng&hl=en&ie=utf-8&oe=utf-8";
//
// using var msg = new HttpRequestMessage(HttpMethod.Get, fullQueryLink);
// msg.Headers.Add("User-Agent",
// "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
// msg.Headers.Add("Cookie", "CONSENT=YES+shp.gws-20210601-0-RC2.en+FX+423;");
//
// using var http = _httpFactory.CreateClient();
// http.DefaultRequestHeaders.Clear();
//
// using var response = await http.SendAsync(msg);
// await using var content = await response.Content.ReadAsStreamAsync();
//
// using var document = await _googleParser.ParseDocumentAsync(content);
// var elems = document.QuerySelectorAll("div.g > div > div");
//
// var resultsElem = document.QuerySelectorAll("#resultStats").FirstOrDefault();
// var totalResults = resultsElem?.TextContent;
// //var time = resultsElem.Children.FirstOrDefault()?.TextContent
// //^ this doesn't work for some reason, <nobr> is completely missing in parsed collection
// if (!elems.Any())
// return default;
//
// var results = elems.Select(elem =>
// {
// var children = elem.Children.ToList();
// if (children.Count < 2)
// return null;
//
// var href = (children[0].QuerySelector("a") as IHtmlAnchorElement)?.Href;
// var name = children[0].QuerySelector("h3")?.TextContent;
//
// if (href is null || name is null)
// return null;
//
// var txt = children[1].TextContent;
//
// if (string.IsNullOrWhiteSpace(txt))
// return null;
//
// return new GoogleSearchResult(name, href, txt);
// })
// .Where(x => x is not null)
// .ToList();
//
// return new(results.AsReadOnly(), fullQueryLink, totalResults);
// }
// }

13
src/NadekoBot/Modules/Searches/Search/IImageSearchResult.cs Normal file
View File

@@ -0,0 +1,13 @@
namespace NadekoBot.Modules.Searches;
public interface IImageSearchResult
{
ISearchResultInformation Info { get; }
IReadOnlyCollection<IImageSearchResultEntry> Entries { get; }
}
public interface IImageSearchResultEntry
{
string Link { get; }
}

8
src/NadekoBot/Modules/Searches/Search/ISearchResult.cs Normal file
View File

@@ -0,0 +1,8 @@
namespace NadekoBot.Modules.Searches;
public interface ISearchResult
{
string? Answer { get; }
IReadOnlyCollection<ISearchResultEntry> Entries { get; }
ISearchResultInformation Info { get; }
}

9
src/NadekoBot/Modules/Searches/Search/ISearchResultEntry.cs Normal file
View File

@@ -0,0 +1,9 @@
namespace NadekoBot.Modules.Searches;
public interface ISearchResultEntry
{
string Title { get; }
string Url { get; }
string DisplayUrl { get; }
string? Description { get; }
}

7
src/NadekoBot/Modules/Searches/Search/ISearchResultInformation.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace NadekoBot.Modules.Searches;
public interface ISearchResultInformation
{
string TotalResults { get; }
string SearchTime { get; }
}

9
src/NadekoBot/Modules/Searches/Search/ISearchService.cs Normal file
View File

@@ -0,0 +1,9 @@
using MorseCode.ITask;
namespace NadekoBot.Modules.Searches;
public interface ISearchService
{
ITask<ISearchResult?> SearchAsync(string query);
ITask<IImageSearchResult?> SearchImagesAsync(string query);
}

10
src/NadekoBot/Modules/Searches/Search/ISearchServiceFactory.cs Normal file
View File

@@ -0,0 +1,10 @@
using NadekoBot.Modules.Searches.Youtube;
namespace NadekoBot.Modules.Searches;
public interface ISearchServiceFactory
{
public ISearchService GetSearchService(string? hint = null);
public ISearchService GetImageSearchService(string? hint = null);
public IYoutubeSearchService GetYoutubeSearchService(string? hint = null);
}

206
src/NadekoBot/Modules/Searches/Search/SearchCommands.cs Normal file
View File

@@ -0,0 +1,206 @@
using NadekoBot.Modules.Searches.Youtube;
using StackExchange.Redis;
using System.Net.Http.Json;
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public partial class Searches
{
public partial class SearchCommands : NadekoModule
{
private readonly ISearchServiceFactory _searchFactory;
private readonly ConnectionMultiplexer _redis;
public SearchCommands(
ISearchServiceFactory searchFactory,
ConnectionMultiplexer redis)
{
_searchFactory = searchFactory;
_redis = redis;
}
[Cmd]
public async partial Task Google([Leftover] string? query = null)
{
query = query?.Trim();
if (string.IsNullOrWhiteSpace(query))
{
await ErrorLocalizedAsync(strs.specify_search_params);
return;
}
_ = ctx.Channel.TriggerTypingAsync();
var search = _searchFactory.GetSearchService();
var data = await search.SearchAsync(query);
if (data is null or { Entries: null or { Count: 0 } })
{
await ReplyErrorLocalizedAsync(strs.no_results);
return;
}
// 3 with an answer
// 4 without an answer
// 5 is ideal but it lookes horrible on mobile
var takeCount = string.IsNullOrWhiteSpace(data.Answer)
? 4
: 3;
var descStr = data.Entries
.Take(takeCount)
.Select(static res => $@"**[{Format.Sanitize(res.Title)}]({res.Url})**
*{Format.EscapeUrl(res.DisplayUrl)}*
{Format.Sanitize(res.Description ?? "-")}")
.Join("\n\n");
if (!string.IsNullOrWhiteSpace(data.Answer))
descStr = Format.Code(data.Answer) + "\n\n" + descStr;
descStr = descStr.TrimTo(4096);
var embed = _eb.Create()
.WithOkColor()
.WithAuthor(ctx.User)
.WithTitle(query.TrimTo(64)!)
.WithDescription(descStr)
.WithFooter(
GetText(strs.results_in(data.Info.TotalResults, data.Info.SearchTime)),
"https://i.imgur.com/G46fm8J.png");
await ctx.Channel.EmbedAsync(embed);
}
[Cmd]
public async partial Task Image([Leftover] string? query = null)
{
query = query?.Trim();
if (string.IsNullOrWhiteSpace(query))
{
await ErrorLocalizedAsync(strs.specify_search_params);
return;
}
_ = ctx.Channel.TriggerTypingAsync();
var search = _searchFactory.GetImageSearchService();
var data = await search.SearchImagesAsync(query);
if (data is null or { Entries: null or { Count: 0 } })
{
await ReplyErrorLocalizedAsync(strs.no_search_results);
return;
}
var embeds = new List<IEmbedBuilder>(4);
IEmbedBuilder CreateEmbed(IImageSearchResultEntry entry)
{
return _eb.Create(ctx)
.WithOkColor()
.WithAuthor(ctx.User)
.WithTitle(query)
.WithUrl("https://google.com")
.WithImageUrl(entry.Link);
}
embeds.Add(CreateEmbed(data.Entries.First())
.WithFooter(
GetText(strs.results_in(data.Info.TotalResults, data.Info.SearchTime)),
"https://i.imgur.com/G46fm8J.png"));
var random = data.Entries.Skip(1)
.Shuffle()
.Take(3)
.ToArray();
foreach (var entry in random)
{
embeds.Add(CreateEmbed(entry));
}
await ctx.Channel.EmbedAsync(null, embeds: embeds);
}
private async Task AddYoutubeUrlToCacheAsync(string query, string url)
{
var db = _redis.GetDatabase();
await db.StringSetAsync($"search:youtube:{query}", url, expiry: 1.Hours());
}
private async Task<VideoInfo?> GetYoutubeUrlFromCacheAsync(string query)
{
var db = _redis.GetDatabase();
var url = await db.StringGetAsync($"search:youtube:{query}");
if (string.IsNullOrWhiteSpace(url))
return null;
return new VideoInfo()
{
Url = url
};
}
[Cmd]
public async partial Task Youtube([Leftover] string? query = null)
{
query = query?.Trim();
if (string.IsNullOrWhiteSpace(query))
{
await ErrorLocalizedAsync(strs.specify_search_params);
return;
}
_ = ctx.Channel.TriggerTypingAsync();
var maybeResult = await GetYoutubeUrlFromCacheAsync(query)
?? await _searchFactory.GetYoutubeSearchService().SearchAsync(query);
if (maybeResult is not {} result || result is {Url: null})
{
await ReplyErrorLocalizedAsync(strs.no_results);
return;
}
await AddYoutubeUrlToCacheAsync(query, result.Url);
await ctx.Channel.SendMessageAsync(result.Url);
}
// [Cmd]
// public async partial Task DuckDuckGo([Leftover] string query = null)
// {
// query = query?.Trim();
// if (!await ValidateQuery(query))
// return;
//
// _ = ctx.Channel.TriggerTypingAsync();
//
// var data = await _service.DuckDuckGoSearchAsync(query);
// if (data is null)
// {
// await ReplyErrorLocalizedAsync(strs.no_results);
// return;
// }
//
// var desc = data.Results.Take(5)
// .Select(res => $@"[**{res.Title}**]({res.Link})
// {res.Text.TrimTo(380 - res.Title.Length - res.Link.Length)}");
//
// var descStr = string.Join("\n\n", desc);
//
// var embed = _eb.Create()
// .WithAuthor(ctx.User.ToString(),
// "https://upload.wikimedia.org/wikipedia/en/9/90/The_DuckDuckGo_Duck.png")
// .WithDescription($"{GetText(strs.search_for)} **{query}**\n\n" + descStr)
// .WithOkColor();
//
// await ctx.Channel.EmbedAsync(embed);
// }
}
}

9
src/NadekoBot/Modules/Searches/Search/SearchServiceBase.cs Normal file
View File

@@ -0,0 +1,9 @@
using MorseCode.ITask;
namespace NadekoBot.Modules.Searches;
public abstract class SearchServiceBase : ISearchService
{
public abstract ITask<ISearchResult?> SearchAsync(string query);
public abstract ITask<IImageSearchResult?> SearchImagesAsync(string query);
}

28
src/NadekoBot/Modules/Searches/Search/Searx/SearxImageSearchResult.cs Normal file
View File

@@ -0,0 +1,28 @@
using System.Globalization;
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public sealed class SearxImageSearchResult : IImageSearchResult
{
public string SearchTime { get; set; } = null!;
public ISearchResultInformation Info
=> new SearxSearchResultInformation()
{
SearchTime = SearchTime,
TotalResults = NumberOfResults.ToString("N", CultureInfo.InvariantCulture)
};
public IReadOnlyCollection<IImageSearchResultEntry> Entries
=> Results;
[JsonPropertyName("results")]
public List<SearxImageSearchResultEntry> Results { get; set; } = new List<SearxImageSearchResultEntry>();
[JsonPropertyName("query")]
public string Query { get; set; } = null!;
[JsonPropertyName("number_of_results")]
public double NumberOfResults { get; set; }
}

14
src/NadekoBot/Modules/Searches/Search/Searx/SearxImageSearchResultEntry.cs Normal file
View File

@@ -0,0 +1,14 @@
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public sealed class SearxImageSearchResultEntry : IImageSearchResultEntry
{
public string Link
=> ImageSource.StartsWith("//")
? "https:" + ImageSource
: ImageSource;
[JsonPropertyName("img_src")]
public string ImageSource { get; set; } = string.Empty;
}

30
src/NadekoBot/Modules/Searches/Search/Searx/SearxInfobox.cs Normal file
View File

@@ -0,0 +1,30 @@
// using System.Text.Json.Serialization;
//
// namespace NadekoBot.Modules.Searches;
//
// public sealed class SearxInfobox
// {
// [JsonPropertyName("infobox")]
// public string Infobox { get; set; }
//
// [JsonPropertyName("id")]
// public string Id { get; set; }
//
// [JsonPropertyName("content")]
// public string Content { get; set; }
//
// [JsonPropertyName("img_src")]
// public string ImgSrc { get; set; }
//
// [JsonPropertyName("urls")]
// public List<SearxUrlData> Urls { get; } = new List<SearxUrlData>();
//
// [JsonPropertyName("engine")]
// public string Engine { get; set; }
//
// [JsonPropertyName("engines")]
// public List<string> Engines { get; } = new List<string>();
//
// [JsonPropertyName("attributes")]
// public List<SearxSearchAttribute> Attributes { get; } = new List<SearxSearchAttribute>();
// }

15
src/NadekoBot/Modules/Searches/Search/Searx/SearxSearchAttribute.cs Normal file
View File

@@ -0,0 +1,15 @@
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public sealed class SearxSearchAttribute
{
[JsonPropertyName("label")]
public string? Label { get; set; }
[JsonPropertyName("value")]
public string? Value { get; set; }
[JsonPropertyName("entity")]
public string? Entity { get; set; }
}

47
src/NadekoBot/Modules/Searches/Search/Searx/SearxSearchResult.cs Normal file
View File

@@ -0,0 +1,47 @@
using System.Globalization;
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public sealed class SearxSearchResult : ISearchResult
{
[JsonPropertyName("query")]
public string Query { get; set; } = null!;
[JsonPropertyName("number_of_results")]
public double NumberOfResults { get; set; }
[JsonPropertyName("results")]
public List<SearxSearchResultEntry> Results { get; set; } = new List<SearxSearchResultEntry>();
[JsonPropertyName("answers")]
public List<string> Answers { get; set; } = new List<string>();
//
// [JsonPropertyName("corrections")]
// public List<object> Corrections { get; } = new List<object>();
// [JsonPropertyName("infoboxes")]
// public List<InfoboxModel> Infoboxes { get; } = new List<InfoboxModel>();
//
// [JsonPropertyName("suggestions")]
// public List<string> Suggestions { get; } = new List<string>();
// [JsonPropertyName("unresponsive_engines")]
// public List<object> UnresponsiveEngines { get; } = new List<object>();
public string SearchTime { get; set; } = null!;
public IReadOnlyCollection<ISearchResultEntry> Entries
=> Results;
public ISearchResultInformation Info
=> new SearxSearchResultInformation()
{
SearchTime = SearchTime,
TotalResults = NumberOfResults.ToString("N", CultureInfo.InvariantCulture)
};
public string? Answer
=> Answers.FirstOrDefault();
}

51
src/NadekoBot/Modules/Searches/Search/Searx/SearxSearchResultEntry.cs Normal file
View File

@@ -0,0 +1,51 @@
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public sealed class SearxSearchResultEntry : ISearchResultEntry
{
public string DisplayUrl
=> Url;
public string Description
=> Content.TrimTo(768)!;
[JsonPropertyName("url")]
public string Url { get; set; } = null!;
[JsonPropertyName("title")]
public string Title { get; set; } = null!;
[JsonPropertyName("content")]
public string? Content { get; set; }
// [JsonPropertyName("engine")]
// public string Engine { get; set; }
//
// [JsonPropertyName("parsed_url")]
// public List<string> ParsedUrl { get; } = new List<string>();
//
// [JsonPropertyName("template")]
// public string Template { get; set; }
//
// [JsonPropertyName("engines")]
// public List<string> Engines { get; } = new List<string>();
//
// [JsonPropertyName("positions")]
// public List<int> Positions { get; } = new List<int>();
//
// [JsonPropertyName("score")]
// public double Score { get; set; }
//
// [JsonPropertyName("category")]
// public string Category { get; set; }
//
// [JsonPropertyName("pretty_url")]
// public string PrettyUrl { get; set; }
//
// [JsonPropertyName("open_group")]
// public bool OpenGroup { get; set; }
//
// [JsonPropertyName("close_group")]
// public bool? CloseGroup { get; set; }
}

7
src/NadekoBot/Modules/Searches/Search/Searx/SearxSearchResultInformation.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace NadekoBot.Modules.Searches;
public sealed class SearxSearchResultInformation : ISearchResultInformation
{
public string TotalResults { get; init; } = string.Empty;
public string SearchTime { get; init; } = string.Empty;
}

76
src/NadekoBot/Modules/Searches/Search/Searx/SearxSearchService.cs Normal file
View File

@@ -0,0 +1,76 @@
using MorseCode.ITask;
using System.Diagnostics;
using System.Globalization;
using System.Text.Json;
namespace NadekoBot.Modules.Searches;
public sealed class SearxSearchService : SearchServiceBase, INService
{
private readonly IHttpClientFactory _http;
private readonly SearchesConfigService _scs;
private static readonly Random _rng = new NadekoRandom();
public SearxSearchService(IHttpClientFactory http, SearchesConfigService scs)
=> (_http, _scs) = (http, scs);
private string GetRandomInstance()
{
var instances = _scs.Data.SearxInstances;
if (instances is null or { Count: 0 })
throw new InvalidOperationException("No searx instances specified in searches.yml");
return instances[_rng.Next(0, instances.Count)];
}
public override async ITask<SearxSearchResult> SearchAsync(string query)
{
ArgumentNullException.ThrowIfNull(query);
var instanceUrl = GetRandomInstance();
Log.Information("Using {Instance} instance for web search...", instanceUrl);
var sw = Stopwatch.StartNew();
using var http = _http.CreateClient();
await using var res = await http.GetStreamAsync($"{instanceUrl}"
+ $"?q={Uri.EscapeDataString(query)}"
+ $"&format=json"
+ $"&strict=2");
sw.Stop();
var dat = await JsonSerializer.DeserializeAsync<SearxSearchResult>(res);
if (dat is null)
return new SearxSearchResult();
dat.SearchTime = sw.Elapsed.TotalSeconds.ToString("N2", CultureInfo.InvariantCulture);
return dat;
}
public override async ITask<SearxImageSearchResult> SearchImagesAsync(string query)
{
ArgumentNullException.ThrowIfNull(query);
var instanceUrl = GetRandomInstance();
Log.Information("Using {Instance} instance for img search...", instanceUrl);
var sw = Stopwatch.StartNew();
using var http = _http.CreateClient();
await using var res = await http.GetStreamAsync($"{instanceUrl}"
+ $"?q={Uri.EscapeDataString(query)}"
+ $"&format=json"
+ $"&category_images=on"
+ $"&strict=2");
sw.Stop();
var dat = await JsonSerializer.DeserializeAsync<SearxImageSearchResult>(res);
if (dat is null)
return new SearxImageSearchResult();
dat.SearchTime = sw.Elapsed.TotalSeconds.ToString("N2", CultureInfo.InvariantCulture);
return dat;
}
}

15
src/NadekoBot/Modules/Searches/Search/Searx/SearxUrlData.cs Normal file
View File

@@ -0,0 +1,15 @@
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public sealed class SearxUrlData
{
[JsonPropertyName("title")]
public string Title { get; set; } = null!;
[JsonPropertyName("url")]
public string Url { get; set; } = null!;
[JsonPropertyName("official")]
public bool? Official { get; set; }
}

6
src/NadekoBot/Modules/Searches/Search/Youtube/IYoutubeSearchService.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace NadekoBot.Modules.Searches.Youtube;
public interface IYoutubeSearchService
{
Task<VideoInfo?> SearchAsync(string query);
}

9
src/NadekoBot/Modules/Searches/Search/Youtube/InvidiousSearchResponse.cs Normal file
View File

@@ -0,0 +1,9 @@
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches;
public sealed class InvidiousSearchResponse
{
[JsonPropertyName("videoId")]
public string VideoId { get; set; } = null!;
}

46
src/NadekoBot/Modules/Searches/Search/Youtube/InvidiousYtSearchService.cs Normal file
View File

@@ -0,0 +1,46 @@
using NadekoBot.Modules.Searches.Youtube;
using System.Net.Http.Json;
namespace NadekoBot.Modules.Searches;
public sealed class InvidiousYtSearchService : IYoutubeSearchService, INService
{
private readonly IHttpClientFactory _http;
private readonly SearchesConfigService _scs;
private readonly NadekoRandom _rng;
public InvidiousYtSearchService(
IHttpClientFactory http,
SearchesConfigService scs)
{
_http = http;
_scs = scs;
_rng = new();
}
public async Task<VideoInfo?> SearchAsync(string query)
{
ArgumentNullException.ThrowIfNull(query);
var instances = _scs.Data.InvidiousInstances;
if (instances is null or { Count: 0 })
{
Log.Warning("Attempted to use Invidious as the .youtube provider but there are no 'invidiousInstances' "
+ "specified in `data/searches.yml`");
return null;
}
var instance = instances[_rng.Next(0, instances.Count)];
using var http = _http.CreateClient();
var res = await http.GetFromJsonAsync<List<InvidiousSearchResponse>>(
$"{instance}/api/v1/search"
+ $"?q={query}"
+ $"&type=video");
if (res is null or {Count: 0})
return null;
return new VideoInfo(res[0].VideoId);
}
}

9
src/NadekoBot/Modules/Searches/Search/Youtube/VideoInfo.cs Normal file
View File

@@ -0,0 +1,9 @@
namespace NadekoBot.Modules.Searches.Youtube;
public readonly struct VideoInfo
{
public VideoInfo(string videoId)
=> Url = $"https://youtube.com/watch?v={videoId}";
public string Url { get; init; }
}

26
src/NadekoBot/Modules/Searches/Search/Youtube/YoutubeDataApiSearchService.cs Normal file
View File

@@ -0,0 +1,26 @@
namespace NadekoBot.Modules.Searches.Youtube;
public sealed class YoutubeDataApiSearchService : IYoutubeSearchService, INService
{
private readonly IGoogleApiService _gapi;
public YoutubeDataApiSearchService(IGoogleApiService gapi)
{
_gapi = gapi;
}
public async Task<VideoInfo?> SearchAsync(string query)
{
ArgumentNullException.ThrowIfNull(query);
var results = await _gapi.GetVideoLinksByKeywordAsync(query);
var first = results.FirstOrDefault();
if (first is null)
return null;
return new()
{
Url = first
};
}
}

7
src/NadekoBot/Modules/Searches/Search/Youtube/YtdlYoutubeSearchService.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace NadekoBot.Modules.Searches.Youtube;
public sealed class YtdlYoutubeSearchService : YoutubedlxServiceBase, INService
{
public override async Task<VideoInfo?> SearchAsync(string query)
=> await InternalGetInfoAsync(query, false);
}

7
src/NadekoBot/Modules/Searches/Search/Youtube/YtdlpYoutubeSearchService.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace NadekoBot.Modules.Searches.Youtube;
public sealed class YtdlpYoutubeSearchService : YoutubedlxServiceBase, INService
{
public override async Task<VideoInfo?> SearchAsync(string query)
=> await InternalGetInfoAsync(query, true);
}

34
src/NadekoBot/Modules/Searches/Search/Youtube/YtdlxServiceBase.cs Normal file
View File

@@ -0,0 +1,34 @@
namespace NadekoBot.Modules.Searches.Youtube;
public abstract class YoutubedlxServiceBase : IYoutubeSearchService
{
private YtdlOperation CreateYtdlOp(bool isYtDlp)
=> new YtdlOperation("-4 "
+ "--geo-bypass "
+ "--encoding UTF8 "
+ "--get-id "
+ "--no-check-certificate "
+ "--default-search "
+ "\"ytsearch:\" -- \"{0}\"",
isYtDlp: isYtDlp);
protected async Task<VideoInfo?> InternalGetInfoAsync(string query, bool isYtDlp)
{
var op = CreateYtdlOp(isYtDlp);
var data = await op.GetDataAsync(query);
var items = data?.Split('\n');
if (items is null or { Length: 0 })
return null;
var id = items.FirstOrDefault(x => x.Length is > 5 and < 15);
if (id is null)
return null;
return new VideoInfo()
{
Url = $"https://youtube.com/watch?v={id}"
};
}
public abstract Task<VideoInfo?> SearchAsync(string query);
}

147
src/NadekoBot/Modules/Searches/Searches.cs
View File

@@ -1,6 +1,4 @@
#nullable disable
using AngleSharp;
using AngleSharp.Html.Dom;
using Microsoft.Extensions.Caching.Memory;
using NadekoBot.Modules.Administration.Services;
using NadekoBot.Modules.Searches.Common;
@@ -11,9 +9,9 @@ using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Drawing.Processing;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using System.Diagnostics.CodeAnalysis;
using System.Net;
using Color = SixLabors.ImageSharp.Color;
using Configuration = AngleSharp.Configuration;
namespace NadekoBot.Modules.Searches;
@@ -92,7 +90,7 @@ public partial class Searches : NadekoModule<SearchesService>
.AddField("🌇 " + Format.Bold(GetText(strs.sunset)), $"{sunset:HH:mm} {timezone}", true)
.WithOkColor()
.WithFooter("Powered by openweathermap.org",
$"http://openweathermap.org/img/w/{data.Weather[0].Icon}.png");
$"https://openweathermap.org/img/w/{data.Weather[0].Icon}.png");
}
await ctx.Channel.EmbedAsync(embed);
@@ -146,22 +144,6 @@ public partial class Searches : NadekoModule<SearchesService>
await ctx.Channel.SendMessageAsync(embed: eb.Build());
}
[Cmd]
public async partial Task Youtube([Leftover] string query = null)
{
if (!await ValidateQuery(query))
return;
var result = (await _google.GetVideoLinksByKeywordAsync(query)).FirstOrDefault();
if (string.IsNullOrWhiteSpace(result))
{
await ReplyErrorLocalizedAsync(strs.no_results);
return;
}
await ctx.Channel.SendMessageAsync(result);
}
[Cmd]
public async partial Task Movie([Leftover] string query = null)
{
@@ -180,7 +162,7 @@ public partial class Searches : NadekoModule<SearchesService>
await ctx.Channel.EmbedAsync(_eb.Create()
.WithOkColor()
.WithTitle(movie.Title)
.WithUrl($"http://www.imdb.com/title/{movie.ImdbId}/")
.WithUrl($"https://www.imdb.com/title/{movie.ImdbId}/")
.WithDescription(movie.Plot.TrimTo(1000))
.AddField("Rating", movie.ImdbRating, true)
.AddField("Genre", movie.Genre, true)
@@ -210,67 +192,13 @@ public partial class Searches : NadekoModule<SearchesService>
return ctx.Channel.EmbedAsync(_eb.Create().WithOkColor().WithImageUrl(url));
}
[Cmd]
public async partial Task Image([Leftover] string query = null)
{
var oterms = query?.Trim();
if (!await ValidateQuery(query))
return;
query = WebUtility.UrlEncode(oterms)?.Replace(' ', '+');
try
{
var res = await _google.GetImageAsync(oterms);
var embed = _eb.Create()
.WithOkColor()
.WithAuthor(GetText(strs.image_search_for) + " " + oterms.TrimTo(50),
"http://i.imgur.com/G46fm8J.png",
$"https://www.google.rs/search?q={query}&source=lnms&tbm=isch")
.WithDescription(res.Link)
.WithImageUrl(res.Link)
.WithTitle(ctx.User.ToString());
await ctx.Channel.EmbedAsync(embed);
}
catch
{
Log.Warning("Falling back to Imgur");
var fullQueryLink = $"http://imgur.com/search?q={query}";
var config = Configuration.Default.WithDefaultLoader();
using var document = await BrowsingContext.New(config).OpenAsync(fullQueryLink);
var elems = document.QuerySelectorAll("a.image-list-link").ToList();
if (!elems.Any())
return;
var img =
elems.ElementAtOrDefault(new NadekoRandom().Next(0, elems.Count))?.Children?.FirstOrDefault() as
IHtmlImageElement;
if (img?.Source is null)
return;
var source = img.Source.Replace("b.", ".", StringComparison.InvariantCulture);
var embed = _eb.Create()
.WithOkColor()
.WithAuthor(GetText(strs.image_search_for) + " " + oterms.TrimTo(50),
"http://s.imgur.com/images/logo-1200-630.jpg?",
fullQueryLink)
.WithDescription(source)
.WithImageUrl(source)
.WithTitle(ctx.User.ToString());
await ctx.Channel.EmbedAsync(embed);
}
}
[Cmd]
public async partial Task Lmgtfy([Leftover] string ffs = null)
{
if (!await ValidateQuery(ffs))
return;
var shortenedUrl = await _google.ShortenUrl($"http://lmgtfy.com/?q={Uri.EscapeDataString(ffs)}");
var shortenedUrl = await _google.ShortenUrl($"https://lmgtfy.com/?q={Uri.EscapeDataString(ffs)}");
await SendConfirmAsync($"<{shortenedUrl}>");
}
@@ -317,69 +245,6 @@ public partial class Searches : NadekoModule<SearchesService>
.AddField(GetText(strs.short_url), $"<{shortLink}>"));
}
[Cmd]
public async partial Task Google([Leftover] string query = null)
{
query = query?.Trim();
if (!await ValidateQuery(query))
return;
_ = ctx.Channel.TriggerTypingAsync();
var data = await _service.GoogleSearchAsync(query);
if (data is null)
{
await ReplyErrorLocalizedAsync(strs.no_results);
return;
}
var desc = data.Results.Take(5)
.Select(res => $@"[**{res.Title}**]({res.Link})
{res.Text.TrimTo(400 - res.Title.Length - res.Link.Length)}");
var descStr = string.Join("\n\n", desc);
var embed = _eb.Create()
.WithAuthor(ctx.User.ToString(), "http://i.imgur.com/G46fm8J.png")
.WithTitle(ctx.User.ToString())
.WithFooter(data.TotalResults)
.WithDescription($"{GetText(strs.search_for)} **{query}**\n\n" + descStr)
.WithOkColor();
await ctx.Channel.EmbedAsync(embed);
}
[Cmd]
public async partial Task DuckDuckGo([Leftover] string query = null)
{
query = query?.Trim();
if (!await ValidateQuery(query))
return;
_ = ctx.Channel.TriggerTypingAsync();
var data = await _service.DuckDuckGoSearchAsync(query);
if (data is null)
{
await ReplyErrorLocalizedAsync(strs.no_results);
return;
}
var desc = data.Results.Take(5)
.Select(res => $@"[**{res.Title}**]({res.Link})
{res.Text.TrimTo(380 - res.Title.Length - res.Link.Length)}");
var descStr = string.Join("\n\n", desc);
var embed = _eb.Create()
.WithAuthor(ctx.User.ToString(),
"https://upload.wikimedia.org/wikipedia/en/9/90/The_DuckDuckGo_Duck.png")
.WithDescription($"{GetText(strs.search_for)} **{query}**\n\n" + descStr)
.WithOkColor();
await ctx.Channel.EmbedAsync(embed);
}
[Cmd]
public async partial Task MagicTheGathering([Leftover] string search)
{
@@ -446,7 +311,7 @@ public partial class Searches : NadekoModule<SearchesService>
using (var http = _httpFactory.CreateClient())
{
var res = await http.GetStringAsync(
$"http://api.urbandictionary.com/v0/define?term={Uri.EscapeDataString(query)}");
$"https://api.urbandictionary.com/v0/define?term={Uri.EscapeDataString(query)}");
try
{
var items = JsonConvert.DeserializeObject<UrbanResponse>(res).List;
@@ -732,7 +597,7 @@ public partial class Searches : NadekoModule<SearchesService>
await ctx.Channel.SendMessageAsync($"https://store.steampowered.com/app/{appId}");
}
private async Task<bool> ValidateQuery(string query)
private async Task<bool> ValidateQuery([MaybeNullWhen(false)] string query)
{
if (!string.IsNullOrWhiteSpace(query))
return true;

172
src/NadekoBot/Modules/Searches/SearchesService.cs
View File

@@ -26,15 +26,6 @@ public class SearchesService : INService
Birds
}
private static readonly HtmlParser _googleParser = new(new()
{
IsScripting = false,
IsEmbedded = false,
IsSupportingProcessingInstructions = false,
IsKeepingSourceReferences = false,
IsNotSupportingFrames = true
});
public List<WoWJoke> WowJokes { get; } = new();
public List<MagicItem> MagicItems { get; } = new();
private readonly IHttpClientFactory _httpFactory;
@@ -161,7 +152,7 @@ public class SearchesService : INService
using var http = _httpFactory.CreateClient();
try
{
var data = await http.GetStringAsync("http://api.openweathermap.org/data/2.5/weather?"
var data = await http.GetStringAsync("https://api.openweathermap.org/data/2.5/weather?"
+ $"q={query}&"
+ "appid=42cd627dd60debf25a5739e50a217d74&"
+ "units=metric");
@@ -440,22 +431,6 @@ public class SearchesService : INService
public async Task<int> GetSteamAppIdByName(string query)
{
const string steamGameIdsKey = "steam_names_to_appid";
// var exists = await db.KeyExistsAsync(steamGameIdsKey);
// if we didn't get steam name to id map already, get it
//if (!exists)
//{
// using (var http = _httpFactory.CreateClient())
// {
// // https://api.steampowered.com/ISteamApps/GetAppList/v2/
// var gamesStr = await http.GetStringAsync("https://api.steampowered.com/ISteamApps/GetAppList/v2/");
// var apps = JsonConvert.DeserializeAnonymousType(gamesStr, new { applist = new { apps = new List<SteamGameId>() } }).applist.apps;
// //await db.HashSetAsync("steam_game_ids", apps.Select(app => new HashEntry(app.Name.Trim().ToLowerInvariant(), app.AppId)).ToArray());
// await db.StringSetAsync("steam_game_ids", gamesStr, TimeSpan.FromHours(24));
// //await db.KeyExpireAsync("steam_game_ids", TimeSpan.FromHours(24), CommandFlags.FireAndForget);
// }
//}
var gamesMap = await _cache.GetOrAddCachedDataAsync(steamGameIdsKey,
async _ =>
@@ -502,150 +477,5 @@ public class SearchesService : INService
}
return gamesMap[key];
//// try finding the game id
//var val = db.HashGet(STEAM_GAME_IDS_KEY, query);
//if (val == default)
// return -1; // not found
//var appid = (int)val;
//return appid;
// now that we have appid, get the game info with that appid
//var gameData = await _cache.GetOrAddCachedDataAsync($"steam_game:{appid}", SteamGameDataFactory, appid, TimeSpan.FromHours(12))
//;
//return gameData;
}
public async Task<GoogleSearchResultData> GoogleSearchAsync(string query)
{
query = WebUtility.UrlEncode(query)?.Replace(' ', '+');
var fullQueryLink = $"https://www.google.ca/search?q={query}&safe=on&lr=lang_eng&hl=en&ie=utf-8&oe=utf-8";
using var msg = new HttpRequestMessage(HttpMethod.Get, fullQueryLink);
msg.Headers.Add("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
msg.Headers.Add("Cookie", "CONSENT=YES+shp.gws-20210601-0-RC2.en+FX+423;");
using var http = _httpFactory.CreateClient();
http.DefaultRequestHeaders.Clear();
using var response = await http.SendAsync(msg);
await using var content = await response.Content.ReadAsStreamAsync();
using var document = await _googleParser.ParseDocumentAsync(content);
var elems = document.QuerySelectorAll("div.g > div > div");
var resultsElem = document.QuerySelectorAll("#resultStats").FirstOrDefault();
var totalResults = resultsElem?.TextContent;
//var time = resultsElem.Children.FirstOrDefault()?.TextContent
//^ this doesn't work for some reason, <nobr> is completely missing in parsed collection
if (!elems.Any())
return default;
var results = elems.Select(elem =>
{
var children = elem.Children.ToList();
if (children.Count < 2)
return null;
var href = (children[0].QuerySelector("a") as IHtmlAnchorElement)?.Href;
var name = children[0].QuerySelector("h3")?.TextContent;
if (href is null || name is null)
return null;
var txt = children[1].TextContent;
if (string.IsNullOrWhiteSpace(txt))
return null;
return new GoogleSearchResult(name, href, txt);
})
.Where(x => x is not null)
.ToList();
return new(results.AsReadOnly(), fullQueryLink, totalResults);
}
public async Task<GoogleSearchResultData> DuckDuckGoSearchAsync(string query)
{
query = WebUtility.UrlEncode(query)?.Replace(' ', '+');
var fullQueryLink = "https://html.duckduckgo.com/html";
using var http = _httpFactory.CreateClient();
http.DefaultRequestHeaders.Clear();
http.DefaultRequestHeaders.Add("User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36");
using var formData = new MultipartFormDataContent();
formData.Add(new StringContent(query), "q");
using var response = await http.PostAsync(fullQueryLink, formData);
var content = await response.Content.ReadAsStringAsync();
using var document = await _googleParser.ParseDocumentAsync(content);
var searchResults = document.QuerySelector(".results");
var elems = searchResults.QuerySelectorAll(".result");
if (!elems.Any())
return default;
var results = elems.Select(elem =>
{
if (elem.QuerySelector(".result__a") is not IHtmlAnchorElement anchor)
return null;
var href = anchor.Href;
var name = anchor.TextContent;
if (string.IsNullOrWhiteSpace(href) || string.IsNullOrWhiteSpace(name))
return null;
var txt = elem.QuerySelector(".result__snippet")?.TextContent;
if (string.IsNullOrWhiteSpace(txt))
return null;
return new GoogleSearchResult(name, href, txt);
})
.Where(x => x is not null)
.ToList();
return new(results.AsReadOnly(), fullQueryLink, "0");
}
//private async Task<SteamGameData> SteamGameDataFactory(int appid)
//{
// using (var http = _httpFactory.CreateClient())
// {
// // https://store.steampowered.com/api/appdetails?appids=
// var responseStr = await http.GetStringAsync($"https://store.steampowered.com/api/appdetails?appids={appid}");
// var data = JsonConvert.DeserializeObject<Dictionary<int, SteamGameData.Container>>(responseStr);
// if (!data.ContainsKey(appid) || !data[appid].Success)
// return null; // for some reason we can't get the game with valid appid. SHould never happen
// return data[appid].Data;
// }
//}
public class GoogleSearchResultData
{
public IReadOnlyList<GoogleSearchResult> Results { get; }
public string FullQueryLink { get; }
public string TotalResults { get; }
public GoogleSearchResultData(
IReadOnlyList<GoogleSearchResult> results,
string fullQueryLink,
string totalResults)
{
Results = results;
FullQueryLink = fullQueryLink;
TotalResults = totalResults;
}
}
}

2
src/NadekoBot/Modules/Searches/StreamNotification/StreamNotificationService.cs
View File

@@ -1,6 +1,4 @@
#nullable disable
using LinqToDB;
using LinqToDB.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Common.ModuleBehaviors;
using NadekoBot.Db;

0
src/NadekoBot/Modules/Searches/Services/AtlExtensions.cs → src/NadekoBot/Modules/Searches/_Common/AtlExtensions.cs
View File

7
src/NadekoBot/Modules/Searches/_Common/Config/ImgSearchEngine.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace NadekoBot.Modules.Searches;
public enum ImgSearchEngine
{
Google,
Searx,
}

63
src/NadekoBot/Modules/Searches/_Common/Config/SearchesConfig.cs Normal file
View File

@@ -0,0 +1,63 @@
using Cloneable;
using NadekoBot.Common.Yml;
namespace NadekoBot.Modules.Searches;
[Cloneable]
public partial class SearchesConfig : ICloneable<SearchesConfig>
{
[Comment(@"Which engine should .search command
'google' requires googleApiKey and google.searchId set in creds.yml
'searx' requires at least one searx instance specified in the 'searxInstances' property below")]
public WebSearchEngine WebSearchEngine { get; set; } = WebSearchEngine.Google;
[Comment(@"Which engine should .image command use
'google' requires googleApiKey and google.imageSearchId set in creds.yml
'searx' requires at least one searx instance specified in the 'searxInstances' property below")]
public ImgSearchEngine ImgSearchEngine { get; set; } = ImgSearchEngine.Google;
[Comment(@"Which search provider will be used for the `.youtube` command.
- `ytDataApiv3` - uses google's official youtube data api. Requires `GoogleApiKey` set in creds and youtube data api enabled in developers console
- `ytdl` - default, uses youtube-dl. Requires `youtube-dl` to be installed and it's path added to env variables. Slow.
- `ytdlp` - recommended easy, uses `yt-dlp`. Requires `yt-dlp` to be installed and it's path added to env variables
- `invidious` - recommended advanced, uses invidious api. Requires at least one invidious instance specified in the `invidiousInstances` property")]
public YoutubeSearcher YtProvider { get; set; } = YoutubeSearcher.Ytdl;
[Comment(@"Set the searx instance urls in case you want to use 'searx' for either img or web search.
Nadeko will use a random one for each request.
Use a fully qualified url. Example: `https://my-searx-instance.mydomain.com`
Instances specified must support 'format=json' query parameter.
- In case you're running your own searx instance, set
search:
formats:
- json
in 'searxng/settings.yml' on your server
- If you're using a public instance, make sure that the instance you're using supports it (they usually don't)")]
public List<string> SearxInstances { get; set; } = new List<string>();
[Comment(@"Set the invidious instance urls in case you want to use 'invidious' for `.youtube` search
Nadeko will use a random one for each request.
These instances may be used for music queue functionality in the future.
Use a fully qualified url. Example: https://my-invidious-instance.mydomain.com
Instances specified must have api available.
You check that by opening an api endpoint in your browser. For example: https://my-invidious-instance.mydomain.com/api/v1/trending")]
public List<string> InvidiousInstances { get; set; } = new List<string>();
}
public enum YoutubeSearcher
{
YtDataApiv3,
Ytdl,
Ytdlp,
Invid,
Invidious = 3
}

31
src/NadekoBot/Modules/Searches/_Common/Config/SearchesConfigService.cs Normal file
View File

@@ -0,0 +1,31 @@
using NadekoBot.Common.Configs;
namespace NadekoBot.Modules.Searches;
public class SearchesConfigService : ConfigServiceBase<SearchesConfig>
{
private static string FILE_PATH = "data/searches.yml";
private static readonly TypedKey<SearchesConfig> _changeKey = new("config.searches.updated");
public override string Name
=> "searches";
public SearchesConfigService(IConfigSeria serializer, IPubSub pubSub)
: base(FILE_PATH, serializer, pubSub, _changeKey)
{
AddParsedProp("webEngine",
sc => sc.WebSearchEngine,
ConfigParsers.InsensitiveEnum,
ConfigPrinters.ToString);
AddParsedProp("imgEngine",
sc => sc.ImgSearchEngine,
ConfigParsers.InsensitiveEnum,
ConfigPrinters.ToString);
AddParsedProp("ytProvider",
sc => sc.YtProvider,
ConfigParsers.InsensitiveEnum,
ConfigPrinters.ToString);
}
}

7
src/NadekoBot/Modules/Searches/_Common/Config/WebSearchEngine.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace NadekoBot.Modules.Searches;
public enum WebSearchEngine
{
Google,
Searx,
}