using NadekoBot.Common; using NadekoBot.Modules.Searches.Common; using NadekoBot.Services; using NadekoBot.Extensions; using Newtonsoft.Json; using Newtonsoft.Json.Linq; using SixLabors.ImageSharp; using SixLabors.ImageSharp.Drawing.Processing; using SixLabors.ImageSharp.PixelFormats; using SixLabors.ImageSharp.Processing; using System; using System.Collections.Generic; using System.IO; using System.Linq; using System.Net; using System.Net.Http; using System.Threading.Tasks; using AngleSharp.Html.Dom; using AngleSharp.Html.Parser; using Serilog; using HorizontalAlignment = SixLabors.Fonts.HorizontalAlignment; using Image = SixLabors.ImageSharp.Image; namespace NadekoBot.Modules.Searches.Services { public class SearchesService : INService { private readonly IHttpClientFactory _httpFactory; private readonly IGoogleApiService _google; private readonly IImageCache _imgs; private readonly IDataCache _cache; private readonly FontProvider _fonts; private readonly IBotCredentials _creds; private readonly NadekoRandom _rng; public List WowJokes { get; } = new List(); public List MagicItems { get; } = new List(); private readonly List _yomamaJokes; public SearchesService(IGoogleApiService google, IDataCache cache, IHttpClientFactory factory, FontProvider fonts, IBotCredentials creds) { _httpFactory = factory; _google = google; _imgs = cache.LocalImages; _cache = cache; _fonts = fonts; _creds = creds; _rng = new NadekoRandom(); //joke commands if (File.Exists("data/wowjokes.json")) { WowJokes = JsonConvert.DeserializeObject>(File.ReadAllText("data/wowjokes.json")); } else Log.Warning("data/wowjokes.json is missing. WOW Jokes are not loaded."); if (File.Exists("data/magicitems.json")) { MagicItems = JsonConvert.DeserializeObject>(File.ReadAllText("data/magicitems.json")); } else Log.Warning("data/magicitems.json is missing. Magic items are not loaded."); if (File.Exists("data/yomama.txt")) { _yomamaJokes = File.ReadAllLines("data/yomama.txt") .Shuffle() .ToList(); } else { _yomamaJokes = new List(); Log.Warning("data/yomama.txt is missing. .yomama command won't work"); } } public async Task GetRipPictureAsync(string text, Uri imgUrl) { byte[] data = await _cache.GetOrAddCachedDataAsync($"nadeko_rip_{text}_{imgUrl}", GetRipPictureFactory, (text, imgUrl), TimeSpan.FromDays(1)).ConfigureAwait(false); return data.ToStream(); } private void DrawAvatar(Image bg, Image avatarImage) => bg.Mutate(x => x.Grayscale().DrawImage(avatarImage, new Point(83, 139), new GraphicsOptions())); public async Task GetRipPictureFactory((string text, Uri avatarUrl) arg) { var (text, avatarUrl) = arg; using (var bg = Image.Load(_imgs.Rip.ToArray())) { var (succ, data) = (false, (byte[])null); //await _cache.TryGetImageDataAsync(avatarUrl); if (!succ) { using (var http = _httpFactory.CreateClient()) { data = await http.GetByteArrayAsync(avatarUrl); using (var avatarImg = Image.Load(data)) { avatarImg.Mutate(x => x .Resize(85, 85) .ApplyRoundedCorners(42)); data = avatarImg.ToStream().ToArray(); DrawAvatar(bg, avatarImg); } await _cache.SetImageDataAsync(avatarUrl, data); } } else { using (var avatarImg = Image.Load(data)) { DrawAvatar(bg, avatarImg); } } bg.Mutate(x => x.DrawText( new TextGraphicsOptions() { TextOptions = new TextOptions { HorizontalAlignment = HorizontalAlignment.Center, WrapTextWidth = 190, }.WithFallbackFonts(_fonts.FallBackFonts) }, text, _fonts.RipFont, SixLabors.ImageSharp.Color.Black, new PointF(25, 225))); //flowa using (var flowers = Image.Load(_imgs.RipOverlay.ToArray())) { bg.Mutate(x => x.DrawImage(flowers, new Point(0, 0), new GraphicsOptions())); } return bg.ToStream().ToArray(); } } public Task GetWeatherDataAsync(string query) { query = query.Trim().ToLowerInvariant(); return _cache.GetOrAddCachedDataAsync($"nadeko_weather_{query}", GetWeatherDataFactory, query, expiry: TimeSpan.FromHours(3)); } private async Task GetWeatherDataFactory(string query) { using (var http = _httpFactory.CreateClient()) { try { var data = await http.GetStringAsync($"http://api.openweathermap.org/data/2.5/weather?" + $"q={query}&" + $"appid=42cd627dd60debf25a5739e50a217d74&" + $"units=metric").ConfigureAwait(false); if (data is null) return null; return JsonConvert.DeserializeObject(data); } catch (Exception ex) { Log.Warning(ex.Message); return null; } } } public Task<((string Address, DateTime Time, string TimeZoneName), TimeErrors?)> GetTimeDataAsync(string arg) { return GetTimeDataFactory(arg); //return _cache.GetOrAddCachedDataAsync($"nadeko_time_{arg}", // GetTimeDataFactory, // arg, // TimeSpan.FromMinutes(1)); } private async Task<((string Address, DateTime Time, string TimeZoneName), TimeErrors?)> GetTimeDataFactory(string query) { query = query.Trim(); if (string.IsNullOrEmpty(query)) { return (default, TimeErrors.InvalidInput); } if (string.IsNullOrWhiteSpace(_creds.LocationIqApiKey) || string.IsNullOrWhiteSpace(_creds.TimezoneDbApiKey)) { return (default, TimeErrors.ApiKeyMissing); } try { using (var _http = _httpFactory.CreateClient()) { var res = await _cache.GetOrAddCachedDataAsync($"geo_{query}", _ => { var url = "https://eu1.locationiq.com/v1/search.php?" + (string.IsNullOrWhiteSpace(_creds.LocationIqApiKey) ? "key=" : $"key={_creds.LocationIqApiKey}&") + $"q={Uri.EscapeDataString(query)}&" + $"format=json"; var res = _http.GetStringAsync(url); return res; }, "", TimeSpan.FromHours(1)); var responses = JsonConvert.DeserializeObject(res); if (responses is null || responses.Length == 0) { Log.Warning("Geocode lookup failed for: {Query}", query); return (default, TimeErrors.NotFound); } var geoData = responses[0]; using (var req = new HttpRequestMessage(HttpMethod.Get, "http://api.timezonedb.com/v2.1/get-time-zone?" + $"key={_creds.TimezoneDbApiKey}&format=json&" + "by=position&" + $"lat={geoData.Lat}&lng={geoData.Lon}")) { using (var geoRes = await _http.SendAsync(req)) { var resString = await geoRes.Content.ReadAsStringAsync(); var timeObj = JsonConvert.DeserializeObject(resString); var time = new DateTime(1970, 1, 1, 0, 0, 0, System.DateTimeKind.Utc).AddSeconds(timeObj.Timestamp); return (( Address: responses[0].DisplayName, Time: time, TimeZoneName: timeObj.TimezoneName ), default); } } } } catch (Exception ex) { Log.Error(ex, "Weather error: {Message}", ex.Message); return (default, TimeErrors.NotFound); } } public enum ImageTag { Food, Dogs, Cats, Birds } public string GetRandomImageUrl(ImageTag tag) { var subpath = tag.ToString().ToLowerInvariant(); int max; switch (tag) { case ImageTag.Food: max = 773; break; case ImageTag.Dogs: max = 750; break; case ImageTag.Cats: max = 773; break; case ImageTag.Birds: max = 578; break; default: max = 100; break; } return $"https://nadeko-pictures.nyc3.digitaloceanspaces.com/{subpath}/" + _rng.Next(1, max).ToString("000") + ".png"; } private readonly object yomamaLock = new object(); private int yomamaJokeIndex = 0; public Task GetYomamaJoke() { string joke; lock (yomamaLock) { if (yomamaJokeIndex >= _yomamaJokes.Count) { yomamaJokeIndex = 0; var newList = _yomamaJokes.ToList(); _yomamaJokes.Clear(); _yomamaJokes.AddRange(newList.Shuffle()); } joke = _yomamaJokes[yomamaJokeIndex++]; } return Task.FromResult(joke); // using (var http = _httpFactory.CreateClient()) // { // var response = await http.GetStringAsync(new Uri("http://api.yomomma.info/")).ConfigureAwait(false); // return JObject.Parse(response)["joke"].ToString() + " 😆"; // } } public async Task<(string Setup, string Punchline)> GetRandomJoke() { using (var http = _httpFactory.CreateClient()) { var res = await http.GetStringAsync("https://official-joke-api.appspot.com/random_joke"); var resObj = JsonConvert.DeserializeAnonymousType(res, new {setup = "", punchline = ""}); return (resObj.setup, resObj.punchline); } } public async Task GetChuckNorrisJoke() { using (var http = _httpFactory.CreateClient()) { var response = await http.GetStringAsync(new Uri("http://api.icndb.com/jokes/random/")).ConfigureAwait(false); return JObject.Parse(response)["value"]["joke"].ToString() + " 😆"; } } public async Task GetMtgCardAsync(string search) { search = search.Trim().ToLowerInvariant(); var data = await _cache.GetOrAddCachedDataAsync($"nadeko_mtg_{search}", GetMtgCardFactory, search, TimeSpan.FromDays(1)).ConfigureAwait(false); if (data is null || data.Length == 0) return null; return data[_rng.Next(0, data.Length)]; } private async Task GetMtgCardFactory(string search) { async Task GetMtgDataAsync(MtgResponse.Data card) { string storeUrl; try { storeUrl = await _google.ShortenUrl($"https://shop.tcgplayer.com/productcatalog/product/show?" + $"newSearch=false&" + $"ProductType=All&" + $"IsProductNameExact=false&" + $"ProductName={Uri.EscapeUriString(card.Name)}").ConfigureAwait(false); } catch { storeUrl = ""; } return new MtgData { Description = card.Text, Name = card.Name, ImageUrl = card.ImageUrl, StoreUrl = storeUrl, Types = string.Join(",\n", card.Types), ManaCost = card.ManaCost, }; } using (var http = _httpFactory.CreateClient()) { http.DefaultRequestHeaders.Clear(); var response = await http.GetStringAsync($"https://api.magicthegathering.io/v1/cards?name={Uri.EscapeUriString(search)}") .ConfigureAwait(false); var responseObject = JsonConvert.DeserializeObject(response); if (responseObject is null) return new MtgData[0]; var cards = responseObject.Cards.Take(5).ToArray(); if (cards.Length == 0) return new MtgData[0]; var tasks = new List>(cards.Length); for (int i = 0; i < cards.Length; i++) { var card = cards[i]; tasks.Add(GetMtgDataAsync(card)); } return await Task.WhenAll(tasks).ConfigureAwait(false); } } public Task GetHearthstoneCardDataAsync(string name) { name = name.ToLowerInvariant(); return _cache.GetOrAddCachedDataAsync($"nadeko_hearthstone_{name}", HearthstoneCardDataFactory, name, TimeSpan.FromDays(1)); } private async Task HearthstoneCardDataFactory(string name) { using (var http = _httpFactory.CreateClient()) { http.DefaultRequestHeaders.Clear(); http.DefaultRequestHeaders.Add("x-rapidapi-key", _creds.RapidApiKey); try { var response = await http.GetStringAsync($"https://omgvamp-hearthstone-v1.p.rapidapi.com/" + $"cards/search/{Uri.EscapeUriString(name)}").ConfigureAwait(false); var objs = JsonConvert.DeserializeObject(response); if (objs is null || objs.Length == 0) return null; var data = objs.FirstOrDefault(x => x.Collectible) ?? objs.FirstOrDefault(x => !string.IsNullOrEmpty(x.PlayerClass)) ?? objs.FirstOrDefault(); if (data is null) return null; if (!string.IsNullOrWhiteSpace(data.Img)) { data.Img = await _google.ShortenUrl(data.Img).ConfigureAwait(false); } if (!string.IsNullOrWhiteSpace(data.Text)) { var converter = new Html2Markdown.Converter(); data.Text = converter.Convert(data.Text); } return data; } catch (Exception ex) { Log.Error(ex.Message); return null; } } } public Task GetMovieDataAsync(string name) { name = name.Trim().ToLowerInvariant(); return _cache.GetOrAddCachedDataAsync($"nadeko_movie_{name}", GetMovieDataFactory, name, TimeSpan.FromDays(1)); } private async Task GetMovieDataFactory(string name) { using (var http = _httpFactory.CreateClient()) { var res = await http.GetStringAsync(string.Format("https://omdbapi.nadeko.bot/?t={0}&y=&plot=full&r=json", name.Trim().Replace(' ', '+'))).ConfigureAwait(false); var movie = JsonConvert.DeserializeObject(res); if (movie?.Title is null) return null; movie.Poster = await _google.ShortenUrl(movie.Poster).ConfigureAwait(false); return movie; } } public async Task GetSteamAppIdByName(string query) { var redis = _cache.Redis; var db = redis.GetDatabase(); const string STEAM_GAME_IDS_KEY = "steam_names_to_appid"; var exists = await db.KeyExistsAsync(STEAM_GAME_IDS_KEY).ConfigureAwait(false); // if we didn't get steam name to id map already, get it //if (!exists) //{ // using (var http = _httpFactory.CreateClient()) // { // // https://api.steampowered.com/ISteamApps/GetAppList/v2/ // var gamesStr = await http.GetStringAsync("https://api.steampowered.com/ISteamApps/GetAppList/v2/").ConfigureAwait(false); // var apps = JsonConvert.DeserializeAnonymousType(gamesStr, new { applist = new { apps = new List() } }).applist.apps; // //await db.HashSetAsync("steam_game_ids", apps.Select(app => new HashEntry(app.Name.Trim().ToLowerInvariant(), app.AppId)).ToArray()).ConfigureAwait(false); // await db.StringSetAsync("steam_game_ids", gamesStr, TimeSpan.FromHours(24)); // //await db.KeyExpireAsync("steam_game_ids", TimeSpan.FromHours(24), CommandFlags.FireAndForget).ConfigureAwait(false); // } //} var gamesMap = await _cache.GetOrAddCachedDataAsync(STEAM_GAME_IDS_KEY, async _ => { using (var http = _httpFactory.CreateClient()) { // https://api.steampowered.com/ISteamApps/GetAppList/v2/ var gamesStr = await http.GetStringAsync("https://api.steampowered.com/ISteamApps/GetAppList/v2/").ConfigureAwait(false); var apps = JsonConvert.DeserializeAnonymousType(gamesStr, new { applist = new { apps = new List() } }).applist.apps; return apps .OrderBy(x => x.Name, StringComparer.OrdinalIgnoreCase) .GroupBy(x => x.Name) .ToDictionary(x => x.Key, x => x.First().AppId); //await db.HashSetAsync("steam_game_ids", apps.Select(app => new HashEntry(app.Name.Trim().ToLowerInvariant(), app.AppId)).ToArray()).ConfigureAwait(false); //await db.StringSetAsync("steam_game_ids", gamesStr, TimeSpan.FromHours(24)); //await db.KeyExpireAsync("steam_game_ids", TimeSpan.FromHours(24), CommandFlags.FireAndForget).ConfigureAwait(false); } }, default(string), TimeSpan.FromHours(24)); if (gamesMap is null) return -1; query = query.Trim(); var keyList = gamesMap.Keys.ToList(); var key = keyList.FirstOrDefault(x => x.Equals(query, StringComparison.OrdinalIgnoreCase)); if (key == default) { key = keyList.FirstOrDefault(x => x.StartsWith(query, StringComparison.OrdinalIgnoreCase)); if (key == default) return -1; } return gamesMap[key]; //// try finding the game id //var val = db.HashGet(STEAM_GAME_IDS_KEY, query); //if (val == default) // return -1; // not found //var appid = (int)val; //return appid; // now that we have appid, get the game info with that appid //var gameData = await _cache.GetOrAddCachedDataAsync($"steam_game:{appid}", SteamGameDataFactory, appid, TimeSpan.FromHours(12)) // .ConfigureAwait(false); //return gameData; } //private async Task SteamGameDataFactory(int appid) //{ // using (var http = _httpFactory.CreateClient()) // { // // https://store.steampowered.com/api/appdetails?appids= // var responseStr = await http.GetStringAsync($"https://store.steampowered.com/api/appdetails?appids={appid}").ConfigureAwait(false); // var data = JsonConvert.DeserializeObject>(responseStr); // if (!data.ContainsKey(appid) || !data[appid].Success) // return null; // for some reason we can't get the game with valid appid. SHould never happen // return data[appid].Data; // } //} public class GoogleSearchResultData { public IReadOnlyList Results { get; } public string FullQueryLink { get; } public string TotalResults { get; } public GoogleSearchResultData(IReadOnlyList results, string fullQueryLink, string totalResults) { Results = results; FullQueryLink = fullQueryLink; TotalResults = totalResults; } } private static readonly HtmlParser _googleParser = new HtmlParser(new HtmlParserOptions() { IsScripting = false, IsEmbedded = false, IsSupportingProcessingInstructions = false, IsKeepingSourceReferences = false, IsNotSupportingFrames = true, }); public async Task GoogleSearchAsync(string query) { query = WebUtility.UrlEncode(query)?.Replace(' ', '+'); var fullQueryLink = $"https://www.google.ca/search?q={ query }&safe=on&lr=lang_eng&hl=en&ie=utf-8&oe=utf-8"; using var msg = new HttpRequestMessage(HttpMethod.Get, fullQueryLink); msg.Headers.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); msg.Headers.Add("Cookie", "CONSENT=YES+shp.gws-20210601-0-RC2.en+FX+423;"); using var http = _httpFactory.CreateClient(); http.DefaultRequestHeaders.Clear(); using var response = await http.SendAsync(msg); var content = await response.Content.ReadAsStreamAsync(); using var document = await _googleParser.ParseDocumentAsync(content); var elems = document.QuerySelectorAll("div.g > div > div"); var resultsElem = document.QuerySelectorAll("#resultStats").FirstOrDefault(); var totalResults = resultsElem?.TextContent; //var time = resultsElem.Children.FirstOrDefault()?.TextContent //^ this doesn't work for some reason, is completely missing in parsed collection if (!elems.Any()) return default; var results = elems.Select(elem => { var children = elem.Children.ToList(); if (children.Count < 2) return null; var href = (children[0].QuerySelector("a") as IHtmlAnchorElement)?.Href; var name = children[0].QuerySelector("h3")?.TextContent; if (href is null || name is null) return null; var txt = children[1].TextContent; if (string.IsNullOrWhiteSpace(txt)) return null; return new GoogleSearchResult(name, href, txt); }) .Where(x => x != null) .ToList(); return new GoogleSearchResultData( results.AsReadOnly(), fullQueryLink, totalResults); } public async Task DuckDuckGoSearchAsync(string query) { query = WebUtility.UrlEncode(query)?.Replace(' ', '+'); var fullQueryLink = $"https://html.duckduckgo.com/html"; using var http = _httpFactory.CreateClient(); http.DefaultRequestHeaders.Clear(); http.DefaultRequestHeaders.Add("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.77 Safari/537.36"); using var formData = new MultipartFormDataContent(); formData.Add(new StringContent(query), "q"); using var response = await http.PostAsync(fullQueryLink, formData); var content = await response.Content.ReadAsStringAsync(); using var document = await _googleParser.ParseDocumentAsync(content); var searchResults = document.QuerySelector(".results"); var elems = searchResults.QuerySelectorAll(".result"); if (!elems.Any()) return default; var results = elems.Select(elem => { var anchor = elem.QuerySelector(".result__a") as IHtmlAnchorElement; if (anchor is null) return null; var href = anchor.Href; var name = anchor.TextContent; if (string.IsNullOrWhiteSpace(href) || string.IsNullOrWhiteSpace(name)) return null; var txt = elem.QuerySelector(".result__snippet")?.TextContent; if (string.IsNullOrWhiteSpace(txt)) return null; return new GoogleSearchResult(name, href, txt); }) .Where(x => x != null) .ToList(); return new GoogleSearchResultData( results.AsReadOnly(), fullQueryLink, "0"); } } }