- Migrated to twitch helix api for stream notifications

- Added 2 new fields to creds.yml -> TwitchClientId and TwitchClientSecret
- Users now have to provide them in order to use twitch stream notifications as Twitch is shutting down old api on 18th of February 2022
- Possibly fixed some issues with streams randomly getting cleared
- Added an incomplete trovo stream notification provider
This commit is contained in:
Kwoth
2022-02-15 17:03:31 +01:00
parent 7ed1b13e85
commit e6e802b563
15 changed files with 340 additions and 330 deletions

View File

@@ -72,9 +72,15 @@ Used for cryptocurrency related commands.")]
public string OsuApiKey { get; set; }
[Comment(@"Optional Trovo client id.
You should only use this if Trovo notifications stopped working or you're getting ratelimit errors.")]
You should use this if Trovo stream notifications stopped working or you're getting ratelimit errors.")]
public string TrovoClientId { get; set; }
[Comment(@"Obtain by creating an application at https://dev.twitch.tv/console/apps")]
public string TwitchClientId { get; set; }
[Comment(@"Obtain by creating an application at https://dev.twitch.tv/console/apps")]
public string TwitchClientSecret { get; set; }
[Comment(@"Command and args which will be used to restart the bot.
Only used if bot is executed directly (NOT through the coordinator)
placeholders:

View File

@@ -22,6 +22,8 @@ public interface IBotCredentials
string CoinmarketcapApiKey { get; }
string TrovoClientId { get; }
string CoordinatorUrl { get; set; }
string TwitchClientId { get; set; }
string TwitchClientSecret { get; set; }
}
public class RestartConfig

View File

@@ -99,7 +99,6 @@ public class UserPunishService : INService, IReadyExecutor
if (p is not null)
{
// todo shouldn't this try to get through api too
var user = await guild.GetUserAsync(userId);
if (user is null)
return null;

View File

@@ -452,6 +452,7 @@ public sealed class MusicService : IMusicService
if (TryGetMusicPlayer(guildId, out var mp))
mp.AutoPlay = newValue;
return newValue;
}

View File

@@ -275,7 +275,6 @@ public sealed class MusicPlayer : IMusicPlayer
_ = OnCompleted?.Invoke(this, track);
// todo update when settings are changed
if (AutoPlay && track.Platform == MusicPlatform.Youtube)
{
try

View File

@@ -16,9 +16,6 @@ public partial class Searches
public StreamNotificationCommands(DbService db)
=> _db = db;
// private static readonly Regex picartoRegex = new Regex(@"picarto.tv/(?<name>.+[^/])/?",
// RegexOptions.Compiled | RegexOptions.IgnoreCase);
[Cmd]
[RequireContext(ContextType.Guild)]
[UserPerm(GuildPerm.ManageMessages)]

View File

@@ -0,0 +1,64 @@
#nullable disable
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches.Common.StreamNotifications.Providers;
public class HelixStreamsResponse
{
public class PaginationData
{
[JsonPropertyName("cursor")]
public string Cursor { get; set; }
}
public class StreamData
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("user_id")]
public string UserId { get; set; }
[JsonPropertyName("user_login")]
public string UserLogin { get; set; }
[JsonPropertyName("user_name")]
public string UserName { get; set; }
[JsonPropertyName("game_id")]
public string GameId { get; set; }
[JsonPropertyName("game_name")]
public string GameName { get; set; }
[JsonPropertyName("type")]
public string Type { get; set; }
[JsonPropertyName("title")]
public string Title { get; set; }
[JsonPropertyName("viewer_count")]
public int ViewerCount { get; set; }
[JsonPropertyName("started_at")]
public DateTime StartedAt { get; set; }
[JsonPropertyName("language")]
public string Language { get; set; }
[JsonPropertyName("thumbnail_url")]
public string ThumbnailUrl { get; set; }
[JsonPropertyName("tag_ids")]
public List<string> TagIds { get; set; }
[JsonPropertyName("is_mature")]
public bool IsMature { get; set; }
}
[JsonPropertyName("data")]
public List<StreamData> Data { get; set; }
[JsonPropertyName("pagination")]
public PaginationData Pagination { get; set; }
}

View File

@@ -0,0 +1,46 @@
#nullable disable
using System.Text.Json.Serialization;
namespace NadekoBot.Modules.Searches.Common.StreamNotifications.Providers;
public class HelixUsersResponse
{
public class User
{
[JsonPropertyName("id")]
public string Id { get; set; }
[JsonPropertyName("login")]
public string Login { get; set; }
[JsonPropertyName("display_name")]
public string DisplayName { get; set; }
[JsonPropertyName("type")]
public string Type { get; set; }
[JsonPropertyName("broadcaster_type")]
public string BroadcasterType { get; set; }
[JsonPropertyName("description")]
public string Description { get; set; }
[JsonPropertyName("profile_image_url")]
public string ProfileImageUrl { get; set; }
[JsonPropertyName("offline_image_url")]
public string OfflineImageUrl { get; set; }
[JsonPropertyName("view_count")]
public int ViewCount { get; set; }
[JsonPropertyName("email")]
public string Email { get; set; }
[JsonPropertyName("created_at")]
public DateTime CreatedAt { get; set; }
}
[JsonPropertyName("data")]
public List<User> Data { get; set; }
}

View File

@@ -3,7 +3,7 @@ using NadekoBot.Db.Models;
namespace NadekoBot.Modules.Searches.Common;
public class StreamData
public record StreamData
{
public FollowedStream.FType StreamType { get; set; }
public string Name { get; set; }

View File

@@ -26,7 +26,7 @@ public class NotifChecker
_key = $"{uniqueCacheKey}_followed_streams_data";
_streamProviders = new()
{
{ FollowedStream.FType.Twitch, new TwitchProvider(httpClientFactory) },
{ FollowedStream.FType.Twitch, new TwitchHelixProvider(httpClientFactory, credsProvider) },
{ FollowedStream.FType.Picarto, new PicartoProvider(httpClientFactory) },
{ FollowedStream.FType.Trovo, new TrovoProvider(httpClientFactory, credsProvider) }
};
@@ -95,14 +95,20 @@ public class NotifChecker
{
// update cached data
var key = newData.CreateKey();
CacheAddData(key, newData, true);
// compare old data with new data
var oldData = oldStreamDataDict[key.Type][key.Name];
// this is the first pass
if (oldData is null)
if (!oldStreamDataDict.TryGetValue(key.Type, out var typeDict)
|| !typeDict.TryGetValue(key.Name, out var oldData)
|| oldData is null)
{
continue;
}
// fill with last known game in case it's empty
if (string.IsNullOrWhiteSpace(newData.Game))
newData.Game = oldData.Game;
CacheAddData(key, newData, true);
// if the stream is offline, we need to check if it was
// marked as offline once previously

View File

@@ -39,11 +39,11 @@ public class PicartoProvider : Provider
return Task.FromResult<StreamData?>(null);
}
public override async Task<StreamData?> GetStreamDataAsync(string id)
public override async Task<StreamData?> GetStreamDataAsync(string login)
{
var data = await GetStreamDataAsync(new List<string>
{
id
login
});
return data.FirstOrDefault();

View File

@@ -42,9 +42,9 @@ public abstract class Provider
/// <summary>
/// Gets stream data of the specified id/username on this <see cref="Platform" />
/// </summary>
/// <param name="id">Name (or id where applicable) of the user on the platform</param>
/// <param name="login">Name (or id where applicable) of the user on the platform</param>
/// <returns><see cref="StreamData" /> of the user. Null if none found</returns>
public abstract Task<StreamData?> GetStreamDataAsync(string id);
public abstract Task<StreamData?> GetStreamDataAsync(string login);
/// <summary>
/// Gets stream data of all specified ids/usernames on this <see cref="Platform" />

View File

@@ -31,7 +31,7 @@ public class TrovoProvider : Provider
return GetStreamDataAsync(match.Groups["channel"].Value);
}
public override async Task<StreamData?> GetStreamDataAsync(string id)
public override async Task<StreamData?> GetStreamDataAsync(string login)
{
using var http = _httpClientFactory.CreateClient();
@@ -53,7 +53,7 @@ public class TrovoProvider : Provider
$"https://open-api.trovo.live/openplatform/channels/id",
new TrovoRequestData()
{
ChannelId = id
ChannelId = login
});
res.EnsureSuccessStatusCode();
@@ -62,8 +62,8 @@ public class TrovoProvider : Provider
if (data is null)
{
Log.Warning("An empty response received while retrieving stream data for trovo.live/{TrovoId}", id);
_failingStreams.TryAdd(id, DateTime.UtcNow);
Log.Warning("An empty response received while retrieving stream data for trovo.live/{TrovoId}", login);
_failingStreams.TryAdd(login, DateTime.UtcNow);
return null;
}
@@ -83,8 +83,8 @@ public class TrovoProvider : Provider
}
catch (Exception ex)
{
Log.Warning(ex, "Error retrieving stream data for trovo.live/{TrovoId}", id);
_failingStreams.TryAdd(id, DateTime.UtcNow);
Log.Warning(ex, "Error retrieving stream data for trovo.live/{TrovoId}", login);
_failingStreams.TryAdd(login, DateTime.UtcNow);
return null;
}
}

View File

@@ -1,167 +1,197 @@
// #nullable disable
// using NadekoBot.Db.Models;
// using System;
// using System.Collections.Generic;
// using System.Linq;
// using System.Net.Http;
// using System.Text.Json;
// using System.Text.Json.Serialization;
// using System.Text.RegularExpressions;
// using System.Threading.Tasks;
// using NadekoBot.Services.Database.Models;
// using NadekoBot.Extensions;
// using Serilog;
// using TwitchLib.Api;
// using JsonSerializer = System.Text.Json.JsonSerializer;
//
// namespace NadekoBot.Modules.Searches.Common.StreamNotifications.Providers
// {
// public sealed class TwitchHelixProvider : Provider
// {
// private readonly IHttpClientFactory _httpClientFactory;
//
// //
// private static Regex Regex { get; } = new Regex(@"twitch.tv/(?<name>.+[^/])/?",
// RegexOptions.Compiled | RegexOptions.IgnoreCase);
//
// public override FollowedStream.FType Platform => FollowedStream.FType.Twitch;
//
// private (string Token, DateTime Expiry) _token = default;
// private readonly TwitchAPI _api;
//
// public TwitchHelixProvider(IHttpClientFactory httpClientFactory)
// {
// _httpClientFactory = httpClientFactory;
// _api = new TwitchAPI();
// }
//
// private async Task EnsureTokenValidAsync()
// => await _api.Auth.GetAccessTokenAsync();
//
// public override Task<bool> IsValidUrl(string url)
// {
// var match = Regex.Match(url);
// if (!match.Success)
// return Task.FromResult(false);
//
// var username = match.Groups["name"].Value;
// return Task.FromResult(true);
// }
//
// public override Task<StreamData?> GetStreamDataByUrlAsync(string url)
// {
// var match = Regex.Match(url);
// if (match.Success)
// {
// var name = match.Groups["name"].Value;
// return GetStreamDataAsync(name);
// }
//
// return Task.FromResult<StreamData?>(null);
// }
//
// public override async Task<StreamData?> GetStreamDataAsync(string id)
// {
// var data = await GetStreamDataAsync(new List<string> {id});
//
// return data.FirstOrDefault();
// }
//
// public override async Task<List<StreamData>> GetStreamDataAsync(List<string> logins)
// {
// if (logins.Count == 0)
// return new List<StreamData>();
//
// await EnsureTokenValidAsync();
//
// using var http = _httpClientFactory.CreateClient();
// http.DefaultRequestHeaders.Clear();
// http.DefaultRequestHeaders.Add("client-id","67w6z9i09xv2uoojdm9l0wsyph4hxo6");
// http.DefaultRequestHeaders.Add("Authorization",$"Bearer {_token.Token}");
//
// var res = new TwitchResponse()
// {
// Data = new List<TwitchResponse.StreamApiData>()
// };
// foreach (var chunk in logins.Chunk(500))
// {
// try
// {
// var str = await http.GetStringAsync($"https://api.twitch.tv/helix/streams" +
// $"?user_login={chunk.JoinWith(',')}" +
// $"&first=100");
//
// res = JsonSerializer.Deserialize<TwitchResponse>(str);
// }
// catch (Exception ex)
// {
// Log.Warning(ex, "Something went wrong retreiving {StreamPlatform} streams", Platform);
// return new List<StreamData>();
// }
//
// if (res.Data.Count == 0)
// {
// return new List<StreamData>();
// }
// }
//
// return res.Data.Select(ToStreamData).ToList();
// }
//
// private StreamData ToStreamData(TwitchResponse.StreamApiData apiData)
// {
// return new StreamData()
// {
// StreamType = FollowedStream.FType.Twitch,
// Name = apiData.UserName,
// UniqueName = apiData.UserId,
// Viewers = apiData.ViewerCount,
// Title = apiData.Title,
// IsLive = apiData.Type == "live",
// Preview = apiData.ThumbnailUrl
// ?.Replace("{width}", "640")
// ?.Replace("{height}", "480"),
// Game = apiData.GameId,
// };
// }
// }
//
// public class TwitchResponse
// {
// [JsonPropertyName("data")]
// public List<StreamApiData> Data { get; set; }
//
// public class StreamApiData
// {
// [JsonPropertyName("id")]
// public string Id { get; set; }
//
// [JsonPropertyName("user_id")]
// public string UserId { get; set; }
//
// [JsonPropertyName("user_name")]
// public string UserName { get; set; }
//
// [JsonPropertyName("game_id")]
// public string GameId { get; set; }
//
// [JsonPropertyName("type")]
// public string Type { get; set; }
//
// [JsonPropertyName("title")]
// public string Title { get; set; }
//
// [JsonPropertyName("viewer_count")]
// public int ViewerCount { get; set; }
//
// [JsonPropertyName("language")]
// public string Language { get; set; }
//
// [JsonPropertyName("thumbnail_url")]
// public string ThumbnailUrl { get; set; }
//
// [JsonPropertyName("started_at")]
// public DateTime StartedAt { get; set; }
// }
// }
// }
using NadekoBot.Db.Models;
using System.Text.RegularExpressions;
using TwitchLib.Api;
using JsonSerializer = System.Text.Json.JsonSerializer;
namespace NadekoBot.Modules.Searches.Common.StreamNotifications.Providers;
public sealed class TwitchHelixProvider : Provider
{
private readonly IHttpClientFactory _httpClientFactory;
private static Regex Regex { get; } = new(@"twitch.tv/(?<name>[\w\d\-_]+)/?",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
public override FollowedStream.FType Platform
=> FollowedStream.FType.Twitch;
private readonly Lazy<TwitchAPI> _api;
private readonly string _clientId;
public TwitchHelixProvider(IHttpClientFactory httpClientFactory, IBotCredsProvider credsProvider)
{
_httpClientFactory = httpClientFactory;
var creds = credsProvider.GetCreds();
_clientId = creds.TwitchClientId;
var clientSecret = creds.TwitchClientSecret;
_api = new(() => new()
{
Helix =
{
Settings =
{
ClientId = _clientId,
Secret = clientSecret
}
}
});
}
private async Task<string?> EnsureTokenValidAsync()
=> await _api.Value.Auth.GetAccessTokenAsync();
public override Task<bool> IsValidUrl(string url)
{
var match = Regex.Match(url);
if (!match.Success)
{
return Task.FromResult(false);
}
return Task.FromResult(true);
}
public override Task<StreamData?> GetStreamDataByUrlAsync(string url)
{
var match = Regex.Match(url);
if (match.Success)
{
var name = match.Groups["name"].Value;
return GetStreamDataAsync(name);
}
return Task.FromResult<StreamData?>(null);
}
public override async Task<StreamData?> GetStreamDataAsync(string login)
{
var data = await GetStreamDataAsync(new List<string>
{
login
});
return data.FirstOrDefault();
}
public override async Task<IReadOnlyCollection<StreamData>> GetStreamDataAsync(List<string> logins)
{
if (logins.Count == 0)
{
return Array.Empty<StreamData>();
}
var token = await EnsureTokenValidAsync();
if (token is null)
{
Log.Warning("Twitch client id and client secret key are not added to creds.yml or incorrect");
return Array.Empty<StreamData>();
}
using var http = _httpClientFactory.CreateClient();
http.DefaultRequestHeaders.Clear();
http.DefaultRequestHeaders.Add("Client-Id", _clientId);
http.DefaultRequestHeaders.Add("Authorization", $"Bearer {token}");
var loginsSet = logins.Select(x => x.ToLowerInvariant())
.Distinct()
.ToHashSet();
var dataDict = new Dictionary<string, StreamData>();
foreach (var chunk in logins.Chunk(100))
{
try
{
var str = await http.GetStringAsync($"https://api.twitch.tv/helix/users"
+ $"?{chunk.Select(x => $"login={x}").Join('&')}"
+ $"&first=100");
var resObj = JsonSerializer.Deserialize<HelixUsersResponse>(str);
if (resObj?.Data is null || resObj.Data.Count == 0)
continue;
foreach (var user in resObj.Data)
{
var lowerLogin = user.Login.ToLowerInvariant();
if (loginsSet.Remove(lowerLogin))
{
dataDict[lowerLogin] = UserToStreamData(user);
}
}
}
catch (Exception ex)
{
Log.Warning(ex, "Something went wrong retreiving {StreamPlatform} streams", Platform);
return new List<StreamData>();
}
}
// any item left over loginsSet is an invalid username
foreach (var login in loginsSet)
{
_failingStreams.TryAdd(login, DateTime.UtcNow);
}
// only get streams for users which exist
foreach (var chunk in dataDict.Keys.Chunk(100))
{
try
{
var str = await http.GetStringAsync($"https://api.twitch.tv/helix/streams"
+ $"?{chunk.Select(x => $"user_login={x}").Join('&')}"
+ "&first=100");
var res = JsonSerializer.Deserialize<HelixStreamsResponse>(str);
if (res?.Data is null || res.Data.Count == 0)
{
continue;
}
foreach (var helixStreamData in res.Data)
{
var login = helixStreamData.UserLogin.ToLowerInvariant();
if (dataDict.TryGetValue(login, out var old))
{
dataDict[login] = FillStreamData(old, helixStreamData);
}
}
}
catch (Exception ex)
{
Log.Warning(ex, "Something went wrong retreiving {StreamPlatform} streams", Platform);
return new List<StreamData>();
}
}
return dataDict.Values;
}
private StreamData UserToStreamData(HelixUsersResponse.User user)
=> new()
{
UniqueName = user.Login,
Name = user.DisplayName,
AvatarUrl = user.ProfileImageUrl,
IsLive = false,
StreamUrl = $"https://twitch.tv/{user.Login}",
StreamType = FollowedStream.FType.Twitch,
Preview = user.OfflineImageUrl
};
private StreamData FillStreamData(StreamData partial, HelixStreamsResponse.StreamData apiData)
=> partial with
{
StreamType = FollowedStream.FType.Twitch,
Viewers = apiData.ViewerCount,
Title = apiData.Title,
IsLive = apiData.Type == "live",
Preview = apiData.ThumbnailUrl
.Replace("{width}", "640")
.Replace("{height}", "480"),
Game = apiData.GameName,
};
}

View File

@@ -1,140 +0,0 @@
using NadekoBot.Db.Models;
using Newtonsoft.Json;
using System.Text.RegularExpressions;
namespace NadekoBot.Modules.Searches.Common.StreamNotifications.Providers;
public class TwitchProvider : Provider
{
private static Regex Regex { get; } = new(@"twitch.tv/(?<name>.+[^/])/?",
RegexOptions.Compiled | RegexOptions.IgnoreCase);
public override FollowedStream.FType Platform
=> FollowedStream.FType.Twitch;
public override IReadOnlyDictionary<string, DateTime> FailingStreams
=> _failingStreams;
private readonly IHttpClientFactory _httpClientFactory;
public TwitchProvider(IHttpClientFactory httpClientFactory)
=> _httpClientFactory = httpClientFactory;
public override void ClearErrorsFor(string login)
=> _failingStreams.TryRemove(login, out _);
public override Task<bool> IsValidUrl(string url)
{
var match = Regex.Match(url);
if (!match.Success)
return Task.FromResult(false);
// var username = match.Groups["name"].Value;
return Task.FromResult(true);
}
public override Task<StreamData?> GetStreamDataByUrlAsync(string url)
{
var match = Regex.Match(url);
if (match.Success)
{
var name = match.Groups["name"].Value;
return GetStreamDataAsync(name);
}
return Task.FromResult<StreamData?>(null);
}
public override async Task<StreamData?> GetStreamDataAsync(string id)
{
var data = await GetStreamDataAsync(new List<string>
{
id
});
return data.FirstOrDefault();
}
public override async Task<IReadOnlyCollection<StreamData>> GetStreamDataAsync(List<string> logins)
{
if (logins.Count == 0)
return new List<StreamData>();
using var http = _httpClientFactory.CreateClient();
http.DefaultRequestHeaders.Add("Client-Id", "67w6z9i09xv2uoojdm9l0wsyph4hxo6");
http.DefaultRequestHeaders.Add("Accept", "application/vnd.twitchtv.v5+json");
var toReturn = new List<StreamData>();
foreach (var login in logins)
{
try
{
// get id based on the username
var idsStr = await http.GetStringAsync($"https://api.twitch.tv/kraken/users?login={login}");
var userData = JsonConvert.DeserializeObject<TwitchUsersResponseV5>(idsStr);
var user = userData?.Users.FirstOrDefault();
// if user can't be found, skip, it means there is no such user
if (user is null)
continue;
// get stream data
var str = await http.GetStringAsync($"https://api.twitch.tv/kraken/streams/{user.Id}");
var resObj = JsonConvert.DeserializeAnonymousType(str,
new
{
Stream = new TwitchResponseV5.Stream()
});
// if stream is null, user is not streaming
if (resObj?.Stream is null)
{
// if user is not streaming, get his offline banner
var chStr = await http.GetStringAsync($"https://api.twitch.tv/kraken/channels/{user.Id}");
var ch = JsonConvert.DeserializeObject<TwitchResponseV5.Channel>(chStr)!;
toReturn.Add(new()
{
StreamType = FollowedStream.FType.Twitch,
Name = ch.DisplayName,
UniqueName = ch.Name,
Title = ch.Status,
IsLive = false,
AvatarUrl = ch.Logo,
StreamUrl = $"https://twitch.tv/{ch.Name}",
Preview = ch.VideoBanner // set video banner as the preview,
});
continue; // move on
}
toReturn.Add(ToStreamData(resObj.Stream));
_failingStreams.TryRemove(login, out _);
}
catch (Exception ex)
{
Log.Warning("Something went wrong retreiving {StreamPlatform} stream data for {Login}: {ErrorMessage}",
Platform,
login,
ex.Message);
_failingStreams.TryAdd(login, DateTime.UtcNow);
}
}
return toReturn;
}
private StreamData ToStreamData(TwitchResponseV5.Stream stream)
=> new()
{
StreamType = FollowedStream.FType.Twitch,
Name = stream.Channel.DisplayName,
UniqueName = stream.Channel.Name,
Viewers = stream.Viewers,
Title = stream.Channel.Status,
IsLive = true,
Preview = stream.Preview.Large,
Game = stream.Channel.Game,
StreamUrl = $"https://twitch.tv/{stream.Channel.Name}",
AvatarUrl = stream.Channel.Logo
};
}