More restructuring

This commit is contained in:
Kwoth
2023-03-26 14:44:25 +02:00
parent 01f70f0a24
commit 308ba36b2e
319 changed files with 681 additions and 218 deletions

View File

@@ -0,0 +1,123 @@
using System.ComponentModel;
using System.Diagnostics;
using System.Text;
using Nadeko.Common;
namespace NadekoBot.Modules.Music.Resolvers;
public sealed class LocalTrackResolver : ILocalTrackResolver
{
private static readonly HashSet<string> _musicExtensions = new[]
{
".MP4", ".MP3", ".FLAC", ".OGG", ".WAV", ".WMA", ".WMV", ".AAC", ".MKV", ".WEBM", ".M4A", ".AA", ".AAX",
".ALAC", ".AIFF", ".MOV", ".FLV", ".OGG", ".M4V"
}.ToHashSet();
public async Task<ITrackInfo?> ResolveByQueryAsync(string query)
{
if (!File.Exists(query))
return null;
var trackDuration = await Ffprobe.GetTrackDurationAsync(query);
return new SimpleTrackInfo(Path.GetFileNameWithoutExtension(query),
$"https://google.com?q={Uri.EscapeDataString(Path.GetFileNameWithoutExtension(query))}",
"https://cdn.discordapp.com/attachments/155726317222887425/261850914783100928/1482522077_music.png",
trackDuration,
MusicPlatform.Local,
$"\"{Path.GetFullPath(query)}\"");
}
public async IAsyncEnumerable<ITrackInfo> ResolveDirectoryAsync(string dirPath)
{
DirectoryInfo dir;
try
{
dir = new(dirPath);
}
catch (Exception ex)
{
Log.Error(ex, "Specified directory {DirectoryPath} could not be opened", dirPath);
yield break;
}
var files = dir.EnumerateFiles()
.Where(x =>
{
if (!x.Attributes.HasFlag(FileAttributes.Hidden | FileAttributes.System)
&& _musicExtensions.Contains(x.Extension.ToUpperInvariant()))
return true;
return false;
})
.ToList();
var firstFile = files.FirstOrDefault()?.FullName;
if (firstFile is null)
yield break;
var firstData = await ResolveByQueryAsync(firstFile);
if (firstData is not null)
yield return firstData;
var fileChunks = files.Skip(1).Chunk(10);
foreach (var chunk in fileChunks)
{
var part = await chunk.Select(x => ResolveByQueryAsync(x.FullName)).WhenAll();
// nullable reference types being annoying
foreach (var p in part)
{
if (p is null)
continue;
yield return p;
}
}
}
}
public static class Ffprobe
{
public static async Task<TimeSpan> GetTrackDurationAsync(string query)
{
query = query.Replace("\"", "");
try
{
using var p = Process.Start(new ProcessStartInfo
{
FileName = "ffprobe",
Arguments =
$"-v error -show_entries format=duration -of default=noprint_wrappers=1:nokey=1 -- \"{query}\"",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
StandardOutputEncoding = Encoding.UTF8,
StandardErrorEncoding = Encoding.UTF8,
CreateNoWindow = true
});
if (p is null)
return TimeSpan.Zero;
var data = await p.StandardOutput.ReadToEndAsync();
if (double.TryParse(data, out var seconds))
return TimeSpan.FromSeconds(seconds);
var errorData = await p.StandardError.ReadToEndAsync();
if (!string.IsNullOrWhiteSpace(errorData))
Log.Warning("Ffprobe warning for file {FileName}: {ErrorMessage}", query, errorData);
return TimeSpan.Zero;
}
catch (Win32Exception)
{
Log.Warning("Ffprobe was likely not installed. Local song durations will show as (?)");
}
catch (Exception ex)
{
Log.Error(ex, "Unknown exception running ffprobe; {ErrorMessage}", ex.Message);
}
return TimeSpan.Zero;
}
}

View File

@@ -0,0 +1,106 @@
#nullable disable
using System.Text.RegularExpressions;
namespace NadekoBot.Modules.Music.Resolvers;
public class RadioResolver : IRadioResolver
{
private readonly Regex _plsRegex = new(@"File1=(?<url>.*?)\n", RegexOptions.Compiled);
private readonly Regex _m3URegex = new(@"(?<url>^[^#].*)", RegexOptions.Compiled | RegexOptions.Multiline);
private readonly Regex _asxRegex = new(@"<ref href=""(?<url>.*?)""", RegexOptions.Compiled);
private readonly Regex _xspfRegex = new(@"<location>(?<url>.*?)</location>", RegexOptions.Compiled);
public async Task<ITrackInfo> ResolveByQueryAsync(string query)
{
if (IsRadioLink(query))
query = await HandleStreamContainers(query);
return new SimpleTrackInfo(query.TrimTo(50),
query,
"https://cdn.discordapp.com/attachments/155726317222887425/261850925063340032/1482522097_radio.png",
TimeSpan.MaxValue,
MusicPlatform.Radio,
query);
}
public static bool IsRadioLink(string query)
=> (query.StartsWith("http", StringComparison.InvariantCulture)
|| query.StartsWith("ww", StringComparison.InvariantCulture))
&& (query.Contains(".pls") || query.Contains(".m3u") || query.Contains(".asx") || query.Contains(".xspf"));
private async Task<string> HandleStreamContainers(string query)
{
string file = null;
try
{
using var http = new HttpClient();
file = await http.GetStringAsync(query);
}
catch
{
return query;
}
if (query.Contains(".pls"))
{
try
{
var m = _plsRegex.Match(file);
var res = m.Groups["url"]?.ToString();
return res?.Trim();
}
catch
{
Log.Warning("Failed reading .pls:\n{PlsFile}", file);
return null;
}
}
if (query.Contains(".m3u"))
{
try
{
var m = _m3URegex.Match(file);
var res = m.Groups["url"]?.ToString();
return res?.Trim();
}
catch
{
Log.Warning("Failed reading .m3u:\n{M3uFile}", file);
return null;
}
}
if (query.Contains(".asx"))
{
try
{
var m = _asxRegex.Match(file);
var res = m.Groups["url"]?.ToString();
return res?.Trim();
}
catch
{
Log.Warning("Failed reading .asx:\n{AsxFile}", file);
return null;
}
}
if (query.Contains(".xspf"))
{
try
{
var m = _xspfRegex.Match(file);
var res = m.Groups["url"]?.ToString();
return res?.Trim();
}
catch
{
Log.Warning("Failed reading .xspf:\n{XspfFile}", file);
return null;
}
}
return query;
}
}

View File

@@ -0,0 +1,85 @@
using Newtonsoft.Json.Linq;
using System.Runtime.CompilerServices;
using System.Text.RegularExpressions;
using Nadeko.Common;
namespace NadekoBot.Modules.Music.Resolvers;
public sealed class SoundcloudResolver : ISoundcloudResolver
{
private readonly SoundCloudApiService _sc;
private readonly ITrackCacher _trackCacher;
private readonly IHttpClientFactory _httpFactory;
public SoundcloudResolver(SoundCloudApiService sc, ITrackCacher trackCacher, IHttpClientFactory httpFactory)
{
_sc = sc;
_trackCacher = trackCacher;
_httpFactory = httpFactory;
}
public bool IsSoundCloudLink(string url)
=> Regex.IsMatch(url, "(.*)(soundcloud.com|snd.sc)(.*)");
public async IAsyncEnumerable<ITrackInfo> ResolvePlaylistAsync(string playlist)
{
playlist = Uri.EscapeDataString(playlist);
using var http = _httpFactory.CreateClient();
var responseString = await http.GetStringAsync($"https://scapi.nadeko.bot/resolve?url={playlist}");
var scvids = JObject.Parse(responseString)["tracks"]?.ToObject<SoundCloudVideo[]>();
if (scvids is null)
yield break;
foreach (var videosChunk in scvids.Where(x => x.Streamable is true).Chunk(5))
{
var cachableTracks = videosChunk.Select(VideoModelToCachedData).ToList();
await cachableTracks.Select(_trackCacher.CacheTrackDataAsync).WhenAll();
foreach (var info in cachableTracks.Select(CachableDataToTrackInfo))
yield return info;
}
}
private ICachableTrackData VideoModelToCachedData(SoundCloudVideo svideo)
=> new CachableTrackData
{
Title = svideo.FullName,
Url = svideo.TrackLink,
Thumbnail = svideo.ArtworkUrl,
TotalDurationMs = svideo.Duration,
Id = svideo.Id.ToString(),
Platform = MusicPlatform.SoundCloud
};
private ITrackInfo CachableDataToTrackInfo(ICachableTrackData trackData)
=> new SimpleTrackInfo(trackData.Title,
trackData.Url,
trackData.Thumbnail,
trackData.Duration,
trackData.Platform,
GetStreamUrl(trackData.Id));
[MethodImpl(MethodImplOptions.AggressiveInlining)]
private string GetStreamUrl(string trackId)
=> $"https://api.soundcloud.com/tracks/{trackId}/stream?client_id=368b0c85751007cd588d869d3ae61ac0";
public async Task<ITrackInfo?> ResolveByQueryAsync(string query)
{
var cached = await _trackCacher.GetCachedDataByQueryAsync(query, MusicPlatform.SoundCloud);
if (cached is not null)
return CachableDataToTrackInfo(cached);
var svideo = !IsSoundCloudLink(query)
? await _sc.GetVideoByQueryAsync(query)
: await _sc.ResolveVideoAsync(query);
if (svideo is null)
return null;
var cachableData = VideoModelToCachedData(svideo);
await _trackCacher.CacheTrackDataByQueryAsync(query, cachableData);
return CachableDataToTrackInfo(cachableData);
}
}

View File

@@ -0,0 +1,56 @@
namespace NadekoBot.Modules.Music;
public sealed class TrackResolveProvider : ITrackResolveProvider
{
private readonly IYoutubeResolver _ytResolver;
private readonly ILocalTrackResolver _localResolver;
private readonly ISoundcloudResolver _soundcloudResolver;
private readonly IRadioResolver _radioResolver;
public TrackResolveProvider(
IYoutubeResolver ytResolver,
ILocalTrackResolver localResolver,
ISoundcloudResolver soundcloudResolver,
IRadioResolver radioResolver)
{
_ytResolver = ytResolver;
_localResolver = localResolver;
_soundcloudResolver = soundcloudResolver;
_radioResolver = radioResolver;
}
public Task<ITrackInfo?> QuerySongAsync(string query, MusicPlatform? forcePlatform)
{
switch (forcePlatform)
{
case MusicPlatform.Radio:
return _radioResolver.ResolveByQueryAsync(query);
case MusicPlatform.Youtube:
return _ytResolver.ResolveByQueryAsync(query);
case MusicPlatform.Local:
return _localResolver.ResolveByQueryAsync(query);
case MusicPlatform.SoundCloud:
return _soundcloudResolver.ResolveByQueryAsync(query);
case null:
var match = _ytResolver.YtVideoIdRegex.Match(query);
if (match.Success)
return _ytResolver.ResolveByIdAsync(match.Groups["id"].Value);
else if (_soundcloudResolver.IsSoundCloudLink(query))
return _soundcloudResolver.ResolveByQueryAsync(query);
else if (Uri.TryCreate(query, UriKind.Absolute, out var uri) && uri.IsFile)
return _localResolver.ResolveByQueryAsync(uri.AbsolutePath);
else if (IsRadioLink(query))
return _radioResolver.ResolveByQueryAsync(query);
else
return _ytResolver.ResolveByQueryAsync(query, false);
default:
Log.Error("Unsupported platform: {MusicPlatform}", forcePlatform);
return Task.FromResult<ITrackInfo?>(null);
}
}
public static bool IsRadioLink(string query)
=> (query.StartsWith("http", StringComparison.InvariantCulture)
|| query.StartsWith("ww", StringComparison.InvariantCulture))
&& (query.Contains(".pls") || query.Contains(".m3u") || query.Contains(".asx") || query.Contains(".xspf"));
}

View File

@@ -0,0 +1,315 @@
using System.Globalization;
using System.Text.RegularExpressions;
using NadekoBot.Modules.Searches;
namespace NadekoBot.Modules.Music;
public sealed class YtdlYoutubeResolver : IYoutubeResolver
{
private static readonly string[] _durationFormats =
{
"ss", "m\\:ss", "mm\\:ss", "h\\:mm\\:ss", "hh\\:mm\\:ss", "hhh\\:mm\\:ss"
};
private static readonly Regex _expiryRegex = new(@"(?:[\?\&]expire\=(?<timestamp>\d+))");
private static readonly Regex _simplePlaylistRegex = new(@"&list=(?<id>[\w\-]{12,})", RegexOptions.Compiled);
public Regex YtVideoIdRegex { get; } =
new(@"(?:youtube\.com\/\S*(?:(?:\/e(?:mbed))?\/|watch\?(?:\S*?&?v\=))|youtu\.be\/)(?<id>[a-zA-Z0-9_-]{6,11})",
RegexOptions.Compiled);
private readonly ITrackCacher _trackCacher;
private readonly YtdlOperation _ytdlPlaylistOperation;
private readonly YtdlOperation _ytdlIdOperation;
private readonly YtdlOperation _ytdlSearchOperation;
private readonly IGoogleApiService _google;
public YtdlYoutubeResolver(ITrackCacher trackCacher, IGoogleApiService google, SearchesConfigService scs)
{
_trackCacher = trackCacher;
_google = google;
_ytdlPlaylistOperation = new("-4 "
+ "--geo-bypass "
+ "--encoding UTF8 "
+ "-f bestaudio "
+ "-e "
+ "--get-url "
+ "--get-id "
+ "--get-thumbnail "
+ "--get-duration "
+ "--no-check-certificate "
+ "-i "
+ "--yes-playlist "
+ "-- \"{0}\"", scs.Data.YtProvider != YoutubeSearcher.Ytdl);
_ytdlIdOperation = new("-4 "
+ "--geo-bypass "
+ "--encoding UTF8 "
+ "-f bestaudio "
+ "-e "
+ "--get-url "
+ "--get-id "
+ "--get-thumbnail "
+ "--get-duration "
+ "--no-check-certificate "
+ "-- \"{0}\"", scs.Data.YtProvider != YoutubeSearcher.Ytdl);
_ytdlSearchOperation = new("-4 "
+ "--geo-bypass "
+ "--encoding UTF8 "
+ "-f bestaudio "
+ "-e "
+ "--get-url "
+ "--get-id "
+ "--get-thumbnail "
+ "--get-duration "
+ "--no-check-certificate "
+ "--default-search "
+ "\"ytsearch:\" -- \"{0}\"", scs.Data.YtProvider != YoutubeSearcher.Ytdl);
}
private YtTrackData ResolveYtdlData(string ytdlOutputString)
{
if (string.IsNullOrWhiteSpace(ytdlOutputString))
return default;
var dataArray = ytdlOutputString.Trim().Split('\n');
if (dataArray.Length < 5)
{
Log.Information("Not enough data received: {YtdlData}", ytdlOutputString);
return default;
}
if (!TimeSpan.TryParseExact(dataArray[4], _durationFormats, CultureInfo.InvariantCulture, out var time))
time = TimeSpan.Zero;
var thumbnail = Uri.IsWellFormedUriString(dataArray[3], UriKind.Absolute) ? dataArray[3].Trim() : string.Empty;
return new(dataArray[0], dataArray[1], thumbnail, dataArray[2], time);
}
private ITrackInfo DataToInfo(in YtTrackData trackData)
=> new RemoteTrackInfo(
trackData.Id,
trackData.Title,
$"https://youtube.com/watch?v={trackData.Id}",
trackData.Thumbnail,
trackData.Duration,
MusicPlatform.Youtube,
CreateCacherFactory(trackData.Id));
private Func<Task<string?>> CreateCacherFactory(string id)
=> () => _trackCacher.GetOrCreateStreamLink(id,
MusicPlatform.Youtube,
async () => await ExtractNewStreamUrlAsync(id));
private static TimeSpan GetExpiry(string streamUrl)
{
var match = _expiryRegex.Match(streamUrl);
if (match.Success && double.TryParse(match.Groups["timestamp"].ToString(), out var timestamp))
{
var realExpiry = timestamp.ToUnixTimestamp() - DateTime.UtcNow;
if (realExpiry > TimeSpan.FromMinutes(60))
return realExpiry.Subtract(TimeSpan.FromMinutes(30));
return realExpiry;
}
return TimeSpan.FromHours(1);
}
private async Task<(string StreamUrl, TimeSpan Expiry)> ExtractNewStreamUrlAsync(string id)
{
var data = await _ytdlIdOperation.GetDataAsync(id);
var trackInfo = ResolveYtdlData(data);
if (string.IsNullOrWhiteSpace(trackInfo.StreamUrl))
return default;
return (trackInfo.StreamUrl!, GetExpiry(trackInfo.StreamUrl!));
}
public async Task<ITrackInfo?> ResolveByIdAsync(string id)
{
id = id.Trim();
var cachedData = await _trackCacher.GetCachedDataByIdAsync(id, MusicPlatform.Youtube);
if (cachedData is null)
{
Log.Information("Resolving youtube track by Id: {YoutubeId}", id);
var data = await _ytdlIdOperation.GetDataAsync(id);
var trackInfo = ResolveYtdlData(data);
if (string.IsNullOrWhiteSpace(trackInfo.Title))
return default;
var toReturn = DataToInfo(in trackInfo);
await Task.WhenAll(_trackCacher.CacheTrackDataAsync(toReturn.ToCachedData(id)),
CacheStreamUrlAsync(trackInfo));
return toReturn;
}
return DataToInfo(new(cachedData.Title, cachedData.Id, cachedData.Thumbnail, null, cachedData.Duration));
}
private Task CacheStreamUrlAsync(YtTrackData trackInfo)
=> _trackCacher.CacheStreamUrlAsync(trackInfo.Id,
MusicPlatform.Youtube,
trackInfo.StreamUrl!,
GetExpiry(trackInfo.StreamUrl!));
public async IAsyncEnumerable<ITrackInfo> ResolveTracksByPlaylistIdAsync(string playlistId)
{
Log.Information("Resolving youtube tracks from playlist: {PlaylistId}", playlistId);
var count = 0;
var ids = await _trackCacher.GetPlaylistTrackIdsAsync(playlistId, MusicPlatform.Youtube);
if (ids.Count > 0)
{
foreach (var id in ids)
{
var trackInfo = await ResolveByIdAsync(id);
if (trackInfo is null)
continue;
yield return trackInfo;
}
yield break;
}
var data = string.Empty;
var trackIds = new List<string>();
await foreach (var line in _ytdlPlaylistOperation.EnumerateDataAsync(playlistId))
{
data += line;
if (++count == 5)
{
var trackData = ResolveYtdlData(data);
data = string.Empty;
count = 0;
if (string.IsNullOrWhiteSpace(trackData.Id))
continue;
var info = DataToInfo(in trackData);
await Task.WhenAll(_trackCacher.CacheTrackDataAsync(info.ToCachedData(trackData.Id)),
CacheStreamUrlAsync(trackData));
trackIds.Add(trackData.Id);
yield return info;
}
else
data += Environment.NewLine;
}
await _trackCacher.CachePlaylistTrackIdsAsync(playlistId, MusicPlatform.Youtube, trackIds);
}
public async IAsyncEnumerable<ITrackInfo> ResolveTracksFromPlaylistAsync(string query)
{
string? playlistId;
// try to match playlist id inside the query, if a playlist url has been queried
var match = _simplePlaylistRegex.Match(query);
if (match.Success)
{
// if it's a success, just return from that playlist using the id
playlistId = match.Groups["id"].ToString();
await foreach (var track in ResolveTracksByPlaylistIdAsync(playlistId))
yield return track;
yield break;
}
// if a query is a search term, try the cache
playlistId = await _trackCacher.GetPlaylistIdByQueryAsync(query, MusicPlatform.Youtube);
if (playlistId is null)
{
// if it's not in the cache
// find playlist id by keyword using google api
try
{
var playlistIds = await _google.GetPlaylistIdsByKeywordsAsync(query);
playlistId = playlistIds.FirstOrDefault();
}
catch (Exception ex)
{
Log.Warning(ex, "Error Getting playlist id via GoogleApi");
}
// if query is not a playlist url
// and query result is not in the cache
// and api returns no values
// it means invalid input has been used,
// or google api key is not provided
if (playlistId is null)
yield break;
}
// cache the query -> playlist id for fast future lookup
await _trackCacher.CachePlaylistIdByQueryAsync(query, MusicPlatform.Youtube, playlistId);
await foreach (var track in ResolveTracksByPlaylistIdAsync(playlistId))
yield return track;
}
public Task<ITrackInfo?> ResolveByQueryAsync(string query)
=> ResolveByQueryAsync(query, true);
public async Task<ITrackInfo?> ResolveByQueryAsync(string query, bool tryResolving)
{
if (tryResolving)
{
var match = YtVideoIdRegex.Match(query);
if (match.Success)
return await ResolveByIdAsync(match.Groups["id"].Value);
}
Log.Information("Resolving youtube song by search term: {YoutubeQuery}", query);
var cachedData = await _trackCacher.GetCachedDataByQueryAsync(query, MusicPlatform.Youtube);
if (cachedData is null || string.IsNullOrWhiteSpace(cachedData.Title))
{
var stringData = await _ytdlSearchOperation.GetDataAsync(query);
var trackData = ResolveYtdlData(stringData);
var trackInfo = DataToInfo(trackData);
await Task.WhenAll(_trackCacher.CacheTrackDataByQueryAsync(query, trackInfo.ToCachedData(trackData.Id)),
CacheStreamUrlAsync(trackData));
return trackInfo;
}
return DataToInfo(new(cachedData.Title, cachedData.Id, cachedData.Thumbnail, null, cachedData.Duration));
}
private readonly struct YtTrackData
{
public readonly string Title;
public readonly string Id;
public readonly string Thumbnail;
public readonly string? StreamUrl;
public readonly TimeSpan Duration;
public YtTrackData(
string title,
string id,
string thumbnail,
string? streamUrl,
TimeSpan duration)
{
Title = title.Trim();
Id = id.Trim();
Thumbnail = thumbnail;
StreamUrl = streamUrl;
Duration = duration;
}
}
}