- Bot now takes shard id (optional) and total shards (optional) command line arguments (changed from shard id and parent process id)

- Sharding with coordinator now works properly
- Documented creds.yml RestartCommand - it has no effect when coordinator is starting the bot
- Regenerated creds_example.yml
- Removed all db migrators as the v3 requires the user to have updated 2.x all the way
- TotalShards in creds.yml gets overriden by coord.yml's TotalShards if the bot is ran through coordinator (more precisely, by the command line argument to the bot)
- Coordinator now runs on http://localhost:3442 by default, you can change this in appsettings.json
    - This is done because of macos https issues
    - Primarily because https for regular users is a massive hassle. Coordinator shouldn't be exposed anyway
- Minor cleanup
This commit is contained in:
Kwoth
2021-06-25 19:14:34 +02:00
parent a1ef862382
commit 670b0aca96
14 changed files with 119 additions and 398 deletions

View File

@@ -79,7 +79,7 @@ namespace NadekoBot.Coordinator
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
Log.Information("Executing");
// Log.Information("Executing");
bool first = true;
while (!stoppingToken.IsCancellationRequested)
@@ -98,7 +98,7 @@ namespace NadekoBot.Coordinator
if (first)
{
Log.Information("Startup order: {StartupOrder}",string.Join(' ', shardIds));
// Log.Information("Startup order: {StartupOrder}",string.Join(' ', shardIds));
first = false;
}
@@ -192,7 +192,7 @@ namespace NadekoBot.Coordinator
FileName = _config.ShardStartCommand,
Arguments = string.Format(_config.ShardStartArgs,
shardId,
Environment.ProcessId),
_config.TotalShards),
EnvironmentVariables =
{
{"NADEKOBOT_IS_COORDINATED", "1"}

View File

@@ -5,16 +5,5 @@
"Microsoft": "Warning",
"Microsoft.Hosting.Lifetime": "Information"
}
},
"AllowedHosts": "*",
"Kestrel": {
"EndpointDefaults": {
"Protocols": "Http2"
},
"Endpoints": {
"Http": {
"Url": "https://localhost:3443"
}
}
}
}

View File

@@ -13,7 +13,7 @@
},
"Endpoints": {
"Http": {
"Url": "http://localhost:3443"
"Url": "http://localhost:3442"
}
}
}

View File

@@ -1,5 +1,5 @@
TotalShards: 1
TotalShards: 3
RecheckIntervalMs: 5000
ShardStartCommand: dotnet
ShardStartArgs: run -p "..\NadekoBot\NadekoBot.csproj" --no-build -- {0}
ShardStartArgs: run -p "..\NadekoBot\NadekoBot.csproj" --no-build -- {0} {1}
UnresponsiveSec: 30

View File

@@ -52,12 +52,12 @@ namespace NadekoBot
public event Func<GuildConfig, Task> JoinedGuild = delegate { return Task.CompletedTask; };
private readonly BotCredsProvider _credsProvider;
public Bot(int shardId)
public Bot(int shardId, int? totalShards)
{
if (shardId < 0)
throw new ArgumentOutOfRangeException(nameof(shardId));
_credsProvider = new BotCredsProvider();
_credsProvider = new BotCredsProvider(totalShards);
_creds = _credsProvider.GetCreds();
_db = new DbService(_creds);
@@ -171,7 +171,6 @@ namespace NadekoBot
Log.Information($"All services loaded in {sw.Elapsed.TotalSeconds:F2}s");
}
// todo remove config migrations
private void ApplyConfigMigrations()
{
// execute all migrators
@@ -180,13 +179,6 @@ namespace NadekoBot
{
migrator.EnsureMigrated();
}
// and then drop the bot config table
// var conn = _db.GetDbContext()._context.Database.GetDbConnection();
// using var deleteBotConfig = conn.CreateCommand();
// deleteBotConfig.CommandText = "DROP TABLE IF EXISTS BotConfig;";
// deleteBotConfig.ExecuteNonQuery();
}
// todo isn't there a built in for loading type readers?

View File

@@ -24,6 +24,12 @@ namespace Nadeko.Common
Type = "sqlite",
ConnectionString = "Data Source=data/NadekoBot.db"
};
CoordinatorUrl = "http://localhost:3442";
RestartCommand = new()
{
};
}
[Comment(@"DO NOT CHANGE")]
@@ -36,7 +42,6 @@ namespace Nadeko.Common
**DO NOT ADD PEOPLE YOU DON'T TRUST**")]
public ICollection<ulong> OwnerIds { get; set; }
// todo update total shards on startup
[Comment(@"The number of shards that the bot will running on.
Leave at 1 if you don't know what you're doing.")]
public int TotalShards { get; set; }
@@ -65,8 +70,9 @@ go to https://www.patreon.com/portal -> my clients -> create client")]
[Comment(@"Database options. Don't change if you don't know what you're doing. Leave null for default values")]
public DbOptions Db { get; set; }
public RestartConfig RestartCommand { get; set; }
[Comment(@"Address and port of the coordinator endpoint. Leave empty for default.
Change only if you've changed the coordinator address or port.")]
public string CoordinatorUrl { get; set; }
[YamlIgnore]
public string PatreonCampaignId => Patreon?.CampaignId;
@@ -94,6 +100,19 @@ Used for cryptocurrency related commands.")]
[Comment(@"Api key used for Osu related commands. Obtain this key at https://osu.ppy.sh/p/api")]
public string OsuApiKey { get; set; }
[Comment(@"Command and args which will be used to restart the bot.
Only used if bot is executed directly (NOT through the coordinator)
placeholders:
{0} -> shard id
{1} -> total shards
Linux default
cmd: dotnet
args: ""NadekoBot.dll -- {0}""
Windows default
cmd: NadekoBot.exe
args: {0}")]
public RestartConfig RestartCommand { get; set; }
public class DbOptions
{

View File

@@ -1,70 +0,0 @@
using System;
using System.Data.Common;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Services;
using Serilog;
namespace NadekoBot.Modules.Xp.Services
{
public sealed class XpConfigMigrator : IConfigMigrator
{
private readonly DbService _db;
private readonly XpConfigService _gss;
public XpConfigMigrator(DbService dbService, XpConfigService gss)
{
_db = dbService;
_gss = gss;
}
public void EnsureMigrated()
{
using var uow = _db.GetDbContext();
using var conn = uow.Database.GetDbConnection();
Migrate(conn);
}
private void Migrate(DbConnection conn)
{
using (var checkTableCommand = conn.CreateCommand())
{
// make sure table still exists
checkTableCommand.CommandText =
"SELECT name FROM sqlite_master WHERE type='table' AND name='BotConfig';";
var checkReader = checkTableCommand.ExecuteReader();
if (!checkReader.HasRows)
return;
}
using (var checkMigratedCommand = conn.CreateCommand())
{
checkMigratedCommand.CommandText =
"UPDATE BotConfig SET HasMigratedXpSettings = 1 WHERE HasMigratedXpSettings = 0;";
var changedRows = checkMigratedCommand.ExecuteNonQuery();
if (changedRows == 0)
return;
}
Log.Information("Migrating Xp settings...");
using var com = conn.CreateCommand();
com.CommandText = $@"SELECT XpPerMessage, XpMinutesTimeout, VoiceXpPerMinute, MaxXpMinutes
FROM BotConfig";
using var reader = com.ExecuteReader();
if (!reader.Read())
return;
_gss.ModifyConfig(ModifyAction(reader));
}
private static Action<XpConfig> ModifyAction(DbDataReader reader)
=> config =>
{
config.XpPerMessage = (int) (long) reader["XpPerMessage"];
config.MessageXpCooldown = (int) (long) reader["XpMinutesTimeout"];
config.VoiceMaxMinutes = (int) (long) reader["MaxXpMinutes"];
config.VoiceXpPerMinute = (double) reader["VoiceXpPerMinute"];
};
}
}

View File

@@ -1,14 +1,35 @@
using NadekoBot;
using System;
using NadekoBot;
using NadekoBot.Services;
using Serilog;
var pid = System.Environment.ProcessId;
var shardId = 0;
if (args.Length == 1)
int.TryParse(args[0], out shardId);
int? totalShards = null; // 0 to read from creds.yml
if (args.Length > 0)
{
if (!int.TryParse(args[0], out shardId))
{
Console.Error.WriteLine("Invalid first argument (shard id)");
return;
}
if (args.Length > 1)
{
if (!int.TryParse(args[1], out var shardCount))
{
Console.Error.WriteLine("Invalid second argument (total shards)");
return;
}
totalShards = shardCount;
}
}
LogSetup.SetupLogger(shardId);
Log.Information($"Pid: {pid}");
await new Bot(shardId).RunAndBlockAsync();
await new Bot(shardId, totalShards).RunAndBlockAsync();

View File

@@ -27,6 +27,7 @@ namespace NadekoBot.Services
string LocationIqApiKey { get; }
string TimezoneDbApiKey { get; }
string CoinmarketcapApiKey { get; }
string CoordinatorUrl { get; set; }
}
// todo move somewhere else
@@ -38,13 +39,7 @@ namespace NadekoBot.Services
public class RestartConfig
{
public RestartConfig(string cmd, string args)
{
this.Cmd = cmd;
this.Args = args;
}
public string Cmd { get; }
public string Args { get; }
public string Cmd { get; set; }
public string Args { get; set; }
}
}

View File

@@ -1,7 +1,9 @@
using Microsoft.Extensions.Configuration;
using System;
using Microsoft.Extensions.Configuration;
using System.IO;
using Microsoft.Extensions.Primitives;
using Nadeko.Common;
using NadekoBot.Common;
using NadekoBot.Common.Yml;
using Serilog;
@@ -10,6 +12,7 @@ namespace NadekoBot.Services
// todo check why is memory usage so unstable
public class BotCredsProvider
{
private readonly int? _totalShards;
private const string _credsFileName = "creds.yml";
private string CredsPath => Path.Combine(Directory.GetCurrentDirectory(), _credsFileName);
private const string _credsExampleFileName = "creds_example.yml";
@@ -28,12 +31,42 @@ namespace NadekoBot.Services
_creds.OwnerIds.Clear();
_config.Bind(_creds);
// todo load defaults for restart command, redis, and some others maybe?
if (string.IsNullOrWhiteSpace(_creds.Token))
{
Log.Error("Token is missing from credentials.json or Environment variables.\n" +
"Add it and restart the program.");
Helpers.ReadErrorAndExit(5);
return;
}
if (string.IsNullOrWhiteSpace(_creds.RestartCommand?.Cmd)
|| string.IsNullOrWhiteSpace(_creds.RestartCommand?.Args))
{
if (Environment.OSVersion.Platform == PlatformID.Unix)
{
_creds.RestartCommand = new RestartConfig()
{
Args = "dotnet",
Cmd = "NadekoBot.dll -- {0}",
};
}
else
{
_creds.RestartCommand = new RestartConfig()
{
Args = "NadekoBot.exe",
Cmd = "{0}",
};
}
}
_creds.TotalShards = _totalShards ?? _creds.TotalShards;
}
}
public BotCredsProvider()
public BotCredsProvider(int? totalShards = null)
{
_totalShards = totalShards;
if (!File.Exists(CredsExamplePath))
{
File.WriteAllText(CredsExamplePath, Yaml.Serializer.Serialize(_creds));

View File

@@ -20,8 +20,11 @@ namespace NadekoBot.Services
public RemoteGrpcCoordinator(IBotCredentials creds, DiscordSocketClient client)
{
// todo should use credentials
var channel = Grpc.Net.Client.GrpcChannel.ForAddress("https://localhost:3443");
var coordUrl = string.IsNullOrWhiteSpace(creds.CoordinatorUrl)
? "http://localhost:3442"
: creds.CoordinatorUrl;
var channel = Grpc.Net.Client.GrpcChannel.ForAddress(coordUrl);
_coordClient = new(channel);
_client = client;
}
@@ -29,8 +32,8 @@ namespace NadekoBot.Services
public bool RestartBot()
{
_coordClient.RestartAllShards(new RestartAllRequest
{
{
});
return true;

View File

@@ -1,122 +0,0 @@
using System.Collections.Generic;
using System.Data.Common;
using System.Globalization;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Common.Configs;
using Serilog;
using SixLabors.ImageSharp.PixelFormats;
namespace NadekoBot.Services
{
public sealed class BotConfigMigrator : IConfigMigrator
{
private readonly DbService _db;
private readonly BotConfigService _bss;
public BotConfigMigrator(DbService dbService, BotConfigService bss)
{
_db = dbService;
_bss = bss;
}
public void EnsureMigrated()
{
using var uow = _db.GetDbContext();
using var conn = uow.Database.GetDbConnection();
// check if bot config exists
using (var checkTableCommand = conn.CreateCommand())
{
// make sure table still exists
checkTableCommand.CommandText =
"SELECT name FROM sqlite_master WHERE type='table' AND name='BotConfig';";
var checkReader = checkTableCommand.ExecuteReader();
if (!checkReader.HasRows)
return;
}
MigrateBotConfig(conn);
using var dropBlockedTable = conn.CreateCommand();
dropBlockedTable.CommandText = "DROP TABLE IF EXISTS BlockedCmdOrMdl;";
dropBlockedTable.ExecuteNonQuery();
}
private void MigrateBotConfig(DbConnection conn)
{
using (var checkMigratedCommand = conn.CreateCommand())
{
checkMigratedCommand.CommandText =
"UPDATE BotConfig SET HasMigratedBotSettings = 1 WHERE HasMigratedBotSettings = 0;";
var changedRows = checkMigratedCommand.ExecuteNonQuery();
if (changedRows == 0)
return;
}
Log.Information("Migrating bot settings...");
var blockedCommands = new HashSet<string>();
using (var cmdCom = conn.CreateCommand())
{
cmdCom.CommandText = $"SELECT Name from BlockedCmdOrMdl WHERE BotConfigId is not NULL";
var cmdReader = cmdCom.ExecuteReader();
while (cmdReader.Read())
blockedCommands.Add(cmdReader.GetString(0));
}
var blockedModules = new HashSet<string>();
using (var mdlCom = conn.CreateCommand())
{
mdlCom.CommandText = $"SELECT Name from BlockedCmdOrMdl WHERE BotConfigId is NULL";
var mdlReader = mdlCom.ExecuteReader();
while (mdlReader.Read())
blockedModules.Add(mdlReader.GetString(0));
}
using var com = conn.CreateCommand();
com.CommandText = $@"SELECT DefaultPrefix, ForwardMessages, ForwardToAllOwners,
OkColor, ErrorColor, ConsoleOutputType, DMHelpString, HelpString, RotatingStatuses, Locale, GroupGreets
FROM BotConfig";
using var reader = com.ExecuteReader();
if (!reader.Read())
return;
_bss.ModifyConfig((x) =>
{
x.Prefix = reader.GetString(0);
x.ForwardMessages = reader.GetBoolean(1);
x.ForwardToAllOwners = reader.GetBoolean(2);
x.Color = new ColorConfig()
{
Ok = Rgba32.TryParseHex(reader.GetString(3), out var okColor)
? okColor
: Rgba32.ParseHex("00e584"),
Error = Rgba32.TryParseHex(reader.GetString(4), out var errorColor)
? errorColor
: Rgba32.ParseHex("ee281f"),
};
x.ConsoleOutputType = (ConsoleOutputType) reader.GetInt32(5);
x.DmHelpText = reader.IsDBNull(6) ? string.Empty : reader.GetString(6);
x.HelpText = reader.IsDBNull(7) ? string.Empty : reader.GetString(7);
x.RotateStatuses = reader.GetBoolean(8);
try
{
x.DefaultLocale = new CultureInfo(reader.GetString(9));
}
catch
{
x.DefaultLocale = new CultureInfo("en-US");
}
x.GroupGreets = reader.GetBoolean(10);
x.Blocked.Commands = blockedCommands;
x.Blocked.Modules = blockedModules;
});
Log.Information("Data written to data/bot.yml");
}
}
}

View File

@@ -1,155 +0,0 @@
using System;
using System.Data.Common;
using Microsoft.EntityFrameworkCore;
using NadekoBot.Modules.Gambling.Common;
using NadekoBot.Modules.Gambling.Services;
using Serilog;
namespace NadekoBot.Services
{
public sealed class GamblingConfigMigrator : IConfigMigrator
{
private readonly DbService _db;
private readonly GamblingConfigService _gss;
public GamblingConfigMigrator(DbService dbService, GamblingConfigService gss)
{
_db = dbService;
_gss = gss;
}
public void EnsureMigrated()
{
using var uow = _db.GetDbContext();
using var conn = uow.Database.GetDbConnection();
Migrate(conn);
}
private void Migrate(DbConnection conn)
{
using (var checkTableCommand = conn.CreateCommand())
{
// make sure table still exists
checkTableCommand.CommandText =
"SELECT name FROM sqlite_master WHERE type='table' AND name='BotConfig';";
var checkReader = checkTableCommand.ExecuteReader();
if (!checkReader.HasRows)
return;
}
using (var checkMigratedCommand = conn.CreateCommand())
{
checkMigratedCommand.CommandText =
"UPDATE BotConfig SET HasMigratedGamblingSettings = 1 WHERE HasMigratedGamblingSettings = 0;";
var changedRows = checkMigratedCommand.ExecuteNonQuery();
if (changedRows == 0)
return;
}
Log.Information("Migrating gambling settings...");
using var com = conn.CreateCommand();
com.CommandText = $@"SELECT CurrencyGenerationChance, CurrencyGenerationCooldown,
CurrencySign, CurrencyName, CurrencyGenerationPassword, MinBet, MaxBet, BetflipMultiplier,
TimelyCurrency, TimelyCurrencyPeriod, CurrencyDropAmount, CurrencyDropAmountMax, DailyCurrencyDecay,
DivorcePriceMultiplier, PatreonCurrencyPerCent, MinWaifuPrice, WaifuGiftMultiplier
FROM BotConfig";
using var reader = com.ExecuteReader();
if (!reader.Read())
return;
using (var itemsCommand = conn.CreateCommand())
{
itemsCommand.CommandText = WaifuItemUpdateQuery;
itemsCommand.ExecuteNonQuery();
}
_gss.ModifyConfig(ModifyAction(reader));
Log.Information("Data written to data/gambling.yml");
}
private static Action<GamblingConfig> ModifyAction(DbDataReader reader)
=> realConfig =>
{
realConfig.Currency.Sign = (string) reader["CurrencySign"];
realConfig.Currency.Name = (string) reader["CurrencyName"];
realConfig.MinBet = (int) (long) reader["MinBet"];
realConfig.MaxBet = (int) (long) reader["MaxBet"];
realConfig.BetFlip = new GamblingConfig.BetFlipConfig()
{
Multiplier = (decimal) (double) reader["BetflipMultiplier"],
};
realConfig.Generation = new GamblingConfig.GenerationConfig()
{
MaxAmount = (int) (reader["CurrencyDropAmountMax"] as long? ?? (long) reader["CurrencyDropAmount"]),
MinAmount = (int) (long) reader["CurrencyDropAmount"],
Chance = (decimal) (double) reader["CurrencyGenerationChance"],
GenCooldown = (int) (long) reader["CurrencyGenerationCooldown"],
HasPassword = reader.GetBoolean(4),
};
realConfig.Timely = new GamblingConfig.TimelyConfig()
{
Amount = (int) (long) reader["TimelyCurrency"],
Cooldown = (int) (long) reader["TimelyCurrencyPeriod"],
};
realConfig.Decay = new GamblingConfig.DecayConfig()
{Percent = (decimal) (double) reader["DailyCurrencyDecay"],};
realConfig.Waifu = new GamblingConfig.WaifuConfig()
{
MinPrice = (int) (long) reader["MinWaifuPrice"],
Multipliers = new GamblingConfig.WaifuConfig.MultipliersData()
{
AllGiftPrices = (decimal) (long) reader["WaifuGiftMultiplier"],
WaifuReset = (int) (long) reader["DivorcePriceMultiplier"]
}
};
realConfig.PatreonCurrencyPerCent = (decimal) (double) reader["PatreonCurrencyPerCent"];
};
private const string WaifuItemUpdateQuery = @"UPDATE WaifuItem
SET Name = CASE ItemEmoji
WHEN '🥔' THEN 'potato'
WHEN '🍪' THEN 'cookie'
WHEN '🥖' THEN 'bread'
WHEN '🍭' THEN 'lollipop'
WHEN '🌹' THEN 'rose'
WHEN '🍺' THEN 'beer'
WHEN '🌮' THEN 'taco'
WHEN '💌' THEN 'loveletter'
WHEN '🥛' THEN 'milk'
WHEN '🍕' THEN 'pizza'
WHEN '🍫' THEN 'chocolate'
WHEN '🍦' THEN 'icecream'
WHEN '🍣' THEN 'sushi'
WHEN '🍚' THEN 'rice'
WHEN '🍉' THEN 'watermelon'
WHEN '🍱' THEN 'bento'
WHEN '🎟' THEN 'movieticket'
WHEN '🍰' THEN 'cake'
WHEN '📔' THEN 'book'
WHEN '🐱' THEN 'cat'
WHEN '🐶' THEN 'dog'
WHEN '🐼' THEN 'panda'
WHEN '💄' THEN 'lipstick'
WHEN '👛' THEN 'purse'
WHEN '📱' THEN 'iphone'
WHEN '👗' THEN 'dress'
WHEN '💻' THEN 'laptop'
WHEN '🎻' THEN 'violin'
WHEN '🎹' THEN 'piano'
WHEN '🚗' THEN 'car'
WHEN '💍' THEN 'ring'
WHEN '🛳' THEN 'ship'
WHEN '🏠' THEN 'house'
WHEN '🚁' THEN 'helicopter'
WHEN '🚀' THEN 'spaceship'
WHEN '🌕' THEN 'moon'
ELSE 'unknown'
END
";
}
}

View File

@@ -4,7 +4,7 @@ version: 1
token: ''
# List of Ids of the users who have bot owner permissions
# **DO NOT ADD PEOPLE YOU DON'T TRUST**
ownerIds:
ownerIds: []
# The number of shards that the bot will running on.
# Leave at 1 if you don't know what you're doing.
totalShards: 1
@@ -39,7 +39,9 @@ db:
type: sqlite
# Connection string. Will default to "Data Source=data/NadekoBot.db"
connectionString: Data Source=data/NadekoBot.db
restartCommand:
# Address and port of the coordinator endpoint. Leave empty for default.
# Change only if you've changed the coordinator address or port.
coordinatorUrl: http://localhost:3442
votesUrl:
votesToken:
# Api key obtained on https://rapidapi.com (go to MyApps -> Add New App -> Enter Name -> Application key)
@@ -55,3 +57,17 @@ timezoneDbApiKey:
coinmarketcapApiKey:
# Api key used for Osu related commands. Obtain this key at https://osu.ppy.sh/p/api
osuApiKey:
# Command and args which will be used to restart the bot.
# Only used if bot is executed directly (NOT through the coordinator)
# placeholders:
# {0} -> shard id
# {1} -> total shards
# Linux default
# cmd: dotnet
# args: "NadekoBot.dll -- {0}"
# Windows default
# cmd: NadekoBot.exe
# args: {0}
restartCommand:
cmd:
args: