liz/PluralKit
liz/PluralKit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: move scheduled tasks to separate project

Browse Source
This commit is contained in:
spiral 2021-10-15 06:27:38 -04:00
parent 362770eef0
commit 3bc451eb4b
No known key found for this signature in database
GPG Key ID: A6059F0CA0E1BD31
13 changed files with 1560 additions and 42 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
Dockerfile
View File

@ -7,6 +7,7 @@ COPY Myriad/Myriad.csproj /app/Myriad/
COPY PluralKit.API/PluralKit.API.csproj /app/PluralKit.API/
COPY PluralKit.Bot/PluralKit.Bot.csproj /app/PluralKit.Bot/
COPY PluralKit.Core/PluralKit.Core.csproj /app/PluralKit.Core/
COPY PluralKit.ScheduledTasks/PluralKit.ScheduledTasks.csproj /app/PluralKit.ScheduledTasks/
COPY PluralKit.Tests/PluralKit.Tests.csproj /app/PluralKit.Tests/
COPY .git/ /app/.git
RUN dotnet restore PluralKit.sln

5
PluralKit.Bot/Bot.cs
View File

@ -260,7 +260,7 @@ namespace PluralKit.Bot
{
// i'm just going to disable this for now we need to find something nicer
// await _errorMessageService.SendErrorMessage(reportChannel.Value,
// sentryEvent.EventId.ToString());
// sentryEvent.EventId.ToString());
}
}
}
@ -272,9 +272,6 @@ namespace PluralKit.Bot
await UpdateBotStatus();
// Clean up message cache in postgres
await _commandMessageService.CleanupOldMessages();
// Collect some stats, submit them to the metrics backend
await _collector.CollectStats();
await Task.WhenAll(((IMetricsRoot)_metrics).ReportRunner.RunAllAsync());

12
PluralKit.Bot/Commands/Misc.cs
View File

@ -85,11 +85,7 @@ namespace PluralKit.Bot
var messagesProxied = _metrics.Snapshot.GetForContext("Bot").Meters.FirstOrDefault(m => m.MultidimensionalName == BotMetrics.MessagesProxied.Name)?.Value;
var commandsRun = _metrics.Snapshot.GetForContext("Bot").Meters.FirstOrDefault(m => m.MultidimensionalName == BotMetrics.CommandsRun.Name)?.Value;
var totalSystems = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.SystemCount.Name)?.Value ?? 0;
var totalMembers = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.MemberCount.Name)?.Value ?? 0;
var totalGroups = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.GroupCount.Name)?.Value ?? 0;
var totalSwitches = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.SwitchCount.Name)?.Value ?? 0;
var totalMessages = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.MessageCount.Name)?.Value ?? 0;
var counts = await _repo.GetStats();
var shardId = ctx.Shard.ShardId;
var shardTotal = ctx.Cluster.Shards.Count;
@ -113,7 +109,11 @@ namespace PluralKit.Bot
.Field(new("CPU usage", $"{_cpu.LastCpuMeasure:P1}", true))
.Field(new("Memory usage", $"{memoryUsage / 1024 / 1024} MiB", true))
.Field(new("Latency", $"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo.ShardLatency.Milliseconds} ms", true))
.Field(new("Total numbers", $"{totalSystems:N0} systems, {totalMembers:N0} members, {totalGroups:N0} groups, {totalSwitches:N0} switches, {totalMessages:N0} messages"))
.Field(new("Total numbers", $"{counts.SystemCount:N0} systems,"
+ $" {counts.MemberCount:N0} members,"
+ $" {counts.GroupCount:N0} groups,"
+ $" {counts.SwitchCount:N0} switches,"
+ $" {counts.MessageCount:N0} messages"))
.Timestamp(Process.GetCurrentProcess().StartTime.ToString("O"))
.Footer(new($"PluralKit {BuildInfoService.Version} • https://github.com/xSke/PluralKit • Last restarted: ")); ;
await ctx.Rest.EditMessage(msg.ChannelId, msg.Id,

13
PluralKit.Bot/Services/CommandMessageService.cs
View File

@ -10,8 +10,6 @@ namespace PluralKit.Bot
{
public class CommandMessageService
{
private static readonly Duration CommandMessageRetention = Duration.FromHours(24);
private readonly IDatabase _db;
private readonly ModelRepository _repo;
private readonly IClock _clock;
@ -35,16 +33,5 @@ namespace PluralKit.Bot
{
return await _repo.GetCommandMessage(messageId);
}
public async Task CleanupOldMessages()
{
var deleteThresholdInstant = _clock.GetCurrentInstant() - CommandMessageRetention;
var deleteThresholdSnowflake = DiscordUtils.InstantToSnowflake(deleteThresholdInstant);
var deletedRows = await _repo.DeleteCommandMessagesBefore(deleteThresholdSnowflake);
_logger.Information("Pruned {DeletedRows} command messages older than retention {Retention} (older than {DeleteThresholdInstant} / {DeleteThresholdSnowflake})",
deletedRows, CommandMessageRetention, deleteThresholdInstant, deleteThresholdSnowflake);
}
}
}

30
PluralKit.Bot/Services/PeriodicStatCollector.cs
View File

@ -20,7 +20,7 @@ namespace PluralKit.Bot
private readonly IDiscordCache _cache;
private readonly CpuStatService _cpu;
private readonly IDatabase _db;
private readonly ModelRepository _repo;
private readonly WebhookCacheService _webhookCache;
@ -28,13 +28,13 @@ namespace PluralKit.Bot
private readonly ILogger _logger;
public PeriodicStatCollector(IMetrics metrics, ILogger logger, WebhookCacheService webhookCache, DbConnectionCountHolder countHolder, CpuStatService cpu, IDatabase db, IDiscordCache cache)
public PeriodicStatCollector(IMetrics metrics, ILogger logger, WebhookCacheService webhookCache, DbConnectionCountHolder countHolder, CpuStatService cpu, ModelRepository repo, IDiscordCache cache)
{
_metrics = metrics;
_webhookCache = webhookCache;
_countHolder = countHolder;
_cpu = cpu;
_db = db;
_repo = repo;
_cache = cache;
_logger = logger.ForContext<PeriodicStatCollector>();
}
@ -63,12 +63,15 @@ namespace PluralKit.Bot
_metrics.Measure.Gauge.SetValue(BotMetrics.Channels, channelCount);
// Aggregate DB stats
var counts = await _db.Execute(c => c.QueryFirstAsync<Counts>("select (select count(*) from systems) as systems, (select count(*) from members) as members, (select count(*) from switches) as switches, (select count(*) from messages) as messages, (select count(*) from groups) as groups"));
_metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.Systems);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.Members);
_metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.Switches);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.Messages);
_metrics.Measure.Gauge.SetValue(CoreMetrics.GroupCount, counts.Groups);
// just fetching from database here - actual updating of the data is done in PluralKit.ScheduledTasks
// if you're not running ScheduledTasks and want up-to-date counts, uncomment the following line:
// await _repo.UpdateStats();
var counts = await _repo.GetStats();
_metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.SystemCount);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.MemberCount);
_metrics.Measure.Gauge.SetValue(CoreMetrics.GroupCount, counts.GroupCount);
_metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.SwitchCount);
_metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.MessageCount);
// Process info
var process = Process.GetCurrentProcess();
@ -88,14 +91,5 @@ namespace PluralKit.Bot
stopwatch.Stop();
_logger.Debug("Updated metrics in {Time}", stopwatch.ElapsedDuration());
}
public class Counts
{
public int Systems { get; }
public int Members { get; }
public int Switches { get; }
public int Messages { get; }
public int Groups { get; }
}
}
}

10
PluralKit.Core/Database/Migrations/19.sql Normal file
View File

@ -0,0 +1,10 @@
-- schema version 19: 2021-10-15 --
-- add stats to info table
alter table info add column system_count int;
alter table info add column member_count int;
alter table info add column group_count int;
alter table info add column switch_count int;
alter table info add column message_count int;
update info set schema_version = 18;

33
PluralKit.Core/Database/Repository/ModelRepository.Stats.cs Normal file
View File

@ -0,0 +1,33 @@
using System.Threading;
using System.Threading.Tasks;
using Dapper;
using SqlKata;
namespace PluralKit.Core
{
public partial class ModelRepository
{
public async Task UpdateStats()
{
await _db.Execute(conn => conn.ExecuteAsync("update info set system_count = (select count(*) from systems)"));
await _db.Execute(conn => conn.ExecuteAsync("update info set member_count = (select count(*) from members)"));
await _db.Execute(conn => conn.ExecuteAsync("update info set group_count = (select count(*) from groups)"));
await _db.Execute(conn => conn.ExecuteAsync("update info set switch_count = (select count(*) from switches)"));
await _db.Execute(conn => conn.ExecuteAsync("update info set message_count = (select count(*) from messages)"));
}
public Task<Counts> GetStats()
=> _db.Execute(conn => conn.QuerySingleAsync<Counts>("select * from info"));
public class Counts
{
public int SystemCount { get; }
public int MemberCount { get; }
public int GroupCount { get; }
public int SwitchCount { get; }
public int MessageCount { get; }
}
}
}

2
PluralKit.Core/Database/Utils/DatabaseMigrator.cs
View File

@ -12,7 +12,7 @@ namespace PluralKit.Core
internal class DatabaseMigrator
{
private const string RootPath = "PluralKit.Core.Database"; // "resource path" root for SQL files
private const int TargetSchemaVersion = 18;
private const int TargetSchemaVersion = 19;
private readonly ILogger _logger;
public DatabaseMigrator(ILogger logger)

15
PluralKit.ScheduledTasks/PluralKit.ScheduledTasks.csproj Normal file
View File

@ -0,0 +1,15 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net5.0</TargetFramework>
</PropertyGroup>
<ItemGroup>
<ProjectReference Include="..\PluralKit.Core\PluralKit.Core.csproj" />
</ItemGroup>
<PropertyGroup>
<RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
</PropertyGroup>
</Project>

40
PluralKit.ScheduledTasks/Startup.cs Normal file
View File

@ -0,0 +1,40 @@
using System;
using System.Threading.Tasks;
using Autofac;
using Microsoft.Extensions.Configuration;
using PluralKit.Core;
namespace PluralKit.ScheduledTasks
{
class Startup
{
static async Task Main(string[] args)
{
// Load configuration and run global init stuff
var config = InitUtils.BuildConfiguration(args).Build();
InitUtils.InitStatic();
var services = BuildContainer(config);
services.Resolve<TaskHandler>().Run();
await Task.Delay(-1);
}
private static IContainer BuildContainer(IConfiguration config)
{
var builder = new ContainerBuilder();
builder.RegisterInstance(config);
builder.RegisterModule(new ConfigModule<CoreConfig>());
builder.RegisterModule(new LoggingModule("ScheduledTasks"));
builder.RegisterModule(new MetricsModule());
builder.RegisterModule<DataStoreModule>();
builder.RegisterType<TaskHandler>().AsSelf().SingleInstance();
return builder.Build();
}
}
}

75
PluralKit.ScheduledTasks/TaskHandler.cs Normal file
View File

@ -0,0 +1,75 @@
using System;
using System.Diagnostics;
using System.Threading;
using System.Threading.Tasks;
using NodaTime;
using NodaTime.Extensions;
using Serilog;
using PluralKit.Core;
namespace PluralKit.ScheduledTasks
{
public class TaskHandler
{
private static readonly Duration CommandMessageRetention = Duration.FromHours(24);
private Timer _periodicTask;
private readonly ILogger _logger;
private readonly IDatabase _db;
private readonly ModelRepository _repo;
public TaskHandler(ILogger logger, IDatabase db, ModelRepository repo)
{
_logger = logger;
_db = db;
_repo = repo;
}
public void Run()
{
_logger.Information("Starting scheduled task runner...");
var timeNow = SystemClock.Instance.GetCurrentInstant();
var timeTillNextWholeMinute = TimeSpan.FromMilliseconds(60000 - timeNow.ToUnixTimeMilliseconds() % 60000 + 250);
_periodicTask = new Timer(_ =>
{
var __ = UpdatePeriodic();
}, null, timeTillNextWholeMinute, TimeSpan.FromMinutes(1));
}
private async Task UpdatePeriodic()
{
_logger.Information("Running per-minute scheduled tasks.");
var stopwatch = new Stopwatch();
stopwatch.Start();
_logger.Information("Updating database stats...");
await _repo.UpdateStats();
// Clean up message cache in postgres
await CleanupOldMessages();
stopwatch.Stop();
_logger.Information("Ran scheduled tasks in {Time}", stopwatch.ElapsedDuration());
}
private async Task CleanupOldMessages()
{
var deleteThresholdInstant = SystemClock.Instance.GetCurrentInstant() - CommandMessageRetention;
var deleteThresholdSnowflake = InstantToSnowflake(deleteThresholdInstant);
var deletedRows = await _repo.DeleteCommandMessagesBefore(deleteThresholdSnowflake);
_logger.Information("Pruned {DeletedRows} command messages older than retention {Retention} (older than {DeleteThresholdInstant} / {DeleteThresholdSnowflake})",
deletedRows, CommandMessageRetention, deleteThresholdInstant, deleteThresholdSnowflake);
}
// we don't have access to PluralKit.Bot here, so this needs to be vendored
public static ulong InstantToSnowflake(Instant time) =>
(ulong)(time - Instant.FromUtc(2015, 1, 1, 0, 0, 0)).TotalMilliseconds << 22;
}
}

1359
PluralKit.ScheduledTasks/packages.lock.json Normal file
View File

File diff suppressed because it is too large Load Diff

7
PluralKit.sln
View File

@ -1,5 +1,6 @@

Microsoft Visual Studio Solution File, Format Version 12.00
#
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.Bot", "PluralKit.Bot\PluralKit.Bot.csproj", "{F2C5562D-FD96-4C11-B54E-93737D127959}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.Core", "PluralKit.Core\PluralKit.Core.csproj", "{5DBE037D-179D-4C05-8A28-35E37129C961}"
@ -10,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.Tests", "PluralKi
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Myriad", "Myriad\Myriad.csproj", "{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}"
EndProject
Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.ScheduledTasks", "PluralKit.ScheduledTasks\PluralKit.ScheduledTasks.csproj", "{374A8EB3-655D-4230-982B-459AE3553991}"
EndProject
Global
GlobalSection(SolutionConfigurationPlatforms) = preSolution
Debug|Any CPU = Debug|Any CPU
@ -36,5 +39,9 @@ Global
{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}.Debug|Any CPU.Build.0 = Debug|Any CPU
{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}.Release|Any CPU.ActiveCfg = Release|Any CPU
{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}.Release|Any CPU.Build.0 = Release|Any CPU
{374A8EB3-655D-4230-982B-459AE3553991}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
{374A8EB3-655D-4230-982B-459AE3553991}.Debug|Any CPU.Build.0 = Debug|Any CPU
{374A8EB3-655D-4230-982B-459AE3553991}.Release|Any CPU.ActiveCfg = Release|Any CPU
{374A8EB3-655D-4230-982B-459AE3553991}.Release|Any CPU.Build.0 = Release|Any CPU
EndGlobalSection
EndGlobal