Merge branch 'main' into feat/apiv2

Dockerfile

@@ -7,6 +7,7 @@ COPY Myriad/Myriad.csproj /app/Myriad/
 COPY PluralKit.API/PluralKit.API.csproj /app/PluralKit.API/
 COPY PluralKit.Bot/PluralKit.Bot.csproj /app/PluralKit.Bot/
 COPY PluralKit.Core/PluralKit.Core.csproj /app/PluralKit.Core/
+COPY PluralKit.ScheduledTasks/PluralKit.ScheduledTasks.csproj /app/PluralKit.ScheduledTasks/
 COPY PluralKit.Tests/PluralKit.Tests.csproj /app/PluralKit.Tests/
 COPY .git/ /app/.git
 RUN dotnet restore PluralKit.sln

PluralKit.Bot/Bot.cs

@@ -257,7 +257,11 @@ namespace PluralKit.Bot
                 {
                     var botPerms = PermissionsIn(reportChannel.Value);
                     if (botPerms.HasFlag(PermissionSet.SendMessages | PermissionSet.EmbedLinks))
-                        await _errorMessageService.SendErrorMessage(reportChannel.Value, sentryEvent.EventId.ToString());
+                    {
+                        // i'm just going to disable this for now we need to find something nicer
+                        // await _errorMessageService.SendErrorMessage(reportChannel.Value,
+                        // sentryEvent.EventId.ToString());
+                    }
                 }
             }
         }
@@ -268,9 +272,6 @@ namespace PluralKit.Bot
 
             await UpdateBotStatus();
 
-            // Clean up message cache in postgres
-            await _commandMessageService.CleanupOldMessages();
-
             // Collect some stats, submit them to the metrics backend
             await _collector.CollectStats();
             await Task.WhenAll(((IMetricsRoot)_metrics).ReportRunner.RunAllAsync());

PluralKit.Bot/Commands/Misc.cs

@@ -85,11 +85,7 @@ namespace PluralKit.Bot
             var messagesProxied = _metrics.Snapshot.GetForContext("Bot").Meters.FirstOrDefault(m => m.MultidimensionalName == BotMetrics.MessagesProxied.Name)?.Value;
             var commandsRun = _metrics.Snapshot.GetForContext("Bot").Meters.FirstOrDefault(m => m.MultidimensionalName == BotMetrics.CommandsRun.Name)?.Value;
 
-            var totalSystems = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.SystemCount.Name)?.Value ?? 0;
+            var counts = await _repo.GetStats();
-            var totalMembers = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.MemberCount.Name)?.Value ?? 0;
-            var totalGroups = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.GroupCount.Name)?.Value ?? 0;
-            var totalSwitches = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.SwitchCount.Name)?.Value ?? 0;
-            var totalMessages = _metrics.Snapshot.GetForContext("Application").Gauges.FirstOrDefault(m => m.MultidimensionalName == CoreMetrics.MessageCount.Name)?.Value ?? 0;
 
             var shardId = ctx.Shard.ShardId;
             var shardTotal = ctx.Cluster.Shards.Count;
@@ -113,7 +109,11 @@ namespace PluralKit.Bot
                 .Field(new("CPU usage", $"{_cpu.LastCpuMeasure:P1}", true))
                 .Field(new("Memory usage", $"{memoryUsage / 1024 / 1024} MiB", true))
                 .Field(new("Latency", $"API: {apiLatency.TotalMilliseconds:F0} ms, shard: {shardInfo.ShardLatency.Milliseconds} ms", true))
-                .Field(new("Total numbers", $"{totalSystems:N0} systems, {totalMembers:N0} members, {totalGroups:N0} groups, {totalSwitches:N0} switches, {totalMessages:N0} messages"))
+                .Field(new("Total numbers", $"{counts.SystemCount:N0} systems,"
+                    + $" {counts.MemberCount:N0} members,"
+                    + $" {counts.GroupCount:N0} groups,"
+                    + $" {counts.SwitchCount:N0} switches,"
+                    + $" {counts.MessageCount:N0} messages"))
                 .Timestamp(Process.GetCurrentProcess().StartTime.ToString("O"))
                 .Footer(new($"PluralKit {BuildInfoService.Version} • https://github.com/xSke/PluralKit • Last restarted: ")); ;
             await ctx.Rest.EditMessage(msg.ChannelId, msg.Id,

PluralKit.Bot/Services/CommandMessageService.cs

@@ -10,8 +10,6 @@ namespace PluralKit.Bot
 {
     public class CommandMessageService
     {
-        private static readonly Duration CommandMessageRetention = Duration.FromHours(24);
-
         private readonly IDatabase _db;
         private readonly ModelRepository _repo;
         private readonly IClock _clock;
@@ -35,16 +33,5 @@ namespace PluralKit.Bot
         {
             return await _repo.GetCommandMessage(messageId);
         }
-
-        public async Task CleanupOldMessages()
-        {
-            var deleteThresholdInstant = _clock.GetCurrentInstant() - CommandMessageRetention;
-            var deleteThresholdSnowflake = DiscordUtils.InstantToSnowflake(deleteThresholdInstant);
-
-            var deletedRows = await _repo.DeleteCommandMessagesBefore(deleteThresholdSnowflake);
-
-            _logger.Information("Pruned {DeletedRows} command messages older than retention {Retention} (older than {DeleteThresholdInstant} / {DeleteThresholdSnowflake})",
-                deletedRows, CommandMessageRetention, deleteThresholdInstant, deleteThresholdSnowflake);
-        }
     }
 }

PluralKit.Bot/Services/PeriodicStatCollector.cs

@@ -20,7 +20,7 @@ namespace PluralKit.Bot
         private readonly IDiscordCache _cache;
         private readonly CpuStatService _cpu;
 
-        private readonly IDatabase _db;
+        private readonly ModelRepository _repo;
 
         private readonly WebhookCacheService _webhookCache;
 
@@ -28,13 +28,13 @@ namespace PluralKit.Bot
 
         private readonly ILogger _logger;
 
-        public PeriodicStatCollector(IMetrics metrics, ILogger logger, WebhookCacheService webhookCache, DbConnectionCountHolder countHolder, CpuStatService cpu, IDatabase db, IDiscordCache cache)
+        public PeriodicStatCollector(IMetrics metrics, ILogger logger, WebhookCacheService webhookCache, DbConnectionCountHolder countHolder, CpuStatService cpu, ModelRepository repo, IDiscordCache cache)
         {
             _metrics = metrics;
             _webhookCache = webhookCache;
             _countHolder = countHolder;
             _cpu = cpu;
-            _db = db;
+            _repo = repo;
             _cache = cache;
             _logger = logger.ForContext<PeriodicStatCollector>();
         }
@@ -63,12 +63,15 @@ namespace PluralKit.Bot
             _metrics.Measure.Gauge.SetValue(BotMetrics.Channels, channelCount);
 
             // Aggregate DB stats
-            var counts = await _db.Execute(c => c.QueryFirstAsync<Counts>("select (select count(*) from systems) as systems, (select count(*) from members) as members, (select count(*) from switches) as switches, (select count(*) from messages) as messages, (select count(*) from groups) as groups"));
+            // just fetching from database here - actual updating of the data is done in PluralKit.ScheduledTasks
-            _metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.Systems);
+            // if you're not running ScheduledTasks and want up-to-date counts, uncomment the following line:
-            _metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.Members);
+            // await _repo.UpdateStats();
-            _metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.Switches);
+            var counts = await _repo.GetStats();
-            _metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.Messages);
+            _metrics.Measure.Gauge.SetValue(CoreMetrics.SystemCount, counts.SystemCount);
-            _metrics.Measure.Gauge.SetValue(CoreMetrics.GroupCount, counts.Groups);
+            _metrics.Measure.Gauge.SetValue(CoreMetrics.MemberCount, counts.MemberCount);
+            _metrics.Measure.Gauge.SetValue(CoreMetrics.GroupCount, counts.GroupCount);
+            _metrics.Measure.Gauge.SetValue(CoreMetrics.SwitchCount, counts.SwitchCount);
+            _metrics.Measure.Gauge.SetValue(CoreMetrics.MessageCount, counts.MessageCount);
 
             // Process info
             var process = Process.GetCurrentProcess();
@@ -88,14 +91,5 @@ namespace PluralKit.Bot
             stopwatch.Stop();
             _logger.Debug("Updated metrics in {Time}", stopwatch.ElapsedDuration());
         }
-
-        public class Counts
-        {
-            public int Systems { get; }
-            public int Members { get; }
-            public int Switches { get; }
-            public int Messages { get; }
-            public int Groups { get; }
-        }
     }
 }

PluralKit.Core/Database/Migrations/19.sql

@@ -0,0 +1,10 @@
+-- schema version 19: 2021-10-15 --
+-- add stats to info table
+
+alter table info add column system_count int;
+alter table info add column member_count int;
+alter table info add column group_count int;
+alter table info add column switch_count int;
+alter table info add column message_count int;
+
+update info set schema_version = 18;

PluralKit.Core/Database/Repository/ModelRepository.Stats.cs

@@ -0,0 +1,33 @@
+using System.Threading;
+using System.Threading.Tasks;
+
+using Dapper;
+
+using SqlKata;
+
+namespace PluralKit.Core
+{
+    public partial class ModelRepository
+    {
+        public async Task UpdateStats()
+        {
+            await _db.Execute(conn => conn.ExecuteAsync("update info set system_count = (select count(*) from systems)"));
+            await _db.Execute(conn => conn.ExecuteAsync("update info set member_count = (select count(*) from members)"));
+            await _db.Execute(conn => conn.ExecuteAsync("update info set group_count = (select count(*) from groups)"));
+            await _db.Execute(conn => conn.ExecuteAsync("update info set switch_count = (select count(*) from switches)"));
+            await _db.Execute(conn => conn.ExecuteAsync("update info set message_count = (select count(*) from messages)"));
+        }
+
+        public Task<Counts> GetStats()
+            => _db.Execute(conn => conn.QuerySingleAsync<Counts>("select * from info"));
+
+        public class Counts
+        {
+            public int SystemCount { get; }
+            public int MemberCount { get; }
+            public int GroupCount { get; }
+            public int SwitchCount { get; }
+            public int MessageCount { get; }
+        }
+    }
+}

PluralKit.Core/Database/Utils/DatabaseMigrator.cs

@@ -12,7 +12,7 @@ namespace PluralKit.Core
     internal class DatabaseMigrator
     {
         private const string RootPath = "PluralKit.Core.Database"; // "resource path" root for SQL files
-        private const int TargetSchemaVersion = 18;
+        private const int TargetSchemaVersion = 19;
         private readonly ILogger _logger;
 
         public DatabaseMigrator(ILogger logger)

PluralKit.ScheduledTasks/PluralKit.ScheduledTasks.csproj

@@ -0,0 +1,15 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net5.0</TargetFramework>
+  </PropertyGroup>
+
+  <ItemGroup>
+    <ProjectReference Include="..\PluralKit.Core\PluralKit.Core.csproj" />
+  </ItemGroup>
+
+  <PropertyGroup>
+      <RestorePackagesWithLockFile>true</RestorePackagesWithLockFile>
+  </PropertyGroup>
+</Project>

PluralKit.ScheduledTasks/Startup.cs

@@ -0,0 +1,40 @@
+using System;
+using System.Threading.Tasks;
+
+using Autofac;
+
+using Microsoft.Extensions.Configuration;
+
+using PluralKit.Core;
+
+namespace PluralKit.ScheduledTasks
+{
+    class Startup
+    {
+        static async Task Main(string[] args)
+        {
+            // Load configuration and run global init stuff
+            var config = InitUtils.BuildConfiguration(args).Build();
+            InitUtils.InitStatic();
+
+            var services = BuildContainer(config);
+            services.Resolve<TaskHandler>().Run();
+
+            await Task.Delay(-1);
+        }
+
+        private static IContainer BuildContainer(IConfiguration config)
+        {
+            var builder = new ContainerBuilder();
+
+            builder.RegisterInstance(config);
+            builder.RegisterModule(new ConfigModule<CoreConfig>());
+            builder.RegisterModule(new LoggingModule("ScheduledTasks"));
+            builder.RegisterModule(new MetricsModule());
+            builder.RegisterModule<DataStoreModule>();
+            builder.RegisterType<TaskHandler>().AsSelf().SingleInstance();
+
+            return builder.Build();
+        }
+    }
+}

PluralKit.ScheduledTasks/TaskHandler.cs

@@ -0,0 +1,75 @@
+using System;
+using System.Diagnostics;
+using System.Threading;
+using System.Threading.Tasks;
+
+using NodaTime;
+using NodaTime.Extensions;
+
+using Serilog;
+
+using PluralKit.Core;
+
+namespace PluralKit.ScheduledTasks
+{
+    public class TaskHandler
+    {
+
+        private static readonly Duration CommandMessageRetention = Duration.FromHours(24);
+        private Timer _periodicTask;
+
+        private readonly ILogger _logger;
+        private readonly IDatabase _db;
+        private readonly ModelRepository _repo;
+
+        public TaskHandler(ILogger logger, IDatabase db, ModelRepository repo)
+        {
+            _logger = logger;
+            _db = db;
+            _repo = repo;
+        }
+
+        public void Run()
+        {
+            _logger.Information("Starting scheduled task runner...");
+            var timeNow = SystemClock.Instance.GetCurrentInstant();
+            var timeTillNextWholeMinute = TimeSpan.FromMilliseconds(60000 - timeNow.ToUnixTimeMilliseconds() % 60000 + 250);
+            _periodicTask = new Timer(_ =>
+            {
+                var __ = UpdatePeriodic();
+            }, null, timeTillNextWholeMinute, TimeSpan.FromMinutes(1));
+        }
+
+        private async Task UpdatePeriodic()
+        {
+            _logger.Information("Running per-minute scheduled tasks.");
+            var stopwatch = new Stopwatch();
+            stopwatch.Start();
+
+            _logger.Information("Updating database stats...");
+            await _repo.UpdateStats();
+
+            // Clean up message cache in postgres
+            await CleanupOldMessages();
+
+            stopwatch.Stop();
+            _logger.Information("Ran scheduled tasks in {Time}", stopwatch.ElapsedDuration());
+        }
+
+        private async Task CleanupOldMessages()
+        {
+            var deleteThresholdInstant = SystemClock.Instance.GetCurrentInstant() - CommandMessageRetention;
+            var deleteThresholdSnowflake = InstantToSnowflake(deleteThresholdInstant);
+
+            var deletedRows = await _repo.DeleteCommandMessagesBefore(deleteThresholdSnowflake);
+
+            _logger.Information("Pruned {DeletedRows} command messages older than retention {Retention} (older than {DeleteThresholdInstant} / {DeleteThresholdSnowflake})",
+                deletedRows, CommandMessageRetention, deleteThresholdInstant, deleteThresholdSnowflake);
+        }
+
+        // we don't have access to PluralKit.Bot here, so this needs to be vendored
+        public static ulong InstantToSnowflake(Instant time) =>
+            (ulong)(time - Instant.FromUtc(2015, 1, 1, 0, 0, 0)).TotalMilliseconds << 22;
+
+    }
+}

PluralKit.sln

@@ -1,5 +1,6 @@
 
 Microsoft Visual Studio Solution File, Format Version 12.00
+# 
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.Bot", "PluralKit.Bot\PluralKit.Bot.csproj", "{F2C5562D-FD96-4C11-B54E-93737D127959}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.Core", "PluralKit.Core\PluralKit.Core.csproj", "{5DBE037D-179D-4C05-8A28-35E37129C961}"
@@ -10,6 +11,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.Tests", "PluralKi
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Myriad", "Myriad\Myriad.csproj", "{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}"
 EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralKit.ScheduledTasks", "PluralKit.ScheduledTasks\PluralKit.ScheduledTasks.csproj", "{374A8EB3-655D-4230-982B-459AE3553991}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -36,5 +39,9 @@ Global
 		{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}.Debug|Any CPU.Build.0 = Debug|Any CPU
 		{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}.Release|Any CPU.ActiveCfg = Release|Any CPU
 		{ACB9BF37-F29C-4068-A7D1-2EFF2C308C4B}.Release|Any CPU.Build.0 = Release|Any CPU
+		{374A8EB3-655D-4230-982B-459AE3553991}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{374A8EB3-655D-4230-982B-459AE3553991}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{374A8EB3-655D-4230-982B-459AE3553991}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{374A8EB3-655D-4230-982B-459AE3553991}.Release|Any CPU.Build.0 = Release|Any CPU
 	EndGlobalSection
 EndGlobal

docs/content/.vuepress/config.js

@@ -64,12 +64,7 @@ module.exports = {
 
   plugins: [
     '@vuepress/plugin-back-to-top',
-    [
+    "plausible-analytics",
-      '@vuepress/google-analytics',
-      {
-        "ga": "UA-173942267-1"
-      }
-    ],
     ["vuepress-plugin-clean-urls", { normalSuffix: "/" }],
   ],
 }

docs/content/privacy-policy.md

@@ -29,4 +29,12 @@ You can export your system information using the `pk;export` command. This does
 
 You can delete your information using `pk;system delete`. This will delete all system information and associated members, switches, and messages. This will not delete your information from the database backups. Contact me if you want that wiped, too.
 
-The bot is [open-source](https://github.com/xSke/PluralKit). While I can't *prove* this is the code that's running on the production server... it is, promise.
+The bot is [open-source](https://github.com/xSke/PluralKit). While I can't *prove* this is the code that's running on the production server... it is, promise.
+
+## Website
+
+We use [Plausible](https://plausible.io) for website analytics. The analytics are publicly accessible at [https://plausible.io/pluralkit.me](https://plausible.io/pluralkit.me).
+
+In short, Plausible logs information related to *your current visit on this website*, not information *about you*. A full list of everything that Plausible logs can be found on their [data policy](https://plausible.io/data-policy) page.
+
+If you do not wish for your visits to be included in the analytics, please follow [Plausible's opt-out guide](https://plausible.io/docs/excluding). <noscript>*JavaScript is disabled in this browser, which automatically blocks Plausible Analytics.*</noscript>

docs/package.json

@@ -9,11 +9,11 @@
   "license": "AGPL-3.0-or-later",
   "devDependencies": {
     "@vuepress/plugin-back-to-top": "1.8.2",
-    "@vuepress/plugin-google-analytics": "1.8.2",
     "markdown-it-custom-header-link": "^1.0.5",
     "vuepress": "1.8.2",
     "vuepress-plugin-clean-urls": "1.1.2",
-    "vuepress-plugin-dehydrate": "1.1.5"
+    "vuepress-plugin-dehydrate": "1.1.5",
+    "vuepress-plugin-plausible-analytics": "^0.2.1"
   },
   "dependencies": {
     "vuepress-theme-default-prefers-color-scheme": "2.0.0"

docs/yarn.lock

@@ -1173,11 +1173,6 @@
   dependencies:
     lodash.debounce "^4.0.8"
 
-"@vuepress/plugin-google-analytics@1.8.2":
-  version "1.8.2"
-  resolved "https://registry.yarnpkg.com/@vuepress/plugin-google-analytics/-/plugin-google-analytics-1.8.2.tgz#96cf65f1f0ecbb3bcf6b0d10089dafe2aea875bc"
-  integrity sha512-BMFayLzT2BvXmnhM9mDHw0UPU7J0pH1X9gQA4HmZxOf7f3+atK5eJGsc1Ia/+1FTG2ESvhFLUU/CC3h5arjEJw==
-
 "@vuepress/plugin-last-updated@1.8.2":
   version "1.8.2"
   resolved "https://registry.yarnpkg.com/@vuepress/plugin-last-updated/-/plugin-last-updated-1.8.2.tgz#7ce689f8d5050cf0213949bc2e5aa879c09ff4b1"
@@ -7678,12 +7673,10 @@ vuepress-plugin-container@^2.0.2:
     "@vuepress/shared-utils" "^1.2.0"
     markdown-it-container "^2.0.0"
 
-vuepress-plugin-dehydrate@1.1.5:
+vuepress-plugin-plausible-analytics@^0.2.1:
-  version "1.1.5"
+  version "0.2.1"
-  resolved "https://registry.yarnpkg.com/vuepress-plugin-dehydrate/-/vuepress-plugin-dehydrate-1.1.5.tgz#c29f08b85f337d3bc7c0ba09fe58cd9ed2f6e6c1"
+  resolved "https://registry.yarnpkg.com/vuepress-plugin-plausible-analytics/-/vuepress-plugin-plausible-analytics-0.2.1.tgz#d6733b17305bd37d5b9797460c8a8d179b79fc2d"
-  integrity sha512-9F2x1vLCK4poPUMkLupD4HsgWdbZ68Escvma+DE1Dk6aAJdH5FGwmfOMxj4sMCBwz7S4s6bTMna+QQgD3+bzBA==
+  integrity sha512-e4ZLAFEbfhjpHv08+QqO1lMZeDa/kcg71yLtzG1LCre7sfihQyuV82ZFU3yGflqDgimcfv2AkZe24fIdoRqYmg==
-  dependencies:
-    "@vuepress/shared-utils" "^1.2.0"
 
 vuepress-plugin-smooth-scroll@^0.0.3:
   version "0.0.3"