Expand pk;stats functionality

PluralKit.Bot/Bot.cs

@@ -113,6 +113,8 @@ namespace PluralKit.Bot
 
             .AddTransient<ProxyCacheService>()
             .AddSingleton<WebhookCacheService>()
+            .AddSingleton<ShardInfoService>()
+            .AddSingleton<CpuStatService>()
 
             .AddTransient<IDataStore, PostgresDataStore>()
 
@@ -162,6 +164,8 @@ namespace PluralKit.Bot
             _client.MessageDeleted += (msg, channel) => HandleEvent(s => s.AddMessageDeleteBreadcrumb(msg, channel), eh => eh.HandleMessageDeleted(msg, channel));
             _client.MessagesBulkDeleted += (msgs, channel) => HandleEvent(s => s.AddMessageBulkDeleteBreadcrumb(msgs, channel), eh => eh.HandleMessagesBulkDelete(msgs, channel));
             
+            _services.GetService<ShardInfoService>().Init(_client);
+
             return Task.CompletedTask;
         }
 

PluralKit.Bot/Commands/MiscCommands.cs

@@ -1,22 +1,32 @@
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Linq;
 using System.Threading.Tasks;
 using App.Metrics;
+
 using Discord;
+
 using Humanizer;
 
+using NodaTime;
+
 using PluralKit.Bot.CommandSystem;
+using PluralKit.Core;
 
 namespace PluralKit.Bot.Commands {
     public class MiscCommands
     {
         private BotConfig _botConfig;
         private IMetrics _metrics;
+        private CpuStatService _cpu;
+        private ShardInfoService _shards;
 
-        public MiscCommands(BotConfig botConfig, IMetrics metrics)
+        public MiscCommands(BotConfig botConfig, IMetrics metrics, CpuStatService cpu, ShardInfoService shards)
         {
             _botConfig = botConfig;
             _metrics = metrics;
+            _cpu = cpu;
+            _shards = shards;
         }
         
         public async Task Invite(Context ctx)
@@ -45,16 +55,43 @@ namespace PluralKit.Bot.Commands {
 
         public async Task Stats(Context ctx)
         {
+            var msg = await ctx.Reply($"...");
+            
             var messagesReceived = _metrics.Snapshot.GetForContext("Bot").Meters.First(m => m.MultidimensionalName == BotMetrics.MessagesReceived.Name).Value;
             var messagesProxied = _metrics.Snapshot.GetForContext("Bot").Meters.First(m => m.MultidimensionalName == BotMetrics.MessagesProxied.Name).Value;
-            
             var commandsRun = _metrics.Snapshot.GetForContext("Bot").Meters.First(m => m.MultidimensionalName == BotMetrics.CommandsRun.Name).Value;
 
-            await ctx.Reply(embed: new EmbedBuilder()
+            var totalSystems = _metrics.Snapshot.GetForContext("Application").Gauges.First(m => m.MultidimensionalName == CoreMetrics.SystemCount.Name).Value;
-                .AddField("Messages processed", $"{messagesReceived.OneMinuteRate:F1}/s ({messagesReceived.FifteenMinuteRate:F1}/s over 15m)")
+            var totalMembers = _metrics.Snapshot.GetForContext("Application").Gauges.First(m => m.MultidimensionalName == CoreMetrics.MemberCount.Name).Value;
-                .AddField("Messages proxied", $"{messagesProxied.OneMinuteRate:F1}/s ({messagesProxied.FifteenMinuteRate:F1}/s over 15m)")
+            var totalSwitches = _metrics.Snapshot.GetForContext("Application").Gauges.First(m => m.MultidimensionalName == CoreMetrics.SwitchCount.Name).Value;
-                .AddField("Commands executed", $"{commandsRun.OneMinuteRate:F1}/s ({commandsRun.FifteenMinuteRate:F1}/s over 15m)")
+            var totalMessages = _metrics.Snapshot.GetForContext("Application").Gauges.First(m => m.MultidimensionalName == CoreMetrics.MessageCount.Name).Value;
-                .Build());
+
+            var shardId = ctx.Shard.ShardId;
+            var shardTotal = ctx.Client.Shards.Count;
+            var shardUpTotal = ctx.Client.Shards.Select(s => s.ConnectionState == ConnectionState.Connected).Count();
+            var shardInfo = _shards.GetShardInfo(ctx.Shard);
+            
+            var process = Process.GetCurrentProcess();
+            var memoryUsage = process.WorkingSet64;
+
+            var shardUptime = SystemClock.Instance.GetCurrentInstant() - shardInfo.LastConnectionTime;
+
+            var embed = new EmbedBuilder()
+                .AddField("Messages processed", $"{messagesReceived.OneMinuteRate * 60:F1}/m ({messagesReceived.FifteenMinuteRate * 60:F1}/m over 15m)", true)
+                .AddField("Messages proxied", $"{messagesProxied.OneMinuteRate * 60:F1}/m ({messagesProxied.FifteenMinuteRate * 60:F1}/m over 15m)", true)
+                .AddField("Commands executed", $"{commandsRun.OneMinuteRate * 60:F1}/m ({commandsRun.FifteenMinuteRate * 60:F1}/m over 15m)", true)
+                .AddField("Current shard", $"Shard #{shardId} (of {shardTotal} total, {shardUpTotal} are up)", true)
+                .AddField("Shard uptime", $"{Formats.DurationFormat.Format(shardUptime)} ({shardInfo.DisconnectionCount} disconnections)", true)
+                .AddField("CPU usage", $"{_cpu.LastCpuMeasure * 100:P1}", true)
+                .AddField("Memory usage", $"{memoryUsage / 1024 / 1024} MiB", true)
+                .AddField("Latency", $"API: {(msg.Timestamp - ctx.Message.Timestamp).TotalMilliseconds:F0} ms, shard: {shardInfo.ShardLatency} ms", true)
+                .AddField("Total numbers", $"{totalSystems} systems, {totalMembers} members, {totalSwitches} switches, {totalMessages} messages");
+
+            await msg.ModifyAsync(f =>
+            {
+                f.Content = Optional<string>.Unspecified;
+                f.Embed = embed.Build();
+            });
         }
         
         public async Task PermCheckGuild(Context ctx)

PluralKit.Bot/Services/CpuStatService.cs

@@ -0,0 +1,47 @@
+using System.Diagnostics;
+using System.Threading.Tasks;
+
+using Serilog;
+
+namespace PluralKit.Bot
+{
+    public class CpuStatService
+    {
+        private ILogger _logger;
+        
+        public double LastCpuMeasure { get; private set; }
+
+        public CpuStatService(ILogger logger)
+        {
+            _logger = logger;
+        }
+
+        /// <summary>
+        /// Gets the current CPU usage. Estimation takes ~5 seconds (of mostly sleeping).
+        /// </summary>
+        public async Task<double> EstimateCpuUsage()
+        {
+            // We get the current processor time, wait 5 seconds, then compare
+            // https://medium.com/@jackwild/getting-cpu-usage-in-net-core-7ef825831b8b
+            
+            _logger.Information("Estimating CPU usage...");
+            var stopwatch = new Stopwatch();
+            
+            stopwatch.Start();
+            var cpuTimeBefore = Process.GetCurrentProcess().TotalProcessorTime;
+            
+            await Task.Delay(5000);
+            
+            stopwatch.Stop();
+            var cpuTimeAfter = Process.GetCurrentProcess().TotalProcessorTime;
+
+            var cpuTimePassed = cpuTimeAfter - cpuTimeBefore;
+            var timePassed = stopwatch.Elapsed;
+
+            var percent = cpuTimePassed / timePassed;
+            _logger.Information("CPU usage measured as {Percent:P}", percent);
+            LastCpuMeasure = percent;
+            return percent;
+        }
+    }
+}

PluralKit.Bot/Services/PeriodicStatCollector.cs

@@ -16,6 +16,7 @@ namespace PluralKit.Bot
     {
         private DiscordShardedClient _client;
         private IMetrics _metrics;
+        private CpuStatService _cpu;
 
         private IDataStore _data;
 
@@ -25,13 +26,14 @@ namespace PluralKit.Bot
 
         private ILogger _logger;
 
-        public PeriodicStatCollector(IDiscordClient client, IMetrics metrics, ILogger logger, WebhookCacheService webhookCache, DbConnectionCountHolder countHolder, IDataStore data)
+        public PeriodicStatCollector(IDiscordClient client, IMetrics metrics, ILogger logger, WebhookCacheService webhookCache, DbConnectionCountHolder countHolder, IDataStore data, CpuStatService cpu)
         {
             _client = (DiscordShardedClient) client;
             _metrics = metrics;
             _webhookCache = webhookCache;
             _countHolder = countHolder;
             _data = data;
+            _cpu = cpu;
             _logger = logger.ForContext<PeriodicStatCollector>();
         }
 
@@ -71,7 +73,7 @@ namespace PluralKit.Bot
             _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessPrivateMemory, process.PrivateMemorySize64);
             _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessThreads, process.Threads.Count);
             _metrics.Measure.Gauge.SetValue(CoreMetrics.ProcessHandles, process.HandleCount);
-            _metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await EstimateCpuUsage());
+            _metrics.Measure.Gauge.SetValue(CoreMetrics.CpuUsage, await _cpu.EstimateCpuUsage());
             
             // Database info
             _metrics.Measure.Gauge.SetValue(CoreMetrics.DatabaseConnections, _countHolder.ConnectionCount);
@@ -82,29 +84,5 @@ namespace PluralKit.Bot
             stopwatch.Stop();
             _logger.Information("Updated metrics in {Time}", stopwatch.ElapsedDuration());
         }
-
-        private async Task<double> EstimateCpuUsage()
-        {
-            // We get the current processor time, wait 5 seconds, then compare
-            // https://medium.com/@jackwild/getting-cpu-usage-in-net-core-7ef825831b8b
-            
-            _logger.Information("Estimating CPU usage...");
-            var stopwatch = new Stopwatch();
-            
-            stopwatch.Start();
-            var cpuTimeBefore = Process.GetCurrentProcess().TotalProcessorTime;
-            
-            await Task.Delay(5000);
-            
-            stopwatch.Stop();
-            var cpuTimeAfter = Process.GetCurrentProcess().TotalProcessorTime;
-
-            var cpuTimePassed = cpuTimeAfter - cpuTimeBefore;
-            var timePassed = stopwatch.Elapsed;
-
-            var percent = cpuTimePassed / timePassed;
-            _logger.Information("CPU usage measured as {Percent:P}", percent);
-            return percent;
-        }
     }
 }

PluralKit.Bot/Services/ShardInfoService.cs

@@ -0,0 +1,58 @@
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+
+using Discord.WebSocket;
+
+using NodaTime;
+
+namespace PluralKit.Bot
+{
+    public class ShardInfoService
+    {
+        public class ShardInfo
+        {
+            public Instant LastConnectionTime;
+            public int DisconnectionCount;
+            public int ShardLatency;
+        }
+
+        private Dictionary<int, ShardInfo> _shardInfo = new Dictionary<int, ShardInfo>();
+        
+        public void Init(DiscordShardedClient client)
+        {
+            for (var i = 0; i < client.Shards.Count; i++) 
+                _shardInfo[i] = new ShardInfo();
+
+            client.ShardConnected += ShardConnected;
+            client.ShardDisconnected += ShardDisconnected;
+            client.ShardReady += ShardReady;
+            client.ShardLatencyUpdated += ShardLatencyUpdated;
+        }
+
+        public ShardInfo GetShardInfo(DiscordSocketClient shard) => _shardInfo[shard.ShardId];
+
+        private Task ShardLatencyUpdated(int oldLatency, int newLatency, DiscordSocketClient shard)
+        {
+            _shardInfo[shard.ShardId].ShardLatency = newLatency;
+            return Task.CompletedTask;
+        }
+
+        private Task ShardReady(DiscordSocketClient shard)
+        {
+            return Task.CompletedTask;
+        }
+
+        private Task ShardDisconnected(Exception e, DiscordSocketClient shard)
+        {
+            _shardInfo[shard.ShardId].DisconnectionCount++;
+            return Task.CompletedTask;
+        }
+
+        private Task ShardConnected(DiscordSocketClient shard)
+        {
+            _shardInfo[shard.ShardId].LastConnectionTime = SystemClock.Instance.GetCurrentInstant();
+            return Task.CompletedTask;
+        }
+    }
+}