Add more performance metrics

PluralKit.Bot/Services/ShardInfoService.cs

@@ -1,6 +1,9 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 
+using App.Metrics;
+
 using DSharpPlus;
 using DSharpPlus.EventArgs;
 
@@ -24,13 +27,15 @@ namespace PluralKit.Bot
             public bool Connected;
         }
 
+        private IMetrics _metrics;
         private ILogger _logger;
         private DiscordShardedClient _client;
         private Dictionary<int, ShardInfo> _shardInfo = new Dictionary<int, ShardInfo>();
         
-        public ShardInfoService(ILogger logger, DiscordShardedClient client)
+        public ShardInfoService(ILogger logger, DiscordShardedClient client, IMetrics metrics)
         {
             _client = client;
+            _metrics = metrics;
             _logger = logger.ForContext<ShardInfoService>();
         }
 
@@ -41,6 +46,13 @@ namespace PluralKit.Bot
             _client.SocketOpened += RefreshShardList;
         }
 
+        private void ReportShardStatus()
+        {
+            foreach (var (id, shard) in _shardInfo)
+                _metrics.Measure.Gauge.SetValue(BotMetrics.ShardLatency, new MetricTags("shard", id.ToString()), shard.ShardLatency.TotalMilliseconds);
+            _metrics.Measure.Gauge.SetValue(BotMetrics.ShardsConnected, _shardInfo.Count(s => s.Value.Connected));
+        }
+
         private async Task RefreshShardList()
         {
             // This callback doesn't actually receive the shard that was opening, so we just try to check we have 'em all (so far)
@@ -95,6 +107,7 @@ namespace PluralKit.Bot
             var info = TryGetShard(e.Client);
             // info.LastConnectionTime = SystemClock.Instance.GetCurrentInstant();
             info.Connected = true;
+            ReportShardStatus();
             return Task.CompletedTask;
         }
 
@@ -105,6 +118,7 @@ namespace PluralKit.Bot
             var info = TryGetShard(e.Client);
             info.LastConnectionTime = SystemClock.Instance.GetCurrentInstant();
             info.Connected = true;
+            ReportShardStatus();
             return Task.CompletedTask;
         }
 
@@ -115,6 +129,7 @@ namespace PluralKit.Bot
             var info = TryGetShard(e.Client);
             info.DisconnectionCount++;
             info.Connected = false;
+            ReportShardStatus();
             return Task.CompletedTask; 
         }
 

PluralKit.Bot/Services/WebhookCacheService.cs

@@ -4,6 +4,8 @@ using System.Linq;
 using System.Net.Http;
 using System.Threading.Tasks;
 
+using App.Metrics;
+
 using DSharpPlus;
 using DSharpPlus.Entities;
 
@@ -18,11 +20,13 @@ namespace PluralKit.Bot
         private DiscordShardedClient _client;
         private ConcurrentDictionary<ulong, Lazy<Task<DiscordWebhook>>> _webhooks;
 
+        private IMetrics _metrics;
         private ILogger _logger;
 
-        public WebhookCacheService(DiscordShardedClient client, ILogger logger)
+        public WebhookCacheService(DiscordShardedClient client, ILogger logger, IMetrics metrics)
         {
             _client = client;
+            _metrics = metrics;
             _logger = logger.ForContext<WebhookCacheService>();
             _webhooks = new ConcurrentDictionary<ulong, Lazy<Task<DiscordWebhook>>>();
         }
@@ -78,6 +82,7 @@ namespace PluralKit.Bot
         private async Task<DiscordWebhook> GetOrCreateWebhook(DiscordChannel channel)
         {
             _logger.Debug("Webhook for channel {Channel} not found in cache, trying to fetch", channel.Id);
+            _metrics.Measure.Meter.Mark(BotMetrics.WebhookCacheMisses);
             return await FindExistingWebhook(channel) ?? await DoCreateWebhook(channel);
         }
 

PluralKit.Bot/Services/WebhookExecutorService.cs

@@ -13,6 +13,7 @@ using DSharpPlus.Exceptions;
 using Humanizer;
 
 using Newtonsoft.Json;
+using Newtonsoft.Json.Serialization;
 
 using Serilog;
 
@@ -72,36 +73,34 @@ namespace PluralKit.Bot
                 await AddAttachmentsToBuilder(dwb, attachmentChunks[0]);
             }
             
-            var timerCtx = _metrics.Measure.Timer.Time(BotMetrics.WebhookResponseTime);
-
             DiscordMessage response;
-            try
-            {
-                response = await webhook.ExecuteAsync(dwb);
-            }
-            catch (JsonReaderException)
-            {
-                // This happens sometimes when we hit a CloudFlare error (or similar) on Discord's end
-                // Nothing we can do about this - happens sometimes under server load, so just drop the message and give up
-                throw new WebhookExecutionErrorOnDiscordsEnd();
-            }
-            catch (NotFoundException e)
-            {
-                var errorText = e.WebResponse?.Response;
-                if (errorText != null && errorText.Contains("10015") && !hasRetried)
+            using (_metrics.Measure.Timer.Time(BotMetrics.WebhookResponseTime)) {
+                try
                 {
-                    // Error 10015 = "Unknown Webhook" - this likely means the webhook was deleted
-                    // but is still in our cache. Invalidate, refresh, try again
-                    _logger.Warning("Error invoking webhook {Webhook} in channel {Channel}", webhook.Id, webhook.ChannelId);
-                    
-                    var newWebhook = await _webhookCache.InvalidateAndRefreshWebhook(channel, webhook);
-                    return await ExecuteWebhookInner(channel, newWebhook, name, avatarUrl, content, attachments, hasRetried: true);
+                    response = await webhook.ExecuteAsync(dwb);
                 }
+                catch (JsonReaderException)
+                {
+                    // This happens sometimes when we hit a CloudFlare error (or similar) on Discord's end
+                    // Nothing we can do about this - happens sometimes under server load, so just drop the message and give up
+                    throw new WebhookExecutionErrorOnDiscordsEnd();
+                }
+                catch (NotFoundException e)
+                {
+                    var errorText = e.WebResponse?.Response;
+                    if (errorText != null && errorText.Contains("10015") && !hasRetried)
+                    {
+                        // Error 10015 = "Unknown Webhook" - this likely means the webhook was deleted
+                        // but is still in our cache. Invalidate, refresh, try again
+                        _logger.Warning("Error invoking webhook {Webhook} in channel {Channel}", webhook.Id, webhook.ChannelId);
+                        
+                        var newWebhook = await _webhookCache.InvalidateAndRefreshWebhook(channel, webhook);
+                        return await ExecuteWebhookInner(channel, newWebhook, name, avatarUrl, content, attachments, hasRetried: true);
+                    }
 
-                throw;
-            }
-
-            timerCtx.Dispose();
+                    throw;
+                }
+            } 
 
             // We don't care about whether the sending succeeds, and we don't want to *wait* for it, so we just fork it off
             var _ = TrySendRemainingAttachments(webhook, name, avatarUrl, attachmentChunks);