Add logging/metrics for error *messages*

PluralKit.Bot/BotMetrics.cs

@@ -23,6 +23,7 @@ namespace PluralKit.Bot
         public static TimerOptions ProxyMembersQueryTime => new TimerOptions { Name = "Proxy member query duration", Context = "Bot", RateUnit = TimeUnit.Seconds, DurationUnit = TimeUnit.Seconds, MeasurementUnit = Unit.Calls };
         public static TimerOptions DiscordApiRequests => new TimerOptions { Name = "Discord API requests", MeasurementUnit = Unit.Requests, Context = "Bot"};
         public static MeterOptions BotErrors => new MeterOptions { Name = "Bot errors", MeasurementUnit = Unit.Errors, RateUnit = TimeUnit.Seconds, Context = "Bot"};
+        public static MeterOptions ErrorMessagesSent => new MeterOptions { Name = "Error messages sent", MeasurementUnit = Unit.Errors, RateUnit = TimeUnit.Seconds, Context = "Bot"};
         public static TimerOptions EventsHandled => new TimerOptions { Name = "Events handled", MeasurementUnit = Unit.Errors, RateUnit = TimeUnit.Seconds, DurationUnit = TimeUnit.Seconds, Context = "Bot"};
     }
 }

PluralKit.Bot/Services/ErrorMessageService.cs

@@ -1,35 +1,73 @@
-using System.Collections.Concurrent;
+using System;
+using System.Collections.Concurrent;
 using System.Threading.Tasks;
 
+using App.Metrics;
+
 using DSharpPlus.Entities;
 
 using NodaTime;
 
+using Serilog;
+
 namespace PluralKit.Bot
 {
     public class ErrorMessageService
     {
         private static readonly Duration MinErrorInterval = Duration.FromSeconds(10);
         private readonly ConcurrentDictionary<ulong, Instant> _lastErrorInChannel = new ConcurrentDictionary<ulong, Instant>();
+        
+        private readonly IMetrics _metrics;
+        private readonly ILogger _logger;
+        
+        public ErrorMessageService(IMetrics metrics, ILogger logger)
+        {
+            _metrics = metrics;
+            _logger = logger;
+        }
 
         public async Task SendErrorMessage(DiscordChannel channel, string errorId)
         {
-            var now = SystemClock.Instance.GetCurrentInstant(); 
+            var now = SystemClock.Instance.GetCurrentInstant();
-            if (_lastErrorInChannel.TryGetValue(channel.Id, out var lastErrorTime))
+            if (!ShouldSendErrorMessage(channel, now))
             {
-                var interval = now - lastErrorTime;
+                _logger.Warning("Rate limited sending error message to {ChannelId} with error code {ErrorId}", channel.Id, errorId);
-                if (interval < MinErrorInterval)
+                _metrics.Measure.Meter.Mark(BotMetrics.ErrorMessagesSent, "throttled");
-                    return;
+                return;
             }
-            _lastErrorInChannel[channel.Id] = now;
+
-            
             var embed = new DiscordEmbedBuilder()
                 .WithColor(new DiscordColor(0xE74C3C))
                 .WithTitle("Internal error occurred")
                 .WithDescription("For support, please send the error code above in **#bug-reports-and-errors** on **[the support server *(click to join)*](https://discord.gg/PczBt78)** with a description of what you were doing at the time.")
                 .WithFooter(errorId)
                 .WithTimestamp(now.ToDateTimeOffset());
-            await channel.SendMessageAsync($"> **Error code:** `{errorId}`", embed: embed.Build());
+
+            try
+            {
+                await channel.SendMessageAsync($"> **Error code:** `{errorId}`", embed: embed.Build());
+                _logger.Information("Sent error message to {ChannelId} with error code {ErrorId}", channel.Id, errorId);
+                _metrics.Measure.Meter.Mark(BotMetrics.ErrorMessagesSent, "sent");
+            }
+            catch (Exception e)
+            {
+                _logger.Error(e, "Error sending error message to {ChannelId}", channel.Id);
+                _metrics.Measure.Meter.Mark(BotMetrics.ErrorMessagesSent, "failed");
+                throw;
+            }
+        }
+
+        private bool ShouldSendErrorMessage(DiscordChannel channel, Instant now)
+        {
+            if (_lastErrorInChannel.TryGetValue(channel.Id, out var lastErrorTime))
+            {
+                var interval = now - lastErrorTime;
+                if (interval < MinErrorInterval)
+                    return false;
+            }
+
+            _lastErrorInChannel[channel.Id] = now;
+            return true;
         }
     }
 }