liz/PluralKit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Converted shard and logclean service

Browse Source
This commit is contained in:
Ske
2021-01-30 01:07:43 +01:00
parent da9d84a197
commit d56e878c28
11 changed files with 264 additions and 153 deletions
Download Patch File Download Diff File Expand all files Collapse all files

165
PluralKit.Bot/Services/LoggerCleanService.cs
View File

@@ -4,31 +4,40 @@ using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using DSharpPlus;
using DSharpPlus.Entities;
using Myriad.Cache;
using Myriad.Extensions;
using Myriad.Rest;
using Myriad.Rest.Exceptions;
using Myriad.Types;
using Dapper;
using NodaTime;
using NodaTime.Extensions;
using NodaTime.Text;
using PluralKit.Core;
using Serilog;
namespace PluralKit.Bot
{
public class LoggerCleanService
{
private static readonly Regex _basicRegex = new Regex("(\\d{17,19})");
private static readonly Regex _dynoRegex = new Regex("Message ID: (\\d{17,19})");
private static readonly Regex _carlRegex = new Regex("ID: (\\d{17,19})");
private static readonly Regex _circleRegex = new Regex("\\(`(\\d{17,19})`\\)");
private static readonly Regex _loggerARegex = new Regex("Message = (\\d{17,19})");
private static readonly Regex _loggerBRegex = new Regex("MessageID:(\\d{17,19})");
private static readonly Regex _auttajaRegex = new Regex("Message (\\d{17,19}) deleted");
private static readonly Regex _mantaroRegex = new Regex("Message \\(?ID:? (\\d{17,19})\\)? created by .* in channel .* was deleted\\.");
private static readonly Regex _pancakeRegex = new Regex("Message from <@(\\d{17,19})> deleted in");
private static readonly Regex _unbelievaboatRegex = new Regex("Message ID: (\\d{17,19})");
private static readonly Regex _vanessaRegex = new Regex("Message sent by <@!?(\\d{17,19})> deleted in");
private static readonly Regex _salRegex = new Regex("\\(ID: (\\d{17,19})\\)");
private static readonly Regex _GearBotRegex = new Regex("\\(``(\\d{17,19})``\\) in <#\\d{17,19}> has been removed.");
private static readonly Regex _GiselleRegex = new Regex("\\*\\*Message ID\\*\\*: `(\\d{17,19})`");
private static readonly Regex _basicRegex = new("(\\d{17,19})");
private static readonly Regex _dynoRegex = new("Message ID: (\\d{17,19})");
private static readonly Regex _carlRegex = new("ID: (\\d{17,19})");
private static readonly Regex _circleRegex = new("\\(`(\\d{17,19})`\\)");
private static readonly Regex _loggerARegex = new("Message = (\\d{17,19})");
private static readonly Regex _loggerBRegex = new("MessageID:(\\d{17,19})");
private static readonly Regex _auttajaRegex = new("Message (\\d{17,19}) deleted");
private static readonly Regex _mantaroRegex = new("Message \\(?ID:? (\\d{17,19})\\)? created by .* in channel .* was deleted\\.");
private static readonly Regex _pancakeRegex = new("Message from <@(\\d{17,19})> deleted in");
private static readonly Regex _unbelievaboatRegex = new("Message ID: (\\d{17,19})");
private static readonly Regex _vanessaRegex = new("Message sent by <@!?(\\d{17,19})> deleted in");
private static readonly Regex _salRegex = new("\\(ID: (\\d{17,19})\\)");
private static readonly Regex _GearBotRegex = new("\\(``(\\d{17,19})``\\) in <#\\d{17,19}> has been removed.");
private static readonly Regex _GiselleRegex = new("\\*\\*Message ID\\*\\*: `(\\d{17,19})`");
private static readonly Dictionary<ulong, LoggerBot> _bots = new[]
{
@@ -57,29 +66,35 @@ namespace PluralKit.Bot
.ToDictionary(b => b.WebhookName);
private readonly IDatabase _db;
private DiscordShardedClient _client;
private readonly DiscordApiClient _client;
private readonly IDiscordCache _cache;
private readonly Bot _bot; // todo: get rid of this nasty
private readonly ILogger _logger;
public LoggerCleanService(IDatabase db, DiscordShardedClient client)
public LoggerCleanService(IDatabase db, DiscordApiClient client, IDiscordCache cache, Bot bot, ILogger logger)
{
_db = db;
_client = client;
_cache = cache;
_bot = bot;
_logger = logger.ForContext<LoggerCleanService>();
}
public ICollection<LoggerBot> Bots => _bots.Values;
public async ValueTask HandleLoggerBotCleanup(Message msg)
{
// TODO: fix!!
/*
if (msg.Channel.Type != ChannelType.Text) return;
if (!msg.Channel.BotHasAllPermissions(Permissions.ManageMessages)) return;
var channel = _cache.GetChannel(msg.ChannelId);
if (channel.Type != Channel.ChannelType.GuildText) return;
if (!_bot.PermissionsIn(channel.Id).HasFlag(PermissionSet.ManageMessages)) return;
// If this message is from a *webhook*, check if the name matches one of the bots we know
// TODO: do we need to do a deeper webhook origin check, or would that be too hard on the rate limit?
// If it's from a *bot*, check the bot ID to see if we know it.
LoggerBot bot = null;
if (msg.WebhookMessage) _botsByWebhookName.TryGetValue(msg.Author.Username, out bot);
else if (msg.Author.IsBot) _bots.TryGetValue(msg.Author.Id, out bot);
if (msg.WebhookId != null) _botsByWebhookName.TryGetValue(msg.Author.Username, out bot);
else if (msg.Author.Bot) _bots.TryGetValue(msg.Author.Id, out bot);
// If we didn't find anything before, or what we found is an unsupported bot, bail
if (bot == null) return;
@@ -96,33 +111,43 @@ namespace PluralKit.Bot
// either way but shouldn't be too much, given it's constrained by user ID and guild.
var fuzzy = bot.FuzzyExtractFunc(msg);
if (fuzzy == null) return;
_logger.Debug("Fuzzy logclean for {BotName} on {MessageId}: {@FuzzyExtractResult}",
bot.Name, msg.Id, fuzzy);
var mid = await _db.Execute(conn =>
conn.QuerySingleOrDefaultAsync<ulong?>(
"select mid from messages where sender = @User and mid > @ApproxID and guild = @Guild limit 1",
new
{
fuzzy.Value.User,
Guild = msg.Channel.GuildId,
Guild = msg.GuildId,
ApproxId = DiscordUtils.InstantToSnowflake(
fuzzy.Value.ApproxTimestamp - TimeSpan.FromSeconds(3))
fuzzy.Value.ApproxTimestamp - Duration.FromSeconds(3))
}));
if (mid == null) return; // If we didn't find a corresponding message, bail
// If we didn't find a corresponding message, bail
if (mid == null)
return;
// Otherwise, we can *reasonably assume* that this is a logged deletion, so delete the log message.
await msg.DeleteAsync();
await _client.DeleteMessage(msg.ChannelId, msg.Id);
}
else if (bot.ExtractFunc != null)
{
// Other bots give us the message ID itself, and we can just extract that from the database directly.
var extractedId = bot.ExtractFunc(msg);
if (extractedId == null) return; // If we didn't find anything, bail.
_logger.Debug("Pure logclean for {BotName} on {MessageId}: {@FuzzyExtractResult}",
bot.Name, msg.Id, extractedId);
var mid = await _db.Execute(conn => conn.QuerySingleOrDefaultAsync<ulong?>(
"select mid from messages where original_mid = @Mid", new {Mid = extractedId.Value}));
if (mid == null) return;
// If we've gotten this far, we found a logged deletion of a trigger message. Just yeet it!
await msg.DeleteAsync();
await _client.DeleteMessage(msg.ChannelId, msg.Id);
} // else should not happen, but idk, it might
}
catch (NotFoundException)
@@ -131,10 +156,9 @@ namespace PluralKit.Bot
// The only thing I can think of that'd cause this are the DeleteAsync() calls which 404 when
// the message doesn't exist anyway - so should be safe to just ignore it, right?
}
*/
}
private static ulong? ExtractAuttaja(DiscordMessage msg)
private static ulong? ExtractAuttaja(Message msg)
{
// Auttaja has an optional "compact mode" that logs without embeds
// That one puts the ID in the message content, non-compact puts it in the embed description.
@@ -146,7 +170,7 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractDyno(DiscordMessage msg)
private static ulong? ExtractDyno(Message msg)
{
// Embed *description* contains "Message sent by [mention] deleted in [channel]", contains message ID in footer per regex
var embed = msg.Embeds.FirstOrDefault();
@@ -155,7 +179,7 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractLoggerA(DiscordMessage msg)
private static ulong? ExtractLoggerA(Message msg)
{
// This is for Logger#6088 (298822483060981760), distinct from Logger#6278 (327424261180620801).
// Embed contains title "Message deleted in [channel]", and an ID field containing both message and user ID (see regex).
@@ -169,7 +193,7 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractLoggerB(DiscordMessage msg)
private static ulong? ExtractLoggerB(Message msg)
{
// This is for Logger#6278 (327424261180620801), distinct from Logger#6088 (298822483060981760).
// Embed title ends with "A Message Was Deleted!", footer contains message ID as per regex.
@@ -179,7 +203,7 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractGenericBot(DiscordMessage msg)
private static ulong? ExtractGenericBot(Message msg)
{
// Embed, title is "Message Deleted", ID plain in footer.
var embed = msg.Embeds.FirstOrDefault();
@@ -188,7 +212,7 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractBlargBot(DiscordMessage msg)
private static ulong? ExtractBlargBot(Message msg)
{
// Embed, title ends with "Message Deleted", contains ID plain in a field.
var embed = msg.Embeds.FirstOrDefault();
@@ -198,7 +222,7 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractMantaro(DiscordMessage msg)
private static ulong? ExtractMantaro(Message msg)
{
// Plain message, "Message (ID: [id]) created by [user] (ID: [id]) in channel [channel] was deleted.
if (!(msg.Content?.Contains("was deleted.") ?? false)) return null;
@@ -206,7 +230,7 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static FuzzyExtractResult? ExtractCarlBot(DiscordMessage msg)
private static FuzzyExtractResult? ExtractCarlBot(Message msg)
{
// Embed, title is "Message deleted in [channel], **user** ID in the footer, timestamp as, well, timestamp in embed.
// This is the *deletion* timestamp, which we can assume is a couple seconds at most after the message was originally sent
@@ -214,17 +238,21 @@ namespace PluralKit.Bot
if (embed?.Footer == null || embed.Timestamp == null || !(embed.Title?.StartsWith("Message deleted in") ?? false)) return null;
var match = _carlRegex.Match(embed.Footer.Text ?? "");
return match.Success
? new FuzzyExtractResult { User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = embed.Timestamp.Value }
? new FuzzyExtractResult
{
User = ulong.Parse(match.Groups[1].Value),
ApproxTimestamp = OffsetDateTimePattern.Rfc3339.Parse(embed.Timestamp).GetValueOrThrow().ToInstant()
}
: (FuzzyExtractResult?) null;
}
private static FuzzyExtractResult? ExtractCircle(DiscordMessage msg)
private static FuzzyExtractResult? ExtractCircle(Message msg)
{
// Like Auttaja, Circle has both embed and compact modes, but the regex works for both.
// Compact: "Message from [user] ([id]) deleted in [channel]", no timestamp (use message time)
// Embed: Message Author field: "[user] ([id])", then an embed timestamp
string stringWithId = msg.Content;
if (msg.Embeds.Count > 0)
if (msg.Embeds.Length > 0)
{
var embed = msg.Embeds.First();
if (embed.Author?.Name == null || !embed.Author.Name.StartsWith("Message Deleted in")) return null;
@@ -236,11 +264,14 @@ namespace PluralKit.Bot
var match = _circleRegex.Match(stringWithId);
return match.Success
? new FuzzyExtractResult {User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = msg.Timestamp}
? new FuzzyExtractResult {
User = ulong.Parse(match.Groups[1].Value),
ApproxTimestamp = msg.Timestamp().ToInstant()
}
: (FuzzyExtractResult?) null;
}
private static FuzzyExtractResult? ExtractPancake(DiscordMessage msg)
private static FuzzyExtractResult? ExtractPancake(Message msg)
{
// Embed, author is "Message Deleted", description includes a mention, timestamp is *message send time* (but no ID)
// so we use the message timestamp to get somewhere *after* the message was proxied
@@ -248,11 +279,15 @@ namespace PluralKit.Bot
if (embed?.Description == null || embed.Author?.Name != "Message Deleted") return null;
var match = _pancakeRegex.Match(embed.Description);
return match.Success
? new FuzzyExtractResult {User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = msg.Timestamp}
? new FuzzyExtractResult
{
User = ulong.Parse(match.Groups[1].Value),
ApproxTimestamp = msg.Timestamp().ToInstant()
}
: (FuzzyExtractResult?) null;
}
private static ulong? ExtractUnbelievaBoat(DiscordMessage msg)
private static ulong? ExtractUnbelievaBoat(Message msg)
{
// Embed author is "Message Deleted", footer contains message ID per regex
var embed = msg.Embeds.FirstOrDefault();
@@ -261,18 +296,22 @@ namespace PluralKit.Bot
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static FuzzyExtractResult? ExtractVanessa(DiscordMessage msg)
private static FuzzyExtractResult? ExtractVanessa(Message msg)
{
// Title is "Message Deleted", embed description contains mention
var embed = msg.Embeds.FirstOrDefault();
if (embed?.Title == null || embed.Title != "Message Deleted" || embed.Description == null) return null;
var match = _vanessaRegex.Match(embed.Description);
return match.Success
? new FuzzyExtractResult {User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = msg.Timestamp}
? new FuzzyExtractResult
{
User = ulong.Parse(match.Groups[1].Value),
ApproxTimestamp = msg.Timestamp().ToInstant()
}
: (FuzzyExtractResult?) null;
}
private static FuzzyExtractResult? ExtractSAL(DiscordMessage msg)
private static FuzzyExtractResult? ExtractSAL(Message msg)
{
// Title is "Message Deleted!", field "Message Author" contains ID
var embed = msg.Embeds.FirstOrDefault();
@@ -281,22 +320,30 @@ namespace PluralKit.Bot
if (authorField == null) return null;
var match = _salRegex.Match(authorField.Value);
return match.Success
? new FuzzyExtractResult {User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = msg.Timestamp}
? new FuzzyExtractResult
{
User = ulong.Parse(match.Groups[1].Value),
ApproxTimestamp = msg.Timestamp().ToInstant()
}
: (FuzzyExtractResult?) null;
}
private static FuzzyExtractResult? ExtractGearBot(DiscordMessage msg)
private static FuzzyExtractResult? ExtractGearBot(Message msg)
{
// Simple text based message log.
// No message ID, but we have timestamp and author ID.
// Not using timestamp here though (seems to be same as message timestamp), might be worth implementing in the future.
var match = _GearBotRegex.Match(msg.Content);
return match.Success
? new FuzzyExtractResult {User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = msg.Timestamp}
? new FuzzyExtractResult
{
User = ulong.Parse(match.Groups[1].Value),
ApproxTimestamp = msg.Timestamp().ToInstant()
}
: (FuzzyExtractResult?) null;
}
private static ulong? ExtractGiselleBot(DiscordMessage msg)
private static ulong? ExtractGiselleBot(Message msg)
{
var embed = msg.Embeds.FirstOrDefault();
if (embed?.Title == null || embed.Title != "🗑 Message Deleted") return null;
@@ -308,11 +355,11 @@ namespace PluralKit.Bot
{
public string Name;
public ulong Id;
public Func<DiscordMessage, ulong?> ExtractFunc;
public Func<DiscordMessage, FuzzyExtractResult?> FuzzyExtractFunc;
public Func<Message, ulong?> ExtractFunc;
public Func<Message, FuzzyExtractResult?> FuzzyExtractFunc;
public string WebhookName;
public LoggerBot(string name, ulong id, Func<DiscordMessage, ulong?> extractFunc = null, Func<DiscordMessage, FuzzyExtractResult?> fuzzyExtractFunc = null, string webhookName = null)
public LoggerBot(string name, ulong id, Func<Message, ulong?> extractFunc = null, Func<Message, FuzzyExtractResult?> fuzzyExtractFunc = null, string webhookName = null)
{
Name = name;
Id = id;
@@ -324,8 +371,8 @@ namespace PluralKit.Bot
public struct FuzzyExtractResult
{
public ulong User;
public DateTimeOffset ApproxTimestamp;
public ulong User { get; set; }
public Instant ApproxTimestamp { get; set; }
}
}
}

86
PluralKit.Bot/Services/ShardInfoService.cs
View File

@@ -1,12 +1,11 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Net.WebSockets;
using System.Threading.Tasks;
using App.Metrics;
using DSharpPlus;
using DSharpPlus.EventArgs;
using Myriad.Gateway;
using NodaTime;
@@ -16,6 +15,8 @@ using Serilog;
namespace PluralKit.Bot
{
// TODO: how much of this do we need now that we have logging in the shard library?
// A lot could probably be cleaned up...
public class ShardInfoService
{
public class ShardInfo
@@ -30,10 +31,10 @@ namespace PluralKit.Bot
private readonly IMetrics _metrics;
private readonly ILogger _logger;
private readonly DiscordShardedClient _client;
private readonly Dictionary<int, ShardInfo> _shardInfo = new Dictionary<int, ShardInfo>();
private readonly Cluster _client;
private readonly Dictionary<int, ShardInfo> _shardInfo = new();
public ShardInfoService(ILogger logger, DiscordShardedClient client, IMetrics metrics)
public ShardInfoService(ILogger logger, Cluster client, IMetrics metrics)
{
_client = client;
_metrics = metrics;
@@ -44,7 +45,7 @@ namespace PluralKit.Bot
{
// We initialize this before any shards are actually created and connected
// This means the client won't know the shard count, so we attach a listener every time a shard gets connected
_client.SocketOpened += (_, __) => RefreshShardList();
_client.ShardCreated += InitializeShard;
}
private void ReportShardStatus()
@@ -54,44 +55,40 @@ namespace PluralKit.Bot
_metrics.Measure.Gauge.SetValue(BotMetrics.ShardsConnected, _shardInfo.Count(s => s.Value.Connected));
}
private async Task RefreshShardList()
private void InitializeShard(Shard shard)
{
// This callback doesn't actually receive the shard that was opening, so we just try to check we have 'em all (so far)
foreach (var (id, shard) in _client.ShardClients)
// Get or insert info in the client dict
if (_shardInfo.TryGetValue(shard.ShardId, out var info))
{
// Get or insert info in the client dict
if (_shardInfo.TryGetValue(id, out var info))
{
// Skip adding listeners if we've seen this shard & already added listeners to it
if (info.HasAttachedListeners) continue;
} else _shardInfo[id] = info = new ShardInfo();
// Skip adding listeners if we've seen this shard & already added listeners to it
if (info.HasAttachedListeners)
return;
} else _shardInfo[shard.ShardId] = info = new ShardInfo();
// Call our own SocketOpened listener manually (and then attach the listener properly)
SocketOpened(shard);
shard.SocketOpened += () => SocketOpened(shard);
// Register listeners for new shards
_logger.Information("Attaching listeners to new shard #{Shard}", shard.ShardId);
shard.Resumed += () => Resumed(shard);
shard.Ready += () => Ready(shard);
shard.SocketClosed += (closeStatus, message) => SocketClosed(shard, closeStatus, message);
shard.HeartbeatReceived += latency => Heartbeated(shard, latency);
// Call our own SocketOpened listener manually (and then attach the listener properly)
await SocketOpened(shard, null);
shard.SocketOpened += SocketOpened;
// Register listeners for new shards
_logger.Information("Attaching listeners to new shard #{Shard}", shard.ShardId);
shard.Resumed += Resumed;
shard.Ready += Ready;
shard.SocketClosed += SocketClosed;
shard.Heartbeated += Heartbeated;
// Register that we've seen it
info.HasAttachedListeners = true;
}
// Register that we've seen it
info.HasAttachedListeners = true;
}
private Task SocketOpened(DiscordClient shard, SocketEventArgs _)
private void SocketOpened(Shard shard)
{
// We do nothing else here, since this kinda doesn't mean *much*? It's only really started once we get Ready/Resumed
// And it doesn't get fired first time around since we don't have time to add the event listener before it's fired'
_logger.Information("Shard #{Shard} opened socket", shard.ShardId);
return Task.CompletedTask;
}
private ShardInfo TryGetShard(DiscordClient shard)
private ShardInfo TryGetShard(Shard shard)
{
// If we haven't seen this shard before, add it to the dict!
// I don't think this will ever occur since the shard number is constant up-front and we handle those
@@ -101,7 +98,7 @@ namespace PluralKit.Bot
return info;
}
private Task Resumed(DiscordClient shard, ReadyEventArgs e)
private void Resumed(Shard shard)
{
_logger.Information("Shard #{Shard} resumed connection", shard.ShardId);
@@ -109,10 +106,9 @@ namespace PluralKit.Bot
// info.LastConnectionTime = SystemClock.Instance.GetCurrentInstant();
info.Connected = true;
ReportShardStatus();
return Task.CompletedTask;
}
private Task Ready(DiscordClient shard, ReadyEventArgs e)
private void Ready(Shard shard)
{
_logger.Information("Shard #{Shard} sent Ready event", shard.ShardId);
@@ -120,30 +116,28 @@ namespace PluralKit.Bot
info.LastConnectionTime = SystemClock.Instance.GetCurrentInstant();
info.Connected = true;
ReportShardStatus();
return Task.CompletedTask;
}
private Task SocketClosed(DiscordClient shard, SocketCloseEventArgs e)
private void SocketClosed(Shard shard, WebSocketCloseStatus closeStatus, string message)
{
_logger.Warning("Shard #{Shard} disconnected ({CloseCode}: {CloseMessage})", shard.ShardId, e.CloseCode, e.CloseMessage);
_logger.Warning("Shard #{Shard} disconnected ({CloseCode}: {CloseMessage})",
shard.ShardId, closeStatus, message);
var info = TryGetShard(shard);
info.DisconnectionCount++;
info.Connected = false;
ReportShardStatus();
return Task.CompletedTask;
}
private Task Heartbeated(DiscordClient shard, HeartbeatEventArgs e)
private void Heartbeated(Shard shard, TimeSpan latency)
{
var latency = Duration.FromMilliseconds(e.Ping);
_logger.Information("Shard #{Shard} received heartbeat (latency: {Latency} ms)", shard.ShardId, latency.Milliseconds);
_logger.Information("Shard #{Shard} received heartbeat (latency: {Latency} ms)",
shard.ShardId, latency.Milliseconds);
var info = TryGetShard(shard);
info.LastHeartbeatTime = e.Timestamp.ToInstant();
info.LastHeartbeatTime = SystemClock.Instance.GetCurrentInstant();
info.Connected = true;
info.ShardLatency = latency;
return Task.CompletedTask;
info.ShardLatency = latency.ToDuration();
}
public ShardInfo GetShardInfo(Shard shard) => _shardInfo[shard.ShardId];