liz/PluralKit
liz/PluralKit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add logger bot cleanup support

Browse Source
This commit is contained in:
Ske 2020-02-15 00:12:03 +01:00
parent 2268a33600
commit e9cc8ed424
10 changed files with 342 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
PluralKit.Bot/Bot.cs
View File

@ -234,6 +234,7 @@ namespace PluralKit.Bot
private Scope _sentryScope;
private ProxyCache _cache;
private LastMessageCacheService _lastMessageCache;
private LoggerCleanService _loggerClean;
// We're defining in the Autofac module that this class is instantiated with one instance per event
// This means that the HandleMessage function will either be called once, or not at all
@ -241,7 +242,7 @@ namespace PluralKit.Bot
// hence, we just store it in a local variable, ignoring it entirely if it's null.
private IUserMessage _msg = null;
public PKEventHandler(ProxyService proxy, ILogger logger, IMetrics metrics, DiscordShardedClient client, DbConnectionFactory connectionFactory, ILifetimeScope services, CommandTree tree, Scope sentryScope, ProxyCache cache, LastMessageCacheService lastMessageCache)
public PKEventHandler(ProxyService proxy, ILogger logger, IMetrics metrics, DiscordShardedClient client, DbConnectionFactory connectionFactory, ILifetimeScope services, CommandTree tree, Scope sentryScope, ProxyCache cache, LastMessageCacheService lastMessageCache, LoggerCleanService loggerClean)
{
_proxy = proxy;
_logger = logger;
@ -253,6 +254,7 @@ namespace PluralKit.Bot
_sentryScope = sentryScope;
_cache = cache;
_lastMessageCache = lastMessageCache;
_loggerClean = loggerClean;
}
public async Task HandleMessage(SocketMessage arg)
@ -266,9 +268,17 @@ namespace PluralKit.Bot
// Ignore system messages (member joined, message pinned, etc)
var msg = arg as SocketUserMessage;
if (msg == null) return;
// Ignore bot messages
if (msg.Author.IsBot || msg.Author.IsWebhook) return;
// Fetch information about the guild early, as we need it for the logger cleanup
GuildConfig cachedGuild = default; // todo: is this default correct?
if (msg.Channel is ITextChannel textChannel) cachedGuild = await _cache.GetGuildDataCached(textChannel.GuildId);
// Pass guild bot/WH messages onto the logger cleanup service, but otherwise ignore
if ((msg.Author.IsBot || msg.Author.IsWebhook) && msg.Channel is ITextChannel)
{
await _loggerClean.HandleLoggerBotCleanup(arg, cachedGuild);
return;
}
// Add message info as Sentry breadcrumb
_msg = msg;
@ -284,9 +294,7 @@ namespace PluralKit.Bot
// Add to last message cache
_lastMessageCache.AddMessage(arg.Channel.Id, arg.Id);
// We fetch information about the sending account *and* guild from the cache
GuildConfig cachedGuild = default; // todo: is this default correct?
if (msg.Channel is ITextChannel textChannel) cachedGuild = await _cache.GetGuildDataCached(textChannel.GuildId);
// We fetch information about the sending account from the cache
var cachedAccount = await _cache.GetAccountDataCached(msg.Author.Id);
// this ^ may be null, do remember that down the line

3
PluralKit.Bot/Commands/CommandTree.cs
View File

@ -56,6 +56,7 @@ namespace PluralKit.Bot
public static Command LogChannel = new Command("log channel", "log channel <channel>", "Designates a channel to post proxied messages to");
public static Command LogEnable = new Command("log enable", "log enable all|<channel> [channel 2] [channel 3...]", "Enables message logging in certain channels");
public static Command LogDisable = new Command("log disable", "log disable all|<channel> [channel 2] [channel 3...]", "Disables message logging in certain channels");
public static Command LogClean = new Command("logclean", "logclean [on|off]", "Toggles whether to clean up other bots' log channels");
public static Command BlacklistAdd = new Command("blacklist add", "blacklist add all|<channel> [channel 2] [channel 3...]", "Adds certain channels to the proxy blacklist");
public static Command BlacklistRemove = new Command("blacklist remove", "blacklist remove all|<channel> [channel 2] [channel 3...]", "Removes certain channels from the proxy blacklist");
public static Command Invite = new Command("invite", "invite", "Gets a link to invite PluralKit to other servers");
@ -127,6 +128,8 @@ namespace PluralKit.Bot
else if (ctx.Match("disable", "off"))
return ctx.Execute<ServerConfig>(LogDisable, m => m.SetLogEnabled(ctx, false));
else return PrintCommandExpectedError(ctx, LogCommands);
if (ctx.Match("logclean"))
return ctx.Execute<ServerConfig>(LogClean, m => m.SetLogCleanup(ctx));
if (ctx.Match("blacklist", "bl"))
if (ctx.Match("enable", "on", "add", "deny"))
return ctx.Execute<ServerConfig>(BlacklistAdd, m => m.SetBlacklisted(ctx, true));

37
PluralKit.Bot/Commands/ServerConfig.cs
View File

@ -11,9 +11,11 @@ namespace PluralKit.Bot
public class ServerConfig
{
private IDataStore _data;
public ServerConfig(IDataStore data)
private LoggerCleanService _cleanService;
public ServerConfig(IDataStore data, LoggerCleanService cleanService)
{
_data = data;
_cleanService = cleanService;
}
public async Task SetLogChannel(Context ctx)
@ -84,5 +86,38 @@ namespace PluralKit.Bot
await _data.SaveGuildConfig(guildCfg);
await ctx.Reply($"{Emojis.Success} Channels {(onBlacklist ? "added to" : "removed from")} the proxy blacklist.");
}
public async Task SetLogCleanup(Context ctx)
{
ctx.CheckGuildContext().CheckAuthorPermission(GuildPermission.ManageGuild, "Manage Server");
var guildCfg = await _data.GetOrCreateGuildConfig(ctx.Guild.Id);
var botList = string.Join(", ", _cleanService.Bots.Select(b => b.Name).OrderBy(x => x.ToLowerInvariant()));
if (ctx.Match("enable", "on", "yes"))
{
guildCfg.LogCleanupEnabled = true;
await _data.SaveGuildConfig(guildCfg);
await ctx.Reply($"{Emojis.Success} Log cleanup has been **enabled** for this server. Messages deleted by PluralKit will now be cleaned up from logging channels managed by the following bots:\n- **{botList}**\n\n{Emojis.Note} Make sure PluralKit has the **Manage Messages** permission in the channels in question.\n{Emojis.Note} Also, make sure to blacklist the logging channel itself from the bots in question to prevent conflicts.");
}
else if (ctx.Match("disable", "off", "no"))
{
guildCfg.LogCleanupEnabled = false;
await _data.SaveGuildConfig(guildCfg);
await ctx.Reply($"{Emojis.Success} Log cleanup has been **disabled** for this server.");
}
else
{
var eb = new EmbedBuilder()
.WithTitle("Log cleanup settings")
.AddField("Supported bots", botList);
if (guildCfg.LogCleanupEnabled)
eb.WithDescription("Log cleanup is currently **on** for this server. To disable it, type `pk;logclean off`.");
else
eb.WithDescription("Log cleanup is currently **off** for this server. To enable it, type `pk;logclean on`.");
await ctx.Reply(embed: eb.Build());
}
}
}
}

1
PluralKit.Bot/Modules.cs
View File

@ -66,6 +66,7 @@ namespace PluralKit.Bot
builder.RegisterType<CpuStatService>().AsSelf().SingleInstance();
builder.RegisterType<PeriodicStatCollector>().AsSelf().SingleInstance();
builder.RegisterType<LastMessageCacheService>().AsSelf().SingleInstance();
builder.RegisterType<LoggerCleanService>().AsSelf().SingleInstance();
// Sentry stuff
builder.Register(_ => new Scope(null)).AsSelf().InstancePerLifetimeScope();

246
PluralKit.Bot/Services/LoggerCleanService.cs Normal file
View File

@ -0,0 +1,246 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using Dapper;
using Discord;
using Discord.WebSocket;
using PluralKit.Core;
namespace PluralKit.Bot
{
public class LoggerCleanService
{
private static Regex _basicRegex = new Regex("(\\d{17,19})");
private static Regex _dynoRegex = new Regex("Message ID: (\\d{17,19})");
private static Regex _carlRegex = new Regex("ID: (\\d{17,19})");
private static Regex _circleRegex = new Regex("\\(`(\\d{17,19})`\\)");
private static Regex _loggerARegex = new Regex("Message = (\\d{17,19})");
private static Regex _loggerBRegex = new Regex("MessageID:(\\d{17,19})");
private static Regex _auttajaRegex = new Regex("Message (\\d{17,19}) deleted");
private static Regex _mantaroRegex = new Regex("Message \\(?ID:? (\\d{17,19})\\)? created by .* in channel .* was deleted\\.");
private static readonly Dictionary<ulong, LoggerBot> _bots = new[]
{
// These are NOT supported at the moment, since they don't put the deleted message ID in the log
new LoggerBot("Carl-bot", 23514896210395136, fuzzyExtractFunc: ExtractCarlBot, webhookName: "Carl-bot Logging"),
new LoggerBot("Circle", 497196352866877441, fuzzyExtractFunc: ExtractCircle),
// There are two "Logger"s. They seem to be entirely unrelated. Don't ask.
new LoggerBot("Logger#6088", 298822483060981760 , ExtractLoggerA, webhookName: "Logger"),
new LoggerBot("Logger#6278", 327424261180620801, ExtractLoggerB),
new LoggerBot("Dyno", 155149108183695360, ExtractDyno, webhookName: "Dyno"),
new LoggerBot("Auttaja", 242730576195354624, ExtractAuttaja),
new LoggerBot("GenericBot", 295329346590343168, ExtractGenericBot),
new LoggerBot("blargbot", 134133271750639616, ExtractBlargBot),
new LoggerBot("Mantaro", 213466096718708737, ExtractMantaro),
}.ToDictionary(b => b.Id);
private static readonly Dictionary<string, LoggerBot> _botsByWebhookName = _bots.Values
.Where(b => b.WebhookName != null)
.ToDictionary(b => b.WebhookName);
private DbConnectionFactory _db;
private DiscordShardedClient _client;
public LoggerCleanService(DbConnectionFactory db, DiscordShardedClient client)
{
_db = db;
_client = client;
}
public ICollection<LoggerBot> Bots => _bots.Values;
public async ValueTask HandleLoggerBotCleanup(SocketMessage msg, GuildConfig cachedGuild)
{
// Bail if not enabled, or if we don't have permission here
if (!cachedGuild.LogCleanupEnabled) return;
if (!(msg.Channel is SocketTextChannel channel)) return;
if (!channel.Guild.GetUser(_client.CurrentUser.Id).GetPermissions(channel).ManageMessages) return;
// If this message is from a *webhook*, check if the name matches one of the bots we know
// TODO: do we need to do a deeper webhook origin check, or would that be too hard on the rate limit?
// If it's from a *bot*, check the bot ID to see if we know it.
LoggerBot bot = null;
if (msg.Author.IsWebhook) _botsByWebhookName.TryGetValue(msg.Author.Username, out bot);
else if (msg.Author.IsBot) _bots.TryGetValue(msg.Author.Id, out bot);
// If we didn't find anything before, or what we found is an unsupported bot, bail
if (bot == null) return;
// We try two ways of extracting the actual message, depending on the bots
if (bot.FuzzyExtractFunc != null)
{
// Some bots (Carl, Circle, etc) only give us a user ID and a rough timestamp, so we try our best to
// "cross-reference" those with the message DB. We know the deletion event happens *after* the message
// was sent, so we're checking for any messages sent in the same guild within 3 seconds before the
// delete event timestamp, which is... good enough, I think? Potential for false positives and negatives
// either way but shouldn't be too much, given it's constrained by user ID and guild.
var fuzzy = bot.FuzzyExtractFunc(msg);
if (fuzzy == null) return;
using var conn = await _db.Obtain();
var mid = await conn.QuerySingleOrDefaultAsync<ulong?>(
"select mid from messages where sender = @User and mid > @ApproxID and guild = @Guild",
new
{
fuzzy.Value.User,
Guild = (msg.Channel as ITextChannel)?.GuildId ?? 0,
ApproxId = SnowflakeUtils.ToSnowflake(fuzzy.Value.ApproxTimestamp - TimeSpan.FromSeconds(3))
});
if (mid == null) return; // If we didn't find a corresponding message, bail
// Otherwise, we can *reasonably assume* that this is a logged deletion, so delete the log message.
await msg.DeleteAsync();
}
else if (bot.ExtractFunc != null)
{
// Other bots give us the message ID itself, and we can just extract that from the database directly.
var extractedId = bot.ExtractFunc(msg);
if (extractedId == null) return; // If we didn't find anything, bail.
using var conn = await _db.Obtain();
// We do this through an inline query instead of through DataStore since we don't need all the joins it does
var mid = await conn.QuerySingleOrDefaultAsync<ulong?>("select mid from messages where original_mid = @Mid", new {Mid = extractedId.Value});
if (mid == null) return;
// If we've gotten this far, we found a logged deletion of a trigger message. Just yeet it!
await msg.DeleteAsync();
} // else should not happen, but idk, it might
}
private static ulong? ExtractAuttaja(SocketMessage msg)
{
// Auttaja has an optional "compact mode" that logs without embeds
// That one puts the ID in the message content, non-compact puts it in the embed description.
// Regex also checks that this is a deletion.
var stringWithId = msg.Content ?? msg.Embeds.FirstOrDefault()?.Description;
if (stringWithId == null) return null;
var match = _auttajaRegex.Match(stringWithId);
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractDyno(SocketMessage msg)
{
// Embed *description* contains "Message sent by [mention] deleted in [channel]", contains message ID in footer per regex
var embed = msg.Embeds.FirstOrDefault();
if (embed?.Footer == null || !(embed.Description?.Contains("deleted in") ?? false)) return null;
var match = _dynoRegex.Match(embed.Footer.Value.Text ?? "");
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractLoggerA(SocketMessage msg)
{
// This is for Logger#6088 (298822483060981760), distinct from Logger#6278 (327424261180620801).
// Embed contains title "Message deleted in [channel]", and an ID field containing both message and user ID (see regex).
var embed = msg.Embeds.FirstOrDefault();
if (embed == null) return null;
if (!embed.Description.StartsWith("Message deleted in")) return null;
var idField = embed.Fields.FirstOrDefault(f => f.Name == "ID");
if (idField.Value == null) return null; // "OrDefault" = all-null object
var match = _loggerARegex.Match(idField.Value);
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractLoggerB(SocketMessage msg)
{
// This is for Logger#6278 (327424261180620801), distinct from Logger#6088 (298822483060981760).
// Embed title ends with "A Message Was Deleted!", footer contains message ID as per regex.
var embed = msg.Embeds.FirstOrDefault();
if (embed?.Footer == null || !(embed.Title?.EndsWith("A Message Was Deleted!") ?? false)) return null;
var match = _loggerBRegex.Match(embed.Footer.Value.Text ?? "");
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractGenericBot(SocketMessage msg)
{
// Embed, title is "Message Deleted", ID plain in footer.
var embed = msg.Embeds.FirstOrDefault();
if (embed?.Footer == null || !(embed.Title?.Contains("Message Deleted") ?? false)) return null;
var match = _basicRegex.Match(embed.Footer.Value.Text ?? "");
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractBlargBot(SocketMessage msg)
{
// Embed, title ends with "Message Deleted", contains ID plain in a field.
var embed = msg.Embeds.FirstOrDefault();
if (embed == null || !(embed.Title?.EndsWith("Message Deleted") ?? false)) return null;
var field = embed.Fields.FirstOrDefault(f => f.Name == "Message ID");
var match = _basicRegex.Match(field.Value ?? "");
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static ulong? ExtractMantaro(SocketMessage msg)
{
// Plain message, "Message (ID: [id]) created by [user] (ID: [id]) in channel [channel] was deleted.
if (!(msg.Content?.Contains("was deleted.") ?? false)) return null;
var match = _mantaroRegex.Match(msg.Content);
return match.Success ? ulong.Parse(match.Groups[1].Value) : (ulong?) null;
}
private static FuzzyExtractResult? ExtractCarlBot(SocketMessage msg)
{
// Embed, title is "Message deleted in [channel], **user** ID in the footer, timestamp as, well, timestamp in embed.
// This is the *deletion* timestamp, which we can assume is a couple seconds at most after the message was originally sent
var embed = msg.Embeds.FirstOrDefault();
if (embed?.Footer == null || embed.Timestamp == null || !(embed.Title?.StartsWith("Message deleted in") ?? false)) return null;
var match = _carlRegex.Match(embed.Footer.Value.Text ?? "");
return match.Success
? new FuzzyExtractResult { User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = embed.Timestamp.Value }
: (FuzzyExtractResult?) null;
}
private static FuzzyExtractResult? ExtractCircle(SocketMessage msg)
{
// Like Auttaja, Circle has both embed and compact modes, but the regex works for both.
// Compact: "Message from [user] ([id]) deleted in [channel]", no timestamp (use message time)
// Embed: Message Author field: "[user] ([id])", then an embed timestamp
string stringWithId = msg.Content;
if (msg.Embeds.Count > 0)
{
var embed = msg.Embeds.First();
if (embed.Author?.Name == null || !embed.Author.Value.Name.StartsWith("Message Deleted in")) return null;
var field = embed.Fields.FirstOrDefault(f => f.Name == "Message Author");
if (field.Value == null) return null;
stringWithId = field.Value;
}
if (stringWithId == null) return null;
var match = _circleRegex.Match(stringWithId);
return match.Success
? new FuzzyExtractResult {User = ulong.Parse(match.Groups[1].Value), ApproxTimestamp = msg.Timestamp}
: (FuzzyExtractResult?) null;
}
public class LoggerBot
{
public string Name;
public ulong Id;
public Func<SocketMessage, ulong?> ExtractFunc;
public Func<SocketMessage, FuzzyExtractResult?> FuzzyExtractFunc;
public string WebhookName;
public LoggerBot(string name, ulong id, Func<SocketMessage, ulong?> extractFunc = null, Func<SocketMessage, FuzzyExtractResult?> fuzzyExtractFunc = null, string webhookName = null)
{
Name = name;
Id = id;
FuzzyExtractFunc = fuzzyExtractFunc;
ExtractFunc = extractFunc;
WebhookName = webhookName;
}
}
public struct FuzzyExtractResult
{
public ulong User;
public DateTimeOffset ApproxTimestamp;
}
}
}

3
PluralKit.Core/Migrations/5.sql Normal file
View File

@ -0,0 +1,3 @@
-- SCHEMA VERSION 4: 2020-02-14
alter table servers add column log_cleanup_enabled bool not null default false;
update info set schema_version = 5;

1
PluralKit.Core/Services/IDataStore.cs
View File

@ -67,6 +67,7 @@ namespace PluralKit.Core {
public ulong? LogChannel { get; set; }
public ISet<ulong> LogBlacklist { get; set; }
public ISet<ulong> Blacklist { get; set; }
public bool LogCleanupEnabled { get; set; }
}
public class SystemGuildSettings

8
PluralKit.Core/Services/PostgresDataStore.cs
View File

@ -363,6 +363,8 @@ namespace PluralKit.Core {
public ulong? LogChannel { get; set; }
public long[] LogBlacklist { get; set; }
public long[] Blacklist { get; set; }
public bool LogCleanupEnabled { get; set; }
public GuildConfig Into() =>
new GuildConfig
@ -370,7 +372,8 @@ namespace PluralKit.Core {
Id = Id,
LogChannel = LogChannel,
LogBlacklist = new HashSet<ulong>(LogBlacklist?.Select(c => (ulong) c) ?? new ulong[] {}),
Blacklist = new HashSet<ulong>(Blacklist?.Select(c => (ulong) c) ?? new ulong[]{})
Blacklist = new HashSet<ulong>(Blacklist?.Select(c => (ulong) c) ?? new ulong[]{}),
LogCleanupEnabled = LogCleanupEnabled
};
}
@ -388,10 +391,11 @@ namespace PluralKit.Core {
public async Task SaveGuildConfig(GuildConfig cfg)
{
using (var conn = await _conn.Obtain())
await conn.ExecuteAsync("insert into servers (id, log_channel, log_blacklist, blacklist) values (@Id, @LogChannel, @LogBlacklist, @Blacklist) on conflict (id) do update set log_channel = @LogChannel, log_blacklist = @LogBlacklist, blacklist = @Blacklist", new
await conn.ExecuteAsync("insert into servers (id, log_channel, log_blacklist, blacklist, log_cleanup_enabled) values (@Id, @LogChannel, @LogBlacklist, @Blacklist, @LogCleanupEnabled) on conflict (id) do update set log_channel = @LogChannel, log_blacklist = @LogBlacklist, blacklist = @Blacklist, log_cleanup_enabled = @LogCleanupEnabled", new
{
cfg.Id,
cfg.LogChannel,
cfg.LogCleanupEnabled,
LogBlacklist = cfg.LogBlacklist.Select(c => (long) c).ToList(),
Blacklist = cfg.Blacklist.Select(c => (long) c).ToList()
});

2
PluralKit.Core/Services/SchemaService.cs
View File

@ -11,7 +11,7 @@ using Serilog;
namespace PluralKit.Core {
public class SchemaService
{
private const int TargetSchemaVersion = 4;
private const int TargetSchemaVersion = 5;
private DbConnectionFactory _conn;
private ILogger _logger;

30
docs/2-user-guide.md
View File

@ -468,6 +468,36 @@ This requires you to have the *Manage Server* permission on the server. For exam
To disable logging, use the `pk;log` command with no channel name.
### Channel blacklisting
It's possible to blacklist a channel from being used for proxying. To do so, use the `pk;blacklist` command, for examplle:
pk;blacklist add #admin-channel #mod-channel #welcome
pk;blacklist add all
pk;blacklist remove #general-two
pk;blacklist remove all
This requires you to have the *Manage Server* permission on the server.
### Log cleanup
Many servers use *logger bots* for keeping track of edited and deleted messages, nickname changes, and other server events. Because
PluralKit deletes messages as part of proxying, this can often clutter up these logs. To remedy this, PluralKit can delete those
log messages from the logger bots. To enable this, use the following command:
pk;logclean on
This requires you to have the *Manage Server* permission on the server. At the moment, log cleanup works with the following bots:
- Auttaja
- blargbot
- Carl-bot
- Circle
- Dyno
- GenericBot
- Logger (#6088 and #6278)
If you want support for another logging bot, [let me know on the support server](https://discord.gg/PczBt78).
Another alternative is to use the **Gabby Gums** logging bot - an invite link for which can be found [on Gabby Gums' support server](https://discord.gg/Xwhk89T).
## Importing and exporting data
If you're a user of another proxy bot (eg. Tupperbox), or you want to import a saved system backup, you can use the importing and exporting commands.