From ba441a15cc64377c3f988610c7652793805fd989 Mon Sep 17 00:00:00 2001 From: Ske Date: Fri, 12 Jun 2020 20:29:50 +0200 Subject: [PATCH] Too many refactors in one: - Allowed adding ephemeral(ish) views and functions - Moved message_count to a concrete database field - Moved most proxy logic to a stored procedure - Moved database files around and refactored schema manager --- PluralKit.API/Startup.cs | 2 +- PluralKit.Bot/Commands/SystemList.cs | 8 +- PluralKit.Bot/Handlers/MessageCreated.cs | 2 +- PluralKit.Bot/Handlers/MessageEdited.cs | 2 +- PluralKit.Bot/Handlers/ReactionAdded.cs | 2 +- PluralKit.Bot/Init.cs | 2 +- PluralKit.Bot/Lists/PKListMember.cs | 1 - PluralKit.Bot/Lists/SortFilterOptions.cs | 57 +++--- PluralKit.Bot/Modules.cs | 2 +- PluralKit.Bot/Proxy/Autoproxier.cs | 93 ---------- PluralKit.Bot/Proxy/ProxyMatch.cs | 4 +- PluralKit.Bot/Proxy/ProxyMatcher.cs | 69 ++++++++ PluralKit.Bot/Proxy/ProxyService.cs | 167 ++++++++++-------- PluralKit.Bot/Proxy/ProxyTagParser.cs | 49 ++--- PluralKit.Bot/Services/EmbedService.cs | 6 +- PluralKit.Bot/Services/LogChannelService.cs | 75 ++++---- .../{ => Database}/Migrations/0.sql | 0 .../{ => Database}/Migrations/1.sql | 0 .../{ => Database}/Migrations/2.sql | 0 .../{ => Database}/Migrations/3.sql | 0 .../{ => Database}/Migrations/4.sql | 0 .../{ => Database}/Migrations/5.sql | 0 .../{ => Database}/Migrations/6.sql | 0 PluralKit.Core/Database/Migrations/7.sql | 33 ++++ PluralKit.Core/Database/ProxyMember.cs | 26 +++ PluralKit.Core/Database/Schemas.cs | 99 +++++++++++ PluralKit.Core/Database/clean.sql | 3 + PluralKit.Core/Database/functions.sql | 85 +++++++++ PluralKit.Core/Database/views.sql | 29 +++ PluralKit.Core/Models/PKMember.cs | 1 + PluralKit.Core/Modules.cs | 2 +- PluralKit.Core/PluralKit.Core.csproj | 3 +- .../PluralKit.Core.csproj.DotSettings | 1 + PluralKit.Core/Services/DataFileService.cs | 3 +- PluralKit.Core/Services/IDataStore.cs | 23 +-- PluralKit.Core/Services/PostgresDataStore.cs | 29 +-- PluralKit.Core/Services/SchemaService.cs | 74 -------- 37 files changed, 554 insertions(+), 398 deletions(-) delete mode 100644 PluralKit.Bot/Proxy/Autoproxier.cs create mode 100644 PluralKit.Bot/Proxy/ProxyMatcher.cs rename PluralKit.Core/{ => Database}/Migrations/0.sql (100%) rename PluralKit.Core/{ => Database}/Migrations/1.sql (100%) rename PluralKit.Core/{ => Database}/Migrations/2.sql (100%) rename PluralKit.Core/{ => Database}/Migrations/3.sql (100%) rename PluralKit.Core/{ => Database}/Migrations/4.sql (100%) rename PluralKit.Core/{ => Database}/Migrations/5.sql (100%) rename PluralKit.Core/{ => Database}/Migrations/6.sql (100%) mode change 100755 => 100644 create mode 100644 PluralKit.Core/Database/Migrations/7.sql create mode 100644 PluralKit.Core/Database/ProxyMember.cs create mode 100644 PluralKit.Core/Database/Schemas.cs create mode 100644 PluralKit.Core/Database/clean.sql create mode 100644 PluralKit.Core/Database/functions.sql create mode 100644 PluralKit.Core/Database/views.sql delete mode 100644 PluralKit.Core/Services/SchemaService.cs diff --git a/PluralKit.API/Startup.cs b/PluralKit.API/Startup.cs index 5f6b3c94..7ea9ad8a 100644 --- a/PluralKit.API/Startup.cs +++ b/PluralKit.API/Startup.cs @@ -86,7 +86,7 @@ namespace PluralKit.API // This method gets called by the runtime. Use this method to configure the HTTP request pipeline. public void Configure(IApplicationBuilder app, IWebHostEnvironment env) { - SchemaService.Initialize(); + Schemas.Initialize(); if (env.IsDevelopment()) { diff --git a/PluralKit.Bot/Commands/SystemList.cs b/PluralKit.Bot/Commands/SystemList.cs index 4ce59c28..da2cccc4 100644 --- a/PluralKit.Bot/Commands/SystemList.cs +++ b/PluralKit.Bot/Commands/SystemList.cs @@ -50,14 +50,16 @@ namespace PluralKit.Bot private async Task> GetMemberList(PKSystem target, SortFilterOptions opts) { - using var conn = await _db.Obtain(); + await using var conn = await _db.Obtain(); var query = opts.BuildQuery(); var args = new {System = target.Id, opts.Filter}; - + _logger.Debug("Executing sort/filter query `{Query}` with arguments {Args}", query, args); + var timeBefore = _clock.GetCurrentInstant(); var results = (await conn.QueryAsync(query, args)).ToList(); var timeAfter = _clock.GetCurrentInstant(); - _logger.Debug("Executing sort/filter query `{Query}` with arguments {Args} returning {ResultCount} results in {QueryTime}", query, args, results.Count, timeAfter - timeBefore); + + _logger.Debug("Executed sort/filter query `{Query}` returning {ResultCount} results in {QueryTime}", query, results.Count, timeAfter - timeBefore); return results; } diff --git a/PluralKit.Bot/Handlers/MessageCreated.cs b/PluralKit.Bot/Handlers/MessageCreated.cs index 96d34c42..34422a9a 100644 --- a/PluralKit.Bot/Handlers/MessageCreated.cs +++ b/PluralKit.Bot/Handlers/MessageCreated.cs @@ -118,7 +118,7 @@ namespace PluralKit.Bot try { - await _proxy.HandleMessageAsync(evt.Client, cachedGuild, cachedAccount, msg, allowAutoproxy: true); + await _proxy.HandleIncomingMessage(evt.Message, allowAutoproxy: true); } catch (PKError e) { diff --git a/PluralKit.Bot/Handlers/MessageEdited.cs b/PluralKit.Bot/Handlers/MessageEdited.cs index 476ab47d..c1cef6ce 100644 --- a/PluralKit.Bot/Handlers/MessageEdited.cs +++ b/PluralKit.Bot/Handlers/MessageEdited.cs @@ -45,7 +45,7 @@ namespace PluralKit.Bot var guild = await _proxyCache.GetGuildDataCached(evt.Channel.GuildId); // Just run the normal message handling stuff, with a flag to disable autoproxying - await _proxy.HandleMessageAsync(evt.Client, guild, account, evt.Message, allowAutoproxy: false); + await _proxy.HandleIncomingMessage(evt.Message, allowAutoproxy: false); } } } \ No newline at end of file diff --git a/PluralKit.Bot/Handlers/ReactionAdded.cs b/PluralKit.Bot/Handlers/ReactionAdded.cs index 7752e82b..34c8005a 100644 --- a/PluralKit.Bot/Handlers/ReactionAdded.cs +++ b/PluralKit.Bot/Handlers/ReactionAdded.cs @@ -58,7 +58,7 @@ namespace PluralKit.Bot private async ValueTask HandleDeleteReaction(MessageReactionAddEventArgs evt, FullMessage msg) { - if (evt.Channel.BotHasAllPermissions(Permissions.ManageMessages)) return; + if (!evt.Channel.BotHasAllPermissions(Permissions.ManageMessages)) return; // Can only delete your own message if (msg.Message.Sender != evt.User.Id) return; diff --git a/PluralKit.Bot/Init.cs b/PluralKit.Bot/Init.cs index f7d4aacf..e815bf36 100644 --- a/PluralKit.Bot/Init.cs +++ b/PluralKit.Bot/Init.cs @@ -35,7 +35,7 @@ namespace PluralKit.Bot // "Connect to the database" (ie. set off database migrations and ensure state) logger.Information("Connecting to database"); - await services.Resolve().ApplyMigrations(); + await services.Resolve().InitializeDatabase(); // Init the bot instance itself, register handlers and such to the client before beginning to connect logger.Information("Initializing bot"); diff --git a/PluralKit.Bot/Lists/PKListMember.cs b/PluralKit.Bot/Lists/PKListMember.cs index 2e91efc9..a16a1566 100644 --- a/PluralKit.Bot/Lists/PKListMember.cs +++ b/PluralKit.Bot/Lists/PKListMember.cs @@ -6,7 +6,6 @@ namespace PluralKit.Bot { public class PKListMember: PKMember { - public int MessageCount { get; set; } public ulong? LastMessage { get; set; } public Instant? LastSwitchTime { get; set; } } diff --git a/PluralKit.Bot/Lists/SortFilterOptions.cs b/PluralKit.Bot/Lists/SortFilterOptions.cs index b212ad65..2422ec79 100644 --- a/PluralKit.Bot/Lists/SortFilterOptions.cs +++ b/PluralKit.Bot/Lists/SortFilterOptions.cs @@ -52,23 +52,16 @@ namespace PluralKit.Bot // For best performance, add index: // - `on switch_members using btree (member asc nulls last) include (switch);` // TODO: add a migration adding this, perhaps lumped with the rest of the DB changes (it's there in prod) - // TODO: also, this should be moved to a view, ideally // Select clause - StringBuilder query = new StringBuilder(); - query.Append("select members.*, message_info.*"); - query.Append(", (select max(switches.timestamp) from switch_members inner join switches on switches.id = switch_members.switch where switch_members.member = members.id) as last_switch_time"); - query.Append(" from members"); - - // Join here to enforce index scan on messages table by member, collect both max and count in one swoop - query.Append(" left join lateral (select count(messages.mid) as message_count, max(messages.mid) as last_message from messages where messages.member = members.id) as message_info on true"); + StringBuilder query = new StringBuilder("select * from member_list"); // Filtering - query.Append(" where members.system = @System"); + query.Append(" where system = @System"); query.Append(PrivacyFilter switch { - PrivacyFilter.PrivateOnly => $" and members.member_privacy = {(int) PrivacyLevel.Private}", - PrivacyFilter.PublicOnly => $" and members.member_privacy = {(int) PrivacyLevel.Public}", + PrivacyFilter.PrivateOnly => $" and member_privacy = {(int) PrivacyLevel.Private}", + PrivacyFilter.PublicOnly => $" and member_privacy = {(int) PrivacyLevel.Public}", _ => "" }); @@ -78,28 +71,12 @@ namespace PluralKit.Bot // Use position rather than ilike to not bother with escaping and such query.Append(" and ("); query.Append( - "position(lower(@Filter) in lower(members.name)) > 0 or position(lower(@Filter) in lower(coalesce(members.display_name, ''))) > 0"); + "position(lower(@Filter) in lower(name)) > 0 or position(lower(@Filter) in lower(coalesce(display_name, ''))) > 0"); if (SearchInDescription) - query.Append(" or position(lower(@Filter) in lower(coalesce(members.description, ''))) > 0"); + query.Append(" or position(lower(@Filter) in lower(coalesce(description, ''))) > 0"); query.Append(")"); } - - // Order clause - query.Append(SortProperty switch - { - // Name/DN order needs `collate "C"` to match legacy .NET behavior (affects sorting of emojis, etc) - SortProperty.Name => " order by members.name collate \"C\"", - SortProperty.DisplayName => " order by members.display_name, members.name collate \"C\"", - SortProperty.Hid => " order by members.hid", - SortProperty.CreationDate => " order by members.created", - SortProperty.Birthdate => - " order by extract(month from members.birthday), extract(day from members.birthday)", - SortProperty.MessageCount => " order by message_count", - SortProperty.LastMessage => " order by last_message", - SortProperty.LastSwitch => " order by last_switch_time", - _ => throw new ArgumentOutOfRangeException($"Couldn't find order clause for sort property {SortProperty}") - }); - + // Order direction var direction = SortProperty switch { @@ -110,8 +87,24 @@ namespace PluralKit.Bot _ => SortDirection.Ascending }; if (Reverse) direction = direction == SortDirection.Ascending ? SortDirection.Descending : SortDirection.Ascending; - query.Append(direction == SortDirection.Ascending ? " asc" : " desc"); - query.Append(" nulls last"); + var order = direction == SortDirection.Ascending ? "asc" : "desc"; + + // Order clause + const string fallback = "name collate \"C\" asc"; // how to handle null values + query.Append(" order by "); + query.Append(SortProperty switch + { + // Name/DN order needs `collate "C"` to match legacy .NET behavior (affects sorting of emojis, etc) + SortProperty.Name => $"name collate \"C\" {order}", + SortProperty.DisplayName => $"display_name collate \"C\" {order}, name collate \"C\" {order}", + SortProperty.Hid => $"hid {order}", + SortProperty.CreationDate => $"created {order}", + SortProperty.Birthdate => $"birthday_md {order} nulls last, {fallback}", + SortProperty.MessageCount => $"message_count {order} nulls last, {fallback}", + SortProperty.LastMessage => $"last_message {order} nulls last, {fallback}", + SortProperty.LastSwitch => $"last_switch_time {order} nulls last, {fallback}", + _ => throw new ArgumentOutOfRangeException($"Couldn't find order clause for sort property {SortProperty}") + }); return query.ToString(); } diff --git a/PluralKit.Bot/Modules.cs b/PluralKit.Bot/Modules.cs index ff8ec351..6c4478ab 100644 --- a/PluralKit.Bot/Modules.cs +++ b/PluralKit.Bot/Modules.cs @@ -84,7 +84,7 @@ namespace PluralKit.Bot .SingleInstance(); // Proxy stuff - builder.RegisterType().AsSelf().SingleInstance(); + builder.RegisterType().AsSelf().SingleInstance(); builder.RegisterType().AsSelf().SingleInstance(); // Utils diff --git a/PluralKit.Bot/Proxy/Autoproxier.cs b/PluralKit.Bot/Proxy/Autoproxier.cs deleted file mode 100644 index 312ea717..00000000 --- a/PluralKit.Bot/Proxy/Autoproxier.cs +++ /dev/null @@ -1,93 +0,0 @@ -#nullable enable -using System; -using System.Linq; -using System.Threading.Tasks; - -using NodaTime; - -using PluralKit.Core; - - -namespace PluralKit.Bot -{ - public class Autoproxier - { - public static readonly string EscapeString = @"\"; - public static readonly Duration AutoproxyExpiryTime = Duration.FromHours(6); - - private IClock _clock; - private IDataStore _data; - - public Autoproxier(IDataStore data, IClock clock) - { - _data = data; - _clock = clock; - } - - public async ValueTask TryAutoproxy(AutoproxyContext ctx) - { - if (IsEscaped(ctx.Content)) - return null; - - var member = await FindAutoproxyMember(ctx); - if (member == null) return null; - - return new ProxyMatch - { - Content = ctx.Content, - Member = member, - ProxyTags = ProxyTagsFor(member) - }; - } - - private async ValueTask FindAutoproxyMember(AutoproxyContext ctx) - { - switch (ctx.Mode) - { - case AutoproxyMode.Off: - return null; - - case AutoproxyMode.Front: - return await _data.GetFirstFronter(ctx.Account.System); - - case AutoproxyMode.Latch: - // Latch mode: find last proxied message, use *that* member - var msg = await _data.GetLastMessageInGuild(ctx.SenderId, ctx.GuildId); - if (msg == null) return null; // No message found - - // If the message is older than 6 hours, ignore it and force the sender to "refresh" a proxy - // This can be revised in the future, it's a preliminary value. - var timestamp = DiscordUtils.SnowflakeToInstant(msg.Message.Mid); - if (_clock.GetCurrentInstant() - timestamp > AutoproxyExpiryTime) return null; - - return msg.Member; - - case AutoproxyMode.Member: - // We already have the member list cached, so: - // O(n) lookup since n is small (max 1500 de jure) and we're more constrained by memory (for a dictionary) here - return ctx.Account.Members.FirstOrDefault(m => m.Id == ctx.AutoproxyMember); - - default: - throw new ArgumentOutOfRangeException($"Unknown autoproxy mode {ctx.Mode}"); - } - } - - private ProxyTag? ProxyTagsFor(PKMember member) - { - if (member.ProxyTags.Count == 0) return null; - return member.ProxyTags.First(); - } - - private bool IsEscaped(string message) => message.TrimStart().StartsWith(EscapeString); - - public struct AutoproxyContext - { - public CachedAccount Account; - public string Content; - public AutoproxyMode Mode; - public int? AutoproxyMember; - public ulong SenderId; - public ulong GuildId; - } - } -} \ No newline at end of file diff --git a/PluralKit.Bot/Proxy/ProxyMatch.cs b/PluralKit.Bot/Proxy/ProxyMatch.cs index 4a316731..d3578248 100644 --- a/PluralKit.Bot/Proxy/ProxyMatch.cs +++ b/PluralKit.Bot/Proxy/ProxyMatch.cs @@ -5,10 +5,10 @@ namespace PluralKit.Bot { public struct ProxyMatch { - public PKMember Member; + public ProxyMember Member; public string? Content; public ProxyTag? ProxyTags; - + public string? ProxyContent { get diff --git a/PluralKit.Bot/Proxy/ProxyMatcher.cs b/PluralKit.Bot/Proxy/ProxyMatcher.cs new file mode 100644 index 00000000..f2079913 --- /dev/null +++ b/PluralKit.Bot/Proxy/ProxyMatcher.cs @@ -0,0 +1,69 @@ +using System.Collections.Generic; +using System.Linq; + +using NodaTime; + +using PluralKit.Core; + +namespace PluralKit.Bot +{ + public class ProxyMatcher + { + public static readonly Duration LatchExpiryTime = Duration.FromHours(6); + + private IClock _clock; + private ProxyTagParser _parser; + + public ProxyMatcher(ProxyTagParser parser, IClock clock) + { + _parser = parser; + _clock = clock; + } + + public bool TryMatch(IReadOnlyCollection members, out ProxyMatch match, string messageContent, + bool hasAttachments, bool allowAutoproxy) + { + if (TryMatchTags(members, messageContent, hasAttachments, out match)) return true; + if (allowAutoproxy && TryMatchAutoproxy(members, messageContent, out match)) return true; + return false; + } + + private bool TryMatchTags(IReadOnlyCollection members, string messageContent, bool hasAttachments, out ProxyMatch match) + { + if (!_parser.TryMatch(members, messageContent, out match)) return false; + + // Edge case: If we got a match with blank inner text, we'd normally just send w/ attachments + // However, if there are no attachments, the user probably intended something else, so we "un-match" and proceed to autoproxy + return hasAttachments || match.Content.Length > 0; + } + + private bool TryMatchAutoproxy(IReadOnlyCollection members, string messageContent, + out ProxyMatch match) + { + match = default; + + // We handle most autoproxy logic in the database function, so we just look for the member that's marked + var info = members.FirstOrDefault(i => i.IsAutoproxyMember); + if (info == null) return false; + + // If we're in latch mode and the latch message is too old, fail the match too + if (info.AutoproxyMode == AutoproxyMode.Latch && info.LatchMessage != null) + { + var timestamp = DiscordUtils.SnowflakeToInstant(info.LatchMessage.Value); + if (_clock.GetCurrentInstant() - timestamp > LatchExpiryTime) return false; + } + + // Match succeeded, build info object and return + match = new ProxyMatch + { + Content = messageContent, + Member = info, + + // We're autoproxying, so not using any proxy tags here + // we just find the first pair of tags (if any), otherwise null + ProxyTags = info.ProxyTags.FirstOrDefault() + }; + return true; + } + } +} \ No newline at end of file diff --git a/PluralKit.Bot/Proxy/ProxyService.cs b/PluralKit.Bot/Proxy/ProxyService.cs index 2f7f7e76..34b21e03 100644 --- a/PluralKit.Bot/Proxy/ProxyService.cs +++ b/PluralKit.Bot/Proxy/ProxyService.cs @@ -1,6 +1,11 @@ using System; +using System.Collections.Generic; +using System.Data; +using System.Linq; using System.Threading.Tasks; +using Dapper; + using DSharpPlus; using DSharpPlus.Entities; using DSharpPlus.Exceptions; @@ -11,112 +16,80 @@ using Serilog; namespace PluralKit.Bot { - public class ProxyService { - public static readonly TimeSpan MessageDeletionDelay = TimeSpan.FromMilliseconds(1000); - + public class ProxyService + { + public static readonly TimeSpan MessageDeletionDelay = TimeSpan.FromMilliseconds(1000); + private LogChannelService _logChannel; + private DbConnectionFactory _db; private IDataStore _data; private ILogger _logger; private WebhookExecutorService _webhookExecutor; - private ProxyTagParser _parser; - private Autoproxier _autoproxier; - - public ProxyService(LogChannelService logChannel, IDataStore data, ILogger logger, WebhookExecutorService webhookExecutor, ProxyTagParser parser, Autoproxier autoproxier) + private readonly ProxyMatcher _matcher; + + public ProxyService(LogChannelService logChannel, IDataStore data, ILogger logger, + WebhookExecutorService webhookExecutor, DbConnectionFactory db, ProxyMatcher matcher) { _logChannel = logChannel; _data = data; _webhookExecutor = webhookExecutor; - _parser = parser; - _autoproxier = autoproxier; + _db = db; + _matcher = matcher; _logger = logger.ForContext(); } - public async Task TryGetMatch(DiscordMessage message, SystemGuildSettings systemGuildSettings, CachedAccount account, bool allowAutoproxy) + public async Task HandleIncomingMessage(DiscordMessage message, bool allowAutoproxy) { - // First, try parsing by tags - if (_parser.TryParse(message.Content, account.Members, out var tagMatch)) - { - // If the content is blank (and we don't have any attachments), someone just sent a message that happens - // to be equal to someone else's tags. This doesn't count! Proceed to autoproxy in that case. - var isEdgeCase = tagMatch.Content.Trim().Length == 0 && message.Attachments.Count == 0; - if (!isEdgeCase) return tagMatch; - } - - // Then, if AP is enabled, try finding an autoproxy match - if (allowAutoproxy) - return await _autoproxier.TryAutoproxy(new Autoproxier.AutoproxyContext - { - Account = account, - AutoproxyMember = systemGuildSettings.AutoproxyMember, - Content = message.Content, - GuildId = message.Channel.GuildId, - Mode = systemGuildSettings.AutoproxyMode, - SenderId = message.Author.Id - }); - - // Didn't find anything :( - return null; + // Quick context checks to quit early + if (!IsMessageValid(message)) return; + + // Fetch members and try to match to a specific member + var members = await FetchProxyMembers(message.Author.Id, message.Channel.GuildId); + if (!_matcher.TryMatch(members, out var match, message.Content, message.Attachments.Count > 0, + allowAutoproxy)) return; + + // Do some quick permission checks before going through with the proxy + // (do channel checks *after* checking other perms to make sure we don't spam errors when eg. channel is blacklisted) + if (!IsProxyValid(message, match)) return; + if (!await CheckBotPermissionsOrError(message.Channel)) return; + if (!CheckProxyNameBoundsOrError(match)) return; + + // Everything's in order, we can execute the proxy! + await ExecuteProxy(message, match); } - - public async Task HandleMessageAsync(DiscordClient client, GuildConfig guild, CachedAccount account, DiscordMessage message, bool allowAutoproxy) + + private async Task ExecuteProxy(DiscordMessage trigger, ProxyMatch match) { - // Early checks - if (message.Channel.Guild == null) return; - if (guild.Blacklist.Contains(message.ChannelId)) return; - var systemSettingsForGuild = account.SettingsForGuild(message.Channel.GuildId); - if (!systemSettingsForGuild.ProxyEnabled) return; - if (!await EnsureBotPermissions(message.Channel)) return; - - // Find a proxy match (either with tags or autoproxy), bail if we couldn't find any - if (!(await TryGetMatch(message, systemSettingsForGuild, account, allowAutoproxy) is { } match)) - return; + // Send the webhook + var id = await _webhookExecutor.ExecuteWebhook(trigger.Channel, match.Member.ProxyName, match.Member.ProxyAvatar, + match.Content, trigger.Attachments); - // Can't proxy a message with no content and no attachment - if (match.Content.Trim().Length == 0 && message.Attachments.Count == 0) - return; - - var memberSettingsForGuild = account.SettingsForMemberGuild(match.Member.Id, message.Channel.GuildId); - - // Find and check proxied name - var proxyName = match.Member.ProxyName(account.System.Tag, memberSettingsForGuild.DisplayName); - if (proxyName.Length < 2) throw Errors.ProxyNameTooShort(proxyName); - if (proxyName.Length > Limits.MaxProxyNameLength) throw Errors.ProxyNameTooLong(proxyName); - - // Find proxy avatar (server avatar -> member avatar -> system avatar) - var proxyAvatar = memberSettingsForGuild.AvatarUrl ?? match.Member.AvatarUrl ?? account.System.AvatarUrl; - - // Execute the webhook! - var hookMessage = await _webhookExecutor.ExecuteWebhook(message.Channel, proxyName, proxyAvatar, - await SanitizeEveryoneMaybe(message, match.ProxyContent), - message.Attachments - ); - - // Store the message in the database, and log it in the log channel (if applicable) - await _data.AddMessage(message.Author.Id, hookMessage, message.Channel.GuildId, message.Channel.Id, message.Id, match.Member); - await _logChannel.LogMessage(client, account.System, match.Member, hookMessage, message.Id, message.Channel, message.Author, match.Content, guild); + // Handle post-proxy actions + await _data.AddMessage(trigger.Author.Id, trigger.Channel.GuildId, trigger.Channel.Id, id, trigger.Id, match.Member.MemberId); + await _logChannel.LogMessage(match, trigger, id); // Wait a second or so before deleting the original message await Task.Delay(MessageDeletionDelay); - try { - await message.DeleteAsync(); + await trigger.DeleteAsync(); } catch (NotFoundException) { // If it's already deleted, we just log and swallow the exception - _logger.Warning("Attempted to delete already deleted proxy trigger message {Message}", message.Id); + _logger.Warning("Attempted to delete already deleted proxy trigger message {Message}", trigger.Id); } } - - private static async Task SanitizeEveryoneMaybe(DiscordMessage message, - string messageContents) + + private async Task> FetchProxyMembers(ulong account, ulong guild) { - var permissions = await message.Channel.PermissionsIn(message.Author); - return (permissions & Permissions.MentionEveryone) == 0 ? messageContents.SanitizeEveryone() : messageContents; + await using var conn = await _db.Obtain(); + var members = await conn.QueryAsync("proxy_info", + new {account_id = account, guild_id = guild}, commandType: CommandType.StoredProcedure); + return members.ToList(); } - private async Task EnsureBotPermissions(DiscordChannel channel) + private async Task CheckBotPermissionsOrError(DiscordChannel channel) { var permissions = channel.BotPermissions(); @@ -141,5 +114,43 @@ namespace PluralKit.Bot return true; } + + private bool CheckProxyNameBoundsOrError(ProxyMatch match) + { + var proxyName = match.Member.ProxyName; + if (proxyName.Length < 2) throw Errors.ProxyNameTooShort(proxyName); + if (proxyName.Length > Limits.MaxProxyNameLength) throw Errors.ProxyNameTooLong(proxyName); + + // TODO: this never returns false as it throws instead, should this happen? + return true; + } + + private bool IsMessageValid(DiscordMessage message) + { + return + // Must be a guild text channel + message.Channel.Type == ChannelType.Text && + + // Must not be a system message + message.MessageType == MessageType.Default && + !(message.Author.IsSystem ?? false) && + + // Must not be a bot or webhook message + !message.WebhookMessage && + !message.Author.IsBot && + + // Must have either an attachment or content (or both, but not neither) + (message.Attachments.Count > 0 || (message.Content != null && message.Content.Trim().Length > 0)); + } + + private bool IsProxyValid(DiscordMessage message, ProxyMatch match) + { + return + // System and member must have proxying enabled in this guild + match.Member.ProxyEnabled && + + // Channel must not be blacklisted here + !match.Member.ChannelBlacklist.Contains(message.ChannelId); + } } -} +} \ No newline at end of file diff --git a/PluralKit.Bot/Proxy/ProxyTagParser.cs b/PluralKit.Bot/Proxy/ProxyTagParser.cs index 47a3a82a..c4f84423 100644 --- a/PluralKit.Bot/Proxy/ProxyTagParser.cs +++ b/PluralKit.Bot/Proxy/ProxyTagParser.cs @@ -1,4 +1,5 @@ #nullable enable +using System; using System.Collections.Generic; using System.Linq; @@ -8,7 +9,7 @@ namespace PluralKit.Bot { public class ProxyTagParser { - public bool TryParse(string input, IEnumerable members, out ProxyMatch result) + public bool TryMatch(IEnumerable members, string input, out ProxyMatch result) { result = default; @@ -19,7 +20,7 @@ namespace PluralKit.Bot // "Flatten" list of members to a list of tag-member pairs // Then order them by "tag specificity" - // (ProxyString length desc = prefix+suffix length desc = inner message asc = more specific proxy first) + // (prefix+suffix length desc = inner message asc = more specific proxy first) var tags = members .SelectMany(member => member.ProxyTags.Select(tag => (tag, member))) .OrderByDescending(p => p.tag.ProxyString.Length); @@ -34,15 +35,10 @@ namespace PluralKit.Bot if (tag.Prefix == null && tag.Suffix == null) continue; // Can we match with these tags? - if (TryMatchTags(input, tag, out result.Content)) + if (TryMatchTagsInner(input, tag, out result.Content)) { - // (see https://github.com/xSke/PluralKit/pull/181) - if (result.Content == "\U0000fe0f") return false; - // If we extracted a leading mention before, add that back now if (leadingMention != null) result.Content = $"{leadingMention} {result.Content}"; - - // We're done! return true; } @@ -53,8 +49,24 @@ namespace PluralKit.Bot return false; } - private bool TryMatchTags(string input, ProxyTag tag, out string content) + public bool TryMatchTags(string input, ProxyTag tag, out string inner) { + // This just wraps TryMatchTagsInner w/ support for leading mentions + var leadingMention = ExtractLeadingMention(ref input); + + inner = ""; + if (!TryMatchTagsInner(input, tag, out var innerRaw)) return false; + + // Add leading mentions back + inner = leadingMention == null ? innerRaw : $"{leadingMention} {innerRaw}"; + return true; + + } + + private bool TryMatchTagsInner(string input, ProxyTag tag, out string inner) + { + inner = ""; + // Normalize null tags to empty strings var prefix = tag.Prefix ?? ""; var suffix = tag.Suffix ?? ""; @@ -66,19 +78,14 @@ namespace PluralKit.Bot // Special case: image-only proxies + proxy tags with spaces // Trim everything, then see if we have a "contentless tag pair" (normally disallowed, but OK if we have an attachment) if (!isMatch && input.Trim() == prefix.TrimEnd() + suffix.TrimStart()) - { - content = ""; return true; - } - - if (isMatch) - { - content = input.Substring(prefix.Length, input.Length - prefix.Length - suffix.Length); - return true; - } - - content = ""; - return false; + if (!isMatch) return false; + + // We got a match, extract inner text + inner = input.Substring(prefix.Length, input.Length - prefix.Length - suffix.Length).Trim(); + + // (see https://github.com/xSke/PluralKit/pull/181) + return inner != "\U0000fe0f"; } private string? ExtractLeadingMention(ref string input) diff --git a/PluralKit.Bot/Services/EmbedService.cs b/PluralKit.Bot/Services/EmbedService.cs index 7a8b42e6..ce9c1bae 100644 --- a/PluralKit.Bot/Services/EmbedService.cs +++ b/PluralKit.Bot/Services/EmbedService.cs @@ -92,9 +92,7 @@ namespace PluralKit.Bot { // for now we just default to a blank color, yolo color = DiscordUtils.Gray; } - - var messageCount = await _data.GetMemberMessageCount(member); - + var guildSettings = guild != null ? await _data.GetMemberGuildSettings(member, guild.Id) : null; var guildDisplayName = guildSettings?.DisplayName; var avatar = guildSettings?.AvatarUrl ?? member.AvatarUrl; @@ -122,7 +120,7 @@ namespace PluralKit.Bot { if (guild != null && guildDisplayName != null) eb.AddField($"Server Nickname (for {guild.Name})", guildDisplayName.Truncate(1024), true); if (member.Birthday != null && member.MemberPrivacy.CanAccess(ctx)) eb.AddField("Birthdate", member.BirthdayString, true); if (!member.Pronouns.EmptyOrNull() && member.MemberPrivacy.CanAccess(ctx)) eb.AddField("Pronouns", member.Pronouns.Truncate(1024), true); - if (messageCount > 0 && member.MemberPrivacy.CanAccess(ctx)) eb.AddField("Message Count", messageCount.ToString(), true); + if (member.MessageCount > 0 && member.MemberPrivacy.CanAccess(ctx)) eb.AddField("Message Count", member.MessageCount.ToString(), true); if (member.HasProxyTags) eb.AddField("Proxy Tags", string.Join('\n', proxyTagsStr).Truncate(1024), true); if (!member.Color.EmptyOrNull() && member.MemberPrivacy.CanAccess(ctx)) eb.AddField("Color", $"#{member.Color}", true); if (!member.Description.EmptyOrNull() && member.MemberPrivacy.CanAccess(ctx)) eb.AddField("Description", member.Description.NormalizeLineEndSpacing(), false); diff --git a/PluralKit.Bot/Services/LogChannelService.cs b/PluralKit.Bot/Services/LogChannelService.cs index 118af35e..cf14124b 100644 --- a/PluralKit.Bot/Services/LogChannelService.cs +++ b/PluralKit.Bot/Services/LogChannelService.cs @@ -1,5 +1,8 @@ +using System.Linq; using System.Threading.Tasks; +using Dapper; + using DSharpPlus; using DSharpPlus.Entities; using DSharpPlus.Exceptions; @@ -10,57 +13,59 @@ using Serilog; namespace PluralKit.Bot { public class LogChannelService { - private EmbedService _embed; - private IDataStore _data; - private ILogger _logger; + private readonly EmbedService _embed; + private readonly DbConnectionFactory _db; + private readonly IDataStore _data; + private readonly ILogger _logger; + private readonly DiscordRestClient _rest; - public LogChannelService(EmbedService embed, ILogger logger, IDataStore data) + public LogChannelService(EmbedService embed, ILogger logger, DiscordRestClient rest, DbConnectionFactory db, IDataStore data) { _embed = embed; + _rest = rest; + _db = db; _data = data; _logger = logger.ForContext(); } - public async Task LogMessage(DiscordClient client, PKSystem system, PKMember member, ulong messageId, ulong originalMsgId, DiscordChannel originalChannel, DiscordUser sender, string content, GuildConfig? guildCfg = null) + public async Task LogMessage(ProxyMatch proxy, DiscordMessage trigger, ulong hookMessage) { - if (guildCfg == null) - guildCfg = await _data.GetOrCreateGuildConfig(originalChannel.GuildId); - - // Bail if logging is disabled either globally or for this channel - if (guildCfg.Value.LogChannel == null) return; - if (guildCfg.Value.LogBlacklist.Contains(originalChannel.Id)) return; + if (proxy.Member.LogChannel == null || proxy.Member.LogBlacklist.Contains(trigger.ChannelId)) return; + + // Find log channel and check if valid + var logChannel = await FindLogChannel(trigger.Channel.GuildId, proxy); + if (logChannel == null || logChannel.Type != ChannelType.Text) return; + + // Check bot permissions + if (!trigger.Channel.BotHasAllPermissions(Permissions.SendMessages | Permissions.EmbedLinks)) return; + + // Send embed! + await using var conn = await _db.Obtain(); + var embed = _embed.CreateLoggedMessageEmbed(await _data.GetSystemById(proxy.Member.SystemId), + await _data.GetMemberById(proxy.Member.MemberId), hookMessage, trigger.Id, trigger.Author, proxy.Content, + trigger.Channel); + var url = $"https://discord.com/channels/{trigger.Channel.GuildId}/{trigger.ChannelId}/{hookMessage}"; + await logChannel.SendMessageAsync(content: url, embed: embed); + } + + private async Task FindLogChannel(ulong guild, ProxyMatch proxy) + { + var logChannel = proxy.Member.LogChannel.Value; - // Bail if we can't find the channel - DiscordChannel channel; try { - channel = await client.GetChannelAsync(guildCfg.Value.LogChannel.Value); + return await _rest.GetChannelAsync(logChannel); } catch (NotFoundException) { - // If it doesn't exist, remove it from the DB - await RemoveLogChannel(guildCfg.Value); - return; + // Channel doesn't exist, let's remove it from the database too + _logger.Warning("Attempted to fetch missing log channel {LogChannel}, removing from database", logChannel); + await using var conn = await _db.Obtain(); + await conn.ExecuteAsync("update servers set log_channel = null where server = @Guild", + new {Guild = guild}); } - - // Bail if it's not a text channel - if (channel.Type != ChannelType.Text) return; - // Bail if we don't have permission to send stuff here - if (!channel.BotHasAllPermissions(Permissions.SendMessages | Permissions.EmbedLinks)) - return; - - var embed = _embed.CreateLoggedMessageEmbed(system, member, messageId, originalMsgId, sender, content, originalChannel); - - var url = $"https://discord.com/channels/{originalChannel.GuildId}/{originalChannel.Id}/{messageId}"; - - await channel.SendMessageAsync(content: url, embed: embed); - } - - private async Task RemoveLogChannel(GuildConfig cfg) - { - cfg.LogChannel = null; - await _data.SaveGuildConfig(cfg); + return null; } } } \ No newline at end of file diff --git a/PluralKit.Core/Migrations/0.sql b/PluralKit.Core/Database/Migrations/0.sql similarity index 100% rename from PluralKit.Core/Migrations/0.sql rename to PluralKit.Core/Database/Migrations/0.sql diff --git a/PluralKit.Core/Migrations/1.sql b/PluralKit.Core/Database/Migrations/1.sql similarity index 100% rename from PluralKit.Core/Migrations/1.sql rename to PluralKit.Core/Database/Migrations/1.sql diff --git a/PluralKit.Core/Migrations/2.sql b/PluralKit.Core/Database/Migrations/2.sql similarity index 100% rename from PluralKit.Core/Migrations/2.sql rename to PluralKit.Core/Database/Migrations/2.sql diff --git a/PluralKit.Core/Migrations/3.sql b/PluralKit.Core/Database/Migrations/3.sql similarity index 100% rename from PluralKit.Core/Migrations/3.sql rename to PluralKit.Core/Database/Migrations/3.sql diff --git a/PluralKit.Core/Migrations/4.sql b/PluralKit.Core/Database/Migrations/4.sql similarity index 100% rename from PluralKit.Core/Migrations/4.sql rename to PluralKit.Core/Database/Migrations/4.sql diff --git a/PluralKit.Core/Migrations/5.sql b/PluralKit.Core/Database/Migrations/5.sql similarity index 100% rename from PluralKit.Core/Migrations/5.sql rename to PluralKit.Core/Database/Migrations/5.sql diff --git a/PluralKit.Core/Migrations/6.sql b/PluralKit.Core/Database/Migrations/6.sql old mode 100755 new mode 100644 similarity index 100% rename from PluralKit.Core/Migrations/6.sql rename to PluralKit.Core/Database/Migrations/6.sql diff --git a/PluralKit.Core/Database/Migrations/7.sql b/PluralKit.Core/Database/Migrations/7.sql new file mode 100644 index 00000000..55bec878 --- /dev/null +++ b/PluralKit.Core/Database/Migrations/7.sql @@ -0,0 +1,33 @@ +-- SCHEMA VERSION 7: 2020-06-12 +-- (in-db message count row) + +-- Add message count row to members table, initialize it with the correct data +alter table members add column message_count int not null default 0; +update members set message_count = (select count(*) from messages where messages.member = members.id); + + +-- Create a trigger function to increment the message count on inserting to the messages table +create function trg_msgcount_increment() returns trigger as $$ +begin + update members set message_count = message_count + 1 where id = NEW.member; + return NEW; +end; +$$ language plpgsql; + +create trigger increment_member_message_count before insert on messages for each row execute procedure trg_msgcount_increment(); + + +-- Create a trigger function to decrement the message count on deleting from the messages table +create function trg_msgcount_decrement() returns trigger as $$ +begin + -- Don't decrement if count <= zero (shouldn't happen, but we don't want negative message counts) + update members set message_count = message_count - 1 where id = OLD.member and message_count > 0; + return OLD; +end; +$$ language plpgsql; + +create trigger decrement_member_message_count before delete on messages for each row execute procedure trg_msgcount_decrement(); + + +-- (update schema ver) +update info set schema_version = 7; \ No newline at end of file diff --git a/PluralKit.Core/Database/ProxyMember.cs b/PluralKit.Core/Database/ProxyMember.cs new file mode 100644 index 00000000..97361cac --- /dev/null +++ b/PluralKit.Core/Database/ProxyMember.cs @@ -0,0 +1,26 @@ +#nullable enable +using System.Collections.Generic; + +namespace PluralKit.Core +{ + /// + /// Model for the `proxy_info` PL/pgSQL function in `functions.sql` + /// + public class ProxyMember + { + public int SystemId { get; set; } + public int MemberId { get; set; } + public bool ProxyEnabled { get; set; } + public AutoproxyMode AutoproxyMode { get; set; } + public bool IsAutoproxyMember { get; set; } + public ulong? LatchMessage { get; set; } + public string ProxyName { get; set; } = ""; + public string? ProxyAvatar { get; set; } + public IReadOnlyCollection ProxyTags { get; set; } = new ProxyTag[0]; + public bool KeepProxy { get; set; } + + public IReadOnlyCollection ChannelBlacklist { get; set; } = new ulong[0]; + public IReadOnlyCollection LogBlacklist { get; set; } = new ulong[0]; + public ulong? LogChannel { get; set; } + } +} \ No newline at end of file diff --git a/PluralKit.Core/Database/Schemas.cs b/PluralKit.Core/Database/Schemas.cs new file mode 100644 index 00000000..2882c2f3 --- /dev/null +++ b/PluralKit.Core/Database/Schemas.cs @@ -0,0 +1,99 @@ +using System; +using System.Data; +using System.IO; +using System.Threading.Tasks; + +using Dapper; + +using Npgsql; + +using Serilog; + +namespace PluralKit.Core +{ + public class Schemas + { + private const string RootPath = "PluralKit.Core.Database"; // "resource path" root for SQL files + private const int TargetSchemaVersion = 7; + + private DbConnectionFactory _conn; + private ILogger _logger; + + public Schemas(DbConnectionFactory conn, ILogger logger) + { + _conn = conn; + _logger = logger.ForContext(); + } + + public static void Initialize() + { + // Without these it'll still *work* but break at the first launch + probably cause other small issues + NpgsqlConnection.GlobalTypeMapper.MapComposite("proxy_tag"); + NpgsqlConnection.GlobalTypeMapper.MapEnum("privacy_level"); + } + + public async Task InitializeDatabase() + { + // Run everything in a transaction + await using var conn = await _conn.Obtain(); + using var tx = conn.BeginTransaction(); + + // Before applying migrations, clean out views/functions to prevent type errors + await ExecuteSqlFile($"{RootPath}.clean.sql", conn, tx); + + // Apply all migrations between the current database version and the target version + await ApplyMigrations(conn, tx); + + // Now, reapply views/functions (we deleted them above, no need to worry about conflicts) + await ExecuteSqlFile($"{RootPath}.views.sql", conn, tx); + await ExecuteSqlFile($"{RootPath}.functions.sql", conn, tx); + + // Finally, commit tx + tx.Commit(); + } + + private async Task ApplyMigrations(IAsyncDbConnection conn, IDbTransaction tx) + { + var currentVersion = await GetCurrentDatabaseVersion(conn); + _logger.Information("Current schema version: {CurrentVersion}", currentVersion); + for (var migration = currentVersion + 1; migration <= TargetSchemaVersion; migration++) + { + _logger.Information("Applying schema migration {MigrationId}", migration); + await ExecuteSqlFile($"{RootPath}.Migrations.{migration}.sql", conn, tx); + } + } + + private async Task ExecuteSqlFile(string resourceName, IDbConnection conn, IDbTransaction tx = null) + { + await using var stream = typeof(Schemas).Assembly.GetManifestResourceStream(resourceName); + if (stream == null) throw new ArgumentException($"Invalid resource name '{resourceName}'"); + + using var reader = new StreamReader(stream); + var query = await reader.ReadToEndAsync(); + + await conn.ExecuteAsync(query, transaction: tx); + + // If the above creates new enum/composite types, we must tell Npgsql to reload the internal type caches + // This will propagate to every other connection as well, since it marks the global type mapper collection dirty. + // TODO: find a way to get around the cast to our internal tracker wrapper... this could break if that ever changes + ((PerformanceTrackingConnection) conn)._impl.ReloadTypes(); + } + + private async Task GetCurrentDatabaseVersion(IDbConnection conn) + { + // First, check if the "info" table exists (it may not, if this is a *really* old database) + var hasInfoTable = + await conn.QuerySingleOrDefaultAsync( + "select count(*) from information_schema.tables where table_name = 'info'") == 1; + + // If we have the table, read the schema version + if (hasInfoTable) + return await conn.QuerySingleOrDefaultAsync("select schema_version from info"); + + // If not, we return version "-1" + // This means migration 0 will get executed, getting us into a consistent state + // Then, migration 1 gets executed, which creates the info table and sets version to 1 + return -1; + } + } +} \ No newline at end of file diff --git a/PluralKit.Core/Database/clean.sql b/PluralKit.Core/Database/clean.sql new file mode 100644 index 00000000..b03b2ee2 --- /dev/null +++ b/PluralKit.Core/Database/clean.sql @@ -0,0 +1,3 @@ +drop view if exists system_last_switch; +drop view if exists member_list; +drop function if exists proxy_info; diff --git a/PluralKit.Core/Database/functions.sql b/PluralKit.Core/Database/functions.sql new file mode 100644 index 00000000..2f80ad9c --- /dev/null +++ b/PluralKit.Core/Database/functions.sql @@ -0,0 +1,85 @@ +-- Giant "mega-function" to find all information relevant for message proxying +-- Returns one row per member, computes several properties from others +create function proxy_info(account_id bigint, guild_id bigint) + returns table + ( + -- Note: table type gets matched *by index*, not *by name* (make sure order here and in `select` match) + system_id int, -- from: systems.id + member_id int, -- from: members.id + proxy_tags proxy_tag[], -- from: members.proxy_tags + keep_proxy bool, -- from: members.keep_proxy + proxy_enabled bool, -- from: system_guild.proxy_enabled + proxy_name text, -- calculated: name we should proxy under + proxy_avatar text, -- calculated: avatar we should proxy with + autoproxy_mode int, -- from: system_guild.autoproxy_mode + is_autoproxy_member bool, -- calculated: should this member be used for AP? + latch_message bigint, -- calculated: last message from this account in this guild + channel_blacklist bigint[], -- from: servers.blacklist + log_blacklist bigint[], -- from: servers.log_blacklist + log_channel bigint -- from: servers.log_channel + ) +as +$$ +select + -- Basic data + systems.id as system_id, + members.id as member_id, + members.proxy_tags as proxy_tags, + members.keep_proxy as keep_proxy, + + -- Proxy info + coalesce(system_guild.proxy_enabled, true) as proxy_enabled, + case + when systems.tag is not null then (coalesce(member_guild.display_name, members.display_name, members.name) || ' ' || systems.tag) + else coalesce(member_guild.display_name, members.display_name, members.name) + end as proxy_name, + coalesce(member_guild.avatar_url, members.avatar_url, systems.avatar_url) as proxy_avatar, + + -- Autoproxy data + coalesce(system_guild.autoproxy_mode, 1) as autoproxy_mode, + + -- Autoproxy logic is essentially: "is this member the one we should autoproxy?" + case + -- Front mode: check if this is the first fronter + when system_guild.autoproxy_mode = 2 then members.id = (select sls.members[1] + from system_last_switch as sls + where sls.system = systems.id) + + -- Latch mode: check if this is the last proxier + when system_guild.autoproxy_mode = 3 then members.id = last_message_in_guild.member + + -- Member mode: check if this is the selected memebr + when system_guild.autoproxy_mode = 4 then members.id = system_guild.autoproxy_member + + -- no autoproxy: then this member definitely shouldn't be autoproxied :) + else false end as is_autoproxy_member, + + last_message_in_guild.mid as latch_message, + + -- Guild info + coalesce(servers.blacklist, array[]::bigint[]) as channel_blacklist, + coalesce(servers.log_blacklist, array[]::bigint[]) as log_blacklist, + servers.log_channel as log_channel +from accounts + -- Fetch guild info + left join servers on servers.id = guild_id + + -- Fetch the system for this account (w/ guild config) + inner join systems on systems.id = accounts.system + left join system_guild on system_guild.system = accounts.system and system_guild.guild = guild_id + + -- Fetch all members from this system (w/ guild config) + inner join members on members.system = systems.id + left join member_guild on member_guild.member = members.id and member_guild.guild = guild_id + + -- Find ID and member for the last message sent in this guild + left join lateral (select mid, member + from messages + where messages.guild = guild_id + and messages.sender = account_id + and system_guild.autoproxy_mode = 3 + order by mid desc + limit 1) as last_message_in_guild on true +where accounts.uid = account_id; +$$ language sql stable + rows 10; \ No newline at end of file diff --git a/PluralKit.Core/Database/views.sql b/PluralKit.Core/Database/views.sql new file mode 100644 index 00000000..928e397f --- /dev/null +++ b/PluralKit.Core/Database/views.sql @@ -0,0 +1,29 @@ +create view system_last_switch as +select systems.id as system, + last_switch.id as switch, + last_switch.timestamp as timestamp, + array(select member from switch_members where switch_members.switch = last_switch.id) as members +from systems + inner join lateral (select * from switches where switches.system = systems.id order by timestamp desc limit 1) as last_switch on true; + +create view member_list as +select members.*, + -- Find last message ID + (select max(messages.mid) from messages where messages.member = members.id) as last_message, + + -- Find last switch timestamp + ( + select max(switches.timestamp) + from switch_members + inner join switches on switches.id = switch_members.switch + where switch_members.member = members.id + ) as last_switch_time, + + -- Extract month/day from birthday and "force" the year identical (just using 4) -> month/day only sorting! + case when members.birthday is not null then + make_date( + 4, + extract(month from members.birthday)::integer, + extract(day from members.birthday)::integer + ) end as birthday_md +from members; \ No newline at end of file diff --git a/PluralKit.Core/Models/PKMember.cs b/PluralKit.Core/Models/PKMember.cs index 2107db5a..01da9e4b 100644 --- a/PluralKit.Core/Models/PKMember.cs +++ b/PluralKit.Core/Models/PKMember.cs @@ -22,6 +22,7 @@ namespace PluralKit.Core { [JsonProperty("proxy_tags")] public ICollection ProxyTags { get; set; } [JsonProperty("keep_proxy")] public bool KeepProxy { get; set; } [JsonProperty("created")] public Instant Created { get; set; } + [JsonProperty("message_count")] public int MessageCount { get; set; } public PrivacyLevel MemberPrivacy { get; set; } diff --git a/PluralKit.Core/Modules.cs b/PluralKit.Core/Modules.cs index 348e7fc5..0a032451 100644 --- a/PluralKit.Core/Modules.cs +++ b/PluralKit.Core/Modules.cs @@ -25,7 +25,7 @@ namespace PluralKit.Core builder.RegisterType().SingleInstance(); builder.RegisterType().AsSelf().SingleInstance(); builder.RegisterType().AsSelf().As(); - builder.RegisterType().AsSelf(); + builder.RegisterType().AsSelf(); builder.Populate(new ServiceCollection().AddMemoryCache()); builder.RegisterType().AsSelf().SingleInstance(); diff --git a/PluralKit.Core/PluralKit.Core.csproj b/PluralKit.Core/PluralKit.Core.csproj index 9f19e5a5..e4dc7642 100644 --- a/PluralKit.Core/PluralKit.Core.csproj +++ b/PluralKit.Core/PluralKit.Core.csproj @@ -33,7 +33,6 @@ - + - diff --git a/PluralKit.Core/PluralKit.Core.csproj.DotSettings b/PluralKit.Core/PluralKit.Core.csproj.DotSettings index 9259ebbb..33bc593a 100644 --- a/PluralKit.Core/PluralKit.Core.csproj.DotSettings +++ b/PluralKit.Core/PluralKit.Core.csproj.DotSettings @@ -1,4 +1,5 @@  True True + True True \ No newline at end of file diff --git a/PluralKit.Core/Services/DataFileService.cs b/PluralKit.Core/Services/DataFileService.cs index cf8993c2..cce2effa 100644 --- a/PluralKit.Core/Services/DataFileService.cs +++ b/PluralKit.Core/Services/DataFileService.cs @@ -30,7 +30,6 @@ namespace PluralKit.Core // Export members var members = new List(); var pkMembers = _data.GetSystemMembers(system); // Read all members in the system - var messageCounts = await _data.GetMemberMessageCountBulk(system); // Count messages proxied by all members in the system await foreach (var member in pkMembers.Select(m => new DataFileMember { @@ -45,7 +44,7 @@ namespace PluralKit.Core ProxyTags = m.ProxyTags, KeepProxy = m.KeepProxy, Created = DateTimeFormats.TimestampExportFormat.Format(m.Created), - MessageCount = messageCounts.Where(x => x.Member == m.Id).Select(x => x.MessageCount).FirstOrDefault() + MessageCount = m.MessageCount })) members.Add(member); // Export switches diff --git a/PluralKit.Core/Services/IDataStore.cs b/PluralKit.Core/Services/IDataStore.cs index 7a371538..f47c8af5 100644 --- a/PluralKit.Core/Services/IDataStore.cs +++ b/PluralKit.Core/Services/IDataStore.cs @@ -41,12 +41,6 @@ namespace PluralKit.Core { public Instant TimespanEnd; } - public struct MemberMessageCount - { - public int Member; - public int MessageCount; - } - public struct FrontBreakdown { public Dictionary MemberSwitchDurations; @@ -208,18 +202,7 @@ namespace PluralKit.Core { /// /// An enumerable of structs representing each member in the system, in no particular order. IAsyncEnumerable GetSystemMembers(PKSystem system, bool orderByName = false); - /// - /// Gets the amount of messages proxied by a given member. - /// - /// The message count of the given member. - Task GetMemberMessageCount(PKMember member); - - /// - /// Collects a breakdown of each member in a system's message count. - /// - /// An enumerable of members along with their message counts. - Task> GetMemberMessageCountBulk(PKSystem system); - + /// /// Creates a member, auto-generating its corresponding IDs. /// @@ -267,9 +250,9 @@ namespace PluralKit.Core { /// The ID of the channel the message was posted to. /// The ID of the message posted by the webhook. /// The ID of the original trigger message containing the proxy tags. - /// The member (and by extension system) that was proxied. + /// The member (and by extension system) that was proxied. /// - Task AddMessage(ulong senderAccount, ulong guildId, ulong channelId, ulong postedMessageId, ulong triggerMessageId, PKMember proxiedMember); + Task AddMessage(ulong senderAccount, ulong guildId, ulong channelId, ulong postedMessageId, ulong triggerMessageId, int proxiedMemberId); /// /// Deletes a message from the data store. diff --git a/PluralKit.Core/Services/PostgresDataStore.cs b/PluralKit.Core/Services/PostgresDataStore.cs index 76be355e..0bcdd5fa 100644 --- a/PluralKit.Core/Services/PostgresDataStore.cs +++ b/PluralKit.Core/Services/PostgresDataStore.cs @@ -231,25 +231,6 @@ namespace PluralKit.Core { await _cache.InvalidateSystem(member.System); } - public async Task GetMemberMessageCount(PKMember member) - { - using (var conn = await _conn.Obtain()) - return await conn.QuerySingleAsync("select count(*) from messages where member = @Id", member); - } - - public async Task> GetMemberMessageCountBulk(PKSystem system) - { - using (var conn = await _conn.Obtain()) - return await conn.QueryAsync( - @"SELECT messages.member, COUNT(messages.member) messagecount - FROM members - JOIN messages - ON members.id = messages.member - WHERE members.system = @System - GROUP BY messages.member", - new { System = system.Id }); - } - public async Task GetSystemMemberCount(PKSystem system, bool includePrivate) { var query = "select count(*) from members where system = @Id"; @@ -264,19 +245,19 @@ namespace PluralKit.Core { using (var conn = await _conn.Obtain()) return await conn.ExecuteScalarAsync("select count(id) from members"); } - public async Task AddMessage(ulong senderId, ulong messageId, ulong guildId, ulong channelId, ulong originalMessage, PKMember member) { + public async Task AddMessage(ulong senderId, ulong guildId, ulong channelId, ulong postedMessageId, ulong triggerMessageId, int proxiedMemberId) { using (var conn = await _conn.Obtain()) // "on conflict do nothing" in the (pretty rare) case of duplicate events coming in from Discord, which would lead to a DB error before await conn.ExecuteAsync("insert into messages(mid, guild, channel, member, sender, original_mid) values(@MessageId, @GuildId, @ChannelId, @MemberId, @SenderId, @OriginalMid) on conflict do nothing", new { - MessageId = messageId, + MessageId = postedMessageId, GuildId = guildId, ChannelId = channelId, - MemberId = member.Id, + MemberId = proxiedMemberId, SenderId = senderId, - OriginalMid = originalMessage + OriginalMid = triggerMessageId }); - _logger.Information("Stored message {Message} in channel {Channel}", messageId, channelId); + _logger.Debug("Stored message {Message} in channel {Channel}", postedMessageId, channelId); } public async Task GetMessage(ulong id) diff --git a/PluralKit.Core/Services/SchemaService.cs b/PluralKit.Core/Services/SchemaService.cs deleted file mode 100644 index c2630ed2..00000000 --- a/PluralKit.Core/Services/SchemaService.cs +++ /dev/null @@ -1,74 +0,0 @@ -using System; -using System.IO; -using System.Threading.Tasks; - -using Dapper; - -using Npgsql; - -using Serilog; - -namespace PluralKit.Core { - public class SchemaService - { - private const int TargetSchemaVersion = 6; - - private DbConnectionFactory _conn; - private ILogger _logger; - - public SchemaService(DbConnectionFactory conn, ILogger logger) - { - _conn = conn; - _logger = logger.ForContext(); - } - - public static void Initialize() - { - // Without these it'll still *work* but break at the first launch + probably cause other small issues - NpgsqlConnection.GlobalTypeMapper.MapComposite("proxy_tag"); - NpgsqlConnection.GlobalTypeMapper.MapEnum("privacy_level"); - } - - public async Task ApplyMigrations() - { - for (var version = 0; version <= TargetSchemaVersion; version++) - await ApplyMigration(version); - } - - private async Task ApplyMigration(int migrationId) - { - // migrationId is the *target* version - using var conn = await _conn.Obtain(); - using var tx = conn.BeginTransaction(); - - // See if we even have the info table... if not, we implicitly define the version as -1 - // This means migration 0 will get executed, which ensures we're at a consistent state. - // *Technically* this also means schema version 0 will be identified as -1, but since we're only doing these - // checks in the above for loop, this doesn't matter. - var hasInfoTable = await conn.QuerySingleOrDefaultAsync("select count(*) from information_schema.tables where table_name = 'info'") == 1; - - int currentVersion; - if (hasInfoTable) - currentVersion = await conn.QuerySingleOrDefaultAsync("select schema_version from info"); - else currentVersion = -1; - - if (currentVersion >= migrationId) - return; // Don't execute the migration if we're already at the target version. - - using var stream = typeof(SchemaService).Assembly.GetManifestResourceStream($"PluralKit.Core.Migrations.{migrationId}.sql"); - if (stream == null) throw new ArgumentException("Invalid migration ID"); - - using var reader = new StreamReader(stream); - var migrationQuery = await reader.ReadToEndAsync(); - - _logger.Information("Current schema version is {CurrentVersion}, applying migration {MigrationId}", currentVersion, migrationId); - await conn.ExecuteAsync(migrationQuery, transaction: tx); - tx.Commit(); - - // If the above migration creates new enum/composite types, we must tell Npgsql to reload the internal type caches - // This will propagate to every other connection as well, since it marks the global type mapper collection dirty. - // TODO: find a way to get around the cast to our internal tracker wrapper... this could break if that ever changes - ((PerformanceTrackingConnection) conn)._impl.ReloadTypes(); - } - } -} \ No newline at end of file