liz/PluralKit
liz/PluralKit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Rework caching

Browse Source 
This does a *lot* of things. Essentially, it replaces the existing
individual proxy- and autoproxy caches on the bot end with a
global cache (in Core) that handles all the caching at once, and
automatically invalidates the cache once something changes in the
datastore.

This allows us to do proxying and autoproxying with *zero database
queries* (best-case).
This commit is contained in:
Ske 2020-02-01 14:40:57 +01:00
parent 125ea81ec3
commit 82dfe43d5a
16 changed files with 254 additions and 255 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

29
PluralKit.Bot/Bot.cs
View File

@ -194,8 +194,6 @@ namespace PluralKit.Bot
private Task HandleEvent(Func<PKEventHandler, Task> handler)
{
_logger.Debug("Received event");
// Inner function so we can await the handler without stalling the entire pipeline
async Task Inner()
{
@ -245,6 +243,7 @@ namespace PluralKit.Bot
private ILifetimeScope _services;
private CommandTree _tree;
private Scope _sentryScope;
private ProxyCache _cache;
// We're defining in the Autofac module that this class is instantiated with one instance per event
// This means that the HandleMessage function will either be called once, or not at all
@ -252,7 +251,7 @@ namespace PluralKit.Bot
// hence, we just store it in a local variable, ignoring it entirely if it's null.
private IUserMessage _msg = null;
public PKEventHandler(ProxyService proxy, ILogger logger, IMetrics metrics, DiscordShardedClient client, DbConnectionFactory connectionFactory, ILifetimeScope services, CommandTree tree, Scope sentryScope)
public PKEventHandler(ProxyService proxy, ILogger logger, IMetrics metrics, DiscordShardedClient client, DbConnectionFactory connectionFactory, ILifetimeScope services, CommandTree tree, Scope sentryScope, ProxyCache cache)
{
_proxy = proxy;
_logger = logger;
@ -262,6 +261,7 @@ namespace PluralKit.Bot
_services = services;
_tree = tree;
_sentryScope = sentryScope;
_cache = cache;
}
public async Task HandleMessage(SocketMessage arg)
@ -288,6 +288,12 @@ namespace PluralKit.Bot
{"message", msg.Id.ToString()},
});
// We fetch information about the sending account *and* guild from the cache
GuildConfig cachedGuild = default; // todo: is this default correct?
if (msg.Channel is ITextChannel textChannel) cachedGuild = await _cache.GetGuildDataCached(textChannel.GuildId);
var cachedAccount = await _cache.GetAccountDataCached(msg.Author.Id);
// this ^ may be null, do remember that down the line
int argPos = -1;
// Check if message starts with the command prefix
if (msg.Content.StartsWith("pk;", StringComparison.InvariantCultureIgnoreCase)) argPos = 3;
@ -306,23 +312,16 @@ namespace PluralKit.Bot
msg.Content.Substring(argPos).TrimStart().Length;
argPos += trimStartLengthDiff;
// If it does, fetch the sender's system (because most commands need that) into the context,
// and start command execution
// Note system may be null if user has no system, hence `OrDefault`
PKSystem system;
using (var conn = await _connectionFactory.Obtain())
system = await conn.QueryFirstOrDefaultAsync<PKSystem>(
"select systems.* from systems, accounts where accounts.uid = @Id and systems.id = accounts.system",
new {Id = msg.Author.Id});
await _tree.ExecuteCommand(new Context(_services, msg, argPos, system));
await _tree.ExecuteCommand(new Context(_services, msg, argPos, cachedAccount?.System));
}
else
else if (cachedAccount != null)
{
// If not, try proxying anyway
// but only if the account data we got before is present
// no data = no account = no system = no proxy!
try
{
await _proxy.HandleMessageAsync(msg);
await _proxy.HandleMessageAsync(cachedGuild, cachedAccount, msg);
}
catch (PKError e)
{

8
PluralKit.Bot/Commands/Autoproxy.cs
View File

@ -11,12 +11,10 @@ namespace PluralKit.Bot.Commands
public class Autoproxy
{
private IDataStore _data;
private AutoproxyCacheService _cache;
public Autoproxy(IDataStore data, AutoproxyCacheService cache)
public Autoproxy(IDataStore data)
{
_data = data;
_cache = cache;
}
public async Task AutoproxyRoot(Context ctx)
@ -51,7 +49,6 @@ namespace PluralKit.Bot.Commands
settings.AutoproxyMode = AutoproxyMode.Off;
settings.AutoproxyMember = null;
await _data.SetSystemGuildSettings(ctx.System, ctx.Guild.Id, settings);
await _cache.FlushCacheForSystem(ctx.System, ctx.Guild.Id);
await ctx.Reply($"{Emojis.Success} Autoproxy turned off in this server.");
}
}
@ -68,7 +65,6 @@ namespace PluralKit.Bot.Commands
settings.AutoproxyMode = AutoproxyMode.Latch;
settings.AutoproxyMember = null;
await _data.SetSystemGuildSettings(ctx.System, ctx.Guild.Id, settings);
await _cache.FlushCacheForSystem(ctx.System, ctx.Guild.Id);
await ctx.Reply($"{Emojis.Success} Autoproxy set to latch mode in this server. Messages will now be autoproxied using the *last-proxied member* in this server.");
}
}
@ -85,7 +81,6 @@ namespace PluralKit.Bot.Commands
settings.AutoproxyMode = AutoproxyMode.Front;
settings.AutoproxyMember = null;
await _data.SetSystemGuildSettings(ctx.System, ctx.Guild.Id, settings);
await _cache.FlushCacheForSystem(ctx.System, ctx.Guild.Id);
await ctx.Reply($"{Emojis.Success} Autoproxy set to front mode in this server. Messages will now be autoproxied using the *current first fronter*, if any.");
}
}
@ -98,7 +93,6 @@ namespace PluralKit.Bot.Commands
settings.AutoproxyMode = AutoproxyMode.Member;
settings.AutoproxyMember = member.Id;
await _data.SetSystemGuildSettings(ctx.System, ctx.Guild.Id, settings);
await _cache.FlushCacheForSystem(ctx.System, ctx.Guild.Id);
await ctx.Reply($"{Emojis.Success} Autoproxy set to **{member.Name}** in this server.");
}

10
PluralKit.Bot/Commands/Member.cs
View File

@ -11,13 +11,10 @@ namespace PluralKit.Bot.Commands
private IDataStore _data;
private EmbedService _embeds;
private ProxyCacheService _proxyCache;
public Member(IDataStore data, EmbedService embeds, ProxyCacheService proxyCache)
public Member(IDataStore data, EmbedService embeds)
{
_data = data;
_embeds = embeds;
_proxyCache = proxyCache;
}
public async Task NewMember(Context ctx) {
@ -51,8 +48,6 @@ namespace PluralKit.Bot.Commands
await ctx.Reply($"{Emojis.Warn} You have reached the per-system member limit ({Limits.MaxMemberCount}). You will be unable to create additional members until existing members are deleted.");
else if (memberCount >= Limits.MaxMembersWarnThreshold)
await ctx.Reply($"{Emojis.Warn} You are approaching the per-system member limit ({memberCount} / {Limits.MaxMemberCount} members). Please review your member list for unused or duplicate members.");
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task MemberRandom(Context ctx)
@ -68,11 +63,8 @@ namespace PluralKit.Bot.Commands
throw Errors.NoMembersError;
var randInt = randGen.Next(members.Count);
await ctx.Reply(embed: await _embeds.CreateMemberEmbed(ctx.System, members[randInt], ctx.Guild, ctx.LookupContextFor(ctx.System)));
}
public async Task ViewMember(Context ctx, PKMember target)
{
var system = await _data.GetSystemById(target.System);

6
PluralKit.Bot/Commands/MemberAvatar.cs
View File

@ -10,12 +10,10 @@ namespace PluralKit.Bot.Commands
public class MemberAvatar
{
private IDataStore _data;
private ProxyCacheService _proxyCache;
public MemberAvatar(IDataStore data, ProxyCacheService proxyCache)
public MemberAvatar(IDataStore data)
{
_data = data;
_proxyCache = proxyCache;
}
public async Task Avatar(Context ctx, PKMember target)
@ -80,8 +78,6 @@ namespace PluralKit.Bot.Commands
await ctx.Reply($"{Emojis.Success} Member avatar changed to attached image. Please note that if you delete the message containing the attachment, the avatar will stop working.");
}
// No-arguments no-attachment case covered by conditional at the very top
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
}
}

13
PluralKit.Bot/Commands/MemberEdit.cs
View File

@ -11,12 +11,10 @@ namespace PluralKit.Bot.Commands
public class MemberEdit
{
private IDataStore _data;
private ProxyCacheService _proxyCache;
public MemberEdit(IDataStore data, ProxyCacheService proxyCache)
public MemberEdit(IDataStore data)
{
_data = data;
_proxyCache = proxyCache;
}
public async Task Name(Context ctx, PKMember target) {
@ -50,8 +48,6 @@ namespace PluralKit.Bot.Commands
if (memberGuildConfig.DisplayName != null)
await ctx.Reply($"{Emojis.Note} Note that this member has a server name set ({memberGuildConfig.DisplayName.SanitizeMentions()}) in this server ({ctx.Guild.Name.SanitizeMentions()}), and will be proxied using that name here.");
}
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task Description(Context ctx, PKMember target) {
@ -141,8 +137,6 @@ namespace PluralKit.Bot.Commands
}
await ctx.Reply(successStr);
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task ServerName(Context ctx, PKMember target)
@ -168,8 +162,6 @@ namespace PluralKit.Bot.Commands
successStr += $"Member server name cleared. This member will now be proxied using their member name \"{target.Name.SanitizeMentions()}\" in this server ({ctx.Guild.Name.SanitizeMentions()}).";
await ctx.Reply(successStr);
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task KeepProxy(Context ctx, PKMember target)
@ -190,7 +182,6 @@ namespace PluralKit.Bot.Commands
await ctx.Reply($"{Emojis.Success} Member proxy tags will now be included in the resulting message when proxying.");
else
await ctx.Reply($"{Emojis.Success} Member proxy tags will now not be included in the resulting message when proxying.");
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task Privacy(Context ctx, PKMember target)
@ -222,8 +213,6 @@ namespace PluralKit.Bot.Commands
if (!await ctx.ConfirmWithReply(target.Hid)) throw Errors.MemberDeleteCancelled;
await _data.DeleteMember(target);
await ctx.Reply($"{Emojis.Success} Member deleted.");
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
}
}

9
PluralKit.Bot/Commands/MemberProxy.cs
View File

@ -8,12 +8,10 @@ namespace PluralKit.Bot.Commands
public class MemberProxy
{
private IDataStore _data;
private ProxyCacheService _proxyCache;
public MemberProxy(IDataStore data, ProxyCacheService proxyCache)
public MemberProxy(IDataStore data)
{
_data = data;
_proxyCache = proxyCache;
}
public async Task Proxy(Context ctx, PKMember target)
@ -117,9 +115,6 @@ namespace PluralKit.Bot.Commands
await _data.SaveMember(target);
await ctx.Reply($"{Emojis.Success} Member proxy tags set to `{requestedTag.ProxyString.SanitizeMentions()}`.");
}
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
}
}
}

10
PluralKit.Bot/Commands/SystemEdit.cs
View File

@ -17,13 +17,11 @@ namespace PluralKit.Bot.Commands
{
private IDataStore _data;
private EmbedService _embeds;
private ProxyCacheService _proxyCache;
public SystemEdit(IDataStore data, EmbedService embeds, ProxyCacheService proxyCache)
public SystemEdit(IDataStore data, EmbedService embeds)
{
_data = data;
_embeds = embeds;
_proxyCache = proxyCache;
}
public async Task Name(Context ctx)
@ -62,8 +60,6 @@ namespace PluralKit.Bot.Commands
await _data.SaveSystem(ctx.System);
await ctx.Reply($"{Emojis.Success} System tag {(newTag != null ? $"changed. Member names will now end with `{newTag.SanitizeMentions()}` when proxied" : "cleared")}.");
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task Avatar(Context ctx)
@ -115,8 +111,6 @@ namespace PluralKit.Bot.Commands
var embed = url != null ? new EmbedBuilder().WithImageUrl(url).Build() : null;
await ctx.Reply($"{Emojis.Success} System avatar changed.", embed: embed);
}
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task Delete(Context ctx) {
@ -128,8 +122,6 @@ namespace PluralKit.Bot.Commands
await _data.DeleteSystem(ctx.System);
await ctx.Reply($"{Emojis.Success} System deleted.");
await _proxyCache.InvalidateResultsForSystem(ctx.System);
}
public async Task SystemProxy(Context ctx)

5
PluralKit.Bot/Modules.cs
View File

@ -63,9 +63,7 @@ namespace PluralKit.Bot
builder.RegisterType<LogChannelService>().AsSelf().SingleInstance();
builder.RegisterType<DataFileService>().AsSelf().SingleInstance();
builder.RegisterType<WebhookExecutorService>().AsSelf().SingleInstance();
builder.RegisterType<ProxyCacheService>().AsSelf().SingleInstance();
builder.RegisterType<WebhookCacheService>().AsSelf().SingleInstance();
builder.RegisterType<AutoproxyCacheService>().AsSelf().SingleInstance();
builder.RegisterType<ShardInfoService>().AsSelf().SingleInstance();
builder.RegisterType<CpuStatService>().AsSelf().SingleInstance();
builder.RegisterType<PeriodicStatCollector>().AsSelf().SingleInstance();
@ -73,9 +71,6 @@ namespace PluralKit.Bot
// Sentry stuff
builder.Register(_ => new Scope(null)).AsSelf().InstancePerLifetimeScope();
// .NET stuff
builder.Populate(new ServiceCollection()
.AddMemoryCache());
// Utils
builder.Register(c => new HttpClient

1
PluralKit.Bot/PluralKit.Bot.csproj
View File

@ -12,7 +12,6 @@
<ItemGroup>
<PackageReference Include="Humanizer.Core" Version="2.7.9" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="3.1.0" />
<PackageReference Include="Sentry" Version="2.0.0-beta7" />
<PackageReference Include="SixLabors.ImageSharp" Version="1.0.0-beta0007" />
</ItemGroup>

73
PluralKit.Bot/Services/AutoproxyCacheService.cs
View File

@ -1,73 +0,0 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Caching.Memory;
namespace PluralKit.Bot
{
public class AutoproxyCacheResult
{
public SystemGuildSettings GuildSettings;
public PKSystem System;
public PKMember AutoproxyMember;
}
public class AutoproxyCacheService
{
private IMemoryCache _cache;
private IDataStore _data;
private DbConnectionFactory _conn;
public AutoproxyCacheService(IMemoryCache cache, DbConnectionFactory conn, IDataStore data)
{
_cache = cache;
_conn = conn;
_data = data;
}
public async Task<AutoproxyCacheResult> GetGuildSettings(ulong account, ulong guild) =>
await _cache.GetOrCreateAsync(GetKey(account, guild), entry => FetchSettings(account, guild, entry));
public async Task FlushCacheForSystem(PKSystem system, ulong guild)
{
foreach (var account in await _data.GetSystemAccounts(system))
FlushCacheFor(account, guild);
}
public void FlushCacheFor(ulong account, ulong guild) =>
_cache.Remove(GetKey(account, guild));
private async Task<AutoproxyCacheResult> FetchSettings(ulong account, ulong guild, ICacheEntry entry)
{
using var conn = await _conn.Obtain();
var data = (await conn.QueryAsync<SystemGuildSettings, PKSystem, PKMember, AutoproxyCacheResult>(
"select system_guild.*, systems.*, members.* from accounts inner join systems on systems.id = accounts.system inner join system_guild on system_guild.system = systems.id left join members on system_guild.autoproxy_member = members.id where accounts.uid = @Uid and system_guild.guild = @Guild",
(guildSettings, system, autoproxyMember) => new AutoproxyCacheResult
{
GuildSettings = guildSettings,
System = system,
AutoproxyMember = autoproxyMember
},
new {Uid = account, Guild = guild})).FirstOrDefault();
if (data != null)
{
// Long expiry for accounts with no system/settings registered
entry.SetSlidingExpiration(TimeSpan.FromMinutes(5));
entry.SetAbsoluteExpiration(TimeSpan.FromHours(1));
}
else
{
// Shorter expiry if they already have settings
entry.SetSlidingExpiration(TimeSpan.FromMinutes(1));
entry.SetAbsoluteExpiration(TimeSpan.FromMinutes(5));
}
return data;
}
private string GetKey(ulong account, ulong guild) => $"_system_guild_{account}_{guild}";
}
}

82
PluralKit.Bot/Services/ProxyCacheService.cs
View File

@ -1,82 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Caching.Memory;
using Serilog;
namespace PluralKit.Bot
{
public class ProxyCacheService
{
public class ProxyDatabaseResult
{
public PKSystem System;
public PKMember Member;
}
private DbConnectionFactory _conn;
private IMemoryCache _cache;
private ILogger _logger;
public ProxyCacheService(DbConnectionFactory conn, IMemoryCache cache, ILogger logger)
{
_conn = conn;
_cache = cache;
_logger = logger;
}
public Task<IEnumerable<ProxyDatabaseResult>> GetResultsFor(ulong account)
{
_logger.Verbose("Looking up members for account {Account} in cache...", account);
return _cache.GetOrCreateAsync(GetKey(account), (entry) => FetchResults(account, entry));
}
public void InvalidateResultsFor(ulong account)
{
_logger.Information("Invalidating proxy cache for account {Account}", account);
_cache.Remove(GetKey(account));
}
public async Task InvalidateResultsForSystem(PKSystem system)
{
_logger.Debug("Invalidating proxy cache for system {System}", system.Id);
using (var conn = await _conn.Obtain())
foreach (var accountId in await conn.QueryAsync<ulong>("select uid from accounts where system = @Id", system))
_cache.Remove(GetKey(accountId));
}
private async Task<IEnumerable<ProxyDatabaseResult>> FetchResults(ulong account, ICacheEntry entry)
{
_logger.Debug("Members for account {Account} not in cache, fetching", account);
using (var conn = await _conn.Obtain())
{
var results = (await conn.QueryAsync<PKMember, PKSystem, ProxyDatabaseResult>(
"select members.*, systems.* from members, systems, accounts where members.system = systems.id and accounts.system = systems.id and accounts.uid = @Uid",
(member, system) =>
new ProxyDatabaseResult {Member = member, System = system}, new {Uid = account})).ToList();
if (results.Count == 0)
{
// Long expiry for accounts with no system registered
entry.SetSlidingExpiration(TimeSpan.FromMinutes(5));
entry.SetAbsoluteExpiration(TimeSpan.FromHours(1));
}
else
{
// Shorter expiry if they already have a system
entry.SetSlidingExpiration(TimeSpan.FromMinutes(1));
entry.SetAbsoluteExpiration(TimeSpan.FromMinutes(5));
}
return results;
}
}
private object GetKey(ulong account)
{
return $"_proxy_account_{account}";
}
}
}

52
PluralKit.Bot/Services/ProxyService.cs
View File

@ -33,23 +33,21 @@ namespace PluralKit.Bot
private EmbedService _embeds;
private ILogger _logger;
private WebhookExecutorService _webhookExecutor;
private ProxyCacheService _cache;
private AutoproxyCacheService _autoproxyCache;
private ProxyCache _cache;
public ProxyService(IDiscordClient client, LogChannelService logChannel, IDataStore data, EmbedService embeds, ILogger logger, ProxyCacheService cache, WebhookExecutorService webhookExecutor, DbConnectionFactory conn, AutoproxyCacheService autoproxyCache)
public ProxyService(IDiscordClient client, LogChannelService logChannel, IDataStore data, EmbedService embeds, ILogger logger, WebhookExecutorService webhookExecutor, DbConnectionFactory conn, ProxyCache cache)
{
_client = client;
_logChannel = logChannel;
_data = data;
_embeds = embeds;
_cache = cache;
_webhookExecutor = webhookExecutor;
_conn = conn;
_autoproxyCache = autoproxyCache;
_cache = cache;
_logger = logger.ForContext<ProxyService>();
}
private ProxyMatch GetProxyTagMatch(string message, IEnumerable<ProxyCacheService.ProxyDatabaseResult> potentialMembers)
private ProxyMatch GetProxyTagMatch(string message, PKSystem system, IEnumerable<PKMember> potentialMembers)
{
// If the message starts with a @mention, and then proceeds to have proxy tags,
// extract the mention and place it inside the inner message
@ -63,7 +61,7 @@ namespace PluralKit.Bot
}
// Flatten and sort by specificity (ProxyString length desc = prefix+suffix length desc = inner message asc = more specific proxy first!)
var ordered = potentialMembers.SelectMany(m => m.Member.ProxyTags.Select(tag => (tag, m))).OrderByDescending(p => p.Item1.ProxyString.Length);
var ordered = potentialMembers.SelectMany(m => m.ProxyTags.Select(tag => (tag, m))).OrderByDescending(p => p.Item1.ProxyString.Length);
foreach (var (tag, match) in ordered)
{
if (tag.Prefix == null && tag.Suffix == null) continue;
@ -84,36 +82,35 @@ namespace PluralKit.Bot
if (isMatch) {
var inner = message.Substring(prefix.Length, message.Length - prefix.Length - suffix.Length);
if (leadingMention != null) inner = $"{leadingMention} {inner}";
return new ProxyMatch { Member = match.Member, System = match.System, InnerText = inner, ProxyTags = tag};
return new ProxyMatch { Member = match, System = system, InnerText = inner, ProxyTags = tag};
}
}
return null;
}
public async Task HandleMessageAsync(IMessage message)
public async Task HandleMessageAsync(GuildConfig guild, CachedAccount account, IMessage message)
{
// Bail early if this isn't in a guild channel
if (!(message.Channel is ITextChannel channel)) return;
// Find a member with proxy tags matching the message
var results = (await _cache.GetResultsFor(message.Author.Id)).ToList();
var match = GetProxyTagMatch(message.Content, results);
var match = GetProxyTagMatch(message.Content, account.System, account.Members);
// O(n) lookup since n is small (max ~100 in prod) and we're more constrained by memory (for a dictionary) here
var systemSettingsForGuild = account.SettingsForGuild(channel.GuildId);
// If we didn't get a match by proxy tags, try to get one by autoproxy
if (match == null) match = await GetAutoproxyMatch(message, channel);
if (match == null) match = await GetAutoproxyMatch(account, systemSettingsForGuild, message, channel);
// If we still haven't found any, just yeet
if (match == null) return;
// Gather all "extra" data from DB at once
var aux = await _data.GetAuxillaryProxyInformation(channel.GuildId, match.System, match.Member);
// And make sure the channel's not blacklisted from proxying.
if (aux.Guild.Blacklist.Contains(channel.Id)) return;
if (guild.Blacklist.Contains(channel.Id)) return;
// Make sure the system hasn't blacklisted the guild either
if (!aux.SystemGuild.ProxyEnabled) return;
if (!systemSettingsForGuild.ProxyEnabled) return;
// We know message.Channel can only be ITextChannel as PK doesn't work in DMs/groups
// Afterwards we ensure the bot has the right permissions, otherwise bail early
@ -123,8 +120,10 @@ namespace PluralKit.Bot
if (match.InnerText.Trim().Length == 0 && message.Attachments.Count == 0)
return;
var memberSettingsForGuild = account.SettingsForMemberGuild(match.Member.Id, channel.GuildId);
// Get variables in order and all
var proxyName = match.Member.ProxyName(match.System.Tag, aux.MemberGuild.DisplayName);
var proxyName = match.Member.ProxyName(match.System.Tag, memberSettingsForGuild.DisplayName);
var avatarUrl = match.Member.AvatarUrl ?? match.System.AvatarUrl;
// If the name's too long (or short), bail
@ -150,7 +149,7 @@ namespace PluralKit.Bot
// Store the message in the database, and log it in the log channel (if applicable)
await _data.AddMessage(message.Author.Id, hookMessageId, channel.GuildId, message.Channel.Id, message.Id, match.Member);
await _logChannel.LogMessage(match.System, match.Member, hookMessageId, message.Id, message.Channel as IGuildChannel, message.Author, match.InnerText, aux.Guild);
await _logChannel.LogMessage(match.System, match.Member, hookMessageId, message.Id, message.Channel as IGuildChannel, message.Author, match.InnerText, guild);
// Wait a second or so before deleting the original message
await Task.Delay(1000);
@ -166,25 +165,21 @@ namespace PluralKit.Bot
}
}
private async Task<ProxyMatch> GetAutoproxyMatch(IMessage message, IGuildChannel channel)
private async Task<ProxyMatch> GetAutoproxyMatch(CachedAccount account, SystemGuildSettings guildSettings, IMessage message, IGuildChannel channel)
{
// For now we use a backslash as an "escape character", subject to change later
if ((message.Content ?? "").TrimStart().StartsWith("\\")) return null;
// Fetch info from the cache, bail if we don't have anything (either no system or no autoproxy settings - AP defaults to off so this works)
var autoproxyCache = await _autoproxyCache.GetGuildSettings(message.Author.Id, channel.GuildId);
if (autoproxyCache == null) return null;
PKMember member = null;
// Figure out which member to proxy as
switch (autoproxyCache.GuildSettings.AutoproxyMode)
switch (guildSettings.AutoproxyMode)
{
case AutoproxyMode.Off:
// Autoproxy off, bail
return null;
case AutoproxyMode.Front:
// Front mode: just use the current first fronter
member = await _data.GetFirstFronter(autoproxyCache.System);
member = await _data.GetFirstFronter(account.System);
break;
case AutoproxyMode.Latch:
// Latch mode: find last proxied message, use *that* member
@ -201,7 +196,8 @@ namespace PluralKit.Bot
break;
case AutoproxyMode.Member:
// Member mode: just use that member
member = autoproxyCache.AutoproxyMember;
// O(n) lookup since n is small (max 1000 de jure) and we're more constrained by memory (for a dictionary) here
member = account.Members.FirstOrDefault(m => m.Id == guildSettings.AutoproxyMember);
break;
}
@ -209,7 +205,7 @@ namespace PluralKit.Bot
if (member == null) return null;
return new ProxyMatch
{
System = autoproxyCache.System,
System = account.System,
Member = member,
// Autoproxying members with no proxy tags is possible, return the correct result
ProxyTags = member.ProxyTags.Count > 0 ? member.ProxyTags.First() : (ProxyTag?) null,

6
PluralKit.Core/Modules.cs
View File

@ -3,8 +3,11 @@ using System;
using App.Metrics;
using Autofac;
using Autofac.Extensions.DependencyInjection;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using NodaTime;
@ -23,6 +26,9 @@ namespace PluralKit.Core
builder.RegisterType<DbConnectionFactory>().AsSelf().SingleInstance();
builder.RegisterType<PostgresDataStore>().AsSelf().As<IDataStore>();
builder.RegisterType<SchemaService>().AsSelf();
builder.Populate(new ServiceCollection().AddMemoryCache());
builder.RegisterType<ProxyCache>().AsSelf().SingleInstance();
}
}

1
PluralKit.Core/PluralKit.Core.csproj
View File

@ -11,6 +11,7 @@
<PackageReference Include="Autofac.Extensions.DependencyInjection" Version="5.0.1" />
<PackageReference Include="Dapper" Version="1.60.6" />
<PackageReference Include="Dapper.Contrib" Version="1.60.1" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="3.1.1" />
<PackageReference Include="Microsoft.Extensions.Configuration" Version="3.1.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.Binder" Version="3.1.0" />
<PackageReference Include="Microsoft.Extensions.Configuration.CommandLine" Version="3.1.0" />

179
PluralKit.Core/ProxyCache.cs Normal file
View File

@ -0,0 +1,179 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using Dapper;
using Microsoft.Extensions.Caching.Memory;
using Serilog;
namespace PluralKit.Core
{
public class ProxyCache
{
// We can NOT depend on IDataStore as that creates a cycle, since it needs access to call the invalidation methods
private IMemoryCache _cache;
private DbConnectionFactory _db;
private ILogger _logger;
public ProxyCache(IMemoryCache cache, DbConnectionFactory db, ILogger logger)
{
_cache = cache;
_db = db;
_logger = logger;
}
public Task InvalidateSystem(PKSystem system) => InvalidateSystem(system.Id);
public async Task InvalidateSystem(int systemId)
{
if (_cache.TryGetValue<CachedAccount>(KeyForSystem(systemId), out var systemCache))
{
// If we have the system cached here, just invalidate for all the accounts we have in the cache
_logger.Debug("Invalidating cache for system {System} and accounts {Accounts}", systemId, systemCache.Accounts);
_cache.Remove(KeyForSystem(systemId));
foreach (var account in systemCache.Accounts)
_cache.Remove(KeyForAccount(account));
return;
}
// If we don't, look up the accounts from the database and invalidate *those*
_cache.Remove(KeyForSystem(systemId));
using var conn = await _db.Obtain();
var accounts = (await conn.QueryAsync<ulong>("select uid from accounts where system = @System", new {System = systemId})).ToArray();
_logger.Debug("Invalidating cache for system {System} and accounts {Accounts}", systemId, accounts);
foreach (var account in accounts)
_cache.Remove(KeyForAccount(account));
}
public void InvalidateGuild(ulong guild)
{
_logger.Debug("Invalidating cache for guild {Guild}", guild);
_cache.Remove(KeyForGuild(guild));
}
public async Task<GuildConfig> GetGuildDataCached(ulong guild)
{
if (_cache.TryGetValue<GuildConfig>(KeyForGuild(guild), out var item))
{
_logger.Verbose("Cache hit for guild {Guild}", guild);
return item;
}
// When changing this, also see PostgresDataStore::GetOrCreateGuildConfig
using var conn = await _db.Obtain();
_logger.Verbose("Cache miss for guild {Guild}", guild);
var guildConfig = (await conn.QuerySingleOrDefaultAsync<PostgresDataStore.DatabaseCompatibleGuildConfig>(
"insert into servers (id) values (@Id) on conflict do nothing; select * from servers where id = @Id",
new {Id = guild})).Into();
_cache.CreateEntry(KeyForGuild(guild))
.SetValue(guildConfig)
.SetSlidingExpiration(TimeSpan.FromMinutes(5))
.SetAbsoluteExpiration(TimeSpan.FromMinutes(30))
.Dispose(); // Don't ask, but this *saves* the entry. Somehow.
return guildConfig;
}
public async Task<CachedAccount> GetAccountDataCached(ulong account)
{
if (_cache.TryGetValue<CachedAccount>(KeyForAccount(account), out var item))
{
_logger.Verbose("Cache hit for account {Account}", account);
return item;
}
_logger.Verbose("Cache miss for account {Account}", account);
var data = await GetAccountData(account);
if (data == null)
{
_logger.Debug("Cached data for account {Account} (no system)", account);
// If we didn't find any value, set a pretty long expiry and the value to null
_cache.CreateEntry(KeyForAccount(account))
.SetValue(null)
.SetSlidingExpiration(TimeSpan.FromMinutes(5))
.SetAbsoluteExpiration(TimeSpan.FromHours(1))
.Dispose(); // Don't ask, but this *saves* the entry. Somehow.
return null;
}
// If we *did* find the value, cache it for *every account in the system* with a shorter expiry
_logger.Debug("Cached data for system {System} and accounts {Account}", data.System.Id, data.Accounts);
foreach (var linkedAccount in data.Accounts)
{
_cache.CreateEntry(KeyForAccount(linkedAccount))
.SetValue(data)
.SetSlidingExpiration(TimeSpan.FromMinutes(5))
.SetAbsoluteExpiration(TimeSpan.FromMinutes(20))
.Dispose(); // Don't ask, but this *saves* the entry. Somehow.
// And also do it for the system itself so we can look up by that
_cache.CreateEntry(KeyForSystem(data.System.Id))
.SetValue(data)
.SetSlidingExpiration(TimeSpan.FromMinutes(5))
.SetAbsoluteExpiration(TimeSpan.FromMinutes(20))
.Dispose(); // Don't ask, but this *saves* the entry. Somehow.
}
return data;
}
private async Task<CachedAccount> GetAccountData(ulong account)
{
using var conn = await _db.Obtain();
// Doing this as two queries instead of a two-step join to avoid sending duplicate rows for the system over the network for each member
// This *may* be less efficient, haven't done too much stuff about this but having the system ID saved is very useful later on
var system = await conn.QuerySingleOrDefaultAsync<PKSystem>("select systems.* from accounts inner join systems on systems.id = accounts.system where accounts.uid = @Account", new { Account = account });
if (system == null) return null; // No system = no members = no cache value
// Fetches:
// - List of accounts in the system
// - List of members in the system
// - List of guild settings for the system (for every guild)
// - List of guild settings for each member (for every guild)
// I'm slightly worried the volume of guild settings will get too much, but for simplicity reasons I decided
// against caching them individually per-guild, since I can't imagine they'll be edited *that* much
var result = await conn.QueryMultipleAsync(@"
select uid from accounts where system = @System;
select * from members where system = @System;
select * from system_guild where system = @System;
select member_guild.* from members inner join member_guild on member_guild.member = members.id where members.system = @System;
", new {System = system.Id});
return new CachedAccount
{
System = system,
Accounts = (await result.ReadAsync<ulong>()).ToArray(),
Members = (await result.ReadAsync<PKMember>()).ToArray(),
SystemGuild = (await result.ReadAsync<SystemGuildSettings>()).ToArray(),
MemberGuild = (await result.ReadAsync<MemberGuildSettings>()).ToArray()
};
}
private string KeyForAccount(ulong account) => $"_account_cache_{account}";
private string KeyForSystem(int system) => $"_system_cache_{system}";
private string KeyForGuild(ulong guild) => $"_guild_cache_{guild}";
}
public class CachedAccount
{
public PKSystem System;
public PKMember[] Members;
public SystemGuildSettings[] SystemGuild;
public MemberGuildSettings[] MemberGuild;
public ulong[] Accounts;
public SystemGuildSettings SettingsForGuild(ulong guild) =>
SystemGuild.FirstOrDefault(s => s.Guild == guild) ?? new SystemGuildSettings();
public MemberGuildSettings SettingsForMemberGuild(int memberId, ulong guild) =>
MemberGuild.FirstOrDefault(m => m.Member == memberId && m.Guild == guild) ?? new MemberGuildSettings();
}
}

25
PluralKit.Core/Stores.cs
View File

@ -6,6 +6,8 @@ using System.Threading.Tasks;
using Dapper;
using NodaTime;
using PluralKit.Core;
using Serilog;
namespace PluralKit {
@ -76,6 +78,7 @@ namespace PluralKit {
public class SystemGuildSettings
{
public ulong Guild { get; set; }
public bool ProxyEnabled { get; set; } = true;
public AutoproxyMode AutoproxyMode { get; set; } = AutoproxyMode.Off;
@ -84,6 +87,8 @@ namespace PluralKit {
public class MemberGuildSettings
{
public int Member { get; set; }
public ulong Guild { get; set; }
public string DisplayName { get; set; }
}
@ -425,11 +430,13 @@ namespace PluralKit {
public class PostgresDataStore: IDataStore {
private DbConnectionFactory _conn;
private ILogger _logger;
private ProxyCache _cache;
public PostgresDataStore(DbConnectionFactory conn, ILogger logger)
public PostgresDataStore(DbConnectionFactory conn, ILogger logger, ProxyCache cache)
{
_conn = conn;
_logger = logger;
_cache = cache;
}
public async Task<IEnumerable<PKMember>> GetConflictingProxies(PKSystem system, ProxyTag tag)
@ -467,6 +474,7 @@ namespace PluralKit {
settings.AutoproxyMode,
settings.AutoproxyMember
});
await _cache.InvalidateSystem(system);
}
public async Task<PKSystem> CreateSystem(string systemName = null) {
@ -481,6 +489,7 @@ namespace PluralKit {
system = await conn.QuerySingleAsync<PKSystem>("insert into systems (hid, name) values (@Hid, @Name) returning *", new { Hid = hid, Name = systemName });
_logger.Information("Created system {System}", system.Id);
// New system has no accounts, therefore nothing gets cached, therefore no need to invalidate caches right here
return system;
}
@ -491,6 +500,7 @@ namespace PluralKit {
await conn.ExecuteAsync("insert into accounts (uid, system) values (@Id, @SystemId) on conflict do nothing", new { Id = accountId, SystemId = system.Id });
_logger.Information("Linked system {System} to account {Account}", system.Id, accountId);
await _cache.InvalidateSystem(system);
}
public async Task RemoveAccount(PKSystem system, ulong accountId) {
@ -498,6 +508,7 @@ namespace PluralKit {
await conn.ExecuteAsync("delete from accounts where uid = @Id and system = @SystemId", new { Id = accountId, SystemId = system.Id });
_logger.Information("Unlinked system {System} from account {Account}", system.Id, accountId);
await _cache.InvalidateSystem(system);
}
public async Task<PKSystem> GetSystemByAccount(ulong accountId) {
@ -526,12 +537,15 @@ namespace PluralKit {
await conn.ExecuteAsync("update systems set name = @Name, description = @Description, tag = @Tag, avatar_url = @AvatarUrl, token = @Token, ui_tz = @UiTz, description_privacy = @DescriptionPrivacy, member_list_privacy = @MemberListPrivacy, front_privacy = @FrontPrivacy, front_history_privacy = @FrontHistoryPrivacy where id = @Id", system);
_logger.Information("Updated system {@System}", system);
await _cache.InvalidateSystem(system);
}
public async Task DeleteSystem(PKSystem system) {
using (var conn = await _conn.Obtain())
await conn.ExecuteAsync("delete from systems where id = @Id", system);
_logger.Information("Deleted system {System}", system.Id);
await _cache.InvalidateSystem(system);
}
public async Task<IEnumerable<ulong>> GetSystemAccounts(PKSystem system)
@ -568,6 +582,7 @@ namespace PluralKit {
});
_logger.Information("Created member {Member}", member.Id);
await _cache.InvalidateSystem(system);
return member;
}
@ -598,6 +613,7 @@ namespace PluralKit {
tx.Commit();
_logger.Information("Created {MemberCount} members for system {SystemID}", names.Count(), system.Hid);
await _cache.InvalidateSystem(system);
return results;
}
}
@ -630,6 +646,7 @@ namespace PluralKit {
await conn.ExecuteAsync("update members set name = @Name, display_name = @DisplayName, description = @Description, color = @Color, avatar_url = @AvatarUrl, birthday = @Birthday, pronouns = @Pronouns, proxy_tags = @ProxyTags, keep_proxy = @KeepProxy, member_privacy = @MemberPrivacy where id = @Id", member);
_logger.Information("Updated member {@Member}", member);
await _cache.InvalidateSystem(member.System);
}
public async Task DeleteMember(PKMember member) {
@ -637,6 +654,7 @@ namespace PluralKit {
await conn.ExecuteAsync("delete from members where id = @Id", member);
_logger.Information("Deleted member {@Member}", member);
await _cache.InvalidateSystem(member.System);
}
public async Task<MemberGuildSettings> GetMemberGuildSettings(PKMember member, ulong guild)
@ -653,6 +671,7 @@ namespace PluralKit {
await conn.ExecuteAsync(
"insert into member_guild (member, guild, display_name) values (@Member, @Guild, @DisplayName) on conflict (member, guild) do update set display_name = @Displayname",
new {Member = member.Id, Guild = guild, DisplayName = settings.DisplayName});
await _cache.InvalidateSystem(member.System);
}
public async Task<ulong> GetMemberMessageCount(PKMember member)
@ -749,7 +768,7 @@ namespace PluralKit {
}
// Same as GuildConfig, but with ISet<ulong> as long[] instead.
private struct DatabaseCompatibleGuildConfig
public struct DatabaseCompatibleGuildConfig
{
public ulong Id { get; set; }
public ulong? LogChannel { get; set; }
@ -768,6 +787,7 @@ namespace PluralKit {
public async Task<GuildConfig> GetOrCreateGuildConfig(ulong guild)
{
// When changing this, also see ProxyCache::GetGuildDataCached
using (var conn = await _conn.Obtain())
{
return (await conn.QuerySingleOrDefaultAsync<DatabaseCompatibleGuildConfig>(
@ -787,6 +807,7 @@ namespace PluralKit {
Blacklist = cfg.Blacklist.Select(c => (long) c).ToList()
});
_logger.Information("Updated guild configuration {@GuildCfg}", cfg);
_cache.InvalidateGuild(cfg.Id);
}
public async Task<AuxillaryProxyInformation> GetAuxillaryProxyInformation(ulong guild, PKSystem system, PKMember member)