liz/PluralKit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Too many refactors in one:

Browse Source 
- Allowed adding ephemeral(ish) views and functions
- Moved message_count to a concrete database field
- Moved most proxy logic to a stored procedure
- Moved database files around and refactored schema manager
This commit is contained in:
Ske
2020-06-12 20:29:50 +02:00
parent 24f1363bb0
commit ba441a15cc
37 changed files with 554 additions and 398 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
PluralKit.Core/Services/DataFileService.cs
View File

@@ -30,7 +30,6 @@ namespace PluralKit.Core
// Export members
var members = new List<DataFileMember>();
var pkMembers = _data.GetSystemMembers(system); // Read all members in the system
var messageCounts = await _data.GetMemberMessageCountBulk(system); // Count messages proxied by all members in the system
await foreach (var member in pkMembers.Select(m => new DataFileMember
{
@@ -45,7 +44,7 @@ namespace PluralKit.Core
ProxyTags = m.ProxyTags,
KeepProxy = m.KeepProxy,
Created = DateTimeFormats.TimestampExportFormat.Format(m.Created),
MessageCount = messageCounts.Where(x => x.Member == m.Id).Select(x => x.MessageCount).FirstOrDefault()
MessageCount = m.MessageCount
})) members.Add(member);
// Export switches

23
PluralKit.Core/Services/IDataStore.cs
View File

@@ -41,12 +41,6 @@ namespace PluralKit.Core {
public Instant TimespanEnd;
}
public struct MemberMessageCount
{
public int Member;
public int MessageCount;
}
public struct FrontBreakdown
{
public Dictionary<PKMember, Duration> MemberSwitchDurations;
@@ -208,18 +202,7 @@ namespace PluralKit.Core {
/// </summary>
/// <returns>An enumerable of <see cref="PKMember"/> structs representing each member in the system, in no particular order.</returns>
IAsyncEnumerable<PKMember> GetSystemMembers(PKSystem system, bool orderByName = false);
/// <summary>
/// Gets the amount of messages proxied by a given member.
/// </summary>
/// <returns>The message count of the given member.</returns>
Task<ulong> GetMemberMessageCount(PKMember member);
/// <summary>
/// Collects a breakdown of each member in a system's message count.
/// </summary>
/// <returns>An enumerable of members along with their message counts.</returns>
Task<IEnumerable<MemberMessageCount>> GetMemberMessageCountBulk(PKSystem system);
/// <summary>
/// Creates a member, auto-generating its corresponding IDs.
/// </summary>
@@ -267,9 +250,9 @@ namespace PluralKit.Core {
/// <param name="channelId">The ID of the channel the message was posted to.</param>
/// <param name="postedMessageId">The ID of the message posted by the webhook.</param>
/// <param name="triggerMessageId">The ID of the original trigger message containing the proxy tags.</param>
/// <param name="proxiedMember">The member (and by extension system) that was proxied.</param>
/// <param name="proxiedMemberId">The member (and by extension system) that was proxied.</param>
/// <returns></returns>
Task AddMessage(ulong senderAccount, ulong guildId, ulong channelId, ulong postedMessageId, ulong triggerMessageId, PKMember proxiedMember);
Task AddMessage(ulong senderAccount, ulong guildId, ulong channelId, ulong postedMessageId, ulong triggerMessageId, int proxiedMemberId);
/// <summary>
/// Deletes a message from the data store.

29
PluralKit.Core/Services/PostgresDataStore.cs
View File

@@ -231,25 +231,6 @@ namespace PluralKit.Core {
await _cache.InvalidateSystem(member.System);
}
public async Task<ulong> GetMemberMessageCount(PKMember member)
{
using (var conn = await _conn.Obtain())
return await conn.QuerySingleAsync<ulong>("select count(*) from messages where member = @Id", member);
}
public async Task<IEnumerable<MemberMessageCount>> GetMemberMessageCountBulk(PKSystem system)
{
using (var conn = await _conn.Obtain())
return await conn.QueryAsync<MemberMessageCount>(
@"SELECT messages.member, COUNT(messages.member) messagecount
FROM members
JOIN messages
ON members.id = messages.member
WHERE members.system = @System
GROUP BY messages.member",
new { System = system.Id });
}
public async Task<int> GetSystemMemberCount(PKSystem system, bool includePrivate)
{
var query = "select count(*) from members where system = @Id";
@@ -264,19 +245,19 @@ namespace PluralKit.Core {
using (var conn = await _conn.Obtain())
return await conn.ExecuteScalarAsync<ulong>("select count(id) from members");
}
public async Task AddMessage(ulong senderId, ulong messageId, ulong guildId, ulong channelId, ulong originalMessage, PKMember member) {
public async Task AddMessage(ulong senderId, ulong guildId, ulong channelId, ulong postedMessageId, ulong triggerMessageId, int proxiedMemberId) {
using (var conn = await _conn.Obtain())
// "on conflict do nothing" in the (pretty rare) case of duplicate events coming in from Discord, which would lead to a DB error before
await conn.ExecuteAsync("insert into messages(mid, guild, channel, member, sender, original_mid) values(@MessageId, @GuildId, @ChannelId, @MemberId, @SenderId, @OriginalMid) on conflict do nothing", new {
MessageId = messageId,
MessageId = postedMessageId,
GuildId = guildId,
ChannelId = channelId,
MemberId = member.Id,
MemberId = proxiedMemberId,
SenderId = senderId,
OriginalMid = originalMessage
OriginalMid = triggerMessageId
});
_logger.Information("Stored message {Message} in channel {Channel}", messageId, channelId);
_logger.Debug("Stored message {Message} in channel {Channel}", postedMessageId, channelId);
}
public async Task<FullMessage> GetMessage(ulong id)

74
PluralKit.Core/Services/SchemaService.cs
View File

@@ -1,74 +0,0 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Dapper;
using Npgsql;
using Serilog;
namespace PluralKit.Core {
public class SchemaService
{
private const int TargetSchemaVersion = 6;
private DbConnectionFactory _conn;
private ILogger _logger;
public SchemaService(DbConnectionFactory conn, ILogger logger)
{
_conn = conn;
_logger = logger.ForContext<SchemaService>();
}
public static void Initialize()
{
// Without these it'll still *work* but break at the first launch + probably cause other small issues
NpgsqlConnection.GlobalTypeMapper.MapComposite<ProxyTag>("proxy_tag");
NpgsqlConnection.GlobalTypeMapper.MapEnum<PrivacyLevel>("privacy_level");
}
public async Task ApplyMigrations()
{
for (var version = 0; version <= TargetSchemaVersion; version++)
await ApplyMigration(version);
}
private async Task ApplyMigration(int migrationId)
{
// migrationId is the *target* version
using var conn = await _conn.Obtain();
using var tx = conn.BeginTransaction();
// See if we even have the info table... if not, we implicitly define the version as -1
// This means migration 0 will get executed, which ensures we're at a consistent state.
// *Technically* this also means schema version 0 will be identified as -1, but since we're only doing these
// checks in the above for loop, this doesn't matter.
var hasInfoTable = await conn.QuerySingleOrDefaultAsync<int>("select count(*) from information_schema.tables where table_name = 'info'") == 1;
int currentVersion;
if (hasInfoTable)
currentVersion = await conn.QuerySingleOrDefaultAsync<int>("select schema_version from info");
else currentVersion = -1;
if (currentVersion >= migrationId)
return; // Don't execute the migration if we're already at the target version.
using var stream = typeof(SchemaService).Assembly.GetManifestResourceStream($"PluralKit.Core.Migrations.{migrationId}.sql");
if (stream == null) throw new ArgumentException("Invalid migration ID");
using var reader = new StreamReader(stream);
var migrationQuery = await reader.ReadToEndAsync();
_logger.Information("Current schema version is {CurrentVersion}, applying migration {MigrationId}", currentVersion, migrationId);
await conn.ExecuteAsync(migrationQuery, transaction: tx);
tx.Commit();
// If the above migration creates new enum/composite types, we must tell Npgsql to reload the internal type caches
// This will propagate to every other connection as well, since it marks the global type mapper collection dirty.
// TODO: find a way to get around the cast to our internal tracker wrapper... this could break if that ever changes
((PerformanceTrackingConnection) conn)._impl.ReloadTypes();
}
}
}