From adea7be07c1cce3ef9d803bf8741e6bcd974eb13 Mon Sep 17 00:00:00 2001 From: Noko Date: Sun, 6 Oct 2019 02:03:28 -0500 Subject: [PATCH] Improve export performance Refactored ExportSystem to: - Only fetch message counts once (instead of a query per member) - Fetch switches using the newly refactored GetTruncatedSwitchList (gets switches and their members in one shot instead of querying for switch members one switch at a time) - Added a new MessageCountsPerMember method to MemberStore to support the above - Modified GetTruncatedSwitchList query to retrieve switches with no members (wasn't important for frontpercent, but is if we're reusing that method here) This doesn't require any index changes beyond those that support GetTruncatedSwitchList, though we can see a small benefit with an index on messages covering the member column (valuable for other reasons outside of these additions). --- PluralKit.Core/DataFiles.cs | 49 +++++++++++++++++++------------------ PluralKit.Core/Stores.cs | 23 +++++++++++++++-- 2 files changed, 46 insertions(+), 26 deletions(-) diff --git a/PluralKit.Core/DataFiles.cs b/PluralKit.Core/DataFiles.cs index 105caad1..80518230 100644 --- a/PluralKit.Core/DataFiles.cs +++ b/PluralKit.Core/DataFiles.cs @@ -26,11 +26,34 @@ namespace PluralKit.Bot public async Task ExportSystem(PKSystem system) { + // Export members var members = new List(); - foreach (var member in await _members.GetBySystem(system)) members.Add(await ExportMember(member)); + var pkMembers = await _members.GetBySystem(system); // Read all members in the system + var messageCounts = await _members.MessageCountsPerMember(system); // Count messages proxied by all members in the system + members.AddRange(pkMembers.Select(m => new DataFileMember + { + Id = m.Hid, + Name = m.Name, + DisplayName = m.DisplayName, + Description = m.Description, + Birthday = m.Birthday != null ? Formats.DateExportFormat.Format(m.Birthday.Value) : null, + Pronouns = m.Pronouns, + Color = m.Color, + AvatarUrl = m.AvatarUrl, + Prefix = m.Prefix, + Suffix = m.Suffix, + Created = Formats.TimestampExportFormat.Format(m.Created), + MessageCount = messageCounts.Where(x => x.Member.Equals(m.Id)).Select(x => x.MessageCount).FirstOrDefault() + })); + // Export switches var switches = new List(); - foreach (var sw in await _switches.GetSwitches(system, 999999)) switches.Add(await ExportSwitch(sw)); + var switchList = await _switches.GetTruncatedSwitchList(system, Instant.FromDateTimeUtc(DateTime.MinValue.ToUniversalTime()), SystemClock.Instance.GetCurrentInstant()); + switches.AddRange(switchList.Select(x => new DataFileSwitch + { + Timestamp = Formats.TimestampExportFormat.Format(x.TimespanStart), + Members = x.Members.Select(m => m.Hid).ToList() // Look up member's HID using the member export from above + })); return new DataFileSystem { @@ -47,28 +70,6 @@ namespace PluralKit.Bot }; } - private async Task ExportMember(PKMember member) => new DataFileMember - { - Id = member.Hid, - Name = member.Name, - DisplayName = member.DisplayName, - Description = member.Description, - Birthday = member.Birthday != null ? Formats.DateExportFormat.Format(member.Birthday.Value) : null, - Pronouns = member.Pronouns, - Color = member.Color, - AvatarUrl = member.AvatarUrl, - Prefix = member.Prefix, - Suffix = member.Suffix, - Created = Formats.TimestampExportFormat.Format(member.Created), - MessageCount = await _members.MessageCount(member) - }; - - private async Task ExportSwitch(PKSwitch sw) => new DataFileSwitch - { - Members = (await _switches.GetSwitchMembers(sw)).Select(m => m.Hid).ToList(), - Timestamp = Formats.TimestampExportFormat.Format(sw.Timestamp) - }; - public async Task ImportSystem(DataFileSystem data, PKSystem system, ulong accountId) { // TODO: make atomic, somehow - we'd need to obtain one IDbConnection and reuse it diff --git a/PluralKit.Core/Stores.cs b/PluralKit.Core/Stores.cs index 81566f82..44aeeff0 100644 --- a/PluralKit.Core/Stores.cs +++ b/PluralKit.Core/Stores.cs @@ -172,6 +172,25 @@ namespace PluralKit { return await conn.QuerySingleAsync("select count(*) from messages where member = @Id", member); } + public struct MessageBreakdownListEntry + { + public int Member; + public int MessageCount; + } + + public async Task> MessageCountsPerMember(PKSystem system) + { + using (var conn = await _conn.Obtain()) + return await conn.QueryAsync( + @"SELECT messages.member, COUNT(messages.member) messagecount + FROM members + JOIN messages + ON members.id = messages.member + WHERE members.system = @System + GROUP BY messages.member", + new { System = system.Id }); + } + public async Task MemberCount(PKSystem system) { using (var conn = await _conn.Obtain()) @@ -362,7 +381,7 @@ namespace PluralKit { var switchMembersEntries = await conn.QueryAsync( @"SELECT switch_members.member, switches.timestamp FROM switches - JOIN switch_members + LEFT JOIN switch_members ON switches.id = switch_members.switch WHERE switches.system = @System AND ( @@ -451,7 +470,7 @@ namespace PluralKit { select new SwitchListEntry { TimespanStart = g.Key, - Members = g.Select(x => memberObjects[x.Member]).ToList() + Members = g.Where(x => x.Member != 0).Select(x => memberObjects[x.Member]).ToList() }; // Loop through every switch that overlaps the range and add it to the output list