Merge pull request #133 from nephanim/perf/export

Improve export performance

PluralKit.Core/DataFiles.cs

@@ -26,11 +26,34 @@ namespace PluralKit.Bot
 
         public async Task<DataFileSystem> ExportSystem(PKSystem system)
         {
+            // Export members
             var members = new List<DataFileMember>();
-            foreach (var member in await _members.GetBySystem(system)) members.Add(await ExportMember(member));
+            var pkMembers = await _members.GetBySystem(system); // Read all members in the system
+            var messageCounts = await _members.MessageCountsPerMember(system); // Count messages proxied by all members in the system
+            members.AddRange(pkMembers.Select(m => new DataFileMember
+            {
+                Id = m.Hid,
+                Name = m.Name,
+                DisplayName = m.DisplayName,
+                Description = m.Description,
+                Birthday = m.Birthday != null ? Formats.DateExportFormat.Format(m.Birthday.Value) : null,
+                Pronouns = m.Pronouns,
+                Color = m.Color,
+                AvatarUrl = m.AvatarUrl,
+                Prefix = m.Prefix,
+                Suffix = m.Suffix,
+                Created = Formats.TimestampExportFormat.Format(m.Created),
+                MessageCount = messageCounts.Where(x => x.Member.Equals(m.Id)).Select(x => x.MessageCount).FirstOrDefault()
+            }));
 
+            // Export switches
             var switches = new List<DataFileSwitch>();
-            foreach (var sw in await _switches.GetSwitches(system, 999999)) switches.Add(await ExportSwitch(sw));
+            var switchList = await _switches.GetTruncatedSwitchList(system, Instant.FromDateTimeUtc(DateTime.MinValue.ToUniversalTime()), SystemClock.Instance.GetCurrentInstant());
+            switches.AddRange(switchList.Select(x => new DataFileSwitch
+            {
+                Timestamp = Formats.TimestampExportFormat.Format(x.TimespanStart),
+                Members = x.Members.Select(m => m.Hid).ToList() // Look up member's HID using the member export from above
+            }));
 
             return new DataFileSystem
             {

PluralKit.Core/Stores.cs

@@ -172,6 +172,25 @@ namespace PluralKit {
                 return await conn.QuerySingleAsync<int>("select count(*) from messages where member = @Id", member);
         }
 
+        public struct MessageBreakdownListEntry
+        {
+            public int Member;
+            public int MessageCount;
+        }
+
+        public async Task<IEnumerable<MessageBreakdownListEntry>> MessageCountsPerMember(PKSystem system)
+        {
+            using (var conn = await _conn.Obtain())
+                return await conn.QueryAsync<MessageBreakdownListEntry>(
+                    @"SELECT messages.member, COUNT(messages.member) messagecount
+                        FROM members
+                        JOIN messages
+                        ON members.id = messages.member
+                        WHERE members.system = @System
+                        GROUP BY messages.member",
+                    new { System = system.Id });
+        }
+
         public async Task<int> MemberCount(PKSystem system)
         {
             using (var conn = await _conn.Obtain())
@@ -362,7 +381,7 @@ namespace PluralKit {
                 var switchMembersEntries = await conn.QueryAsync<SwitchMembersListEntry>(
                     @"SELECT switch_members.member, switches.timestamp
                         FROM switches
-                        JOIN switch_members
+                        LEFT JOIN switch_members
                         ON switches.id = switch_members.switch
                         WHERE switches.system = @System
                         AND (
@@ -451,7 +470,7 @@ namespace PluralKit {
                 select new SwitchListEntry
                 {
                     TimespanStart = g.Key,
-                    Members = g.Select(x => memberObjects[x.Member]).ToList()
+                    Members = g.Where(x => x.Member != 0).Select(x => memberObjects[x.Member]).ToList()
                 };
 
             // Loop through every switch that overlaps the range and add it to the output list