liz/PluralKit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add front history pagination; upgrade more store methods

Browse Source
This commit is contained in:
Ske
2020-01-18 00:58:35 +01:00
parent 8a689ac0f2
commit b347d2d557
8 changed files with 178 additions and 109 deletions
Download Patch File Download Diff File Expand all files Collapse all files

28
PluralKit.Core/DataFiles.cs
View File

@@ -25,9 +25,10 @@ namespace PluralKit.Bot
{
// Export members
var members = new List<DataFileMember>();
var pkMembers = await _data.GetSystemMembers(system); // Read all members in the system
var pkMembers = _data.GetSystemMembers(system); // Read all members in the system
var messageCounts = await _data.GetMemberMessageCountBulk(system); // Count messages proxied by all members in the system
members.AddRange(pkMembers.Select(m => new DataFileMember
await foreach (var member in pkMembers.Select(m => new DataFileMember
{
Id = m.Hid,
Name = m.Name,
@@ -41,7 +42,7 @@ namespace PluralKit.Bot
KeepProxy = m.KeepProxy,
Created = Formats.TimestampExportFormat.Format(m.Created),
MessageCount = messageCounts.Where(x => x.Member == m.Id).Select(x => x.MessageCount).FirstOrDefault()
}));
})) members.Add(member);
// Export switches
var switches = new List<DataFileSwitch>();
@@ -96,18 +97,25 @@ namespace PluralKit.Bot
await _data.AddAccount(system, accountId);
// Determine which members already exist and which ones need to be created
var existingMembers = await _data.GetSystemMembers(system);
var membersByHid = new Dictionary<string, PKMember>();
var membersByName = new Dictionary<string, PKMember>();
await foreach (var member in _data.GetSystemMembers(system))
{
membersByHid[member.Hid] = member;
membersByName[member.Name] = member;
}
foreach (var d in data.Members)
{
// Try to look up the member with the given ID
var match = existingMembers.FirstOrDefault(m => m.Hid.Equals(d.Id));
if (match == null)
match = existingMembers.FirstOrDefault(m => m.Name.Equals(d.Name)); // Try with the name instead
PKMember match = null;
if (membersByHid.TryGetValue(d.Id, out var matchByHid)) match = matchByHid; // Try to look up the member with the given ID
else if (membersByName.TryGetValue(d.Id, out var matchByName)) match = matchByName; // Try with the name instead
if (match != null)
{
dataFileToMemberMapping.Add(d.Id, match); // Relate the data file ID to the PKMember for importing switches
result.ModifiedNames.Add(d.Name);
}
}
else
{
unmappedMembers.Add(d); // Track members that weren't found so we can create them all
@@ -117,7 +125,7 @@ namespace PluralKit.Bot
// If creating the unmatched members would put us over the member limit, abort before creating any members
// new total: # in the system + (# in the file - # in the file that already exist)
if (data.Members.Count - dataFileToMemberMapping.Count + existingMembers.Count() > Limits.MaxMemberCount)
if (data.Members.Count - dataFileToMemberMapping.Count + membersByHid.Count > Limits.MaxMemberCount)
{
result.Success = false;
result.Message = $"Import would exceed the maximum number of members ({Limits.MaxMemberCount}).";

65
PluralKit.Core/Stores.cs
View File

@@ -203,7 +203,7 @@ namespace PluralKit {
/// Gets all members inside a given system.
/// </summary>
/// <returns>An enumerable of <see cref="PKMember"/> structs representing each member in the system, in no particular order.</returns>
Task<IEnumerable<PKMember>> GetSystemMembers(PKSystem system);
IAsyncEnumerable<PKMember> GetSystemMembers(PKSystem system, bool orderByName = false);
/// <summary>
/// Gets the amount of messages proxied by a given member.
/// </summary>
@@ -292,6 +292,12 @@ namespace PluralKit {
/// <returns>An enumerable of the *count* latest switches in the system, in latest-first order. May contain fewer elements than requested.</returns>
IAsyncEnumerable<PKSwitch> GetSwitches(PKSystem system);
/// <summary>
/// Gets the total amount of switches in a given system.
/// </summary>
Task<int> GetSwitchCount(PKSystem system);
/// <summary>
/// Gets the latest (temporally; closest to now) switch of a given system.
/// </summary>
@@ -388,7 +394,7 @@ namespace PluralKit {
/// </summary>
Task SaveGuildConfig(GuildConfig cfg);
Task<AuxillaryProxyInformation> GetAuxillaryProxyInformation(ulong guild, PKSystem system, PKMember member);
Task<AuxillaryProxyInformation> GetAuxillaryProxyInformation(ulong guild, PKSystem system, PKMember member);
}
public class PostgresDataStore: IDataStore {
@@ -585,9 +591,11 @@ namespace PluralKit {
return await conn.QueryFirstOrDefaultAsync<PKMember>("select * from members where lower(name) = lower(@Name) and system = @SystemID", new { Name = name, SystemID = system.Id });
}
public async Task<IEnumerable<PKMember>> GetSystemMembers(PKSystem system) {
using (var conn = await _conn.Obtain())
return await conn.QueryAsync<PKMember>("select * from members where system = @SystemID", new { SystemID = system.Id });
public IAsyncEnumerable<PKMember> GetSystemMembers(PKSystem system, bool orderByName)
{
var sql = "select * from members where system = @SystemID";
if (orderByName) sql += " order by lower(name) asc";
return _conn.QueryStreamAsync<PKMember>(sql, new { SystemID = system.Id });
}
public async Task SaveMember(PKMember member) {
@@ -867,24 +875,30 @@ namespace PluralKit {
new {System = system.Id});
}
public async Task<IEnumerable<SwitchMembersListEntry>> GetSwitchMembersList(PKSystem system, Instant start, Instant end)
public async Task<int> GetSwitchCount(PKSystem system)
{
using var conn = await _conn.Obtain();
return await conn.QuerySingleAsync<int>("select count(*) from switches where system = @Id", system);
}
public async IAsyncEnumerable<SwitchMembersListEntry> GetSwitchMembersList(PKSystem system, Instant start, Instant end)
{
// Wrap multiple commands in a single transaction for performance
using (var conn = await _conn.Obtain())
using (var tx = conn.BeginTransaction())
{
// Find the time of the last switch outside the range as it overlaps the range
// If no prior switch exists, the lower bound of the range remains the start time
var lastSwitch = await conn.QuerySingleOrDefaultAsync<Instant>(
@"SELECT COALESCE(MAX(timestamp), @Start)
using var conn = await _conn.Obtain();
using var tx = conn.BeginTransaction();
// Find the time of the last switch outside the range as it overlaps the range
// If no prior switch exists, the lower bound of the range remains the start time
var lastSwitch = await conn.QuerySingleOrDefaultAsync<Instant>(
@"SELECT COALESCE(MAX(timestamp), @Start)
FROM switches
WHERE switches.system = @System
AND switches.timestamp < @Start",
new { System = system.Id, Start = start });
new { System = system.Id, Start = start });
// Then collect the time and members of all switches that overlap the range
var switchMembersEntries = await conn.QueryAsync<SwitchMembersListEntry>(
@"SELECT switch_members.member, switches.timestamp
// Then collect the time and members of all switches that overlap the range
var switchMembersEntries = conn.QueryStreamAsync<SwitchMembersListEntry>(
@"SELECT switch_members.member, switches.timestamp
FROM switches
LEFT JOIN switch_members
ON switches.id = switch_members.switch
@@ -895,12 +909,13 @@ namespace PluralKit {
)
AND switches.timestamp < @End
ORDER BY switches.timestamp DESC",
new { System = system.Id, Start = start, End = end, LastSwitch = lastSwitch });
new { System = system.Id, Start = start, End = end, LastSwitch = lastSwitch });
// Commit and return the list
tx.Commit();
return switchMembersEntries;
}
// Yield each value here
await foreach (var entry in switchMembersEntries)
yield return entry;
// Don't really need to worry about the transaction here, we're not doing any *writes*
}
public IAsyncEnumerable<PKMember> GetSwitchMembers(PKSwitch sw)
@@ -938,8 +953,10 @@ namespace PluralKit {
public async Task<IEnumerable<SwitchListEntry>> GetPeriodFronters(PKSystem system, Instant periodStart, Instant periodEnd)
{
// TODO: IAsyncEnumerable-ify this one
// Returns the timestamps and member IDs of switches overlapping the range, in chronological (newest first) order
var switchMembers = await GetSwitchMembersList(system, periodStart, periodEnd);
var switchMembers = await GetSwitchMembersList(system, periodStart, periodEnd).ToListAsync();
// query DB for all members involved in any of the switches above and collect into a dictionary for future use
// this makes sure the return list has the same instances of PKMember throughout, which is important for the dictionary
@@ -950,7 +967,7 @@ namespace PluralKit {
memberObjects = (
await conn.QueryAsync<PKMember>(
"select * from members where id = any(@Switches)", // lol postgres specific `= any()` syntax
new { Switches = switchMembers.Select(m => m.Member).Distinct().ToList() })
new { Switches = switchMembers.Select(m => m.Member).Distinct() })
).ToDictionary(m => m.Id);
}

10
PluralKit.Core/Utils.cs
View File

@@ -676,7 +676,15 @@ namespace PluralKit
{
using var conn = await connFactory.Obtain();
var reader = await conn.ExecuteReaderAsync(sql, param);
await using var reader = (DbDataReader) await conn.ExecuteReaderAsync(sql, param);
var parser = reader.GetRowParser<T>();
while (reader.Read())
yield return parser(reader);
}
public static async IAsyncEnumerable<T> QueryStreamAsync<T>(this IDbConnection conn, string sql, object param)
{
await using var reader = (DbDataReader) await conn.ExecuteReaderAsync(sql, param);
var parser = reader.GetRowParser<T>();
while (reader.Read())
yield return parser(reader);