feat: upgrade to .NET 6, refactor everything
This commit is contained in:
spiral
@@ -1,160 +1,150 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Threading.Tasks;
|
||||
using Dapper;
|
||||
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
using Autofac;
|
||||
|
||||
using Dapper;
|
||||
|
||||
using Serilog;
|
||||
|
||||
namespace PluralKit.Core
|
||||
namespace PluralKit.Core;
|
||||
|
||||
public partial class BulkImporter: IAsyncDisposable
|
||||
{
|
||||
public partial class BulkImporter: IAsyncDisposable
|
||||
private readonly Dictionary<string, GroupId> _existingGroupHids = new();
|
||||
private readonly Dictionary<string, GroupId> _existingGroupNames = new();
|
||||
|
||||
private readonly Dictionary<string, MemberId> _existingMemberHids = new();
|
||||
private readonly Dictionary<string, MemberId> _existingMemberNames = new();
|
||||
private readonly Dictionary<string, GroupId> _knownGroupIdentifiers = new();
|
||||
private readonly Dictionary<string, MemberId> _knownMemberIdentifiers = new();
|
||||
|
||||
private readonly ImportResultNew _result = new();
|
||||
private ILogger _logger { get; init; }
|
||||
private ModelRepository _repo { get; init; }
|
||||
|
||||
private PKSystem _system { get; set; }
|
||||
private IPKConnection _conn { get; init; }
|
||||
private IPKTransaction _tx { get; init; }
|
||||
|
||||
private Func<string, Task> _confirmFunc { get; init; }
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
private ILogger _logger { get; init; }
|
||||
private ModelRepository _repo { get; init; }
|
||||
|
||||
private PKSystem _system { get; set; }
|
||||
private IPKConnection _conn { get; init; }
|
||||
private IPKTransaction _tx { get; init; }
|
||||
|
||||
private Func<string, Task> _confirmFunc { get; init; }
|
||||
|
||||
private readonly Dictionary<string, MemberId> _existingMemberHids = new();
|
||||
private readonly Dictionary<string, MemberId> _existingMemberNames = new();
|
||||
private readonly Dictionary<string, MemberId> _knownMemberIdentifiers = new();
|
||||
|
||||
private readonly Dictionary<string, GroupId> _existingGroupHids = new();
|
||||
private readonly Dictionary<string, GroupId> _existingGroupNames = new();
|
||||
private readonly Dictionary<string, GroupId> _knownGroupIdentifiers = new();
|
||||
|
||||
private ImportResultNew _result = new();
|
||||
|
||||
internal static async Task<ImportResultNew> PerformImport(IPKConnection conn, IPKTransaction tx, ModelRepository repo, ILogger logger,
|
||||
DispatchService dispatch, ulong userId, PKSystem? system, JObject importFile, Func<string, Task> confirmFunc)
|
||||
// try rolling back the transaction
|
||||
// this will throw if the transaction was committed, but that's fine
|
||||
// so we just catch InvalidOperationException
|
||||
try
|
||||
{
|
||||
await using var importer = new BulkImporter()
|
||||
{
|
||||
_logger = logger,
|
||||
_repo = repo,
|
||||
_system = system,
|
||||
_conn = conn,
|
||||
_tx = tx,
|
||||
_confirmFunc = confirmFunc,
|
||||
};
|
||||
|
||||
if (system == null)
|
||||
{
|
||||
system = await repo.CreateSystem(null, importer._conn);
|
||||
await repo.AddAccount(system.Id, userId, importer._conn);
|
||||
importer._result.CreatedSystem = system.Hid;
|
||||
importer._system = system;
|
||||
}
|
||||
|
||||
// Fetch all members in the system and log their names and hids
|
||||
var members = await conn.QueryAsync<PKMember>("select id, hid, name from members where system = @System",
|
||||
new { System = system.Id });
|
||||
foreach (var m in members)
|
||||
{
|
||||
importer._existingMemberHids[m.Hid] = m.Id;
|
||||
importer._existingMemberNames[m.Name] = m.Id;
|
||||
}
|
||||
|
||||
// same as above for groups
|
||||
var groups = await conn.QueryAsync<PKGroup>("select id, hid, name from groups where system = @System",
|
||||
new { System = system.Id });
|
||||
foreach (var g in groups)
|
||||
{
|
||||
importer._existingGroupHids[g.Hid] = g.Id;
|
||||
importer._existingGroupNames[g.Name] = g.Id;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (importFile.ContainsKey("tuppers"))
|
||||
await importer.ImportTupperbox(importFile);
|
||||
else if (importFile.ContainsKey("switches"))
|
||||
await importer.ImportPluralKit(importFile);
|
||||
else
|
||||
throw new ImportException("File type is unknown.");
|
||||
importer._result.Success = true;
|
||||
await tx.CommitAsync();
|
||||
|
||||
_ = dispatch.Dispatch(system.Id, new UpdateDispatchData()
|
||||
{
|
||||
Event = DispatchEvent.SUCCESSFUL_IMPORT
|
||||
});
|
||||
}
|
||||
catch (ImportException e)
|
||||
{
|
||||
importer._result.Success = false;
|
||||
importer._result.Message = e.Message;
|
||||
}
|
||||
catch (ArgumentNullException)
|
||||
{
|
||||
importer._result.Success = false;
|
||||
}
|
||||
|
||||
return importer._result;
|
||||
}
|
||||
|
||||
private (MemberId?, bool) TryGetExistingMember(string hid, string name)
|
||||
{
|
||||
if (_existingMemberHids.TryGetValue(hid, out var byHid)) return (byHid, true);
|
||||
if (_existingMemberNames.TryGetValue(name, out var byName)) return (byName, false);
|
||||
return (null, false);
|
||||
}
|
||||
|
||||
private (GroupId?, bool) TryGetExistingGroup(string hid, string name)
|
||||
{
|
||||
if (_existingGroupHids.TryGetValue(hid, out var byHid)) return (byHid, true);
|
||||
if (_existingGroupNames.TryGetValue(name, out var byName)) return (byName, false);
|
||||
return (null, false);
|
||||
}
|
||||
|
||||
private async Task AssertMemberLimitNotReached(int newMembers)
|
||||
{
|
||||
var memberLimit = _system.MemberLimitOverride ?? Limits.MaxMemberCount;
|
||||
var existingMembers = await _repo.GetSystemMemberCount(_system.Id);
|
||||
if (existingMembers + newMembers > memberLimit)
|
||||
throw new ImportException($"Import would exceed the maximum number of members ({memberLimit}).");
|
||||
}
|
||||
|
||||
private async Task AssertGroupLimitNotReached(int newGroups)
|
||||
{
|
||||
var limit = _system.GroupLimitOverride ?? Limits.MaxGroupCount;
|
||||
var existing = await _repo.GetSystemGroupCount(_system.Id);
|
||||
if (existing + newGroups > limit)
|
||||
throw new ImportException($"Import would exceed the maximum number of groups ({limit}).");
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
// try rolling back the transaction
|
||||
// this will throw if the transaction was committed, but that's fine
|
||||
// so we just catch InvalidOperationException
|
||||
try
|
||||
{
|
||||
await _tx.RollbackAsync();
|
||||
}
|
||||
catch (InvalidOperationException) { }
|
||||
}
|
||||
|
||||
private class ImportException: Exception
|
||||
{
|
||||
public ImportException(string Message) : base(Message) { }
|
||||
await _tx.RollbackAsync();
|
||||
}
|
||||
catch (InvalidOperationException) { }
|
||||
}
|
||||
|
||||
public record ImportResultNew
|
||||
internal static async Task<ImportResultNew> PerformImport(IPKConnection conn, IPKTransaction tx,
|
||||
ModelRepository repo, ILogger logger, DispatchService dispatch, ulong userId,
|
||||
PKSystem? system, JObject importFile, Func<string, Task> confirmFunc)
|
||||
{
|
||||
public int Added = 0;
|
||||
public int Modified = 0;
|
||||
public bool Success;
|
||||
public string? CreatedSystem;
|
||||
public string? Message;
|
||||
await using var importer = new BulkImporter
|
||||
{
|
||||
_logger = logger,
|
||||
_repo = repo,
|
||||
_system = system,
|
||||
_conn = conn,
|
||||
_tx = tx,
|
||||
_confirmFunc = confirmFunc
|
||||
};
|
||||
|
||||
if (system == null)
|
||||
{
|
||||
system = await repo.CreateSystem(null, importer._conn);
|
||||
await repo.AddAccount(system.Id, userId, importer._conn);
|
||||
importer._result.CreatedSystem = system.Hid;
|
||||
importer._system = system;
|
||||
}
|
||||
|
||||
// Fetch all members in the system and log their names and hids
|
||||
var members = await conn.QueryAsync<PKMember>("select id, hid, name from members where system = @System",
|
||||
new { System = system.Id });
|
||||
foreach (var m in members)
|
||||
{
|
||||
importer._existingMemberHids[m.Hid] = m.Id;
|
||||
importer._existingMemberNames[m.Name] = m.Id;
|
||||
}
|
||||
|
||||
// same as above for groups
|
||||
var groups = await conn.QueryAsync<PKGroup>("select id, hid, name from groups where system = @System",
|
||||
new { System = system.Id });
|
||||
foreach (var g in groups)
|
||||
{
|
||||
importer._existingGroupHids[g.Hid] = g.Id;
|
||||
importer._existingGroupNames[g.Name] = g.Id;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
if (importFile.ContainsKey("tuppers"))
|
||||
await importer.ImportTupperbox(importFile);
|
||||
else if (importFile.ContainsKey("switches"))
|
||||
await importer.ImportPluralKit(importFile);
|
||||
else
|
||||
throw new ImportException("File type is unknown.");
|
||||
importer._result.Success = true;
|
||||
await tx.CommitAsync();
|
||||
|
||||
_ = dispatch.Dispatch(system.Id, new UpdateDispatchData { Event = DispatchEvent.SUCCESSFUL_IMPORT });
|
||||
}
|
||||
catch (ImportException e)
|
||||
{
|
||||
importer._result.Success = false;
|
||||
importer._result.Message = e.Message;
|
||||
}
|
||||
catch (ArgumentNullException)
|
||||
{
|
||||
importer._result.Success = false;
|
||||
}
|
||||
|
||||
return importer._result;
|
||||
}
|
||||
|
||||
private (MemberId?, bool) TryGetExistingMember(string hid, string name)
|
||||
{
|
||||
if (_existingMemberHids.TryGetValue(hid, out var byHid)) return (byHid, true);
|
||||
if (_existingMemberNames.TryGetValue(name, out var byName)) return (byName, false);
|
||||
return (null, false);
|
||||
}
|
||||
|
||||
private (GroupId?, bool) TryGetExistingGroup(string hid, string name)
|
||||
{
|
||||
if (_existingGroupHids.TryGetValue(hid, out var byHid)) return (byHid, true);
|
||||
if (_existingGroupNames.TryGetValue(name, out var byName)) return (byName, false);
|
||||
return (null, false);
|
||||
}
|
||||
|
||||
private async Task AssertMemberLimitNotReached(int newMembers)
|
||||
{
|
||||
var memberLimit = _system.MemberLimitOverride ?? Limits.MaxMemberCount;
|
||||
var existingMembers = await _repo.GetSystemMemberCount(_system.Id);
|
||||
if (existingMembers + newMembers > memberLimit)
|
||||
throw new ImportException($"Import would exceed the maximum number of members ({memberLimit}).");
|
||||
}
|
||||
|
||||
private async Task AssertGroupLimitNotReached(int newGroups)
|
||||
{
|
||||
var limit = _system.GroupLimitOverride ?? Limits.MaxGroupCount;
|
||||
var existing = await _repo.GetSystemGroupCount(_system.Id);
|
||||
if (existing + newGroups > limit)
|
||||
throw new ImportException($"Import would exceed the maximum number of groups ({limit}).");
|
||||
}
|
||||
|
||||
private class ImportException: Exception
|
||||
{
|
||||
public ImportException(string Message) : base(Message) { }
|
||||
}
|
||||
}
|
||||
|
||||
public record ImportResultNew
|
||||
{
|
||||
public int Added = 0;
|
||||
public string? CreatedSystem;
|
||||
public string? Message;
|
||||
public int Modified = 0;
|
||||
public bool Success;
|
||||
}
|
||||
@@ -1,8 +1,4 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Dapper;
|
||||
|
||||
@@ -12,245 +8,254 @@ using NodaTime;
|
||||
|
||||
using NpgsqlTypes;
|
||||
|
||||
namespace PluralKit.Core
|
||||
namespace PluralKit.Core;
|
||||
|
||||
public partial class BulkImporter
|
||||
{
|
||||
public partial class BulkImporter
|
||||
private async Task<ImportResultNew> ImportPluralKit(JObject importFile)
|
||||
{
|
||||
private async Task<ImportResultNew> ImportPluralKit(JObject importFile)
|
||||
var patch = SystemPatch.FromJSON(importFile);
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
{
|
||||
var patch = SystemPatch.FromJSON(importFile);
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in export file is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
if (err.Text != null)
|
||||
throw new ImportException(err.Text);
|
||||
throw new ImportException($"Field {err.Key} in export file is invalid.");
|
||||
}
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
await _repo.UpdateSystem(_system.Id, patch, _conn);
|
||||
|
||||
var members = importFile.Value<JArray>("members");
|
||||
var groups = importFile.Value<JArray>("groups");
|
||||
var switches = importFile.Value<JArray>("switches");
|
||||
|
||||
var newMembers = members.Count(m =>
|
||||
{
|
||||
var (found, _) = TryGetExistingMember(m.Value<string>("id"), m.Value<string>("name"));
|
||||
return found == null;
|
||||
});
|
||||
await AssertMemberLimitNotReached(newMembers);
|
||||
|
||||
if (groups != null)
|
||||
{
|
||||
var newGroups = groups.Count(g =>
|
||||
{
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in export file is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
else if (err.Text != null)
|
||||
throw new ImportException(err.Text);
|
||||
else
|
||||
throw new ImportException($"Field {err.Key} in export file is invalid.");
|
||||
}
|
||||
|
||||
await _repo.UpdateSystem(_system.Id, patch, _conn);
|
||||
|
||||
var members = importFile.Value<JArray>("members");
|
||||
var groups = importFile.Value<JArray>("groups");
|
||||
var switches = importFile.Value<JArray>("switches");
|
||||
|
||||
var newMembers = members.Count(m =>
|
||||
{
|
||||
var (found, _) = TryGetExistingMember(m.Value<string>("id"), m.Value<string>("name"));
|
||||
var (found, _) = TryGetExistingGroup(g.Value<string>("id"), g.Value<string>("name"));
|
||||
return found == null;
|
||||
});
|
||||
await AssertMemberLimitNotReached(newMembers);
|
||||
|
||||
if (groups != null)
|
||||
{
|
||||
var newGroups = groups.Count(g =>
|
||||
{
|
||||
var (found, _) = TryGetExistingGroup(g.Value<string>("id"), g.Value<string>("name"));
|
||||
return found == null;
|
||||
});
|
||||
await AssertGroupLimitNotReached(newGroups);
|
||||
}
|
||||
|
||||
foreach (JObject member in members)
|
||||
await ImportMember(member);
|
||||
|
||||
if (groups != null)
|
||||
foreach (JObject group in groups)
|
||||
await ImportGroup(group);
|
||||
|
||||
if (switches.Any(sw => sw.Value<JArray>("members").Any(m => !_knownMemberIdentifiers.ContainsKey((string)m))))
|
||||
throw new ImportException("One or more switches include members that haven't been imported.");
|
||||
|
||||
await ImportSwitches(switches);
|
||||
|
||||
return _result;
|
||||
await AssertGroupLimitNotReached(newGroups);
|
||||
}
|
||||
|
||||
private async Task ImportMember(JObject member)
|
||||
foreach (JObject member in members)
|
||||
await ImportMember(member);
|
||||
|
||||
if (groups != null)
|
||||
foreach (JObject group in groups)
|
||||
await ImportGroup(group);
|
||||
|
||||
if (switches.Any(sw =>
|
||||
sw.Value<JArray>("members").Any(m => !_knownMemberIdentifiers.ContainsKey((string)m))))
|
||||
throw new ImportException("One or more switches include members that haven't been imported.");
|
||||
|
||||
await ImportSwitches(switches);
|
||||
|
||||
return _result;
|
||||
}
|
||||
|
||||
private async Task ImportMember(JObject member)
|
||||
{
|
||||
var id = member.Value<string>("id");
|
||||
var name = member.Value<string>("name");
|
||||
|
||||
var (found, isHidExisting) = TryGetExistingMember(id, name);
|
||||
var isNewMember = found == null;
|
||||
var referenceName = isHidExisting ? id : name;
|
||||
|
||||
if (isNewMember)
|
||||
_result.Added++;
|
||||
else
|
||||
_result.Modified++;
|
||||
|
||||
_logger.Debug(
|
||||
"Importing member with identifier {FileId} to system {System} (is creating new member? {IsCreatingNewMember})",
|
||||
referenceName, _system.Id, isNewMember
|
||||
);
|
||||
|
||||
var patch = MemberPatch.FromJSON(member);
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
{
|
||||
var id = member.Value<string>("id");
|
||||
var name = member.Value<string>("name");
|
||||
|
||||
var (found, isHidExisting) = TryGetExistingMember(id, name);
|
||||
var isNewMember = found == null;
|
||||
var referenceName = isHidExisting ? id : name;
|
||||
|
||||
if (isNewMember)
|
||||
_result.Added++;
|
||||
else
|
||||
_result.Modified++;
|
||||
|
||||
_logger.Debug(
|
||||
"Importing member with identifier {FileId} to system {System} (is creating new member? {IsCreatingNewMember})",
|
||||
referenceName, _system.Id, isNewMember
|
||||
);
|
||||
|
||||
var patch = MemberPatch.FromJSON(member);
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
{
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in member {name} is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
else if (err.Text != null)
|
||||
throw new ImportException($"member {name}: {err.Text}");
|
||||
else
|
||||
throw new ImportException($"Field {err.Key} in member {name} is invalid.");
|
||||
}
|
||||
|
||||
MemberId? memberId = found;
|
||||
|
||||
if (isNewMember)
|
||||
{
|
||||
var newMember = await _repo.CreateMember(_system.Id, patch.Name.Value, _conn);
|
||||
memberId = newMember.Id;
|
||||
}
|
||||
|
||||
_knownMemberIdentifiers[id] = memberId.Value;
|
||||
|
||||
await _repo.UpdateMember(memberId.Value, patch, _conn);
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in member {name} is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
if (err.Text != null)
|
||||
throw new ImportException($"member {name}: {err.Text}");
|
||||
throw new ImportException($"Field {err.Key} in member {name} is invalid.");
|
||||
}
|
||||
|
||||
private async Task ImportGroup(JObject group)
|
||||
var memberId = found;
|
||||
|
||||
if (isNewMember)
|
||||
{
|
||||
var id = group.Value<string>("id");
|
||||
var name = group.Value<string>("name");
|
||||
var newMember = await _repo.CreateMember(_system.Id, patch.Name.Value, _conn);
|
||||
memberId = newMember.Id;
|
||||
}
|
||||
|
||||
var (found, isHidExisting) = TryGetExistingGroup(id, name);
|
||||
var isNewGroup = found == null;
|
||||
var referenceName = isHidExisting ? id : name;
|
||||
_knownMemberIdentifiers[id] = memberId.Value;
|
||||
|
||||
_logger.Debug(
|
||||
"Importing group with identifier {FileId} to system {System} (is creating new group? {IsCreatingNewGroup})",
|
||||
referenceName, _system.Id, isNewGroup
|
||||
);
|
||||
await _repo.UpdateMember(memberId.Value, patch, _conn);
|
||||
}
|
||||
|
||||
var patch = GroupPatch.FromJson(group);
|
||||
private async Task ImportGroup(JObject group)
|
||||
{
|
||||
var id = group.Value<string>("id");
|
||||
var name = group.Value<string>("name");
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
var (found, isHidExisting) = TryGetExistingGroup(id, name);
|
||||
var isNewGroup = found == null;
|
||||
var referenceName = isHidExisting ? id : name;
|
||||
|
||||
_logger.Debug(
|
||||
"Importing group with identifier {FileId} to system {System} (is creating new group? {IsCreatingNewGroup})",
|
||||
referenceName, _system.Id, isNewGroup
|
||||
);
|
||||
|
||||
var patch = GroupPatch.FromJson(group);
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
{
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in group {name} is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
if (err.Text != null)
|
||||
throw new ImportException($"group {name}: {err.Text}");
|
||||
throw new ImportException($"Field {err.Key} in group {name} is invalid.");
|
||||
}
|
||||
|
||||
var groupId = found;
|
||||
|
||||
if (isNewGroup)
|
||||
{
|
||||
var newGroup = await _repo.CreateGroup(_system.Id, patch.Name.Value, _conn);
|
||||
groupId = newGroup.Id;
|
||||
}
|
||||
|
||||
_knownGroupIdentifiers[id] = groupId.Value;
|
||||
|
||||
await _repo.UpdateGroup(groupId.Value, patch, _conn);
|
||||
|
||||
var groupMembers = group.Value<JArray>("members");
|
||||
var currentGroupMembers = (await _conn.QueryAsync<MemberId>(
|
||||
"select member_id from group_members where group_id = @groupId",
|
||||
new { groupId = groupId.Value }
|
||||
)).ToList();
|
||||
|
||||
await using (var importer =
|
||||
_conn.BeginBinaryImport("copy group_members (group_id, member_id) from stdin (format binary)"))
|
||||
{
|
||||
foreach (var memberIdentifier in groupMembers)
|
||||
{
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in group {name} is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
else if (err.Text != null)
|
||||
throw new ImportException($"group {name}: {err.Text}");
|
||||
else
|
||||
throw new ImportException($"Field {err.Key} in group {name} is invalid.");
|
||||
if (!_knownMemberIdentifiers.TryGetValue(memberIdentifier.ToString(), out var memberId))
|
||||
throw new Exception(
|
||||
$"Attempted to import group member with member identifier {memberIdentifier} but could not find a recently imported member with this id!");
|
||||
|
||||
if (currentGroupMembers.Contains(memberId))
|
||||
continue;
|
||||
|
||||
await importer.StartRowAsync();
|
||||
await importer.WriteAsync(groupId.Value.Value, NpgsqlDbType.Integer);
|
||||
await importer.WriteAsync(memberId.Value, NpgsqlDbType.Integer);
|
||||
}
|
||||
|
||||
GroupId? groupId = found;
|
||||
await importer.CompleteAsync();
|
||||
}
|
||||
}
|
||||
|
||||
if (isNewGroup)
|
||||
private async Task ImportSwitches(JArray switches)
|
||||
{
|
||||
var existingSwitches =
|
||||
(await _conn.QueryAsync<PKSwitch>("select * from switches where system = @System",
|
||||
new { System = _system.Id })).ToList();
|
||||
var existingTimestamps = existingSwitches.Select(sw => sw.Timestamp).ToImmutableHashSet();
|
||||
var lastSwitchId = existingSwitches.Count != 0
|
||||
? existingSwitches.Select(sw => sw.Id).Max()
|
||||
: (SwitchId?)null;
|
||||
|
||||
if (switches.Count > 10000)
|
||||
throw new ImportException("Too many switches present in import file.");
|
||||
|
||||
// Import switch definitions
|
||||
var importedSwitches = new Dictionary<Instant, JArray>();
|
||||
await using (var importer =
|
||||
_conn.BeginBinaryImport("copy switches (system, timestamp) from stdin (format binary)"))
|
||||
{
|
||||
foreach (var sw in switches)
|
||||
{
|
||||
var newGroup = await _repo.CreateGroup(_system.Id, patch.Name.Value, _conn);
|
||||
groupId = newGroup.Id;
|
||||
var timestampString = sw.Value<string>("timestamp");
|
||||
var timestamp = DateTimeFormats.TimestampExportFormat.Parse(timestampString);
|
||||
if (!timestamp.Success)
|
||||
throw new ImportException($"Switch timestamp {timestampString} is not an valid timestamp.");
|
||||
|
||||
// Don't import duplicate switches
|
||||
if (existingTimestamps.Contains(timestamp.Value)) continue;
|
||||
|
||||
// Otherwise, write to importer
|
||||
await importer.StartRowAsync();
|
||||
await importer.WriteAsync(_system.Id.Value, NpgsqlDbType.Integer);
|
||||
await importer.WriteAsync(timestamp.Value, NpgsqlDbType.Timestamp);
|
||||
|
||||
var members = sw.Value<JArray>("members");
|
||||
if (members.Count > Limits.MaxSwitchMemberCount)
|
||||
throw new ImportException(
|
||||
$"Switch with timestamp {timestampString} contains too many members ({members.Count} > 100).");
|
||||
|
||||
// Note that we've imported a switch with this timestamp
|
||||
importedSwitches[timestamp.Value] = sw.Value<JArray>("members");
|
||||
}
|
||||
|
||||
_knownGroupIdentifiers[id] = groupId.Value;
|
||||
// Commit the import
|
||||
await importer.CompleteAsync();
|
||||
}
|
||||
|
||||
await _repo.UpdateGroup(groupId.Value, patch, _conn);
|
||||
// Now, fetch all the switches we just added (so, now we get their IDs too)
|
||||
// IDs are sequential, so any ID in this system, with a switch ID > the last max, will be one we just added
|
||||
var justAddedSwitches = await _conn.QueryAsync<PKSwitch>(
|
||||
"select * from switches where system = @System and id > @LastSwitchId",
|
||||
new { System = _system.Id, LastSwitchId = lastSwitchId?.Value ?? -1 });
|
||||
|
||||
var groupMembers = group.Value<JArray>("members");
|
||||
var currentGroupMembers = (await _conn.QueryAsync<MemberId>(
|
||||
"select member_id from group_members where group_id = @groupId",
|
||||
new { groupId = groupId.Value }
|
||||
)).ToList();
|
||||
|
||||
await using (var importer = _conn.BeginBinaryImport("copy group_members (group_id, member_id) from stdin (format binary)"))
|
||||
// Lastly, import the switch members
|
||||
await using (var importer =
|
||||
_conn.BeginBinaryImport("copy switch_members (switch, member) from stdin (format binary)"))
|
||||
{
|
||||
foreach (var justAddedSwitch in justAddedSwitches)
|
||||
{
|
||||
foreach (var memberIdentifier in groupMembers)
|
||||
if (!importedSwitches.TryGetValue(justAddedSwitch.Timestamp, out var switchMembers))
|
||||
throw new Exception(
|
||||
$"Found 'just-added' switch (by ID) with timestamp {justAddedSwitch.Timestamp}, but this did not correspond to a timestamp we just added a switch entry of! :/");
|
||||
|
||||
// We still assume timestamps are unique and non-duplicate, so:
|
||||
foreach (var memberIdentifier in switchMembers)
|
||||
{
|
||||
if (!_knownMemberIdentifiers.TryGetValue(memberIdentifier.ToString(), out var memberId))
|
||||
throw new Exception($"Attempted to import group member with member identifier {memberIdentifier} but could not find a recently imported member with this id!");
|
||||
|
||||
if (currentGroupMembers.Contains(memberId))
|
||||
continue;
|
||||
if (!_knownMemberIdentifiers.TryGetValue((string)memberIdentifier, out var memberId))
|
||||
throw new Exception(
|
||||
$"Attempted to import switch with member identifier {memberIdentifier} but could not find an entry in the id map for this! :/");
|
||||
|
||||
await importer.StartRowAsync();
|
||||
await importer.WriteAsync(groupId.Value.Value, NpgsqlDbType.Integer);
|
||||
await importer.WriteAsync(justAddedSwitch.Id.Value, NpgsqlDbType.Integer);
|
||||
await importer.WriteAsync(memberId.Value, NpgsqlDbType.Integer);
|
||||
}
|
||||
|
||||
await importer.CompleteAsync();
|
||||
}
|
||||
|
||||
}
|
||||
private async Task ImportSwitches(JArray switches)
|
||||
{
|
||||
var existingSwitches = (await _conn.QueryAsync<PKSwitch>("select * from switches where system = @System", new { System = _system.Id })).ToList();
|
||||
var existingTimestamps = existingSwitches.Select(sw => sw.Timestamp).ToImmutableHashSet();
|
||||
var lastSwitchId = existingSwitches.Count != 0 ? existingSwitches.Select(sw => sw.Id).Max() : (SwitchId?)null;
|
||||
|
||||
if (switches.Count > 10000)
|
||||
throw new ImportException($"Too many switches present in import file.");
|
||||
|
||||
// Import switch definitions
|
||||
var importedSwitches = new Dictionary<Instant, JArray>();
|
||||
await using (var importer = _conn.BeginBinaryImport("copy switches (system, timestamp) from stdin (format binary)"))
|
||||
{
|
||||
foreach (var sw in switches)
|
||||
{
|
||||
var timestampString = sw.Value<string>("timestamp");
|
||||
var timestamp = DateTimeFormats.TimestampExportFormat.Parse(timestampString);
|
||||
if (!timestamp.Success) throw new ImportException($"Switch timestamp {timestampString} is not an valid timestamp.");
|
||||
|
||||
// Don't import duplicate switches
|
||||
if (existingTimestamps.Contains(timestamp.Value)) continue;
|
||||
|
||||
// Otherwise, write to importer
|
||||
await importer.StartRowAsync();
|
||||
await importer.WriteAsync(_system.Id.Value, NpgsqlDbType.Integer);
|
||||
await importer.WriteAsync(timestamp.Value, NpgsqlDbType.Timestamp);
|
||||
|
||||
var members = sw.Value<JArray>("members");
|
||||
if (members.Count > Limits.MaxSwitchMemberCount)
|
||||
throw new ImportException($"Switch with timestamp {timestampString} contains too many members ({members.Count} > 100).");
|
||||
|
||||
// Note that we've imported a switch with this timestamp
|
||||
importedSwitches[timestamp.Value] = sw.Value<JArray>("members");
|
||||
}
|
||||
|
||||
// Commit the import
|
||||
await importer.CompleteAsync();
|
||||
}
|
||||
|
||||
// Now, fetch all the switches we just added (so, now we get their IDs too)
|
||||
// IDs are sequential, so any ID in this system, with a switch ID > the last max, will be one we just added
|
||||
var justAddedSwitches = await _conn.QueryAsync<PKSwitch>(
|
||||
"select * from switches where system = @System and id > @LastSwitchId",
|
||||
new { System = _system.Id, LastSwitchId = lastSwitchId?.Value ?? -1 });
|
||||
|
||||
// Lastly, import the switch members
|
||||
await using (var importer = _conn.BeginBinaryImport("copy switch_members (switch, member) from stdin (format binary)"))
|
||||
{
|
||||
foreach (var justAddedSwitch in justAddedSwitches)
|
||||
{
|
||||
if (!importedSwitches.TryGetValue(justAddedSwitch.Timestamp, out var switchMembers))
|
||||
throw new Exception($"Found 'just-added' switch (by ID) with timestamp {justAddedSwitch.Timestamp}, but this did not correspond to a timestamp we just added a switch entry of! :/");
|
||||
|
||||
// We still assume timestamps are unique and non-duplicate, so:
|
||||
foreach (var memberIdentifier in switchMembers)
|
||||
{
|
||||
if (!_knownMemberIdentifiers.TryGetValue((string)memberIdentifier, out var memberId))
|
||||
throw new Exception($"Attempted to import switch with member identifier {memberIdentifier} but could not find an entry in the id map for this! :/");
|
||||
|
||||
await importer.StartRowAsync();
|
||||
await importer.WriteAsync(justAddedSwitch.Id.Value, NpgsqlDbType.Integer);
|
||||
await importer.WriteAsync(memberId.Value, NpgsqlDbType.Integer);
|
||||
}
|
||||
}
|
||||
|
||||
await importer.CompleteAsync();
|
||||
}
|
||||
await importer.CompleteAsync();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,123 +1,124 @@
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
|
||||
using Newtonsoft.Json.Linq;
|
||||
|
||||
using NodaTime;
|
||||
|
||||
namespace PluralKit.Core
|
||||
namespace PluralKit.Core;
|
||||
|
||||
public partial class BulkImporter
|
||||
{
|
||||
public partial class BulkImporter
|
||||
private async Task<ImportResultNew> ImportTupperbox(JObject importFile)
|
||||
{
|
||||
private async Task<ImportResultNew> ImportTupperbox(JObject importFile)
|
||||
var tuppers = importFile.Value<JArray>("tuppers");
|
||||
var newMembers = tuppers.Count(t => !_existingMemberNames.TryGetValue("name", out var memberId));
|
||||
await AssertMemberLimitNotReached(newMembers);
|
||||
|
||||
string lastSetTag = null;
|
||||
var multipleTags = false;
|
||||
var hasGroup = false;
|
||||
|
||||
foreach (JObject tupper in tuppers)
|
||||
(lastSetTag, multipleTags, hasGroup) = await ImportTupper(tupper, lastSetTag);
|
||||
|
||||
if (multipleTags || hasGroup)
|
||||
{
|
||||
var tuppers = importFile.Value<JArray>("tuppers");
|
||||
var newMembers = tuppers.Count(t => !_existingMemberNames.TryGetValue("name", out var memberId));
|
||||
await AssertMemberLimitNotReached(newMembers);
|
||||
var issueStr =
|
||||
$"{Emojis.Warn} The following potential issues were detected converting your Tupperbox input file:";
|
||||
if (hasGroup)
|
||||
issueStr +=
|
||||
"\n- PluralKit does not support member groups. Members will be imported without groups.";
|
||||
if (multipleTags)
|
||||
issueStr +=
|
||||
"\n- PluralKit does not support per-member system tags. Since you had multiple members with distinct tags, those tags will be applied to the members' *display names*/nicknames instead.";
|
||||
|
||||
string lastSetTag = null;
|
||||
bool multipleTags = false;
|
||||
bool hasGroup = false;
|
||||
|
||||
foreach (JObject tupper in tuppers)
|
||||
(lastSetTag, multipleTags, hasGroup) = await ImportTupper(tupper, lastSetTag);
|
||||
|
||||
if (multipleTags || hasGroup)
|
||||
{
|
||||
var issueStr =
|
||||
$"{Emojis.Warn} The following potential issues were detected converting your Tupperbox input file:";
|
||||
if (hasGroup)
|
||||
issueStr +=
|
||||
"\n- PluralKit does not support member groups. Members will be imported without groups.";
|
||||
if (multipleTags)
|
||||
issueStr +=
|
||||
"\n- PluralKit does not support per-member system tags. Since you had multiple members with distinct tags, those tags will be applied to the members' *display names*/nicknames instead.";
|
||||
|
||||
await _confirmFunc(issueStr);
|
||||
_result.Success = true;
|
||||
}
|
||||
|
||||
return _result;
|
||||
await _confirmFunc(issueStr);
|
||||
_result.Success = true;
|
||||
}
|
||||
|
||||
private async Task<(string lastSetTag, bool multipleTags, bool hasGroup)> ImportTupper(JObject tupper, string lastSetTag)
|
||||
return _result;
|
||||
}
|
||||
|
||||
private async Task<(string lastSetTag, bool multipleTags, bool hasGroup)> ImportTupper(
|
||||
JObject tupper, string lastSetTag)
|
||||
{
|
||||
if (!tupper.ContainsKey("name") || tupper["name"].Type == JTokenType.Null)
|
||||
throw new ImportException("Field 'name' cannot be null.");
|
||||
|
||||
var hasGroup = tupper.ContainsKey("group_id") && tupper["group_id"].Type != JTokenType.Null;
|
||||
var multipleTags = false;
|
||||
|
||||
var name = tupper.Value<string>("name");
|
||||
var patch = new MemberPatch();
|
||||
|
||||
patch.Name = name;
|
||||
if (tupper.ContainsKey("avatar_url") && tupper["avatar_url"].Type != JTokenType.Null)
|
||||
patch.AvatarUrl = tupper.Value<string>("avatar_url").NullIfEmpty();
|
||||
if (tupper.ContainsKey("brackets"))
|
||||
{
|
||||
if (!tupper.ContainsKey("name") || tupper["name"].Type == JTokenType.Null)
|
||||
throw new ImportException("Field 'name' cannot be null.");
|
||||
|
||||
var hasGroup = tupper.ContainsKey("group_id") && tupper["group_id"].Type != JTokenType.Null;
|
||||
var multipleTags = false;
|
||||
|
||||
var name = tupper.Value<string>("name");
|
||||
var patch = new MemberPatch();
|
||||
|
||||
patch.Name = name;
|
||||
if (tupper.ContainsKey("avatar_url") && tupper["avatar_url"].Type != JTokenType.Null) patch.AvatarUrl = tupper.Value<string>("avatar_url").NullIfEmpty();
|
||||
if (tupper.ContainsKey("brackets"))
|
||||
{
|
||||
var brackets = tupper.Value<JArray>("brackets");
|
||||
if (brackets.Count % 2 != 0)
|
||||
throw new ImportException($"Field 'brackets' in tupper {name} is invalid.");
|
||||
var tags = new List<ProxyTag>();
|
||||
for (var i = 0; i < brackets.Count / 2; i++)
|
||||
tags.Add(new ProxyTag((string)brackets[i * 2], (string)brackets[i * 2 + 1]));
|
||||
patch.ProxyTags = tags.ToArray();
|
||||
}
|
||||
// todo: && if is new member
|
||||
if (tupper.ContainsKey("posts")) patch.MessageCount = tupper.Value<int>("posts");
|
||||
if (tupper.ContainsKey("show_brackets")) patch.KeepProxy = tupper.Value<bool>("show_brackets");
|
||||
if (tupper.ContainsKey("birthday") && tupper["birthday"].Type != JTokenType.Null)
|
||||
{
|
||||
var parsed = DateTimeFormats.TimestampExportFormat.Parse(tupper.Value<string>("birthday"));
|
||||
if (!parsed.Success)
|
||||
throw new ImportException($"Field 'birthday' in tupper {name} is invalid.");
|
||||
patch.Birthday = LocalDate.FromDateTime(parsed.Value.ToDateTimeUtc());
|
||||
}
|
||||
if (tupper.ContainsKey("description")) patch.Description = tupper.Value<string>("description");
|
||||
if (tupper.ContainsKey("nick")) patch.DisplayName = tupper.Value<string>("nick");
|
||||
if (tupper.ContainsKey("tag") && tupper["tag"].Type != JTokenType.Null)
|
||||
{
|
||||
var tag = tupper.Value<string>("tag");
|
||||
if (tag != lastSetTag)
|
||||
{
|
||||
lastSetTag = tag;
|
||||
multipleTags = true;
|
||||
}
|
||||
patch.DisplayName = $"{name} {tag}";
|
||||
}
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
{
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in tupper {name} is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
else if (err.Text != null)
|
||||
throw new ImportException($"tupper {name}: {err.Text}");
|
||||
else
|
||||
throw new ImportException($"Field {err.Key} in tupper {name} is invalid.");
|
||||
}
|
||||
|
||||
var isNewMember = false;
|
||||
if (!_existingMemberNames.TryGetValue(name, out var memberId))
|
||||
{
|
||||
var newMember = await _repo.CreateMember(_system.Id, name, _conn);
|
||||
memberId = newMember.Id;
|
||||
isNewMember = true;
|
||||
_result.Added++;
|
||||
}
|
||||
else
|
||||
_result.Modified++;
|
||||
|
||||
_logger.Debug("Importing member with identifier {FileId} to system {System} (is creating new member? {IsCreatingNewMember})",
|
||||
name, _system.Id, isNewMember);
|
||||
|
||||
await _repo.UpdateMember(memberId, patch, _conn);
|
||||
|
||||
return (lastSetTag, multipleTags, hasGroup);
|
||||
var brackets = tupper.Value<JArray>("brackets");
|
||||
if (brackets.Count % 2 != 0)
|
||||
throw new ImportException($"Field 'brackets' in tupper {name} is invalid.");
|
||||
var tags = new List<ProxyTag>();
|
||||
for (var i = 0; i < brackets.Count / 2; i++)
|
||||
tags.Add(new ProxyTag((string)brackets[i * 2], (string)brackets[i * 2 + 1]));
|
||||
patch.ProxyTags = tags.ToArray();
|
||||
}
|
||||
|
||||
// todo: && if is new member
|
||||
if (tupper.ContainsKey("posts")) patch.MessageCount = tupper.Value<int>("posts");
|
||||
if (tupper.ContainsKey("show_brackets")) patch.KeepProxy = tupper.Value<bool>("show_brackets");
|
||||
if (tupper.ContainsKey("birthday") && tupper["birthday"].Type != JTokenType.Null)
|
||||
{
|
||||
var parsed = DateTimeFormats.TimestampExportFormat.Parse(tupper.Value<string>("birthday"));
|
||||
if (!parsed.Success)
|
||||
throw new ImportException($"Field 'birthday' in tupper {name} is invalid.");
|
||||
patch.Birthday = LocalDate.FromDateTime(parsed.Value.ToDateTimeUtc());
|
||||
}
|
||||
|
||||
if (tupper.ContainsKey("description")) patch.Description = tupper.Value<string>("description");
|
||||
if (tupper.ContainsKey("nick")) patch.DisplayName = tupper.Value<string>("nick");
|
||||
if (tupper.ContainsKey("tag") && tupper["tag"].Type != JTokenType.Null)
|
||||
{
|
||||
var tag = tupper.Value<string>("tag");
|
||||
if (tag != lastSetTag)
|
||||
{
|
||||
lastSetTag = tag;
|
||||
multipleTags = true;
|
||||
}
|
||||
|
||||
patch.DisplayName = $"{name} {tag}";
|
||||
}
|
||||
|
||||
patch.AssertIsValid();
|
||||
if (patch.Errors.Count > 0)
|
||||
{
|
||||
var err = patch.Errors[0];
|
||||
if (err is FieldTooLongError)
|
||||
throw new ImportException($"Field {err.Key} in tupper {name} is too long "
|
||||
+ $"({(err as FieldTooLongError).ActualLength} > {(err as FieldTooLongError).MaxLength}).");
|
||||
if (err.Text != null)
|
||||
throw new ImportException($"tupper {name}: {err.Text}");
|
||||
throw new ImportException($"Field {err.Key} in tupper {name} is invalid.");
|
||||
}
|
||||
|
||||
var isNewMember = false;
|
||||
if (!_existingMemberNames.TryGetValue(name, out var memberId))
|
||||
{
|
||||
var newMember = await _repo.CreateMember(_system.Id, name, _conn);
|
||||
memberId = newMember.Id;
|
||||
isNewMember = true;
|
||||
_result.Added++;
|
||||
}
|
||||
else
|
||||
{
|
||||
_result.Modified++;
|
||||
}
|
||||
|
||||
_logger.Debug(
|
||||
"Importing member with identifier {FileId} to system {System} (is creating new member? {IsCreatingNewMember})",
|
||||
name, _system.Id, isNewMember);
|
||||
|
||||
await _repo.UpdateMember(memberId, patch, _conn);
|
||||
|
||||
return (lastSetTag, multipleTags, hasGroup);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user