liz/PluralKit
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: split out messages table from main database

Browse Source
This commit is contained in:
spiral
2022-11-23 09:17:19 +00:00
parent 09ac002d26
commit bf7747ab34
14 changed files with 119 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

31
PluralKit.Core/Database/Database.cs
View File

@@ -25,6 +25,7 @@ internal partial class Database: IDatabase
private readonly DbConnectionCountHolder _countHolder;
private readonly DatabaseMigrator _migrator;
private readonly string _connectionString;
private readonly string _messagesConnectionString;
public Database(CoreConfig config, DbConnectionCountHolder countHolder, ILogger logger,
IMetrics metrics, DatabaseMigrator migrator)
@@ -35,20 +36,26 @@ internal partial class Database: IDatabase
_migrator = migrator;
_logger = logger.ForContext<Database>();
var connectionString = new NpgsqlConnectionStringBuilder(_config.Database)
string connectionString(string src)
{
Pooling = true,
Enlist = false,
NoResetOnClose = true,
var builder = new NpgsqlConnectionStringBuilder(src)
{
Pooling = true,
Enlist = false,
NoResetOnClose = true,
// Lower timeout than default (15s -> 2s), should ideally fail-fast instead of hanging
Timeout = 2
};
// Lower timeout than default (15s -> 2s), should ideally fail-fast instead of hanging
Timeout = 2
};
if (_config.DatabasePassword != null)
connectionString.Password = _config.DatabasePassword;
if (_config.DatabasePassword != null)
builder.Password = _config.DatabasePassword;
_connectionString = connectionString.ConnectionString;
return builder.ConnectionString;
}
_connectionString = connectionString(_config.Database);
_messagesConnectionString = connectionString(_config.MessagesDatabase ?? _config.Database);
}
private static readonly PostgresCompiler _compiler = new();
@@ -88,14 +95,14 @@ internal partial class Database: IDatabase
}
// TODO: make sure every SQL query is behind a logged query method
public async Task<IPKConnection> Obtain()
public async Task<IPKConnection> Obtain(bool messages = false)
{
// Mark the request (for a handle, I guess) in the metrics
_metrics.Measure.Meter.Mark(CoreMetrics.DatabaseRequests);
// Create a connection and open it
// We wrap it in PKConnection for tracing purposes
var conn = new PKConnection(new NpgsqlConnection(_connectionString), _countHolder, _logger, _metrics);
var conn = new PKConnection(new NpgsqlConnection(messages ? _messagesConnectionString : _connectionString), _countHolder, _logger, _metrics);
await conn.OpenAsync();
return conn;
}

11
PluralKit.Core/Database/DatabaseQueries.cs
View File

@@ -31,10 +31,17 @@ internal partial class Database: IDatabase
yield return val;
}
public async Task<int> ExecuteQuery(Query q, string extraSql = "", [CallerMemberName] string queryName = "")
public async Task<T> QueryFirst<T>(string q, object param = null, [CallerMemberName] string queryName = "", bool messages = false)
{
using var conn = await Obtain(messages);
using (_metrics.Measure.Timer.Time(CoreMetrics.DatabaseQuery, new MetricTags("Query", queryName)))
return await conn.QueryFirstOrDefaultAsync<T>(q, param);
}
public async Task<int> ExecuteQuery(Query q, string extraSql = "", [CallerMemberName] string queryName = "", bool messages = false)
{
var query = _compiler.Compile(q);
using var conn = await Obtain();
using var conn = await Obtain(messages);
using (_metrics.Measure.Timer.Time(CoreMetrics.DatabaseQuery, new MetricTags("Query", queryName)))
return await conn.ExecuteAsync(query.Sql + $" {extraSql}", query.NamedBindings);
}

5
PluralKit.Core/Database/IDatabase.cs
View File

@@ -7,15 +7,16 @@ namespace PluralKit.Core;
public interface IDatabase
{
Task ApplyMigrations();
Task<IPKConnection> Obtain();
Task<IPKConnection> Obtain(bool messages = false);
Task Execute(Func<IPKConnection, Task> func);
Task<T> Execute<T>(Func<IPKConnection, Task<T>> func);
IAsyncEnumerable<T> Execute<T>(Func<IPKConnection, IAsyncEnumerable<T>> func);
Task<int> ExecuteQuery(Query q, string extraSql = "", [CallerMemberName] string queryName = "");
Task<int> ExecuteQuery(Query q, string extraSql = "", [CallerMemberName] string queryName = "", bool messages = false);
Task<int> ExecuteQuery(IPKConnection? conn, Query q, string extraSql = "",
[CallerMemberName] string queryName = "");
Task<T> QueryFirst<T>(string q, object param = null, [CallerMemberName] string queryName = "", bool messages = false);
Task<T> QueryFirst<T>(Query q, string extraSql = "", [CallerMemberName] string queryName = "");
Task<T> QueryFirst<T>(IPKConnection? conn, Query q, string extraSql = "",

8
PluralKit.Core/Database/Repository/ModelRepository.Member.cs
View File

@@ -89,4 +89,12 @@ public partial class ModelRepository
if (oldMember != null)
_ = _dispatch.Dispatch(oldMember.System, oldMember.Uuid, DispatchEvent.DELETE_MEMBER);
}
public async Task<bool> IsMemberOwnedByAccount(MemberId id, ulong userId)
{
return await _db.QueryFirst<bool>(
"select true from accounts, members where members.id = @member and accounts.uid = @account and members.system = accounts.system",
new { member = id, account = userId }
);
}
}

50
PluralKit.Core/Database/Repository/ModelRepository.Message.cs
View File

@@ -20,29 +20,38 @@ public partial class ModelRepository
_logger.Debug("Stored message {@StoredMessage} in channel {Channel}", msg, msg.Channel);
// "on conflict do nothing" in the (pretty rare) case of duplicate events coming in from Discord, which would lead to a DB error before
return _db.ExecuteQuery(query, "on conflict do nothing");
return _db.ExecuteQuery(query, "on conflict do nothing", messages: true);
}
// todo: add a Mapper to QuerySingle and move this to SqlKata
public async Task<FullMessage?> GetMessage(IPKConnection conn, ulong id)
public async Task<PKMessage?> GetMessage(ulong id)
{
FullMessage Mapper(PKMessage msg, PKMember member, PKSystem system) =>
new() { Message = msg, System = system, Member = member };
return await _db.QueryFirst<PKMessage?>(
"select * from messages where mid = @Id",
new { Id = id },
messages: true
);
}
var query = "select * from messages"
+ " left join members on messages.member = members.id"
+ " left join systems on members.system = systems.id"
+ " where (mid = @Id or original_mid = @Id)";
public async Task<FullMessage?> GetFullMessage(ulong id)
{
var rawMessage = await GetMessage(id);
if (rawMessage == null) return null;
var result = await conn.QueryAsync<PKMessage, PKMember, PKSystem, FullMessage>(
query, Mapper, new { Id = id });
return result.FirstOrDefault();
var member = rawMessage.Member == null ? null : await GetMember(rawMessage.Member.Value);
var system = member == null ? null : await GetSystem(member.System);
return new FullMessage
{
Message = rawMessage,
Member = member,
System = system,
};
}
public async Task DeleteMessage(ulong id)
{
var query = new Query("messages").AsDelete().Where("mid", id);
var rowCount = await _db.ExecuteQuery(query);
var rowCount = await _db.ExecuteQuery(query, messages: true);
if (rowCount > 0)
_logger.Information("Deleted message {MessageId} from database", id);
}
@@ -52,22 +61,9 @@ public partial class ModelRepository
// Npgsql doesn't support ulongs in general - we hacked around it for plain ulongs but tbh not worth it for collections of ulong
// Hence we map them to single longs, which *are* supported (this is ok since they're Technically (tm) stored as signed longs in the db anyway)
var query = new Query("messages").AsDelete().WhereIn("mid", ids.Select(id => (long)id).ToArray());
var rowCount = await _db.ExecuteQuery(query);
var rowCount = await _db.ExecuteQuery(query, messages: true);
if (rowCount > 0)
_logger.Information("Bulk deleted messages ({FoundCount} found) from database: {MessageIds}", rowCount,
ids);
}
public Task<PKMessage?> GetLastMessage(ulong guildId, ulong channelId, ulong accountId)
{
// Want to index scan on the (guild, sender, mid) index so need the additional constraint
var query = new Query("messages")
.Where("guild", guildId)
.Where("channel", channelId)
.Where("sender", accountId)
.OrderByDesc("mid")
.Limit(1);
return _db.QueryFirst<PKMessage?>(query);
}
}