Add basic database schema migration system

PluralKit.Bot/Bot.cs

@@ -51,12 +51,12 @@ namespace PluralKit.Bot
                 var logger = services.GetRequiredService<ILogger>().ForContext<Initialize>();
                 var coreConfig = services.GetRequiredService<CoreConfig>();
                 var botConfig = services.GetRequiredService<BotConfig>();
+                var schema = services.GetRequiredService<SchemaService>();
 
                 using (Sentry.SentrySdk.Init(coreConfig.SentryUrl))
                 {
                     logger.Information("Connecting to database");
-                    using (var conn = await services.GetRequiredService<DbConnectionFactory>().Obtain())
-                        await Schema.CreateTables(conn);
+                    await schema.ApplyMigrations();
 
                     logger.Information("Connecting to Discord");
                     var client = services.GetRequiredService<IDiscordClient>() as DiscordShardedClient;
@@ -83,6 +83,7 @@ namespace PluralKit.Bot
 
             .AddSingleton<DbConnectionCountHolder>()
             .AddTransient<DbConnectionFactory>()
+            .AddTransient<SchemaService>()
 
             .AddSingleton<IDiscordClient, DiscordShardedClient>(_ => new DiscordShardedClient(new DiscordSocketConfig
             {

PluralKit.Core/Migrations/0.sql

@@ -1,3 +1,9 @@
+-- SCHEMA VERSION 0, 2019-12-26
+-- "initial version", considered a "starting point" for the migrations
+
+-- also the assumed database layout of someone either migrating from an older version of PK or starting a new instance,
+-- so everything here *should* be idempotent given a schema version older than this or nonexistent.
+
 -- Create proxy_tag compound type if it doesn't exist
 do $$ begin
     create type proxy_tag as (
@@ -78,10 +84,6 @@ create table if not exists switches
     system    serial    not null references systems (id) on delete cascade,
     timestamp timestamp not null default (current_timestamp at time zone 'utc')
 );
-CREATE INDEX IF NOT EXISTS idx_switches_system
-ON switches USING btree (
-	system ASC NULLS LAST
-) INCLUDE ("timestamp");
 
 create table if not exists switch_members
 (
@@ -89,12 +91,6 @@ create table if not exists switch_members
     switch serial not null references switches (id) on delete cascade,
     member serial not null references members (id) on delete cascade
 );
-CREATE INDEX IF NOT EXISTS idx_switch_members_switch
-ON switch_members USING btree (
-	switch ASC NULLS LAST
-) INCLUDE (member);
-
-create index if not exists idx_message_member on messages (member);
 
 create table if not exists webhooks
 (
@@ -109,4 +105,8 @@ create table if not exists servers
     log_channel   bigint,
     log_blacklist bigint[] not null default array[]::bigint[],
     blacklist     bigint[] not null default array[]::bigint[] 
-);
+);
+
+create index if not exists idx_switches_system on switches using btree (system asc nulls last) include ("timestamp");
+create index if not exists idx_switch_members_switch on switch_members using btree (switch asc nulls last) include (member);
+create index if not exists idx_message_member on messages (member);

PluralKit.Core/Migrations/1.sql

@@ -0,0 +1,15 @@
+-- SCHEMA VERSION 1: 2019-12-26
+-- First version introducing the migration system, therefore we add the info/version table
+
+create table info
+(
+    id int primary key not null default 1, -- enforced only equal to 1
+    
+    schema_version int,    
+    
+    constraint singleton check (id = 1) -- enforce singleton table/row
+);
+
+-- We do an insert here since we *just* added the table
+-- Future migrations should do an update at the end
+insert into info (schema_version) values (1);

PluralKit.Core/PluralKit.Core.csproj

@@ -28,8 +28,7 @@
     </ItemGroup>
 
     <ItemGroup>
-      <None Remove="db_schema.sql" />
-      <EmbeddedResource Include="db_schema.sql" />
+      <EmbeddedResource Include="Migrations\*.sql" />
     </ItemGroup>
 
 </Project>

PluralKit.Core/Schema.cs

@@ -1,19 +0,0 @@
-using System.Data;
-using System.IO;
-using System.Threading.Tasks;
-using Dapper;
-
-namespace PluralKit {
-    public static class Schema {
-        public static async Task CreateTables(IDbConnection connection)
-        {
-            // Load the schema from disk (well, embedded resource) and execute the commands in there
-            using (var stream = typeof(Schema).Assembly.GetManifestResourceStream("PluralKit.Core.db_schema.sql"))
-            using (var reader = new StreamReader(stream))
-            {
-                var result = await reader.ReadToEndAsync();
-                await connection.ExecuteAsync(result);
-            }
-        }
-    }
-}

PluralKit.Core/SchemaService.cs

@@ -0,0 +1,62 @@
+using System;
+using System.Data;
+using System.IO;
+using System.Threading.Tasks;
+using Dapper;
+
+using Npgsql;
+
+using Serilog;
+
+namespace PluralKit {
+    public class SchemaService
+    {
+        private const int TargetSchemaVersion = 1;
+
+        private DbConnectionFactory _conn;
+        private ILogger _logger;
+        
+        public SchemaService(DbConnectionFactory conn, ILogger logger)
+        {
+            _conn = conn;
+            _logger = logger.ForContext<SchemaService>();
+        }
+
+        public async Task ApplyMigrations()
+        {
+            for (var version = 0; version <= TargetSchemaVersion; version++) 
+                await ApplyMigration(version);
+        }
+
+        private async Task ApplyMigration(int migrationId)
+        {
+            // migrationId is the *target* version
+            using var conn = await _conn.Obtain();
+            using var tx = conn.BeginTransaction();
+            
+            // See if we even have the info table... if not, we implicitly define the version as -1
+            // This means migration 0 will get executed, which ensures we're at a consistent state.
+            // *Technically* this also means schema version 0 will be identified as -1, but since we're only doing these
+            // checks in the above for loop, this doesn't matter.
+            var hasInfoTable = await conn.QuerySingleOrDefaultAsync<int>("select count(*) from information_schema.tables where table_name = 'info'") == 1;
+
+            int currentVersion;
+            if (hasInfoTable)
+                currentVersion = await conn.QuerySingleOrDefaultAsync<int>("select schema_version from info");
+            else currentVersion = -1;
+            
+            if (currentVersion >= migrationId)
+                return; // Don't execute the migration if we're already at the target version.
+
+            using var stream = typeof(SchemaService).Assembly.GetManifestResourceStream($"PluralKit.Core.Migrations.{migrationId}.sql");
+            if (stream == null) throw new ArgumentException("Invalid migration ID");
+            
+            using var reader = new StreamReader(stream);
+            var migrationQuery = await reader.ReadToEndAsync();
+            
+            _logger.Information("Current schema version is {CurrentVersion}, applying migration {MigrationId}", currentVersion, migrationId);
+            await conn.ExecuteAsync(migrationQuery, transaction: tx);
+            tx.Commit();
+        }
+    }
+}