feat(webhooks): init, add service/models, add JSON to patch objects

PluralKit.Core/Database/Migrations/20.sql

@@ -0,0 +1,7 @@
+-- schema version 20: insert date
+-- add outgoing webhook to systems
+
+alter table systems add column webhook_url text;
+alter table systems add column webhook_token text;
+
+update info set schema_version = 20;

PluralKit.Core/Database/Repository/ModelRepository.Group.cs

@@ -8,6 +8,12 @@ namespace PluralKit.Core
 {
     public partial class ModelRepository
     {
+        public Task<PKGroup?> GetGroup(GroupId id)
+        {
+            var query = new Query("groups").Where("id", id);
+            return _db.QueryFirst<PKGroup?>(query);
+        }
+
         public Task<PKGroup?> GetGroupByName(SystemId system, string name)
         {
             var query = new Query("groups").Where("system", system).WhereRaw("lower(name) = lower(?)", name.ToLower());

PluralKit.Core/Database/Repository/ModelRepository.Switch.cs

@@ -100,6 +100,9 @@ namespace PluralKit.Core
             return _db.QueryStream<PKSwitch>(query);
         }
 
+        public Task<PKSwitch?> GetSwitch(SwitchId id)
+            => _db.QueryFirst<PKSwitch?>(new Query("switches").Where("id", id));
+
         public Task<PKSwitch> GetSwitchByUuid(Guid uuid)
         {
             var query = new Query("switches").Where("uuid", uuid);

PluralKit.Core/Database/Repository/ModelRepository.cs

@@ -6,10 +6,12 @@ namespace PluralKit.Core
     {
         private readonly ILogger _logger;
         private readonly IDatabase _db;
-        public ModelRepository(ILogger logger, IDatabase db)
+        private readonly DispatchService _dispatch;
+        public ModelRepository(ILogger logger, IDatabase db, DispatchService dispatch)
         {
             _logger = logger.ForContext<ModelRepository>();
             _db = db;
+            _dispatch = dispatch;
         }
     }
 }

PluralKit.Core/Database/Utils/DatabaseMigrator.cs

@@ -12,7 +12,7 @@ namespace PluralKit.Core
     internal class DatabaseMigrator
     {
         private const string RootPath = "PluralKit.Core.Database"; // "resource path" root for SQL files
-        private const int TargetSchemaVersion = 19;
+        private const int TargetSchemaVersion = 20;
         private readonly ILogger _logger;
 
         public DatabaseMigrator(ILogger logger)

PluralKit.Core/Dispatch/DispatchModels.cs

@@ -0,0 +1,109 @@
+using System;
+using System.Net;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+
+using NodaTime;
+
+namespace PluralKit.Core
+{
+    public enum DispatchEvent
+    {
+        PING,
+        UPDATE_SYSTEM,
+        CREATE_MEMBER,
+        UPDATE_MEMBER,
+        DELETE_MEMBER,
+        CREATE_GROUP,
+        UPDATE_GROUP,
+        UPDATE_GROUP_MEMBERS,
+        DELETE_GROUP,
+        LINK_ACCOUNT,
+        UNLINK_ACCOUNT,
+        UPDATE_SYSTEM_GUILD,
+        UPDATE_MEMBER_GUILD,
+        CREATE_MESSAGE,
+        CREATE_SWITCH,
+        UPDATE_SWITCH,
+        UPDATE_SWITCH_MEMBERS,
+        DELETE_SWITCH,
+        DELETE_ALL_SWITCHES,
+    }
+
+    public struct UpdateDispatchData
+    {
+        public DispatchEvent Event;
+        public string SystemId;
+        public string? EntityId;
+        public ulong? GuildId;
+        public string SigningToken;
+        public JObject? EventData;
+    }
+
+    public static class DispatchExt
+    {
+        public static StringContent GetPayloadBody(this UpdateDispatchData data)
+        {
+            var o = new JObject();
+
+            o.Add("type", data.Event.ToString());
+            o.Add("signing_token", data.SigningToken);
+            o.Add("system_id", data.SystemId);
+            if (data.EntityId != null)
+                o.Add("id", data.EntityId);
+            if (data.GuildId != null)
+                o.Add("guild_id", data.GuildId);
+            if (data.EventData != null)
+                o.Add("data", data.EventData);
+
+            return new StringContent(JsonConvert.SerializeObject(o));
+        }
+
+        public static JObject ToDispatchJson(this PKMessage msg, string memberRef)
+        {
+            var o = new JObject();
+
+            o.Add("timestamp", Instant.FromUnixTimeMilliseconds((long)(msg.Mid >> 22) + 1420070400000).FormatExport());
+            o.Add("id", msg.Mid.ToString());
+            o.Add("original", msg.OriginalMid.ToString());
+            o.Add("sender", msg.Sender.ToString());
+            o.Add("channel", msg.Channel.ToString());
+            o.Add("member", memberRef);
+
+            return o;
+        }
+
+        public static async Task<bool> ValidateUri(string uri)
+        {
+            IPHostEntry host = null;
+
+            try
+            {
+                host = await Dns.GetHostEntryAsync(uri);
+            }
+            catch (Exception) { }
+
+            if (host == null || host.AddressList.Length == 0)
+                return false;
+
+#pragma warning disable CS0618
+
+            foreach (var address in host.AddressList)
+            {
+                if ((address.Address & 0x7f000000) == 0x7f000000) // 127.0/8
+                    return false;
+                if ((address.Address & 0x0a000000) == 0x0a000000) // 10.0/8
+                    return false;
+                if ((address.Address & 0xa9fe0000) == 0xa9fe0000) // 169.254/16
+                    return false;
+                if ((address.Address & 0xac100000) == 0xac100000) // 172.16/12
+                    return false;
+            }
+
+            return true;
+        }
+    }
+}

PluralKit.Core/Dispatch/DispatchService.cs

@@ -0,0 +1,209 @@
+using System;
+using System.Linq;
+using System.Collections.Generic;
+using System.Net.Http;
+using System.Threading.Tasks;
+
+using Autofac;
+
+using Serilog;
+
+namespace PluralKit.Core
+{
+    public class DispatchService
+    {
+        private readonly ILogger _logger;
+        private readonly ILifetimeScope _provider;
+        private readonly HttpClient _client = new();
+        public DispatchService(ILogger logger, ILifetimeScope provider, CoreConfig cfg)
+        {
+            _logger = logger;
+            _provider = provider;
+        }
+
+        private async Task DoPostRequest(SystemId system, string webhookUrl, HttpContent content)
+        {
+            if (!await DispatchExt.ValidateUri(webhookUrl))
+            {
+                _logger.Warning("Failed to dispatch webhook for system {SystemId}: URL is invalid or points to a private address", system);
+                return;
+            }
+
+            try
+            {
+                await _client.PostAsync(webhookUrl, content);
+            }
+            catch (HttpRequestException e)
+            {
+                _logger.Error("Could not dispatch webhook request!", e);
+            }
+        }
+
+        public async Task Dispatch(SystemId systemId, UpdateDispatchData data)
+        {
+            if (data.EventData != null && data.EventData.Count == 0)
+                return;
+
+            var repo = _provider.Resolve<ModelRepository>();
+            var system = await repo.GetSystem(systemId);
+            if (system.WebhookUrl == null)
+                return;
+
+            data.SigningToken = system.WebhookToken;
+            data.SystemId = system.Uuid.ToString();
+
+            _logger.Debug("Dispatching webhook for system {SystemId}", systemId);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(MemberId memberId, UpdateDispatchData data)
+        {
+            if (data.EventData != null && data.EventData.Count == 0)
+                return;
+
+            var repo = _provider.Resolve<ModelRepository>();
+            var member = await repo.GetMember(memberId);
+            var system = await repo.GetSystem(member.System);
+            if (system.WebhookUrl == null)
+                return;
+
+            data.SigningToken = system.WebhookToken;
+            data.SystemId = system.Uuid.ToString();
+            data.EntityId = member.Uuid.ToString();
+
+            _logger.Debug("Dispatching webhook for member {MemberId} (system {SystemId})", memberId, system.Id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(GroupId groupId, UpdateDispatchData data)
+        {
+            if (data.EventData != null && data.EventData.Count == 0)
+                return;
+
+            var repo = _provider.Resolve<ModelRepository>();
+            var group = await repo.GetGroup(groupId);
+            var system = await repo.GetSystem(group.System);
+            if (system.WebhookUrl == null)
+                return;
+
+            data.SigningToken = system.WebhookToken;
+            data.SystemId = system.Uuid.ToString();
+            data.EntityId = group.Uuid.ToString();
+
+            _logger.Debug("Dispatching webhook for group {GroupId} (system {SystemId})", groupId, system.Id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(Dictionary<GroupId, MemberId> dict, DispatchEvent evt)
+        {
+            var repo = _provider.Resolve<ModelRepository>();
+            var g = await repo.GetGroup(dict.Keys.FirstOrDefault());
+            var system = await repo.GetSystem(g.System);
+            if (system.WebhookUrl == null)
+                return;
+
+            var data = new UpdateDispatchData();
+            data.Event = DispatchEvent.UPDATE_GROUP_MEMBERS;
+            data.SystemId = system.Uuid.ToString();
+
+
+            _logger.Debug("Dispatching webhook for group member update (system {SystemId})", system.Id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(SwitchId swId, UpdateDispatchData data)
+        {
+            var repo = _provider.Resolve<ModelRepository>();
+            var sw = await repo.GetSwitch(swId);
+            var system = await repo.GetSystem(sw.System);
+            if (system.WebhookUrl == null)
+                return;
+
+            data.SigningToken = system.WebhookToken;
+            data.SystemId = system.Uuid.ToString();
+            data.EntityId = sw.Uuid.ToString();
+
+            _logger.Debug("Dispatching webhook for switch {SwitchId} (system {SystemId})", sw.Id, system.Id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(SystemId systemId, PKMessage newMessage)
+        {
+            var repo = _provider.Resolve<ModelRepository>();
+            var system = await repo.GetSystem(systemId);
+            if (system.WebhookUrl == null)
+                return;
+
+            var member = await repo.GetMember(newMessage.Member);
+
+            var data = new UpdateDispatchData();
+            data.Event = DispatchEvent.CREATE_MESSAGE;
+            data.SystemId = system.Uuid.ToString();
+            data.EventData = newMessage.ToDispatchJson(member.Uuid.ToString());
+
+            _logger.Debug("Dispatching webhook for message create (system {SystemId})", system.Id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(SystemId systemId, ulong guild_id, SystemGuildPatch patch)
+        {
+            var repo = _provider.Resolve<ModelRepository>();
+            var system = await repo.GetSystem(systemId);
+            if (system.WebhookUrl == null)
+                return;
+
+            string memberRef = null;
+            if (patch.AutoproxyMember.Value != null)
+            {
+                var member = await repo.GetMember(patch.AutoproxyMember.Value.Value);
+                memberRef = member.Uuid.ToString();
+            }
+
+            var data = new UpdateDispatchData();
+            data.Event = DispatchEvent.UPDATE_SYSTEM_GUILD;
+            data.SystemId = system.Uuid.ToString();
+            data.GuildId = guild_id;
+            data.EventData = patch.ToJson(memberRef);
+
+            _logger.Debug("Dispatching webhook for system {SystemId} in guild {GuildId}", system.Id, guild_id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(MemberId memberId, ulong guild_id, MemberGuildPatch patch)
+        {
+            var repo = _provider.Resolve<ModelRepository>();
+            var member = await repo.GetMember(memberId);
+            var system = await repo.GetSystem(member.System);
+            if (system.WebhookUrl == null)
+                return;
+
+            var data = new UpdateDispatchData();
+            data.Event = DispatchEvent.UPDATE_MEMBER_GUILD;
+            data.SystemId = system.Uuid.ToString();
+            data.EntityId = member.Uuid.ToString();
+            data.GuildId = guild_id;
+            data.EventData = patch.ToJson();
+
+            _logger.Debug("Dispatching webhook for member {MemberId} (system {SystemId}) in guild {GuildId}", member.Id, system.Id, guild_id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+
+        public async Task Dispatch(SystemId systemId, Guid uuid, DispatchEvent evt)
+        {
+            var repo = _provider.Resolve<ModelRepository>();
+            var system = await repo.GetSystem(systemId);
+            if (system.WebhookUrl == null)
+                return;
+
+            var data = new UpdateDispatchData();
+            data.Event = evt;
+
+            data.SigningToken = system.WebhookToken;
+            data.SystemId = system.Uuid.ToString();
+            data.EntityId = uuid.ToString();
+
+            _logger.Debug("Dispatching webhook for entity delete (system {SystemId})", system.Id);
+            await DoPostRequest(system.Id, system.WebhookUrl, data.GetPayloadBody());
+        }
+    }
+}

PluralKit.Core/Models/PKSystem.cs

@@ -46,6 +46,8 @@ namespace PluralKit.Core
         public string BannerImage { get; }
         public string Color { get; }
         public string Token { get; }
+        public string WebhookUrl { get; }
+        public string WebhookToken { get; }
         public Instant Created { get; }
         public string UiTz { get; set; }
         public bool PingsEnabled { get; }
@@ -100,6 +102,10 @@ namespace PluralKit.Core
 
                         if (ctx == LookupContext.ByOwner)
                         {
+                            // todo: should this be moved to a different JSON model?
+                            o.Add("webhook_url", system.WebhookUrl);
+                            o.Add("webhook_token", system.WebhookToken);
+
                             var p = new JObject();
 
                             p.Add("description_privacy", system.DescriptionPrivacy.ToJsonString());

PluralKit.Core/Models/Patch/GroupPatch.cs

@@ -91,5 +91,51 @@ namespace PluralKit.Core
 
             return patch;
         }
+
+        public JObject ToJson()
+        {
+            var o = new JObject();
+
+            if (Name.IsPresent)
+                o.Add("name", Name.Value);
+            if (Hid.IsPresent)
+                o.Add("id", Hid.Value);
+            if (DisplayName.IsPresent)
+                o.Add("display_name", DisplayName.Value);
+            if (Description.IsPresent)
+                o.Add("description", Description.Value);
+            if (Icon.IsPresent)
+                o.Add("icon", Icon.Value);
+            if (BannerImage.IsPresent)
+                o.Add("banner", BannerImage.Value);
+            if (Color.IsPresent)
+                o.Add("color", Color.Value);
+
+            if (
+                   DescriptionPrivacy.IsPresent
+                || IconPrivacy.IsPresent
+                || ListPrivacy.IsPresent
+                || Visibility.IsPresent
+            )
+            {
+                var p = new JObject();
+
+                if (DescriptionPrivacy.IsPresent)
+                    p.Add("description_privacy", DescriptionPrivacy.Value.ToJsonString());
+
+                if (IconPrivacy.IsPresent)
+                    p.Add("icon_privacy", IconPrivacy.Value.ToJsonString());
+
+                if (ListPrivacy.IsPresent)
+                    p.Add("list_privacy", ListPrivacy.Value.ToJsonString());
+
+                if (Visibility.IsPresent)
+                    p.Add("visibilithy", Visibility.Value.ToJsonString());
+
+                o.Add("privacy", p);
+            }
+
+            return o;
+        }
     }
 }

PluralKit.Core/Models/Patch/MemberGuildPatch.cs

@@ -38,5 +38,18 @@ namespace PluralKit.Core
 
             return patch;
         }
+
+        public JObject ToJson()
+        {
+            var o = new JObject();
+
+            if (DisplayName.IsPresent)
+                o.Add("display_name", DisplayName.Value);
+
+            if (AvatarUrl.IsPresent)
+                o.Add("avatar_url", AvatarUrl.Value);
+
+            return o;
+        }
     }
 }

PluralKit.Core/Models/Patch/MemberPatch.cs

@@ -192,5 +192,77 @@ namespace PluralKit.Core
 
             return patch;
         }
+
+        public JObject ToJson()
+        {
+            var o = new JObject();
+
+            if (Name.IsPresent)
+                o.Add("name", Name.Value);
+            if (Hid.IsPresent)
+                o.Add("id", Hid.Value);
+            if (DisplayName.IsPresent)
+                o.Add("display_name", DisplayName.Value);
+            if (AvatarUrl.IsPresent)
+                o.Add("avatar_url", AvatarUrl.Value);
+            if (BannerImage.IsPresent)
+                o.Add("banner", BannerImage.Value);
+            if (Color.IsPresent)
+                o.Add("color", Color.Value);
+            if (Birthday.IsPresent)
+                o.Add("birthday", Birthday.Value?.FormatExport());
+            if (Pronouns.IsPresent)
+                o.Add("pronouns", Pronouns.Value);
+            if (Description.IsPresent)
+                o.Add("description", Description.Value);
+            if (ProxyTags.IsPresent)
+            {
+                var tagArray = new JArray();
+                foreach (var tag in ProxyTags.Value)
+                    tagArray.Add(new JObject { { "prefix", tag.Prefix }, { "suffix", tag.Suffix } });
+                o.Add("proxy_tags", tagArray);
+            }
+
+            if (KeepProxy.IsPresent)
+                o.Add("keep_proxy", KeepProxy.Value);
+
+            if (
+                   Visibility.IsPresent
+                || NamePrivacy.IsPresent
+                || DescriptionPrivacy.IsPresent
+                || PronounPrivacy.IsPresent
+                || BirthdayPrivacy.IsPresent
+                || AvatarPrivacy.IsPresent
+                || MetadataPrivacy.IsPresent
+            )
+            {
+                var p = new JObject();
+
+                if (Visibility.IsPresent)
+                    p.Add("visibility", Visibility.Value.ToJsonString());
+
+                if (NamePrivacy.IsPresent)
+                    p.Add("name_privacy", NamePrivacy.Value.ToJsonString());
+
+                if (DescriptionPrivacy.IsPresent)
+                    p.Add("description_privacy", DescriptionPrivacy.Value.ToJsonString());
+
+                if (PronounPrivacy.IsPresent)
+                    p.Add("pronoun_privacy", PronounPrivacy.Value.ToJsonString());
+
+                if (BirthdayPrivacy.IsPresent)
+                    p.Add("birthday_privacy", BirthdayPrivacy.Value.ToJsonString());
+
+                if (AvatarPrivacy.IsPresent)
+                    p.Add("avatar_privacy", AvatarPrivacy.Value.ToJsonString());
+
+                if (MetadataPrivacy.IsPresent)
+                    p.Add("metadata_privacy", MetadataPrivacy.Value.ToJsonString());
+
+                o.Add("privacy", p);
+            }
+
+            return o;
+        }
     }
 }

PluralKit.Core/Models/Patch/SystemGuildPatch.cs

@@ -55,5 +55,27 @@ namespace PluralKit.Core
 
             return patch;
         }
+
+        public JObject ToJson(string memberRef)
+        {
+            var o = new JObject();
+
+            if (ProxyEnabled.IsPresent)
+                o.Add("proxying_enabled", ProxyEnabled.Value);
+
+            if (AutoproxyMode.IsPresent)
+                o.Add("autoproxy_mode", AutoproxyMode.Value.ToString().ToLower());
+
+            if (AutoproxyMember.IsPresent)
+                o.Add("autoproxy_member", memberRef);
+
+            if (Tag.IsPresent)
+                o.Add("tag", Tag.Value);
+
+            if (TagEnabled.IsPresent)
+                o.Add("tag_enabled", TagEnabled.Value);
+
+            return o;
+        }
     }
 }

PluralKit.Core/Models/Patch/SystemPatch.cs

@@ -126,5 +126,57 @@ namespace PluralKit.Core
 
             return patch;
         }
+
+        public JObject ToJson()
+        {
+            var o = new JObject();
+
+            if (Name.IsPresent)
+                o.Add("name", Name.Value);
+            if (Hid.IsPresent)
+                o.Add("id", Hid.Value);
+            if (Description.IsPresent)
+                o.Add("description", Description.Value);
+            if (Tag.IsPresent)
+                o.Add("tag", Tag.Value);
+            if (AvatarUrl.IsPresent)
+                o.Add("avatar_url", AvatarUrl.Value);
+            if (BannerImage.IsPresent)
+                o.Add("banner", BannerImage.Value);
+            if (Color.IsPresent)
+                o.Add("color", Color.Value);
+            if (UiTz.IsPresent)
+                o.Add("timezone", UiTz.Value);
+
+            if (
+                   DescriptionPrivacy.IsPresent
+                || MemberListPrivacy.IsPresent
+                || GroupListPrivacy.IsPresent
+                || FrontPrivacy.IsPresent
+                || FrontHistoryPrivacy.IsPresent
+            )
+            {
+                var p = new JObject();
+
+                if (DescriptionPrivacy.IsPresent)
+                    p.Add("description_privacy", DescriptionPrivacy.Value.ToJsonString());
+
+                if (MemberListPrivacy.IsPresent)
+                    p.Add("member_list_privacy", MemberListPrivacy.Value.ToJsonString());
+
+                if (GroupListPrivacy.IsPresent)
+                    p.Add("group_list_privacy", GroupListPrivacy.Value.ToJsonString());
+
+                if (FrontPrivacy.IsPresent)
+                    p.Add("front_privacy", FrontPrivacy.Value.ToJsonString());
+
+                if (FrontHistoryPrivacy.IsPresent)
+                    p.Add("front_history_privacy", FrontHistoryPrivacy.Value.ToJsonString());
+
+                o.Add("privacy", p);
+            }
+
+            return o;
+        }
     }
 }

PluralKit.Core/Modules/DataStoreModule.cs

@@ -14,6 +14,8 @@ namespace PluralKit.Core
             builder.RegisterType<Database>().As<IDatabase>().SingleInstance();
             builder.RegisterType<ModelRepository>().AsSelf().SingleInstance();
 
+            builder.RegisterType<DispatchService>().AsSelf().SingleInstance();
+
             builder.Populate(new ServiceCollection().AddMemoryCache());
         }
     }