From 4a2f1d045b1d092f2e2827dd40f6272f126135ca Mon Sep 17 00:00:00 2001 From: NGPixel Date: Mon, 23 Dec 2019 00:12:19 -0500 Subject: [PATCH] feat: azure blob storage provider + s3 rename fix --- package.json | 1 + server/modules/storage/azure/definition.yml | 42 +++++- server/modules/storage/azure/storage.js | 109 +++++++++++++-- server/modules/storage/s3/common.js | 4 +- yarn.lock | 144 +++++++++++++++++++- 5 files changed, 283 insertions(+), 17 deletions(-) diff --git a/package.json b/package.json index ee303b54..e934f1b5 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ }, "dependencies": { "@aoberoi/passport-slack": "1.0.5", + "@azure/storage-blob": "12.0.1", "@bugsnag/js": "6.4.3", "@exlinc/keycloak-passport": "1.0.2", "algoliasearch": "3.35.1", diff --git a/server/modules/storage/azure/definition.yml b/server/modules/storage/azure/definition.yml index 24360b49..c1741cad 100644 --- a/server/modules/storage/azure/definition.yml +++ b/server/modules/storage/azure/definition.yml @@ -4,7 +4,43 @@ description: Azure Blob Storage by Microsoft provides massively scalable object author: requarks.io logo: https://static.requarks.io/logo/azure.svg website: https://azure.microsoft.com/services/storage/blobs/ +isAvailable: true +supportedModes: + - push +defaultMode: push +schedule: false props: - accountName: String - accountKey: String - container: String + accountName: + type: String + title: Account Name + default: '' + hint: Your unique account name. + order: 1 + accountKey: + type: String + title: Account Access Key + default: '' + hint: Either key 1 or key 2. + order: 2 + containerName: + type: String + title: Container Name + default: 'wiki' + hint: Will automatically be created if it doesn't exist yet. + order: 3 + storageTier: + type: String + title: Storage Tier + hint: Represents the access tier on a blob. Use Cool for lower storage costs but at higher retrieval costs. + order: 4 + default: 'Cool' + enum: + - 'Hot' + - 'Cool' +actions: + - handler: exportAll + label: Export All + hint: Output all content from the DB to Azure Blog Storage, overwriting any existing data. If you enabled Azure Blog Storage after content was created or you temporarily disabled Git, you'll want to execute this action to add the missing content. + - handler: importAll + label: Import Everything + hint: Will import all content currently in Azure Blog Storage. Useful for importing or restoring content from a previously backed up state. diff --git a/server/modules/storage/azure/storage.js b/server/modules/storage/azure/storage.js index ab25ce97..c1f0ca9f 100644 --- a/server/modules/storage/azure/storage.js +++ b/server/modules/storage/azure/storage.js @@ -1,3 +1,14 @@ +const { BlobServiceClient, StorageSharedKeyCredential } = require('@azure/storage-blob') +const pageHelper = require('../../../helpers/page.js') + +/* global WIKI */ + +const getFilePath = (page, pathKey) => { + const fileName = `${page[pathKey]}.${pageHelper.getFileExtension(page.contentType)}` + const withLocaleCode = WIKI.config.lang.namespacing && WIKI.config.lang.code !== page.localeCode + return withLocaleCode ? `${page.localeCode}/${fileName}` : fileName +} + module.exports = { async activated() { @@ -6,18 +17,98 @@ module.exports = { }, async init() { - + WIKI.logger.info(`(STORAGE/AZURE) Initializing...`) + const { accountName, accountKey, containerName } = this.config + this.client = new BlobServiceClient( + `https://${accountName}.blob.core.windows.net`, + new StorageSharedKeyCredential(accountName, accountKey) + ) + this.container = this.client.getContainerClient(containerName) + try { + await this.container.create() + } catch (err) { + if (err.statusCode !== 409) { + WIKI.logger.warn(err) + throw err + } + } + WIKI.logger.info(`(STORAGE/AZURE) Initialization completed.`) }, - async created() { - + async created (page) { + WIKI.logger.info(`(STORAGE/AZURE) Creating file ${page.path}...`) + const filePath = getFilePath(page, 'path') + const pageContent = page.injectMetadata() + const blockBlobClient = this.container.getBlockBlobClient(filePath) + await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier }) }, - async updated() { - + async updated (page) { + WIKI.logger.info(`(STORAGE/AZURE) Updating file ${page.path}...`) + const filePath = getFilePath(page, 'path') + const pageContent = page.injectMetadata() + const blockBlobClient = this.container.getBlockBlobClient(filePath) + await blockBlobClient.upload(pageContent, pageContent.length, { tier: this.config.storageTier }) }, - async deleted() { - + async deleted (page) { + WIKI.logger.info(`(STORAGE/AZURE) Deleting file ${page.path}...`) + const filePath = getFilePath(page, 'path') + const blockBlobClient = this.container.getBlockBlobClient(filePath) + await blockBlobClient.delete({ + deleteSnapshots: 'include' + }) }, - async renamed() { - + async renamed(page) { + WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`) + let sourceFilePath = getFilePath(page, 'path') + let destinationFilePath = getFilePath(page, 'destinationPath') + if (WIKI.config.lang.namespacing) { + if (WIKI.config.lang.code !== page.localeCode) { + sourceFilePath = `${page.localeCode}/${sourceFilePath}` + } + if (WIKI.config.lang.code !== page.destinationLocaleCode) { + destinationFilePath = `${page.destinationLocaleCode}/${destinationFilePath}` + } + } + const sourceBlockBlobClient = this.container.getBlockBlobClient(sourceFilePath) + const destBlockBlobClient = this.container.getBlockBlobClient(destinationFilePath) + await destBlockBlobClient.syncCopyFromURL(sourceBlockBlobClient.url) + await sourceBlockBlobClient.delete({ + deleteSnapshots: 'include' + }) + }, + /** + * ASSET UPLOAD + * + * @param {Object} asset Asset to upload + */ + async assetUploaded (asset) { + WIKI.logger.info(`(STORAGE/AZURE) Creating new file ${asset.path}...`) + const blockBlobClient = this.container.getBlockBlobClient(asset.path) + await blockBlobClient.upload(asset.data, asset.data.length, { tier: this.config.storageTier }) + }, + /** + * ASSET DELETE + * + * @param {Object} asset Asset to delete + */ + async assetDeleted (asset) { + WIKI.logger.info(`(STORAGE/AZURE) Deleting file ${asset.path}...`) + const blockBlobClient = this.container.getBlockBlobClient(asset.path) + await blockBlobClient.delete({ + deleteSnapshots: 'include' + }) + }, + /** + * ASSET RENAME + * + * @param {Object} asset Asset to rename + */ + async assetRenamed (asset) { + WIKI.logger.info(`(STORAGE/AZURE) Renaming file from ${asset.path} to ${asset.destinationPath}...`) + const sourceBlockBlobClient = this.container.getBlockBlobClient(asset.path) + const destBlockBlobClient = this.container.getBlockBlobClient(asset.destinationPath) + await destBlockBlobClient.syncCopyFromURL(sourceBlockBlobClient.url) + await sourceBlockBlobClient.delete({ + deleteSnapshots: 'include' + }) } } diff --git a/server/modules/storage/s3/common.js b/server/modules/storage/s3/common.js index 268517dc..2d88fb17 100644 --- a/server/modules/storage/s3/common.js +++ b/server/modules/storage/s3/common.js @@ -57,8 +57,8 @@ module.exports = class S3CompatibleStorage { } async renamed(page) { WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file ${page.path} to ${page.destinationPath}...`) - let sourceFilePath = `${page.path}.${page.getFileExtension()}` - let destinationFilePath = `${page.destinationPath}.${page.getFileExtension()}` + let sourceFilePath = getFilePath(page, 'path') + let destinationFilePath = getFilePath(page, 'destinationPath') if (WIKI.config.lang.namespacing) { if (WIKI.config.lang.code !== page.localeCode) { sourceFilePath = `${page.localeCode}/${sourceFilePath}` diff --git a/yarn.lock b/yarn.lock index c47b78ad..c7dde81d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -45,6 +45,82 @@ resolved "https://registry.yarnpkg.com/@apollographql/graphql-playground-html/-/graphql-playground-html-1.6.24.tgz#3ce939cb127fb8aaa3ffc1e90dff9b8af9f2e3dc" integrity sha512-8GqG48m1XqyXh4mIZrtB5xOhUwSsh1WsrrsaZQOEYYql3YN9DEu9OOSg0ILzXHZo/h2Q74777YE4YzlArQzQEQ== +"@azure/abort-controller@^1.0.0": + version "1.0.1" + resolved "https://registry.yarnpkg.com/@azure/abort-controller/-/abort-controller-1.0.1.tgz#8510935b25ac051e58920300e9d7b511ca6e656a" + integrity sha512-wP2Jw6uPp8DEDy0n4KNidvwzDjyVV2xnycEIq7nPzj1rHyb/r+t3OPeNT1INZePP2wy5ZqlwyuyOMTi0ePyY1A== + dependencies: + tslib "^1.9.3" + +"@azure/core-asynciterator-polyfill@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@azure/core-asynciterator-polyfill/-/core-asynciterator-polyfill-1.0.0.tgz#dcccebb88406e5c76e0e1d52e8cc4c43a68b3ee7" + integrity sha512-kmv8CGrPfN9SwMwrkiBK9VTQYxdFQEGe0BmQk+M8io56P9KNzpAxcWE/1fxJj7uouwN4kXF0BHW8DNlgx+wtCg== + +"@azure/core-auth@^1.0.0": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@azure/core-auth/-/core-auth-1.0.2.tgz#7377c0cacf0e3c988ce321295bf5d2c174e0e288" + integrity sha512-zhPJObdrhz2ymIqGL1x8i3meEuaLz0UPjH9mOq9RGOlJB2Pb6K6xPtkHbRsfElgoO9USR4hH2XU5pLa4/JHHIw== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-tracing" "1.0.0-preview.7" + "@opentelemetry/types" "^0.2.0" + tslib "^1.9.3" + +"@azure/core-http@^1.0.0": + version "1.0.2" + resolved "https://registry.yarnpkg.com/@azure/core-http/-/core-http-1.0.2.tgz#e281861492cf96459edd035c98ae234401cf8bfd" + integrity sha512-wjtY0Ks0+/4SEcIiMAvXdYvZZpyYj0rgs4dbd1MfgBlXgvvzpDOsPpcvI6ty5pyPBrjAyqrpyQeFud6hqsAnDQ== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-auth" "^1.0.0" + "@azure/core-tracing" "1.0.0-preview.7" + "@azure/logger" "^1.0.0" + "@opentelemetry/types" "^0.2.0" + "@types/node-fetch" "^2.5.0" + "@types/tunnel" "^0.0.1" + form-data "^3.0.0" + node-fetch "^2.6.0" + process "^0.11.10" + tough-cookie "^3.0.1" + tslib "^1.9.3" + tunnel "^0.0.6" + uuid "^3.3.2" + xml2js "^0.4.19" + +"@azure/core-lro@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@azure/core-lro/-/core-lro-1.0.0.tgz#9837398e03aa04b5b0b09158f4338861348dcce4" + integrity sha512-l4abIb8S9qmlv3bJkonLvgGSVQcSXq5jByA7Z28GRGJaQN/mSFal9YQOuLvVag+JXQJsoftuxJFrZiggF2TwOg== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-http" "^1.0.0" + events "^3.0.0" + tslib "^1.9.3" + +"@azure/core-paging@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@azure/core-paging/-/core-paging-1.0.0.tgz#3aa3855582154260326feea97f9f8322cbfe56d9" + integrity sha512-CzaT7LwxU97PZ+/Pn7uAbNGXY2mJ/3b56kmLsZzbR9stfrNfzlILxR94WHG/D1jZEQOk4lUNiaqJ2zP7nSGJhA== + dependencies: + "@azure/core-asynciterator-polyfill" "^1.0.0" + +"@azure/core-tracing@1.0.0-preview.7": + version "1.0.0-preview.7" + resolved "https://registry.yarnpkg.com/@azure/core-tracing/-/core-tracing-1.0.0-preview.7.tgz#e9ee9c88f0dcf50d8e5b468fc827203165ecbc3f" + integrity sha512-pkFCw6OiJrpR+aH1VQe6DYm3fK2KWCC5Jf3m/Pv1RxF08M1Xm08RCyQ5Qe0YyW5L16yYT2nnV48krVhYZ6SGFA== + dependencies: + "@opencensus/web-types" "0.0.7" + "@opentelemetry/types" "^0.2.0" + tslib "^1.9.3" + +"@azure/logger@^1.0.0": + version "1.0.0" + resolved "https://registry.yarnpkg.com/@azure/logger/-/logger-1.0.0.tgz#48b371dfb34288c8797e5c104f6c4fb45bf1772c" + integrity sha512-g2qLDgvmhyIxR3JVS8N67CyIOeFRKQlX/llxYJQr1OSGQqM3HTpVP8MjmjcEKbL/OIt2N9C9UFaNQuKOw1laOA== + dependencies: + tslib "^1.9.3" + "@azure/ms-rest-azure-env@^1.1.2": version "1.1.2" resolved "https://registry.yarnpkg.com/@azure/ms-rest-azure-env/-/ms-rest-azure-env-1.1.2.tgz#8505873afd4a1227ec040894a64fdd736b4a101f" @@ -73,6 +149,21 @@ "@azure/ms-rest-js" "^1.8.7" adal-node "^0.1.28" +"@azure/storage-blob@12.0.1": + version "12.0.1" + resolved "https://registry.yarnpkg.com/@azure/storage-blob/-/storage-blob-12.0.1.tgz#9478f62e475347b0bf5edd773826fd9f07e4784a" + integrity sha512-NxMpOnN2jmWXgCZJqlGLuN5axXpi379PmVuCXWX6WLDwoY/jSJ1WOwWbS9h0Lgs/Lnd7rPrIijJEwUNQeGEtFw== + dependencies: + "@azure/abort-controller" "^1.0.0" + "@azure/core-http" "^1.0.0" + "@azure/core-lro" "^1.0.0" + "@azure/core-paging" "^1.0.0" + "@azure/core-tracing" "1.0.0-preview.7" + "@azure/logger" "^1.0.0" + "@opentelemetry/types" "^0.2.0" + events "^3.0.0" + tslib "^1.9.3" + "@babel/cli@^7.7.5": version "7.7.5" resolved "https://registry.yarnpkg.com/@babel/cli/-/cli-7.7.5.tgz#25702cc65418efc06989af3727897b9f4c8690b6" @@ -1337,6 +1428,16 @@ resolved "https://registry.yarnpkg.com/@mdi/font/-/font-4.7.95.tgz#46fddf35aad64dd623a8b1837f78ca4ed7bc48b1" integrity sha512-/SWooHIFz2dXkQJk3VhEXSbBplOU1lIkGSELAmw0peFEgR8KPqyM//M3vD8WDZETuEOSRVhVqLevP3okrsM5dw== +"@opencensus/web-types@0.0.7": + version "0.0.7" + resolved "https://registry.yarnpkg.com/@opencensus/web-types/-/web-types-0.0.7.tgz#4426de1fe5aa8f624db395d2152b902874f0570a" + integrity sha512-xB+w7ZDAu3YBzqH44rCmG9/RlrOmFuDPt/bpf17eJr8eZSrLt7nc7LnWdxM9Mmoj/YKMHpxRg28txu3TcpiL+g== + +"@opentelemetry/types@^0.2.0": + version "0.2.0" + resolved "https://registry.yarnpkg.com/@opentelemetry/types/-/types-0.2.0.tgz#2a0afd40fa7026e39ea56a454642bda72b172f80" + integrity sha512-GtwNB6BNDdsIPAYEdpp3JnOGO/3AJxjPvny53s3HERBdXSJTGQw8IRhiaTEX0b3w9P8+FwFZde4k+qkjn67aVw== + "@panter/vue-i18next@0.15.1": version "0.15.1" resolved "https://registry.yarnpkg.com/@panter/vue-i18next/-/vue-i18next-0.15.1.tgz#3d27e57fa702611df52bb73042729df5c0ccde2b" @@ -1645,6 +1746,13 @@ resolved "https://registry.yarnpkg.com/@types/minimatch/-/minimatch-3.0.3.tgz#3dca0e3f33b200fc7d1139c0cd96c1268cadfd9d" integrity sha512-tHq6qdbT9U1IRSGf14CL0pUlULksvY9OZ+5eEgl1N7t+OA3tGvNpxJCzuKQlsNgCVwbAs670L1vcVQi8j9HjnA== +"@types/node-fetch@^2.5.0": + version "2.5.4" + resolved "https://registry.yarnpkg.com/@types/node-fetch/-/node-fetch-2.5.4.tgz#5245b6d8841fc3a6208b82291119bc11c4e0ce44" + integrity sha512-Oz6id++2qAOFuOlE1j0ouk1dzl3mmI1+qINPNBhi9nt/gVOz0G+13Ao6qjhdF0Ys+eOkhu6JnFmt38bR3H0POQ== + dependencies: + "@types/node" "*" + "@types/node@*", "@types/node@>=6": version "12.7.4" resolved "https://registry.yarnpkg.com/@types/node/-/node-12.7.4.tgz#64db61e0359eb5a8d99b55e05c729f130a678b04" @@ -1728,6 +1836,13 @@ dependencies: "@types/node" "*" +"@types/tunnel@^0.0.1": + version "0.0.1" + resolved "https://registry.yarnpkg.com/@types/tunnel/-/tunnel-0.0.1.tgz#0d72774768b73df26f25df9184273a42da72b19c" + integrity sha512-AOqu6bQu5MSWwYvehMXLukFHnupHrpZ8nvgae5Ggie9UwzDR1CCwoXgSSWNZJuyOlCdfdsWMA5F2LlmvyoTv8A== + dependencies: + "@types/node" "*" + "@types/uglify-js@*": version "3.0.4" resolved "https://registry.yarnpkg.com/@types/uglify-js/-/uglify-js-3.0.4.tgz#96beae23df6f561862a830b4288a49e86baac082" @@ -4100,7 +4215,7 @@ colorspace@1.1.x: color "3.0.x" text-hex "1.0.x" -combined-stream@^1.0.6, combined-stream@~1.0.6: +combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: version "1.0.8" resolved "https://registry.yarnpkg.com/combined-stream/-/combined-stream-1.0.8.tgz#c3d45a8b34fd730631a110a8a2520682b31d5a7f" integrity sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg== @@ -6100,6 +6215,15 @@ form-data@^2.3.1, form-data@^2.3.2: combined-stream "^1.0.6" mime-types "^2.1.12" +form-data@^3.0.0: + version "3.0.0" + resolved "https://registry.yarnpkg.com/form-data/-/form-data-3.0.0.tgz#31b7e39c85f1355b7139ee0c647cf0de7f83c682" + integrity sha512-CKMFDglpbMi6PyN+brwB9Q/GOw0eAnsrEZDgcsH5Krhz5Od/haKHAX0NmQfha2zPPz0JpWzA7GJHGSnvCRLWsg== + dependencies: + asynckit "^0.4.0" + combined-stream "^1.0.8" + mime-types "^2.1.12" + form-data@~2.3.2: version "2.3.3" resolved "https://registry.yarnpkg.com/form-data/-/form-data-2.3.3.tgz#dcce52c05f644f298c6a7ab936bd724ceffbf3a6" @@ -7071,6 +7195,11 @@ invert-kv@^2.0.0: resolved "https://registry.yarnpkg.com/invert-kv/-/invert-kv-2.0.0.tgz#7393f5afa59ec9ff5f67a27620d11c226e3eec02" integrity sha512-wPVv/y/QQ/Uiirj/vh3oP+1Ww+AWehmi1g5fFWGPF6IpCBCDVrhgHRMvrLfdYcwDh3QJbGXDW4JAuzxElLSqKA== +ip-regex@^2.1.0: + version "2.1.0" + resolved "https://registry.yarnpkg.com/ip-regex/-/ip-regex-2.1.0.tgz#fa78bf5d2e6913c911ce9f819ee5146bb6d844e9" + integrity sha1-+ni/XS5pE8kRzp+BnuUUa7bYROk= + ipaddr.js@1.9.0: version "1.9.0" resolved "https://registry.yarnpkg.com/ipaddr.js/-/ipaddr.js-1.9.0.tgz#37df74e430a0e47550fe54a2defe30d8acd95f65" @@ -9180,7 +9309,7 @@ node-fetch@1.7.3: encoding "^0.1.11" is-stream "^1.0.1" -node-fetch@^2.1.2, node-fetch@^2.2.0, node-fetch@^2.3.0: +node-fetch@^2.1.2, node-fetch@^2.2.0, node-fetch@^2.3.0, node-fetch@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.0.tgz#e633456386d4aa55863f676a7ab0daa8fdecb0fd" integrity sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA== @@ -13491,6 +13620,15 @@ tough-cookie@^2.3.3, tough-cookie@^2.3.4, tough-cookie@^2.4.3: psl "^1.1.28" punycode "^2.1.1" +tough-cookie@^3.0.1: + version "3.0.1" + resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-3.0.1.tgz#9df4f57e739c26930a018184887f4adb7dca73b2" + integrity sha512-yQyJ0u4pZsv9D4clxO69OEjLWYw+jbgspjTue4lTQZLfV0c5l1VmK2y1JK8E9ahdpltPOaAThPcp5nKPUgSnsg== + dependencies: + ip-regex "^2.1.0" + psl "^1.1.28" + punycode "^2.1.1" + tough-cookie@~2.4.3: version "2.4.3" resolved "https://registry.yarnpkg.com/tough-cookie/-/tough-cookie-2.4.3.tgz#53f36da3f47783b0925afa06ff9f3b165280f781" @@ -13552,7 +13690,7 @@ tunnel-agent@^0.6.0: dependencies: safe-buffer "^5.0.1" -tunnel@0.0.6: +tunnel@0.0.6, tunnel@^0.0.6: version "0.0.6" resolved "https://registry.yarnpkg.com/tunnel/-/tunnel-0.0.6.tgz#72f1314b34a5b192db012324df2cc587ca47f92c" integrity sha512-1h/Lnq9yajKY2PEbBadPXj3VxsDDu844OnaAo52UVmIzIvwwtBPIuNvkjuzBlTWpfJyUbG3ez0KSBibQkj4ojg==