From b2ff064d34c8251fe717eee5a5c6603b075c0b1c Mon Sep 17 00:00:00 2001
From: Regev Brody <regevbr@gmail.com>
Date: Sun, 12 Jul 2020 19:19:01 +0300
Subject: [PATCH] fix: stream assets from storage local locations (#2087)

---
 server/models/assets.js                    | 62 +++++++++++++++-------
 server/models/storage.js                   | 17 ++++++
 server/modules/storage/azure/storage.js    |  3 ++
 server/modules/storage/box/storage.js      |  3 ++
 server/modules/storage/disk/storage.js     |  4 +-
 server/modules/storage/dropbox/storage.js  |  3 ++
 server/modules/storage/gdrive/storage.js   |  3 ++
 server/modules/storage/git/storage.js      |  3 ++
 server/modules/storage/onedrive/storage.js |  3 ++
 server/modules/storage/s3/common.js        |  3 ++
 server/modules/storage/sftp/storage.js     |  3 ++
 11 files changed, 86 insertions(+), 21 deletions(-)

diff --git a/server/models/assets.js b/server/models/assets.js
index 514c29bd..396299fb 100644
--- a/server/models/assets.js
+++ b/server/models/assets.js
@@ -6,6 +6,7 @@ const path = require('path')
 const fs = require('fs-extra')
 const _ = require('lodash')
 const assetHelper = require('../helpers/asset')
+const Promise = require('bluebird')
 
 /**
  * Users model
@@ -150,32 +151,53 @@ module.exports = class Asset extends Model {
   }
 
   static async getAsset(assetPath, res) {
-    let assetExists = await WIKI.models.assets.getAssetFromCache(assetPath, res)
-    if (!assetExists) {
-      await WIKI.models.assets.getAssetFromDb(assetPath, res)
+    try {
+      const fileHash = assetHelper.generateHash(assetPath)
+      const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`)
+      if (await WIKI.models.assets.getAssetFromCache(assetPath, cachePath, res)) {
+        return
+      }
+      if (await WIKI.models.assets.getAssetFromStorage(assetPath, res)) {
+        return
+      }
+      await WIKI.models.assets.getAssetFromDb(assetPath, fileHash, cachePath, res)
+    } catch (err) {
+      if (err.code === `ECONNABORTED` || err.code === `EPIPE`) {
+        return
+      }
+      WIKI.logger.error(err)
+      res.sendStatus(500)
     }
   }
 
-  static async getAssetFromCache(assetPath, res) {
-    const fileHash = assetHelper.generateHash(assetPath)
-    const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`)
-
-    return new Promise((resolve, reject) => {
-      res.type(path.extname(assetPath))
-      res.sendFile(cachePath, { dotfiles: 'deny' }, err => {
-        if (err) {
-          resolve(false)
-        } else {
-          resolve(true)
-        }
-      })
-    })
+  static async getAssetFromCache(assetPath, cachePath, res) {
+    try {
+      await fs.access(cachePath, fs.constants.R_OK)
+    } catch (err) {
+      return false
+    }
+    const sendFile = Promise.promisify(res.sendFile, {context: res})
+    res.type(path.extname(assetPath))
+    await sendFile(cachePath, { dotfiles: 'deny' })
+    return true
   }
 
-  static async getAssetFromDb(assetPath, res) {
-    const fileHash = assetHelper.generateHash(assetPath)
-    const cachePath = path.resolve(WIKI.ROOTPATH, WIKI.config.dataPath, `cache/${fileHash}.dat`)
+  static async getAssetFromStorage(assetPath, res) {
+    const localLocations = await WIKI.models.storage.getLocalLocations({
+      asset: {
+        path: assetPath,
+      }
+    })
+    for (let location of _.filter(localLocations, location => Boolean(location.path))) {
+      const assetExists = await WIKI.models.assets.getAssetFromCache(assetPath, location.path, res)
+      if (assetExists) {
+        return true
+      }
+    }
+    return false
+  }
 
+  static async getAssetFromDb(assetPath, fileHash, cachePath, res) {
     const asset = await WIKI.models.assets.query().where('hash', fileHash).first()
     if (asset) {
       const assetData = await WIKI.models.knex('assetData').where('id', asset.id).first()
diff --git a/server/models/storage.js b/server/models/storage.js
index cbfc58b0..a982b7ec 100644
--- a/server/models/storage.js
+++ b/server/models/storage.js
@@ -199,6 +199,23 @@ module.exports = class Storage extends Model {
     }
   }
 
+  static async getLocalLocations({ asset }) {
+    const locations = []
+    const promises = this.targets.map(async (target) => {
+      try {
+        const path = await target.fn.getLocalLocation(asset)
+        locations.push({
+          path,
+          key: target.key
+        })
+      } catch (err) {
+        WIKI.logger.warn(err)
+      }
+    })
+    await Promise.all(promises)
+    return locations
+  }
+
   static async executeAction(targetKey, handler) {
     try {
       const target = _.find(this.targets, ['key', targetKey])
diff --git a/server/modules/storage/azure/storage.js b/server/modules/storage/azure/storage.js
index 1228d767..69a453aa 100644
--- a/server/modules/storage/azure/storage.js
+++ b/server/modules/storage/azure/storage.js
@@ -114,6 +114,9 @@ module.exports = {
     await sourceBlockBlobClient.delete({
       deleteSnapshots: 'include'
     })
+  },
+  async getLocalLocation () {
+
   },
   /**
    * HANDLERS
diff --git a/server/modules/storage/box/storage.js b/server/modules/storage/box/storage.js
index ab25ce97..8ae6cc29 100644
--- a/server/modules/storage/box/storage.js
+++ b/server/modules/storage/box/storage.js
@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }
diff --git a/server/modules/storage/disk/storage.js b/server/modules/storage/disk/storage.js
index 6fc916ac..34117378 100644
--- a/server/modules/storage/disk/storage.js
+++ b/server/modules/storage/disk/storage.js
@@ -116,7 +116,9 @@ module.exports = {
     WIKI.logger.info(`(STORAGE/DISK) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
     await fs.move(path.join(this.config.path, asset.path), path.join(this.config.path, asset.destinationPath), { overwrite: true })
   },
-
+  async getLocalLocation (asset) {
+    return path.join(this.config.path, asset.path)
+  },
   /**
    * HANDLERS
    */
diff --git a/server/modules/storage/dropbox/storage.js b/server/modules/storage/dropbox/storage.js
index ab25ce97..8ae6cc29 100644
--- a/server/modules/storage/dropbox/storage.js
+++ b/server/modules/storage/dropbox/storage.js
@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }
diff --git a/server/modules/storage/gdrive/storage.js b/server/modules/storage/gdrive/storage.js
index ab25ce97..8ae6cc29 100644
--- a/server/modules/storage/gdrive/storage.js
+++ b/server/modules/storage/gdrive/storage.js
@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }
diff --git a/server/modules/storage/git/storage.js b/server/modules/storage/git/storage.js
index 7f5c1e66..4da51a01 100644
--- a/server/modules/storage/git/storage.js
+++ b/server/modules/storage/git/storage.js
@@ -363,6 +363,9 @@ module.exports = {
       '--author': `"${asset.moveAuthorName} <${asset.moveAuthorEmail}>"`
     })
   },
+  async getLocalLocation (asset) {
+    return path.join(this.repoPath, asset.path)
+  },
   /**
    * HANDLERS
    */
diff --git a/server/modules/storage/onedrive/storage.js b/server/modules/storage/onedrive/storage.js
index ab25ce97..8ae6cc29 100644
--- a/server/modules/storage/onedrive/storage.js
+++ b/server/modules/storage/onedrive/storage.js
@@ -19,5 +19,8 @@ module.exports = {
   },
   async renamed() {
 
+  },
+  async getLocalLocation () {
+
   }
 }
diff --git a/server/modules/storage/s3/common.js b/server/modules/storage/s3/common.js
index 66068c4b..b11bce00 100644
--- a/server/modules/storage/s3/common.js
+++ b/server/modules/storage/s3/common.js
@@ -119,6 +119,9 @@ module.exports = class S3CompatibleStorage {
     WIKI.logger.info(`(STORAGE/${this.storageName}) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
     await this.s3.copyObject({ CopySource: asset.path, Key: asset.destinationPath }).promise()
     await this.s3.deleteObject({ Key: asset.path }).promise()
+  }
+  async getLocalLocation () {
+
   }
   /**
    * HANDLERS
diff --git a/server/modules/storage/sftp/storage.js b/server/modules/storage/sftp/storage.js
index 5ded65c5..0684f4cb 100644
--- a/server/modules/storage/sftp/storage.js
+++ b/server/modules/storage/sftp/storage.js
@@ -103,6 +103,9 @@ module.exports = {
     WIKI.logger.info(`(STORAGE/SFTP) Renaming file from ${asset.path} to ${asset.destinationPath}...`)
     await this.ensureDirectory(asset.destinationPath)
     await this.sftp.rename(path.posix.join(this.config.basePath, asset.path), path.posix.join(this.config.basePath, asset.destinationPath))
+  },
+  async getLocalLocation () {
+
   },
   /**
    * HANDLERS