liz/wikijs-fork
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Added thumbnail generation + insert image files display + Create page fix

Browse Source
This commit is contained in:
NGPixel
2016-09-25 00:08:49 -04:00
parent 567e1307da
commit c0be18a8d8
18 changed files with 3767 additions and 74 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent.js
+96 -4
View File
@@ -39,6 +39,7 @@ global.WSInternalKey = process.argv[2];
winston.info('[AGENT] Background Agent is initializing...');
var appconfig = require('./models/config')('./config.yml');
let lcdata = require('./models/localdata').init(appconfig, 'agent');
global.git = require('./models/git').init(appconfig);
global.entries = require('./models/entries').init(appconfig);
@@ -50,8 +51,12 @@ var Promise = require('bluebird');
var fs = Promise.promisifyAll(require("fs-extra"));
var path = require('path');
var cron = require('cron').CronJob;
var wsClient = require('socket.io-client');
global.ws = wsClient('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
var readChunk = require('read-chunk');
var fileType = require('file-type');
global.ws = require('socket.io-client')('http://localhost:' + appconfig.wsPort, { reconnectionAttempts: 10 });
const mimeImgTypes = ['image/png', 'image/jpg']
// ----------------------------------------
// Start Cron
@@ -75,6 +80,7 @@ var job = new cron({
let jobs = [];
let repoPath = path.resolve(ROOTPATH, appconfig.datadir.repo);
let dataPath = path.resolve(ROOTPATH, appconfig.datadir.db);
let uploadsPath = path.join(repoPath, 'uploads');
// ----------------------------------------
@@ -151,11 +157,97 @@ var job = new cron({
return Promise.map(ls, (f) => {
return fs.statAsync(path.join(uploadsPath, f)).then((s) => { return { filename: f, stat: s }; });
}).filter((s) => { return s.stat.isDirectory(); }).then((arrStats) => {
}).filter((s) => { return s.stat.isDirectory(); }).then((arrDirs) => {
let folderNames = _.map(arrDirs, 'filename');
folderNames.unshift('');
ws.emit('uploadsSetFolders', {
auth: WSInternalKey,
content: _.map(arrStats, 'filename')
content: folderNames
});
let allFiles = [];
// Travel each directory
return Promise.map(folderNames, (fldName) => {
let fldPath = path.join(uploadsPath, fldName);
return fs.readdirAsync(fldPath).then((fList) => {
return Promise.map(fList, (f) => {
let fPath = path.join(fldPath, f);
let fPathObj = path.parse(fPath);
return fs.statAsync(fPath)
.then((s) => {
if(!s.isFile()) { return false; }
// Get MIME info
let mimeInfo = fileType(readChunk.sync(fPath, 0, 262));
// Images
if(s.size < 3145728) { // ignore files larger than 3MB
if(_.includes(['image/png', 'image/jpeg', 'image/gif', 'image/webp'], mimeInfo.mime)) {
return lcdata.getImageMetadata(fPath).then((mData) => {
let cacheThumbnailPath = path.parse(path.join(dataPath, 'thumbs', fldName, fPathObj.name + '.png'));
let cacheThumbnailPathStr = path.format(cacheThumbnailPath);
mData = _.pick(mData, ['format', 'width', 'height', 'density', 'hasAlpha', 'orientation']);
mData.category = 'image';
mData.mime = mimeInfo.mime;
mData.folder = fldName;
mData.filename = f;
mData.basename = fPathObj.name;
mData.filesize = s.size;
mData.uploadedOn = moment().utc();
allFiles.push(mData);
// Generate thumbnail
return fs.statAsync(cacheThumbnailPathStr).then((st) => {
return st.isFile();
}).catch((err) => {
return false;
}).then((thumbExists) => {
return (thumbExists) ? true : fs.ensureDirAsync(cacheThumbnailPath.dir).then(() => {
return lcdata.generateThumbnail(fPath, cacheThumbnailPathStr);
});
});
})
}
}
// Other Files
allFiles.push({
category: 'file',
mime: mimeInfo.mime,
folder: fldName,
filename: f,
basename: fPathObj.name,
filesize: s.size,
uploadedOn: moment().utc()
});
});
}, {concurrency: 3});
});
}, {concurrency: 1}).finally(() => {
ws.emit('uploadsSetFiles', {
auth: WSInternalKey,
content: allFiles
});
});
return true;
});
assets/css/app.css
+1 -1
View File
File diff suppressed because one or more lines are too long
assets/js/app.js
+1 -1
View File
File diff suppressed because one or more lines are too long
assets/js/libs.js
+30 -30
View File
File diff suppressed because one or more lines are too long
client/js/components/editor-image.js
+4 -1
View File
@@ -46,7 +46,10 @@ let vueImage = new Vue({
vueImage.isLoadingText = 'Fetching images...';
Vue.nextTick(() => {
socket.emit('uploadsGetImages', { folder: vueImage.currentFolder }, (data) => {
vueImage.images = data;
vueImage.images = _.map(data, (f) => {
f.thumbpath = (f.folder === '') ? f.basename + '.png' : _.join([ f.folder, f.basename + '.png' ], '/');
return f;
});
vueImage.isLoading = false;
});
});
client/scss/app.scss
+1
View File
@@ -7,6 +7,7 @@
$blue: #039BE5;
$turquoise: #00ACC1;
$green: #7CB342;
$purple: #673AB7;
$warning: $orange;
client/scss/layout/_base.scss
+1 -1
View File
@@ -9,7 +9,7 @@ #root {
padding-top: 52px;
}
//$family-sans-serif: "Roboto", "Helvetica", "Arial", sans-serif;
$family-monospace: monospace;
$family-sans-serif: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Helvetica, Arial, sans-serif, "Apple Color Emoji", "Segoe UI Emoji", "Segoe UI Symbol";
[v-cloak] {
client/scss/layout/_content.scss
+1 -1
View File
@@ -35,7 +35,7 @@ .mkcontent {
border-bottom: 1px dotted $grey-light;
padding-bottom: 4px;
font-weight: 400;
color: desaturate(darken($purple, 15%), 10%);
color: desaturate($purple, 20%);
}
a.toc-anchor {
controllers/uploads.js
+21
View File
@@ -5,11 +5,32 @@ var router = express.Router();
var _ = require('lodash');
var validPathRe = new RegExp("^([a-z0-9\\/-]+\\.[a-z0-9]+)$");
var validPathThumbsRe = new RegExp("^([a-z0-9\\/-]+\\.png)$");
// ==========================================
// SERVE UPLOADS FILES
// ==========================================
router.get('/t/*', (req, res, next) => {
let fileName = req.params[0];
if(!validPathThumbsRe.test(fileName)) {
return res.sendStatus(404).end();
}
//todo: Authentication-based access
res.sendFile(fileName, {
root: lcdata.getThumbsPath(),
dotfiles: 'deny'
}, (err) => {
if (err) {
res.status(err.status).end();
}
});
});
router.get('/*', (req, res, next) => {
let fileName = req.params[0];
lib/mimes.json
+3427
View File
File diff suppressed because it is too large Load Diff
models/entries.js
+3 -5
View File
@@ -5,8 +5,6 @@ var Promise = require('bluebird'),
fs = Promise.promisifyAll(require("fs-extra")),
_ = require('lodash'),
farmhash = require('farmhash'),
BSONModule = require('bson'),
BSON = new BSONModule.BSONPure.BSON(),
moment = require('moment');
/**
@@ -82,7 +80,7 @@ module.exports = {
// Load from cache
return fs.readFileAsync(cpath).then((contents) => {
return BSON.deserialize(contents);
return JSON.parse(contents);
}).catch((err) => {
winston.error('Corrupted cache file. Deleting it...');
fs.unlinkSync(cpath);
@@ -156,7 +154,7 @@ module.exports = {
// Cache to disk
if(options.cache) {
let cacheData = BSON.serialize(_.pick(pageData, ['html', 'meta', 'tree', 'parent']), false, false, false);
let cacheData = JSON.stringify(_.pick(pageData, ['html', 'meta', 'tree', 'parent']), false, false, false);
return fs.writeFileAsync(cpath, cacheData).catch((err) => {
winston.error('Unable to write to cache! Performance may be affected.');
return true;
@@ -257,7 +255,7 @@ module.exports = {
* @return {String} The full cache path.
*/
getCachePath(entryPath) {
return path.join(this._cachePath, farmhash.fingerprint32(entryPath) + '.bson');
return path.join(this._cachePath, farmhash.fingerprint32(entryPath) + '.json');
},
/**
models/localdata.js
+146 -5
View File
@@ -2,6 +2,8 @@
var fs = require('fs'),
path = require('path'),
loki = require('lokijs'),
Promise = require('bluebird'),
_ = require('lodash');
/**
@@ -12,7 +14,9 @@ var fs = require('fs'),
module.exports = {
_uploadsPath: './repo/uploads',
_uploadsThumbsPath: './data/thumbs',
_uploadsFolders: [],
_uploadsDb: null,
/**
* Initialize Local Data Storage model
@@ -20,22 +24,88 @@ module.exports = {
* @param {Object} appconfig The application config
* @return {Object} Local Data Storage model instance
*/
init(appconfig, skipFolderCreation = false) {
init(appconfig, mode = 'server') {
let self = this;
self._uploadsPath = path.join(ROOTPATH, appconfig.datadir.db, 'uploads');
self._uploadsPath = path.resolve(ROOTPATH, appconfig.datadir.repo, 'uploads');
self._uploadsThumbsPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'thumbs');
// Create data directories
if(!skipFolderCreation) {
self.createBaseDirectories(appconfig);
// Start in full or bare mode
switch(mode) {
case 'agent':
//todo
break;
case 'server':
self.createBaseDirectories(appconfig);
break;
case 'ws':
self.initDb(appconfig);
break;
}
return self;
},
/**
* Initialize Uploads DB
*
* @param {Object} appconfig The application config
* @return {boolean} Void
*/
initDb(appconfig) {
let self = this;
let dbReadyResolve;
let dbReady = new Promise((resolve, reject) => {
dbReadyResolve = resolve;
});
// Initialize Loki.js
let dbModel = {
Store: new loki(path.join(appconfig.datadir.db, 'uploads.db'), {
env: 'NODEJS',
autosave: true,
autosaveInterval: 15000
}),
onReady: dbReady
};
// Load Models
dbModel.Store.loadDatabase({}, () => {
dbModel.Files = dbModel.Store.getCollection('Files');
if(!dbModel.Files) {
dbModel.Files = dbModel.Store.addCollection('Files', {
indices: ['category', 'folder']
});
}
dbReadyResolve();
});
self._uploadsDb = dbModel;
return true;
},
/**
* Gets the thumbnails folder path.
*
* @return {String} The thumbs path.
*/
getThumbsPath() {
return this._uploadsThumbsPath;
},
/**
* Creates a base directories (Synchronous).
*
@@ -99,6 +169,77 @@ module.exports = {
*/
getUploadsFolders() {
return this._uploadsFolders;
},
/**
* Sets the uploads files.
*
* @param {Array<Object>} arrFiles The uploads files
* @return {Void} Void
*/
setUploadsFiles(arrFiles) {
let self = this;
if(_.isArray(arrFiles) && arrFiles.length > 0) {
self._uploadsDb.Files.clear();
self._uploadsDb.Files.insert(arrFiles);
self._uploadsDb.Files.ensureIndex('category', true);
self._uploadsDb.Files.ensureIndex('folder', true);
}
return;
},
/**
* Gets the uploads files.
*
* @param {String} cat Category type
* @param {String} fld Folder
* @return {Array<Object>} The files matching the query
*/
getUploadsFiles(cat, fld) {
return this._uploadsDb.Files.find({
'$and': [{ 'category' : cat },{ 'folder' : fld }]
});
},
/**
* Generate thumbnail of image
*
* @param {String} sourcePath The source path
* @return {Promise<Object>} Promise returning the resized image info
*/
generateThumbnail(sourcePath, destPath) {
let sharp = require('sharp');
return sharp(sourcePath)
.withoutEnlargement()
.resize(150,150)
.background('white')
.embed()
.flatten()
.toFormat('png')
.toFile(destPath);
},
/**
* Gets the image metadata.
*
* @param {String} sourcePath The source path
* @return {Object} The image metadata.
*/
getImageMetadata(sourcePath) {
let sharp = require('sharp');
return sharp(sourcePath).metadata();
}
};
models/search.js
+1 -1
View File
@@ -22,7 +22,7 @@ module.exports = {
init(appconfig) {
let self = this;
let dbPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'search-index');
let dbPath = path.resolve(ROOTPATH, appconfig.datadir.db, 'search');
searchIndex({
deletable: true,
package.json
+11 -9
View File
@@ -49,31 +49,33 @@
"express-brute": "^1.0.0",
"express-brute-loki": "^1.0.0",
"express-session": "^1.14.1",
"express-validator": "^2.20.8",
"express-validator": "^2.20.10",
"farmhash": "^1.2.1",
"file-type": "^3.8.0",
"fs-extra": "^0.30.0",
"git-wrapper2-promise": "^0.2.9",
"highlight.js": "^9.6.0",
"i18next": "^3.4.2",
"highlight.js": "^9.7.0",
"i18next": "^3.4.3",
"i18next-express-middleware": "^1.0.2",
"i18next-node-fs-backend": "^0.1.2",
"js-yaml": "^3.6.1",
"lodash": "^4.15.0",
"lodash": "^4.16.1",
"lokijs": "^1.4.1",
"markdown-it": "^8.0.0",
"markdown-it-abbr": "^1.0.4",
"markdown-it-anchor": "^2.5.0",
"markdown-it-attrs": "^0.7.0",
"markdown-it-attrs": "^0.7.1",
"markdown-it-emoji": "^1.2.0",
"markdown-it-expand-tabs": "^1.0.11",
"markdown-it-external-links": "0.0.5",
"markdown-it-footnote": "^3.0.1",
"markdown-it-task-lists": "^1.4.1",
"moment": "^2.15.0",
"moment": "^2.15.1",
"moment-timezone": "^0.5.5",
"passport": "^0.3.2",
"passport-local": "^1.0.0",
"pug": "^2.0.0-beta6",
"read-chunk": "^2.0.0",
"remove-markdown": "^0.1.0",
"search-index": "^0.8.15",
"serve-favicon": "^2.3.0",
@@ -89,7 +91,7 @@
"devDependencies": {
"ace-builds": "^1.2.5",
"babel-preset-es2015": "^6.14.0",
"bulma": "^0.1.2",
"bulma": "^0.2.0",
"chai": "^3.5.0",
"chai-as-promised": "^5.3.0",
"codacy-coverage": "^2.0.0",
@@ -107,7 +109,7 @@
"gulp-uglify": "^2.0.0",
"gulp-zip": "^3.2.0",
"istanbul": "^0.4.5",
"jquery": "^3.1.0",
"jquery": "^3.1.1",
"jquery-smooth-scroll": "^2.0.0",
"merge-stream": "^1.0.0",
"mocha": "^3.0.2",
@@ -115,7 +117,7 @@
"nodemon": "^1.10.2",
"sticky-js": "^1.0.5",
"twemoji-awesome": "^1.0.4",
"vue": "^1.0.26"
"vue": "^1.0.27"
},
"snyk": true
}
server.js
+6 -3
View File
@@ -7,7 +7,7 @@
global.ROOTPATH = __dirname;
// ----------------------------------------
// Load global modules
// Load Winston
// ----------------------------------------
var _isDebug = process.env.NODE_ENV === 'development';
@@ -24,9 +24,12 @@ winston.add(winston.transports.Console, {
winston.info('[SERVER] Requarks Wiki is initializing...');
var appconfig = require('./models/config')('./config.yml');
let lcdata = require('./models/localdata').init(appconfig, false);
// ----------------------------------------
// Load global modules
// ----------------------------------------
var appconfig = require('./models/config')('./config.yml');
global.lcdata = require('./models/localdata').init(appconfig, 'server');
global.db = require('./models/db')(appconfig);
global.git = require('./models/git').init(appconfig, false);
global.entries = require('./models/entries').init(appconfig);
views/modals/editor-image.pug
+3 -7
View File
@@ -37,17 +37,13 @@
p.menu-label
| Folders
ul.menu-list
li
a(v-on:click="selectFolder('')", v-bind:class="{ 'is-active': currentFolder === '' }")
span.icon.is-small: i.fa.fa-folder-o
span /
li(v-for="fld in folders")
a(v-on:click="selectFolder(fld)", v-bind:class="{ 'is-active': currentFolder === fld }")
span.icon.is-small: i.fa.fa-folder
span / {{ fld }}
span /{{ fld }}
.column
figure.image.is-128x128
img(src='http://placehold.it/128x128')
figure.image.is-128x128(v-for="img in images")
img(v-bind:src="'/uploads/t/' + img.thumbpath")
.modal(v-bind:class="{ 'is-active': newFolderShow }")
.modal-background
views/pages/create.pug
+4 -1
View File
@@ -21,4 +21,7 @@ block content
.editor-area
textarea#mk-editor= pageData.markdown
include ../modals/create-discard.pug
include ../modals/create-discard.pug
include ../modals/editor-link.pug
include ../modals/editor-image.pug
include ../modals/editor-codeblock.pug
ws-server.js
+10 -4
View File
@@ -33,20 +33,20 @@ if(!process.argv[2] || process.argv[2].length !== 40) {
global.internalAuth = require('./lib/internalAuth').init(process.argv[2]);;
// ----------------------------------------
// Load modules
// Load global modules
// ----------------------------------------
winston.info('[WS] WS Server is initializing...');
var appconfig = require('./models/config')('./config.yml');
let lcdata = require('./models/localdata').init(appconfig, true);
let lcdata = require('./models/localdata').init(appconfig, 'ws');
global.entries = require('./models/entries').init(appconfig);
global.mark = require('./models/markdown');
global.search = require('./models/search').init(appconfig);
// ----------------------------------------
// Load modules
// Load local modules
// ----------------------------------------
var _ = require('lodash');
@@ -141,8 +141,14 @@ io.on('connection', (socket) => {
cb(lcdata.getUploadsFolders());
});
socket.on('uploadsSetFiles', (data, cb) => {
if(internalAuth.validateKey(data.auth)) {
lcdata.setUploadsFiles(data.content);
}
});
socket.on('uploadsGetImages', (data, cb) => {
cb([]);
cb(lcdata.getUploadsFiles('image', data.folder));
});
});