From f6d8e3ab00370a78c0c788ad1e37a7ff77a53555 Mon Sep 17 00:00:00 2001 From: Wu Cheng-Han Date: Mon, 2 Jan 2017 10:59:53 +0800 Subject: [PATCH] Remove LZString compression for data storage --- lib/models/note.js | 17 ++++++----------- lib/models/revision.js | 11 +++++------ lib/realtime.js | 10 ++++------ lib/response.js | 14 +++++++------- lib/workers/dmpWorker.js | 13 ++++++------- 5 files changed, 28 insertions(+), 37 deletions(-) diff --git a/lib/models/note.js b/lib/models/note.js index 5727046..81de991 100644 --- a/lib/models/note.js +++ b/lib/models/note.js @@ -124,8 +124,6 @@ module.exports = function (sequelize, DataTypes) { var body = fs.readFileSync(filePath, 'utf8'); var contentLength = body.length; var title = Note.parseNoteTitle(body); - body = LZString.compressToBase64(body); - title = LZString.compressToBase64(title); if (fsModifiedTime.isAfter(dbModifiedTime) && note.content !== body) { note.update({ title: title, @@ -135,14 +133,14 @@ module.exports = function (sequelize, DataTypes) { sequelize.models.Revision.saveNoteRevision(note, function (err, revision) { if (err) return _callback(err, null); // update authorship on after making revision of docs - var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var patch = dmp.patch_fromText(revision.patch); var operations = Note.transformPatchToOperations(patch, contentLength); - var authorship = note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : []; + var authorship = note.authorship; for (var i = 0; i < operations.length; i++) { authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship); } note.update({ - authorship: LZString.compressToBase64(JSON.stringify(authorship)) + authorship: JSON.stringify(authorship) }).then(function (note) { return callback(null, note.id); }).catch(function (err) { @@ -264,10 +262,7 @@ module.exports = function (sequelize, DataTypes) { return markdown.substr(0, 100).replace(/(?:\r\n|\r|\n)/g, ' '); }, decodeTitle: function (title) { - var decodedTitle = LZString.decompressFromBase64(title); - if (decodedTitle) title = decodedTitle; - else title = 'Untitled'; - return title; + return title ? title : 'Untitled'; }, generateWebTitle: function (title) { title = !title || title == "Untitled" ? "HackMD - Collaborative markdown notes" : title + " - HackMD"; @@ -496,8 +491,8 @@ module.exports = function (sequelize, DataTypes) { if (Note.checkFileExist(filePath)) { var fsCreatedTime = moment(fs.statSync(filePath).ctime); body = fs.readFileSync(filePath, 'utf8'); - note.title = LZString.compressToBase64(Note.parseNoteTitle(body)); - note.content = LZString.compressToBase64(body); + note.title = Note.parseNoteTitle(body); + note.content = body; if (filePath !== config.defaultnotepath) { note.createdAt = fsCreatedTime; } diff --git a/lib/models/revision.js b/lib/models/revision.js index 8b8eba9..6f44cf1 100644 --- a/lib/models/revision.js +++ b/lib/models/revision.js @@ -2,7 +2,6 @@ // external modules var Sequelize = require("sequelize"); -var LZString = require('lz-string'); var async = require('async'); var moment = require('moment'); var childProcess = require('child_process'); @@ -214,7 +213,7 @@ module.exports = function (sequelize, DataTypes) { Revision.create({ noteId: note.id, lastContent: note.content, - length: LZString.decompressFromBase64(note.content).length, + length: note.content.length, authorship: note.authorship }).then(function (revision) { Revision.finishSaveNoteRevision(note, revision, callback); @@ -223,8 +222,8 @@ module.exports = function (sequelize, DataTypes) { }); } else { var latestRevision = revisions[0]; - var lastContent = LZString.decompressFromBase64(latestRevision.content || latestRevision.lastContent); - var content = LZString.decompressFromBase64(note.content); + var lastContent = latestRevision.content || latestRevision.lastContent; + var content = note.content; sendDmpWorker({ msg: 'create patch', lastDoc: lastContent, @@ -244,9 +243,9 @@ module.exports = function (sequelize, DataTypes) { } else { Revision.create({ noteId: note.id, - patch: LZString.compressToBase64(patch), + patch: patch, content: note.content, - length: LZString.decompressFromBase64(note.content).length, + length: note.content.length, authorship: note.authorship }).then(function (revision) { // clear last revision content to reduce db size diff --git a/lib/realtime.js b/lib/realtime.js index c66fea0..a662dee 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -152,12 +152,10 @@ function finishUpdateNote(note, _note, callback) { if (!note || !note.server) return callback(null, null); var body = note.server.document; var title = note.title = models.Note.parseNoteTitle(body); - title = LZString.compressToBase64(title); - body = LZString.compressToBase64(body); var values = { title: title, content: body, - authorship: LZString.compressToBase64(JSON.stringify(note.authorship)), + authorship: note.authorship, lastchangeuserId: note.lastchangeuser, lastchangeAt: Date.now() }; @@ -459,7 +457,7 @@ function startConnection(socket) { var lastchangeuser = note.lastchangeuserId; var lastchangeuserprofile = note.lastchangeuser ? models.User.getProfile(note.lastchangeuser) : null; - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var createtime = note.createdAt; var updatetime = note.lastchangeAt; var server = new ot.EditorSocketIOServer(body, [], noteId, ifMayEdit, operationCallback); @@ -479,7 +477,7 @@ function startConnection(socket) { notes[noteId] = { id: noteId, alias: note.alias, - title: LZString.decompressFromBase64(note.title), + title: note.title, owner: owner, ownerprofile: ownerprofile, permission: note.permission, @@ -491,7 +489,7 @@ function startConnection(socket) { updatetime: moment(updatetime).valueOf(), server: server, authors: authors, - authorship: note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : [] + authorship: note.authorship }; return finishConnection(socket, notes[noteId], users[socket.id]); diff --git a/lib/response.js b/lib/response.js index 2b38cf2..f591d5e 100755 --- a/lib/response.js +++ b/lib/response.js @@ -75,7 +75,7 @@ function showIndex(req, res, next) { } function responseHackMD(res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; try { meta = models.Note.parseMeta(metaMarked(body).meta); @@ -191,7 +191,7 @@ function showPublishNote(req, res, next) { if (!note) { return response.errorNotFound(res); } - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; var markdown = null; try { @@ -248,7 +248,7 @@ function actionSlide(req, res, note) { } function actionDownload(req, res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var title = models.Note.decodeTitle(note.title); var filename = title; filename = encodeURIComponent(filename); @@ -265,7 +265,7 @@ function actionDownload(req, res, note) { } function actionInfo(req, res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; var markdown = null; try { @@ -297,7 +297,7 @@ function actionInfo(req, res, note) { } function actionPDF(req, res, note) { - var body = LZString.decompressFromBase64(note.content); + var body = note.content; try { body = metaMarked(body).markdown; } catch(err) { @@ -479,7 +479,7 @@ function githubActionGist(req, res, note) { if (!error && httpResponse.statusCode == 200) { var access_token = body.access_token; if (access_token) { - var content = LZString.decompressFromBase64(note.content); + var content = note.content; var title = models.Note.decodeTitle(note.title); var filename = title.replace('/', ' ') + '.md'; var gist = { @@ -579,7 +579,7 @@ function showPublishSlide(req, res, next) { if (!note) { return response.errorNotFound(res); } - var body = LZString.decompressFromBase64(note.content); + var body = note.content; var meta = null; var markdown = null; try { diff --git a/lib/workers/dmpWorker.js b/lib/workers/dmpWorker.js index fae3619..5b4b6aa 100644 --- a/lib/workers/dmpWorker.js +++ b/lib/workers/dmpWorker.js @@ -1,5 +1,4 @@ // external modules -var LZString = require('lz-string'); var DiffMatchPatch = require('diff-match-patch'); var dmp = new DiffMatchPatch(); @@ -80,10 +79,10 @@ function getRevision(revisions, count) { for (var i = 0; i < count; i++) { var revision = revisions[i]; if (i == 0) { - startContent = LZString.decompressFromBase64(revision.content || revision.lastContent); + startContent = revision.content || revision.lastContent; } if (i != count - 1) { - var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var patch = dmp.patch_fromText(revision.patch); applyPatches = applyPatches.concat(patch); } lastPatch = revision.patch; @@ -105,11 +104,11 @@ function getRevision(revisions, count) { for (var i = l; i >= count - 1; i--) { var revision = revisions[i]; if (i == l) { - startContent = LZString.decompressFromBase64(revision.lastContent); + startContent = revision.lastContent; authorship = revision.authorship; } if (revision.patch) { - var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); + var patch = dmp.patch_fromText(revision.patch); applyPatches = applyPatches.concat(patch); } lastPatch = revision.patch; @@ -123,8 +122,8 @@ function getRevision(revisions, count) { } var data = { content: finalContent, - patch: dmp.patch_fromText(LZString.decompressFromBase64(lastPatch)), - authorship: authorship ? JSON.parse(LZString.decompressFromBase64(authorship)) : null + patch: dmp.patch_fromText(lastPatch), + authorship: authorship }; var ms_end = (new Date()).getTime(); if (config.debug) {