depau/HackMD
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Remove LZString compression for data storage

Browse source
This commit is contained in:
Wu Cheng-Han 2017-01-02 10:59:53 +08:00
parent c904083d1f
commit f6d8e3ab00
5 changed files with 28 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
lib/models/note.js
View file

@ -124,8 +124,6 @@ module.exports = function (sequelize, DataTypes) {
var body = fs.readFileSync(filePath, 'utf8'); var body = fs.readFileSync(filePath, 'utf8');
var contentLength = body.length; var contentLength = body.length;
var title = Note.parseNoteTitle(body); var title = Note.parseNoteTitle(body);
body = LZString.compressToBase64(body);
title = LZString.compressToBase64(title);
if (fsModifiedTime.isAfter(dbModifiedTime) && note.content !== body) { if (fsModifiedTime.isAfter(dbModifiedTime) && note.content !== body) {
note.update({ note.update({
title: title, title: title,
@ -135,14 +133,14 @@ module.exports = function (sequelize, DataTypes) {
sequelize.models.Revision.saveNoteRevision(note, function (err, revision) { sequelize.models.Revision.saveNoteRevision(note, function (err, revision) {
if (err) return _callback(err, null); if (err) return _callback(err, null);
// update authorship on after making revision of docs // update authorship on after making revision of docs
var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); var patch = dmp.patch_fromText(revision.patch);
var operations = Note.transformPatchToOperations(patch, contentLength); var operations = Note.transformPatchToOperations(patch, contentLength);
var authorship = note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : []; var authorship = note.authorship;
for (var i = 0; i < operations.length; i++) { for (var i = 0; i < operations.length; i++) {
authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship); authorship = Note.updateAuthorshipByOperation(operations[i], null, authorship);
} }
note.update({ note.update({
authorship: LZString.compressToBase64(JSON.stringify(authorship)) authorship: JSON.stringify(authorship)
}).then(function (note) { }).then(function (note) {
return callback(null, note.id); return callback(null, note.id);
}).catch(function (err) { }).catch(function (err) {
@ -264,10 +262,7 @@ module.exports = function (sequelize, DataTypes) {
return markdown.substr(0, 100).replace(/(?:\r\n|\r|\n)/g, ' '); return markdown.substr(0, 100).replace(/(?:\r\n|\r|\n)/g, ' ');
}, },
decodeTitle: function (title) { decodeTitle: function (title) {
var decodedTitle = LZString.decompressFromBase64(title); return title ? title : 'Untitled';
if (decodedTitle) title = decodedTitle;
else title = 'Untitled';
return title;
}, },
generateWebTitle: function (title) { generateWebTitle: function (title) {
title = !title || title == "Untitled" ? "HackMD - Collaborative markdown notes" : title + " - HackMD"; title = !title || title == "Untitled" ? "HackMD - Collaborative markdown notes" : title + " - HackMD";
@ -496,8 +491,8 @@ module.exports = function (sequelize, DataTypes) {
if (Note.checkFileExist(filePath)) { if (Note.checkFileExist(filePath)) {
var fsCreatedTime = moment(fs.statSync(filePath).ctime); var fsCreatedTime = moment(fs.statSync(filePath).ctime);
body = fs.readFileSync(filePath, 'utf8'); body = fs.readFileSync(filePath, 'utf8');
note.title = LZString.compressToBase64(Note.parseNoteTitle(body)); note.title = Note.parseNoteTitle(body);
note.content = LZString.compressToBase64(body); note.content = body;
if (filePath !== config.defaultnotepath) { if (filePath !== config.defaultnotepath) {
note.createdAt = fsCreatedTime; note.createdAt = fsCreatedTime;
} }

11
lib/models/revision.js
View file

@ -2,7 +2,6 @@
// external modules // external modules
var Sequelize = require("sequelize"); var Sequelize = require("sequelize");
var LZString = require('lz-string');
var async = require('async'); var async = require('async');
var moment = require('moment'); var moment = require('moment');
var childProcess = require('child_process'); var childProcess = require('child_process');
@ -214,7 +213,7 @@ module.exports = function (sequelize, DataTypes) {
Revision.create({ Revision.create({
noteId: note.id, noteId: note.id,
lastContent: note.content, lastContent: note.content,
length: LZString.decompressFromBase64(note.content).length, length: note.content.length,
authorship: note.authorship authorship: note.authorship
}).then(function (revision) { }).then(function (revision) {
Revision.finishSaveNoteRevision(note, revision, callback); Revision.finishSaveNoteRevision(note, revision, callback);
@ -223,8 +222,8 @@ module.exports = function (sequelize, DataTypes) {
}); });
} else { } else {
var latestRevision = revisions[0]; var latestRevision = revisions[0];
var lastContent = LZString.decompressFromBase64(latestRevision.content || latestRevision.lastContent); var lastContent = latestRevision.content || latestRevision.lastContent;
var content = LZString.decompressFromBase64(note.content); var content = note.content;
sendDmpWorker({ sendDmpWorker({
msg: 'create patch', msg: 'create patch',
lastDoc: lastContent, lastDoc: lastContent,
@ -244,9 +243,9 @@ module.exports = function (sequelize, DataTypes) {
} else { } else {
Revision.create({ Revision.create({
noteId: note.id, noteId: note.id,
patch: LZString.compressToBase64(patch), patch: patch,
content: note.content, content: note.content,
length: LZString.decompressFromBase64(note.content).length, length: note.content.length,
authorship: note.authorship authorship: note.authorship
}).then(function (revision) { }).then(function (revision) {
// clear last revision content to reduce db size // clear last revision content to reduce db size

10
lib/realtime.js
View file

@ -152,12 +152,10 @@ function finishUpdateNote(note, _note, callback) {
if (!note || !note.server) return callback(null, null); if (!note || !note.server) return callback(null, null);
var body = note.server.document; var body = note.server.document;
var title = note.title = models.Note.parseNoteTitle(body); var title = note.title = models.Note.parseNoteTitle(body);
title = LZString.compressToBase64(title);
body = LZString.compressToBase64(body);
var values = { var values = {
title: title, title: title,
content: body, content: body,
authorship: LZString.compressToBase64(JSON.stringify(note.authorship)), authorship: note.authorship,
lastchangeuserId: note.lastchangeuser, lastchangeuserId: note.lastchangeuser,
lastchangeAt: Date.now() lastchangeAt: Date.now()
}; };
@ -459,7 +457,7 @@ function startConnection(socket) {
var lastchangeuser = note.lastchangeuserId; var lastchangeuser = note.lastchangeuserId;
var lastchangeuserprofile = note.lastchangeuser ? models.User.getProfile(note.lastchangeuser) : null; var lastchangeuserprofile = note.lastchangeuser ? models.User.getProfile(note.lastchangeuser) : null;
var body = LZString.decompressFromBase64(note.content); var body = note.content;
var createtime = note.createdAt; var createtime = note.createdAt;
var updatetime = note.lastchangeAt; var updatetime = note.lastchangeAt;
var server = new ot.EditorSocketIOServer(body, [], noteId, ifMayEdit, operationCallback); var server = new ot.EditorSocketIOServer(body, [], noteId, ifMayEdit, operationCallback);
@ -479,7 +477,7 @@ function startConnection(socket) {
notes[noteId] = { notes[noteId] = {
id: noteId, id: noteId,
alias: note.alias, alias: note.alias,
title: LZString.decompressFromBase64(note.title), title: note.title,
owner: owner, owner: owner,
ownerprofile: ownerprofile, ownerprofile: ownerprofile,
permission: note.permission, permission: note.permission,
@ -491,7 +489,7 @@ function startConnection(socket) {
updatetime: moment(updatetime).valueOf(), updatetime: moment(updatetime).valueOf(),
server: server, server: server,
authors: authors, authors: authors,
authorship: note.authorship ? JSON.parse(LZString.decompressFromBase64(note.authorship)) : [] authorship: note.authorship
}; };
return finishConnection(socket, notes[noteId], users[socket.id]); return finishConnection(socket, notes[noteId], users[socket.id]);

14
lib/response.js
View file

@ -75,7 +75,7 @@ function showIndex(req, res, next) {
} }
function responseHackMD(res, note) { function responseHackMD(res, note) {
var body = LZString.decompressFromBase64(note.content); var body = note.content;
var meta = null; var meta = null;
try { try {
meta = models.Note.parseMeta(metaMarked(body).meta); meta = models.Note.parseMeta(metaMarked(body).meta);
@ -191,7 +191,7 @@ function showPublishNote(req, res, next) {
if (!note) { if (!note) {
return response.errorNotFound(res); return response.errorNotFound(res);
} }
var body = LZString.decompressFromBase64(note.content); var body = note.content;
var meta = null; var meta = null;
var markdown = null; var markdown = null;
try { try {
@ -248,7 +248,7 @@ function actionSlide(req, res, note) {
} }
function actionDownload(req, res, note) { function actionDownload(req, res, note) {
var body = LZString.decompressFromBase64(note.content); var body = note.content;
var title = models.Note.decodeTitle(note.title); var title = models.Note.decodeTitle(note.title);
var filename = title; var filename = title;
filename = encodeURIComponent(filename); filename = encodeURIComponent(filename);
@ -265,7 +265,7 @@ function actionDownload(req, res, note) {
} }
function actionInfo(req, res, note) { function actionInfo(req, res, note) {
var body = LZString.decompressFromBase64(note.content); var body = note.content;
var meta = null; var meta = null;
var markdown = null; var markdown = null;
try { try {
@ -297,7 +297,7 @@ function actionInfo(req, res, note) {
} }
function actionPDF(req, res, note) { function actionPDF(req, res, note) {
var body = LZString.decompressFromBase64(note.content); var body = note.content;
try { try {
body = metaMarked(body).markdown; body = metaMarked(body).markdown;
} catch(err) { } catch(err) {
@ -479,7 +479,7 @@ function githubActionGist(req, res, note) {
if (!error && httpResponse.statusCode == 200) { if (!error && httpResponse.statusCode == 200) {
var access_token = body.access_token; var access_token = body.access_token;
if (access_token) { if (access_token) {
var content = LZString.decompressFromBase64(note.content); var content = note.content;
var title = models.Note.decodeTitle(note.title); var title = models.Note.decodeTitle(note.title);
var filename = title.replace('/', ' ') + '.md'; var filename = title.replace('/', ' ') + '.md';
var gist = { var gist = {
@ -579,7 +579,7 @@ function showPublishSlide(req, res, next) {
if (!note) { if (!note) {
return response.errorNotFound(res); return response.errorNotFound(res);
} }
var body = LZString.decompressFromBase64(note.content); var body = note.content;
var meta = null; var meta = null;
var markdown = null; var markdown = null;
try { try {

13
lib/workers/dmpWorker.js
View file

@ -1,5 +1,4 @@
// external modules // external modules
var LZString = require('lz-string');
var DiffMatchPatch = require('diff-match-patch'); var DiffMatchPatch = require('diff-match-patch');
var dmp = new DiffMatchPatch(); var dmp = new DiffMatchPatch();
@ -80,10 +79,10 @@ function getRevision(revisions, count) {
for (var i = 0; i < count; i++) { for (var i = 0; i < count; i++) {
var revision = revisions[i]; var revision = revisions[i];
if (i == 0) { if (i == 0) {
startContent = LZString.decompressFromBase64(revision.content || revision.lastContent); startContent = revision.content || revision.lastContent;
} }
if (i != count - 1) { if (i != count - 1) {
var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); var patch = dmp.patch_fromText(revision.patch);
applyPatches = applyPatches.concat(patch); applyPatches = applyPatches.concat(patch);
} }
lastPatch = revision.patch; lastPatch = revision.patch;
@ -105,11 +104,11 @@ function getRevision(revisions, count) {
for (var i = l; i >= count - 1; i--) { for (var i = l; i >= count - 1; i--) {
var revision = revisions[i]; var revision = revisions[i];
if (i == l) { if (i == l) {
startContent = LZString.decompressFromBase64(revision.lastContent); startContent = revision.lastContent;
authorship = revision.authorship; authorship = revision.authorship;
} }
if (revision.patch) { if (revision.patch) {
var patch = dmp.patch_fromText(LZString.decompressFromBase64(revision.patch)); var patch = dmp.patch_fromText(revision.patch);
applyPatches = applyPatches.concat(patch); applyPatches = applyPatches.concat(patch);
} }
lastPatch = revision.patch; lastPatch = revision.patch;
@ -123,8 +122,8 @@ function getRevision(revisions, count) {
} }
var data = { var data = {
content: finalContent, content: finalContent,
patch: dmp.patch_fromText(LZString.decompressFromBase64(lastPatch)), patch: dmp.patch_fromText(lastPatch),
authorship: authorship ? JSON.parse(LZString.decompressFromBase64(authorship)) : null authorship: authorship
}; };
var ms_end = (new Date()).getTime(); var ms_end = (new Date()).getTime();
if (config.debug) { if (config.debug) {