From 5aec047a3e0fcb27b06c54dc692cfa78145057df Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Thu, 6 Sep 2018 15:00:02 +0100 Subject: [PATCH 01/54] Some minor improvements for setup script Since we use `yarn` for our container setup and try to enforce dependencies, we should also use yarn in the setup script. Signed-off-by: Sheogorath --- bin/setup | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/bin/setup b/bin/setup index 122cb7a..38aa05a 100755 --- a/bin/setup +++ b/bin/setup @@ -8,11 +8,12 @@ if [ -d .git ]; then cd "$(git rev-parse --show-toplevel)" fi -if ! type npm > /dev/null +if ! type yarn > /dev/null then cat << EOF -npm is not installed, please install Node.js and npm. +yarn is not installed, please install Node.js, npm and yarn. Read more on Node.js official website: https://nodejs.org +And for yarn package manager at: https://yarnpkg.com/en/ Setup will not be run EOF exit 0 @@ -27,8 +28,9 @@ if [ ! -f .sequelizerc ]; then cp .sequelizerc.example .sequelizerc fi -echo "install npm packages" -BUILD_ASSETS=false npm install +echo "install packages" +yarn install --pure-lockfile +yarn install --production=false --pure-lockfile cat << EOF From db59bb99dc59d56e1f807a09845750d05fef43b1 Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Tue, 25 Sep 2018 00:31:21 +0200 Subject: [PATCH 02/54] Run db migrations on start We should force db migrations to run on every start. This will minimize the impact of breaking migrations in future. While it may causes some issues with the next start since CodiMD won't start when the migrations fail. Signed-off-by: Sheogorath --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index b257a11..f0781c2 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,7 @@ "dev": "webpack --config webpack.config.js --progress --colors --watch", "build": "webpack --config webpack.production.js --progress --colors --bail", "postinstall": "bin/heroku", - "start": "node app.js", + "start": "sequelize db:migrate && node app.js", "doctoc": "doctoc --title='# Table of Contents' README.md" }, "dependencies": { From 5b789025f314e4bb154df266b2e6dd148747efa2 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sat, 27 Oct 2018 16:55:14 -0700 Subject: [PATCH 03/54] Add download action to published notes Signed-off-by: Alex Garcia --- lib/response.js | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/response.js b/lib/response.js index 4f572e4..3e38fe6 100644 --- a/lib/response.js +++ b/lib/response.js @@ -468,6 +468,8 @@ function publishNoteActions (req, res, next) { findNote(req, res, function (note) { var action = req.params.action switch (action) { + case 'download': + actionDownload(req, res, note) case 'edit': res.redirect(config.serverURL + '/' + (note.alias ? note.alias : models.Note.encodeNoteId(note.id))) break From fcf08f89c3fe5f7708db7748e209a13b30c1e121 Mon Sep 17 00:00:00 2001 From: Alex Garcia Date: Sat, 27 Oct 2018 17:54:01 -0700 Subject: [PATCH 04/54] forgot break statement Signed-off-by: Alex Garcia --- lib/response.js | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/response.js b/lib/response.js index 3e38fe6..a400d02 100644 --- a/lib/response.js +++ b/lib/response.js @@ -470,6 +470,7 @@ function publishNoteActions (req, res, next) { switch (action) { case 'download': actionDownload(req, res, note) + break case 'edit': res.redirect(config.serverURL + '/' + (note.alias ? note.alias : models.Note.encodeNoteId(note.id))) break From 0915b3300068b86ce9848f7bef25508fcc60fb2f Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Sun, 28 Oct 2018 10:01:37 +0100 Subject: [PATCH 05/54] Add documentation about editor modes in features page Codemirror provides various modes via keymapping. These are already available by a menu in the interface. But they aren't mentioned anywhere. This patch provides some documentation about the editor modes and their implications. Since they are a feature, the documentation is done on the features page. Signed-off-by: Sheogorath --- public/docs/features.md | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/public/docs/features.md b/public/docs/features.md index 3d79003..31c08d2 100644 --- a/public/docs/features.md +++ b/public/docs/features.md @@ -88,9 +88,23 @@ You can hover and click to anchor on it. Edit: === +## Editor Modes: +You can look in the bottom right section of the editor area, there you'll find a button with `sublime` on it. +When you click it, you can select 3 editor modes: + +- sublime (default) +- emacs +- vim + ## Shortcut Keys: -Just like Sublime text, which is pretty quick and convenient. -> For more infomation, see [here](https://codemirror.net/demo/sublime.html). +The shortcut keys depend on your selected editor mode. By default they are just like Sublime text, which is pretty quick and convenient. +> For more information, see [here](https://codemirror.net/demo/sublime.html). + +For emacs: +> For more information, see [here](https://codemirror.net/demo/emacs.html). + +For vim: +> For more information, see [here](https://codemirror.net/demo/vim.html). ## Auto-Complete: This editor provides full auto-complete hints in markdown. From 77b2757a160a0a7de387cd11dbc947be812d1543 Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Wed, 31 Oct 2018 00:25:46 +0100 Subject: [PATCH 06/54] Upgrade some package versions `npm audit` reports a ton of issues on CodiMD. Most of them are minor issues, but these are still things that should be fixed. This changes were created by running `npm audit fix`. Signed-off-by: Sheogorath --- package.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/package.json b/package.json index 524f87e..304a526 100644 --- a/package.json +++ b/package.json @@ -19,7 +19,7 @@ "Idle.Js": "git+https://github.com/shawnmclean/Idle.js", "archiver": "^2.1.1", "async": "^2.1.4", - "aws-sdk": "^2.7.20", + "aws-sdk": "^2.345.0", "azure-storage": "^2.7.0", "base64url": "^3.0.0", "blueimp-md5": "^2.6.0", @@ -62,7 +62,7 @@ "jsdom-nogyp": "^0.8.3", "keymaster": "^1.6.2", "list.js": "^1.5.0", - "lodash": "^4.17.4", + "lodash": "^4.17.11", "lz-string": "git+https://github.com/hackmdio/lz-string.git", "markdown-it": "^8.2.2", "markdown-it-abbr": "^1.0.4", @@ -98,7 +98,7 @@ "passport-ldapauth": "^2.0.0", "passport-local": "^1.0.0", "passport-oauth2": "^1.4.0", - "passport-saml": "^0.31.0", + "passport-saml": "^0.35.0", "passport-twitter": "^1.0.4", "passport.socketio": "^3.7.0", "pdfobject": "^2.0.201604172", @@ -108,7 +108,7 @@ "randomcolor": "^0.5.3", "raphael": "git+https://github.com/dmitrybaranovskiy/raphael", "readline-sync": "^1.4.7", - "request": "^2.79.0", + "request": "^2.88.0", "reveal.js": "~3.6.0", "scrypt": "^6.0.3", "select2": "^3.5.2-browserify", From 59b3885ddaa97e5a22022587a023d6db3c7caad7 Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Fri, 21 Sep 2018 01:23:39 +0200 Subject: [PATCH 07/54] Use OS based tmp dir We should use the official OS temp directory instead of an own one, to not run into conflicts. Also various dependencies already use the OS temp directory, which makes it pointless to use a different for our internal purposes then. This commit provides the changes needed to use the OS tmp directory by default. Signed-off-by: Sheogorath --- lib/config/default.js | 4 +++- tmp/.keep | 0 2 files changed, 3 insertions(+), 1 deletion(-) delete mode 100644 tmp/.keep diff --git a/lib/config/default.js b/lib/config/default.js index c3ada98..15f11aa 100644 --- a/lib/config/default.js +++ b/lib/config/default.js @@ -1,5 +1,7 @@ 'use strict' +const os = require('os') + module.exports = { domain: '', urlPath: '', @@ -39,7 +41,7 @@ module.exports = { dhParamPath: '', // other path viewPath: './public/views', - tmpPath: './tmp', + tmpPath: os.tmpdir(), defaultNotePath: './public/default.md', docsPath: './public/docs', uploadsPath: './public/uploads', diff --git a/tmp/.keep b/tmp/.keep deleted file mode 100644 index e69de29..0000000 From d188b3526ab45f989e09f2a1c1f6e7a7eacf1605 Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Wed, 31 Oct 2018 15:12:34 +0100 Subject: [PATCH 08/54] Again: Replace emoji-plugin regex The Regex introduced in the last commit[1], was already working quite good. But still resulted in false positives for all URL that contained a second `:`. To fix this once and for all, we craft a simple, but long regex based on all emoji names and use this to match them. We could probably optimize it, but that should also be something the regex engine itself can and should do. [1]: 7e45533c75a3697c916e52e5f4ddff42a38bd3d5 (in this source tree) Signed-off-by: Sheogorath --- public/js/extra.js | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/public/js/extra.js b/public/js/extra.js index 4db36ff..ed1470b 100644 --- a/public/js/extra.js +++ b/public/js/extra.js @@ -1147,15 +1147,14 @@ const pdfPlugin = new Plugin( const emojijsPlugin = new Plugin( // regexp to match emoji shortcodes :something: - /:([^\s:]+):/, + // We generate an universal regex that guaranteed only contains the + // emojies we have available. This should prevent all false-positives + new RegExp(':(' + window.emojify.emojiNames.map((item) => { return RegExp.escape(item) }).join('|') + '):', 'i'), (match, utils) => { - const emoji = match[1] ? match[1].toLowerCase() : undefined - if (window.emojify.emojiNames.includes(emoji)) { - const div = $(``) - return div[0].outerHTML - } - return match[0] + const emoji = match[1].toLowerCase() + const div = $(``) + return div[0].outerHTML } ) From d24fb48f16bff4d0e6b1bfdeffd2b41c7e61a347 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Couralet?= Date: Wed, 7 Nov 2018 11:32:20 +0000 Subject: [PATCH 09/54] Fix menu when gitlab is enabled MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédric Couralet --- app.js | 1 + public/views/codimd/header.ejs | 12 ++++++------ 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/app.js b/app.js index 7795dd8..50b6122 100644 --- a/app.js +++ b/app.js @@ -198,6 +198,7 @@ app.locals.authProviders = { email: config.isEmailEnable, allowEmailRegister: config.allowEmailRegister } +app.locals.enableGitlabSnippets = (!config.gitlab.scope || config.gitlab.scope === 'api') app.use(require('./lib/web/baseRouter')) app.use(require('./lib/web/statusRouter')) diff --git a/public/views/codimd/header.ejs b/public/views/codimd/header.ejs index d8df33b..56a42eb 100644 --- a/public/views/codimd/header.ejs +++ b/public/views/codimd/header.ejs @@ -32,7 +32,7 @@
  • <%= __('Slide Mode') %>
    • - <% if((typeof github !== 'undefined' && github) || (typeof dropbox !== 'undefined' && dropbox) || (typeof gitlab !== 'undefined' && gitlab && (!gitlab.scope || gitlab.scope === 'api'))) { %> + <% if((typeof github !== 'undefined' && github) || (typeof dropbox !== 'undefined' && dropbox) || enableGitlabSnippets) { %>
  • <%= __('Export') %>
  • Dropbox @@ -41,7 +41,7 @@
  • Gist
    • <% } %> - <% if(typeof gitlab !== 'undefined' && gitlab && (!gitlab.scope || gitlab.scope === 'api')) { %> + <% if(enableGitlabSnippets) { %>
  • Snippet
    • <% } %> @@ -52,7 +52,7 @@
  • Gist
    • - <% if(typeof gitlab !== 'undefined' && gitlab && (!gitlab.scope || gitlab.scope === 'api')) { %> + <% if(enableGitlabSnippets) { %>
  • Snippet
    • <% } %> @@ -134,7 +134,7 @@
  • <%= __('Slide Mode') %>
    • - <% if((typeof github !== 'undefined' && github) || (typeof dropbox !== 'undefined' && dropbox) || (typeof gitlab !== 'undefined' && gitlab && (!gitlab.scope || gitlab.scope === 'api'))) { %> + <% if((typeof github !== 'undefined' && github) || (typeof dropbox !== 'undefined' && dropbox) || enableGitlabSnippets ) { %>
  • <%= __('Export') %>
  • Dropbox @@ -143,7 +143,7 @@
  • Gist
    • <% } %> - <% if(typeof gitlab !== 'undefined' && gitlab && (!gitlab.scope || gitlab.scope === 'api')) { %> + <% if(enableGitlabSnippets) { %>
  • Snippet
    • <% } %> @@ -154,7 +154,7 @@
  • Gist
    • - <% if(typeof gitlab !== 'undefined' && gitlab && (!gitlab.scope || gitlab.scope === 'api')) { %> + <% if(enableGitlabSnippets) { %>
  • Snippet
    • <% } %> From 67f8a64f2b1ea653b567a48ac6aa760b03c7189c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?C=C3=A9dric=20Couralet?= Date: Wed, 7 Nov 2018 12:12:50 +0000 Subject: [PATCH 10/54] Fix menu for github and dropbox MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Cédric Couralet --- app.js | 6 +++++- lib/config/index.js | 2 ++ public/views/codimd/header.ejs | 8 ++++---- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/app.js b/app.js index 50b6122..622d866 100644 --- a/app.js +++ b/app.js @@ -198,7 +198,11 @@ app.locals.authProviders = { email: config.isEmailEnable, allowEmailRegister: config.allowEmailRegister } -app.locals.enableGitlabSnippets = (!config.gitlab.scope || config.gitlab.scope === 'api') + +// Export/Import menu items +app.locals.enableDropBoxSave = config.isDropboxEnable +app.locals.enableGitHubGist = config.isGitHubEnable +app.locals.enableGitlabSnippets = config.isGitlabSnippetsEnable app.use(require('./lib/web/baseRouter')) app.use(require('./lib/web/statusRouter')) diff --git a/lib/config/index.js b/lib/config/index.js index f8b68e3..501fdca 100644 --- a/lib/config/index.js +++ b/lib/config/index.js @@ -110,6 +110,8 @@ if (config.gitlab && config.gitlab.version !== 'v4' && config.gitlab.version !== logger.warn('config.js contains wrong version (' + config.gitlab.version + ') for gitlab api; it should be \'v3\' or \'v4\'. Defaulting to v4') config.gitlab.version = 'v4' } +// If gitlab scope is api, enable snippets Export/import +config.isGitlabSnippetsEnable = (!config.gitlab.scope || config.gitlab.scope === 'api') // Only update i18n files in development setups config.updateI18nFiles = (env === Environment.development) diff --git a/public/views/codimd/header.ejs b/public/views/codimd/header.ejs index 56a42eb..b83838e 100644 --- a/public/views/codimd/header.ejs +++ b/public/views/codimd/header.ejs @@ -32,12 +32,12 @@
  • <%= __('Slide Mode') %>
    • - <% if((typeof github !== 'undefined' && github) || (typeof dropbox !== 'undefined' && dropbox) || enableGitlabSnippets) { %> + <% if(enableGitHubGist || enableDropBoxSave || enableGitlabSnippets) { %>
  • <%= __('Export') %>
  • Dropbox
    • - <% if(typeof github !== 'undefined' && github) { %> + <% if(enableGitHubGist) { %>
  • Gist
    • <% } %> @@ -134,12 +134,12 @@
  • <%= __('Slide Mode') %>
    • - <% if((typeof github !== 'undefined' && github) || (typeof dropbox !== 'undefined' && dropbox) || enableGitlabSnippets ) { %> + <% if(enableGitHubGist || enableDropBoxSave || enableGitlabSnippets) { %>
  • <%= __('Export') %>
  • Dropbox
    • - <% if(typeof github !== 'undefined' && github) { %> + <% if(enableGitHubGist) { %>
  • Gist
    • <% } %> From c59b94a37b9d44f6c56b12f61cbfb80ff5f0db50 Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Sat, 10 Nov 2018 20:24:41 +0100 Subject: [PATCH 11/54] Remove the xss library from webpack We can load the xss functions directly from the library instead of loading them through the expose loader of webpack, this should simplify the setup and maybe even improve speed a bit. Signed-off-by: Sheogorath --- public/js/render.js | 7 +++++-- public/js/reveal-markdown.js | 4 ++-- webpack.common.js | 6 ------ 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/public/js/render.js b/public/js/render.js index 23b8934..ff5e2bf 100644 --- a/public/js/render.js +++ b/public/js/render.js @@ -1,6 +1,8 @@ /* eslint-env browser, jquery */ -/* global filterXSS */ // allow some attributes + +var filterXSS = require('xss') + var whiteListAttr = ['id', 'class', 'style'] window.whiteListAttr = whiteListAttr // allow link starts with '.', '/' and custom protocol with '://', exclude link starts with javascript:// @@ -71,5 +73,6 @@ function preventXSS (html) { window.preventXSS = preventXSS module.exports = { - preventXSS: preventXSS + preventXSS: preventXSS, + escapeAttrValue: filterXSS.escapeAttrValue } diff --git a/public/js/reveal-markdown.js b/public/js/reveal-markdown.js index d15b5eb..ad5bfd0 100644 --- a/public/js/reveal-markdown.js +++ b/public/js/reveal-markdown.js @@ -1,6 +1,6 @@ /* eslint-env browser, jquery */ -import { preventXSS } from './render' +import { preventXSS, escapeAttrValue } from './render' import { md } from './extra' /** @@ -259,7 +259,7 @@ import { md } from './extra' while ((matchesClass = mardownClassRegex.exec(classes))) { var name = matchesClass[1] var value = matchesClass[2] - if (name.substr(0, 5) === 'data-' || window.whiteListAttr.indexOf(name) !== -1) { elementTarget.setAttribute(name, window.filterXSS.escapeAttrValue(value)) } + if (name.substr(0, 5) === 'data-' || window.whiteListAttr.indexOf(name) !== -1) { elementTarget.setAttribute(name, escapeAttrValue(value)) } } return true } diff --git a/webpack.common.js b/webpack.common.js index 1fbf247..1e9c070 100644 --- a/webpack.common.js +++ b/webpack.common.js @@ -202,7 +202,6 @@ module.exports = { 'babel-polyfill', 'script-loader!jquery-ui-resizable', 'script-loader!js-url', - 'expose-loader?filterXSS!xss', 'script-loader!Idle.Js', 'expose-loader?LZString!lz-string', 'script-loader!codemirror', @@ -253,7 +252,6 @@ module.exports = { 'script-loader!handlebars', 'expose-loader?hljs!highlight.js', 'expose-loader?emojify!emojify.js', - 'expose-loader?filterXSS!xss', 'script-loader!Idle.Js', 'script-loader!gist-embed', 'expose-loader?LZString!lz-string', @@ -273,7 +271,6 @@ module.exports = { ], pretty: [ 'babel-polyfill', - 'expose-loader?filterXSS!xss', 'flowchart.js', 'js-sequence-diagrams', 'expose-loader?RevealMarkdown!reveal-markdown', @@ -298,7 +295,6 @@ module.exports = { 'script-loader!handlebars', 'expose-loader?hljs!highlight.js', 'expose-loader?emojify!emojify.js', - 'expose-loader?filterXSS!xss', 'script-loader!gist-embed', 'flowchart.js', 'js-sequence-diagrams', @@ -310,7 +306,6 @@ module.exports = { slide: [ 'babel-polyfill', 'bootstrap-tooltip', - 'expose-loader?filterXSS!xss', 'flowchart.js', 'js-sequence-diagrams', 'expose-loader?RevealMarkdown!reveal-markdown', @@ -338,7 +333,6 @@ module.exports = { 'script-loader!handlebars', 'expose-loader?hljs!highlight.js', 'expose-loader?emojify!emojify.js', - 'expose-loader?filterXSS!xss', 'script-loader!gist-embed', 'flowchart.js', 'js-sequence-diagrams', From 1f8e8b476feba62df9c61b7ccf5c1c0e61d0bcd6 Mon Sep 17 00:00:00 2001 From: Daan Sprenkels Date: Sat, 10 Nov 2018 23:36:33 +0100 Subject: [PATCH 12/54] Add an etherpad migration guide In this guide I share how a migration from etherpad to codimd can be done. I am not completely sure if the script that is included is completely error-free. Readers/reviewers should be aware that there may be bugs.may be bugs. Signed-off-by: Daan Sprenkels --- docs/guides/migrate-etherpad.md | 131 ++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 docs/guides/migrate-etherpad.md diff --git a/docs/guides/migrate-etherpad.md b/docs/guides/migrate-etherpad.md new file mode 100644 index 0000000..c3783c8 --- /dev/null +++ b/docs/guides/migrate-etherpad.md @@ -0,0 +1,131 @@ +Pad migration guide from etherpad-lite +=== + +The goal of this migration is to do a "dumb" import from all the pads in Etherpad, to notes in +CodiMD. In particular, the url locations of the pads in Etherpad will be lost. Furthermore, any +metadata in Etherpad, such as revisions, author data and also formatted text will not be migrated +to CodiMD (only the plain text contents). + +Note that this guide is not really meant as a support guide. I migrated my own Etherpad to CodiMD, +and it turned out to be quite easy in my opinion. In this guide I share my experience. Stuff may +require some creativity to work properly in your case. When I wrote this guide, I was using +[Etherpad 1.7.0] and [CodiMD 1.2.1]. Good luck! + +[Etherpad 1.7.0]: https://github.com/ether/etherpad-lite/tree/1.7.0 +[CodiMD 1.2.1]: https://github.com/hackmdio/codimd/tree/1.2.1 + +## 0. Requirements + +- `curl` +- running Etherpad server +- running CodiMD server +- [codimd-cli] + +[codimd-cli]: https://github.com/hackmdio/codimd-cli/blob/master/bin/codimd + +## 1. Retrieve the list of pads + +First, compose a list of all the pads that you want to have migrated from your Etherpad. Other than +the admin interface, Etherpad does not have a dedicated function to dump a list of all the pads. +However, the Etherpad wiki explains how to list all the pads by [talking directly to the +database][howtolistallpads]. + +You will end up with a file containing a pad name on each line: + +``` +date-ideas +groceries +london +weddingchecklist +(...) +``` + +[howtolistallpads]: https://github.com/ether/etherpad-lite/wiki/How-to-list-all-pads/49701ecdcbe07aea7ad27ffa23aed0d99c2e17db + +## 2. Run the migration + +Download [codimd-cli] and put the script in the same directory as the file containing the pad names. +Add to this directory the file listed below, I called it `migrate-etherpad.sh`. Modify at least the +configuration settings `ETHERPAD_SERVER` and `CODIMD_SERVER`. + +```shell +#!/bin/sh + +# migrate-etherpad.sh +# +# Description: Migrate pads from etherpad to codimd +# Author: Daan Sprenkels + +# This script uses the codimd command line script[1] to import a list of pads from +# [1]: https://github.com/hackmdio/codimd-cli/blob/master/bin/codimd + +# The base url to where etherpad is hosted +ETHERPAD_SERVER="https://etherpad.example.com" + +# The base url where codimd is hosted +CODIMD_SERVER="https://codimd.example.com" + +# Write a list of pads and the urls which they were migrated to +REDIRECTS_FILE="redirects.txt" + + +# Fail if not called correctly +if (( $# != 1 )); then + echo "Usage: $0 PAD_NAMES_FILE" + exit 2 +fi + +# Do the migration +for PAD_NAME in $1; do + # Download the pad + PAD_FILE="$(mktemp)" + curl "$ETHERPAD_SERVER/p/$PAD_NAME/export/txt" >"$PAD_FILE" + + # Import the pad into codimd + OUTPUT="$(./codimd import "$PAD_FILE")" + echo "$PAD_NAME -> $OUTPUT" >>"$REDIRECTS_FILE" +done +``` + +Call this file like this: + +```shell +./migrate-etherpad.sh pad_names.txt +``` + +This will download all the pads in `pad_names.txt` and put them on CodiMD. They will get assigned +random ids, so you won't be able to find them. The script will save the mappings to a file though +(in my case `redirects.txt`). You can use this file to redirect your users when they visit your +etherpad using a `301 Permanent Redirect` status code (see the next section). + +## 3. Setup redirects (optional) + +I got a `redirects.txt` file that looked a bit like this: + +``` +date-ideas -> Found. Redirecting to https://codimd.example.com/mPt0KfiKSBOTQ3mNcdfn +groceries -> Found. Redirecting to https://codimd.example.com/UukqgwLfhYyUUtARlcJ2_y +london -> Found. Redirecting to https://codimd.example.com/_d3wa-BE8t4Swv5w7O2_9R +weddingchecklist -> Found. Redirecting to https://codimd.example.com/XcQGqlBjl0u40wfT0N8TzQ +(...) +``` + +Using some `sed` magic, I changed it to an nginx config snippet: + +``` +location = /p/date-ideas { + return 301 https://codimd.example.com/mPt0M1KfiKSBOTQ3mNcdfn; +} +location = /p/groceries { + return 301 https://codimd.example.com/UukqgwLfhYyUUtARlcJ2_y; +} +location = /p/london { + return 301 https://codimd.example.com/_d3wa-BE8t4Swv5w7O2_9R; +} +location = /p/weddingchecklist { + return 301 https://codimd.example.com/XcQGqlBjl0u40wfT0N8TzQ; +} +``` + +I put this file into my `etherpad.example.com` nginx config, such that all the users would be +redirected accordingly. From bcc914a7735290b86e0df428aed75dabdf9f1eca Mon Sep 17 00:00:00 2001 From: Sheogorath Date: Fri, 5 Oct 2018 19:33:40 +0200 Subject: [PATCH 13/54] Add full version string Currently we only provide the version from `package.json`. This means that during updates of instances, e.g. the demo instance, which runs latest master instead of a stable release, changes are not reflected to the webclient. This patch adds a fullversion string that contains the current commit and this way makes that clients are notified about changes. Signed-off-by: Sheogorath --- README.md | 2 ++ app.js | 1 + lib/config/environment.js | 1 + lib/config/index.js | 11 +++++++++-- lib/config/utils.js | 33 +++++++++++++++++++++++++++++++++ lib/realtime.js | 2 +- lib/web/statusRouter.js | 2 +- locales/en.json | 5 +++-- public/views/index/body.ejs | 2 +- 9 files changed, 52 insertions(+), 7 deletions(-) diff --git a/README.md b/README.md index 78c4fc7..5e49f2e 100644 --- a/README.md +++ b/README.md @@ -260,6 +260,7 @@ There are some config settings you need to change in the files below. | `CMD_HSTS_PRELOAD` | `true` | whether to allow preloading of the site's HSTS status (e.g. into browsers) | | `CMD_CSP_ENABLE` | `true` | whether to enable Content Security Policy (directives cannot be configured with environment variables) | | `CMD_CSP_REPORTURI` | `https://.report-uri.com/r/d/csp/enforce` | Allows to add a URL for CSP reports in case of violations | +| `CMD_SOURCE_URL` | `https://github.com/hackmdio/codimd/tree/` | Provides the link to the source code of CodiMD on the entry page (Please, make sure you change this when you run a modified version) | ***Note:** Due to the rename process we renamed all `HMD_`-prefix variables to be `CMD_`-prefixed. The old ones continue to work.* @@ -310,6 +311,7 @@ There are some config settings you need to change in the files below. | `minio` | `{ "accessKey": "YOUR_MINIO_ACCESS_KEY", "secretKey": "YOUR_MINIO_SECRET_KEY", "endpoint": "YOUR_MINIO_HOST", port: 9000, secure: true }` | When `imageUploadType` is set to `minio`, you need to set this key. Also checkout our [Minio Image Upload Guide](docs/guides/minio-image-upload.md) | | `s3` | `{ "accessKeyId": "YOUR_S3_ACCESS_KEY_ID", "secretAccessKey": "YOUR_S3_ACCESS_KEY", "region": "YOUR_S3_REGION" }` | When `imageuploadtype` be set to `s3`, you would also need to setup this key, check our [S3 Image Upload Guide](docs/guides/s3-image-upload.md) | | `s3bucket` | `YOUR_S3_BUCKET_NAME` | bucket name when `imageUploadType` is set to `s3` or `minio` | +| `sourceURL` | `https://github.com/hackmdio/codimd/tree/` | Provides the link to the source code of CodiMD on the entry page (Please, make sure you change this when you run a modified version) | 1: relative paths are based on CodiMD's base directory diff --git a/app.js b/app.js index 622d866..c2e958a 100644 --- a/app.js +++ b/app.js @@ -178,6 +178,7 @@ app.set('view engine', 'ejs') // set generally available variables for all views app.locals.useCDN = config.useCDN app.locals.serverURL = config.serverURL +app.locals.sourceURL = config.sourceURL app.locals.allowAnonymous = config.allowAnonymous app.locals.allowAnonymousEdits = config.allowAnonymousEdits app.locals.allowPDFExport = config.allowPDFExport diff --git a/lib/config/environment.js b/lib/config/environment.js index 6737637..0c7c9a4 100644 --- a/lib/config/environment.js +++ b/lib/config/environment.js @@ -3,6 +3,7 @@ const {toBooleanConfig, toArrayConfig, toIntegerConfig} = require('./utils') module.exports = { + sourceURL: process.env.CMD_SOURCE_URL, domain: process.env.CMD_DOMAIN, urlPath: process.env.CMD_URL_PATH, host: process.env.CMD_HOST, diff --git a/lib/config/index.js b/lib/config/index.js index 501fdca..4e1fa50 100644 --- a/lib/config/index.js +++ b/lib/config/index.js @@ -8,6 +8,7 @@ const {merge} = require('lodash') const deepFreeze = require('deep-freeze') const {Environment, Permission} = require('./enum') const logger = require('../logger') +const {getGitCommit, getGitHubURL} = require('./utils') const appRootPath = path.resolve(__dirname, '../../') const env = process.env.NODE_ENV || Environment.development @@ -16,11 +17,17 @@ const debugConfig = { } // Get version string from package.json -const {version} = require(path.join(appRootPath, 'package.json')) +const {version, repository} = require(path.join(appRootPath, 'package.json')) + +const commitID = getGitCommit(appRootPath) +const sourceURL = getGitHubURL(repository.url, commitID || version) +const fullversion = commitID ? `${version}-${commitID}` : version const packageConfig = { version: version, - minimumCompatibleVersion: '0.5.0' + minimumCompatibleVersion: '0.5.0', + fullversion: fullversion, + sourceURL: sourceURL } const configFilePath = path.resolve(appRootPath, process.env.CMD_CONFIG_FILE || diff --git a/lib/config/utils.js b/lib/config/utils.js index b2406cf..dcc8363 100644 --- a/lib/config/utils.js +++ b/lib/config/utils.js @@ -1,5 +1,8 @@ 'use strict' +const fs = require('fs') +const path = require('path') + exports.toBooleanConfig = function toBooleanConfig (configValue) { if (configValue && typeof configValue === 'string') { return (configValue === 'true') @@ -20,3 +23,33 @@ exports.toIntegerConfig = function toIntegerConfig (configValue) { } return configValue } + +exports.getGitCommit = function getGitCommit (repodir) { + if (!fs.existsSync(repodir + '/.git/HEAD')) { + return undefined + } + let reference = fs.readFileSync(repodir + '/.git/HEAD', 'utf8') + if (reference.startsWith('ref: ')) { + reference = reference.substr(5).replace('\n', '') + reference = fs.readFileSync(path.resolve(repodir + '/.git', reference), 'utf8') + } + reference = reference.substr(5).replace('\n', '') + return reference +} + +exports.getGitHubURL = function getGitHubURL (repo, reference) { + // if it's not a github reference, we handle handle that anyway + if (!repo.startsWith('https://github.com') && !repo.startsWith('git@github.com')) { + return repo + } + if (repo.startsWith('git@github.com') || repo.startsWith('ssh://git@github.com')) { + repo = repo.replace(/^(ssh:\/\/)?git@github.com:/, 'https://github.com/') + } + + if (repo.endsWith('.git')) { + repo = repo.replace(/\.git$/, '/') + } else if (!repo.endsWith('/')) { + repo = repo + '/' + } + return repo + 'tree/' + reference +} diff --git a/lib/realtime.js b/lib/realtime.js index f6c62d4..8541baf 100644 --- a/lib/realtime.js +++ b/lib/realtime.js @@ -887,7 +887,7 @@ function connection (socket) { // check version socket.on('version', function () { socket.emit('version', { - version: config.version, + version: config.fullversion, minimumCompatibleVersion: config.minimumCompatibleVersion }) }) diff --git a/lib/web/statusRouter.js b/lib/web/statusRouter.js index fb2609e..2b9cb65 100644 --- a/lib/web/statusRouter.js +++ b/lib/web/statusRouter.js @@ -96,7 +96,7 @@ statusRouter.get('/config', function (req, res) { domain: config.domain, urlpath: config.urlPath, debug: config.debug, - version: config.version, + version: config.fullversion, DROPBOX_APP_KEY: config.dropbox.appKey, allowedUploadMimeTypes: config.allowedUploadMimeTypes } diff --git a/locales/en.json b/locales/en.json index 100f4f5..ead7ce2 100644 --- a/locales/en.json +++ b/locales/en.json @@ -112,5 +112,6 @@ "This will delete your account, all notes that are owned by you and remove all references to your account from other notes.": "This will delete your account, all notes that are owned by you and remove all references to your account from other notes.", "Delete user": "Delete user", "Export user data": "Export user data", - "Help us translating on %s": "Help us translating on %s" -} \ No newline at end of file + "Help us translating on %s": "Help us translating on %s", + "Source Code": "Source Code" +} diff --git a/public/views/index/body.ejs b/public/views/index/body.ejs index 907cc1a..43582d7 100644 --- a/public/views/index/body.ejs +++ b/public/views/index/body.ejs @@ -150,7 +150,7 @@

    - Powered by CodiMD | <%= __('Releases') %><% if(privacyStatement) { %> | <%= __('Privacy') %><% } %><% if(termsOfUse) { %> | <%= __('Terms of Use') %><% } %> + Powered by CodiMD | <%= __('Releases') %>| <%= __('Source Code') %><% if(privacyStatement) { %> | <%= __('Privacy') %><% } %><% if(termsOfUse) { %> | <%= __('Terms of Use') %><% } %>