From 92482cbe6b1003d4c89ba12b628f163483752acb Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 00:48:09 -0300 Subject: [PATCH 01/11] Add local community link parser plugin for Markdown-It --- src/shared/markdown.ts | 75 +++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index 8f4d5c23..c837dba6 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -14,6 +14,7 @@ import markdown_it_sub from "markdown-it-sub"; import markdown_it_sup from "markdown-it-sup"; import Renderer from "markdown-it/lib/renderer"; import Token from "markdown-it/lib/token"; +import { getHttpBase } from "./env"; export let Tribute: any; @@ -72,6 +73,76 @@ const html5EmbedConfig = { }, }; +function localCommunityLinkParser(md) { + const pattern = + /(!\b[^@\s]+@[^@\s]+\.[^.\s]+\b)|\/c\/([^@\s]+)(@[^@\s]+\.[^.\s]+\b)?/g; + + md.core.ruler.push("replace-text", state => { + const tokens = state.tokens; + + for (let i = 0; i < tokens.length; i++) { + if (tokens[i].type === "inline") { + const token = tokens[i]; + + const originalContent = token.content; + + let lastIndex = 0; + originalContent.replace( + pattern, + (match, fullDomainMatch, name, domainTld, index) => { + let url; + // ex: !Testing@example.com + if (fullDomainMatch) { + const [name, domain, tld] = fullDomainMatch + .slice(1) + .split("@") + .join(".") + .split("."); + url = `${getHttpBase()}/c/${name}@${domain}.${tld}`; + } else { + // ex: /c/Testing or /c/Testing@example.com + url = `${getHttpBase()}/c/${name}${domainTld || ""}`; + } + + const beforeContent = originalContent.slice(lastIndex, index); + lastIndex = index + match.length; + + const beforeToken = new state.Token("text", "", 0); + beforeToken.content = beforeContent; + + const linkOpenToken = new state.Token("link_open", "a", 1); + linkOpenToken.attrs = [["href", url]]; + + const textToken = new state.Token("text", "", 0); + textToken.content = match; + + const linkCloseToken = new state.Token("link_close", "a", -1); + + const afterContent = originalContent.slice(lastIndex); + const afterToken = new state.Token("text", "", 0); + afterToken.content = afterContent; + + tokens.splice(i, 1); + + tokens.splice( + i, + 0, + beforeToken, + linkOpenToken, + textToken, + linkCloseToken, + afterToken + ); + + // Update i to skip the newly added tokens + i += 4; + } + ); + } + } + }); +} + export function setupMarkdown() { const markdownItConfig: MarkdownIt.Options = { html: false, @@ -88,7 +159,8 @@ export function setupMarkdown() { .use(markdown_it_sup) .use(markdown_it_footnote) .use(markdown_it_html5_embed, html5EmbedConfig) - .use(markdown_it_container, "spoiler", spoilerConfig); + .use(markdown_it_container, "spoiler", spoilerConfig) + .use(localCommunityLinkParser); // .use(markdown_it_emoji, { // defs: emojiDefs, // }); @@ -99,6 +171,7 @@ export function setupMarkdown() { .use(markdown_it_footnote) .use(markdown_it_html5_embed, html5EmbedConfig) .use(markdown_it_container, "spoiler", spoilerConfig) + .use(localCommunityLinkParser) // .use(markdown_it_emoji, { // defs: emojiDefs, // }) From 02717be15c7aa94f6ef5e6fd1ecb97ac61d85ad6 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 01:07:33 -0300 Subject: [PATCH 02/11] Add community link class --- src/shared/markdown.ts | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index c837dba6..6e6b504f 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -111,7 +111,10 @@ function localCommunityLinkParser(md) { beforeToken.content = beforeContent; const linkOpenToken = new state.Token("link_open", "a", 1); - linkOpenToken.attrs = [["href", url]]; + linkOpenToken.attrs = [ + ["href", url], + ["class", "community-link"], + ]; const textToken = new state.Token("text", "", 0); textToken.content = match; From 07235d7a5b3385e45d71102404f515a7bb54fe40 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 13:44:43 -0300 Subject: [PATCH 03/11] Update getHttpBase dependency reference --- src/shared/markdown.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index 6e6b504f..d35b469e 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -8,13 +8,13 @@ import { CustomEmojiView } from "lemmy-js-client"; import { default as MarkdownIt } from "markdown-it"; import markdown_it_container from "markdown-it-container"; // import markdown_it_emoji from "markdown-it-emoji/bare"; +import { getHttpBase } from "@utils/env"; import markdown_it_footnote from "markdown-it-footnote"; import markdown_it_html5_embed from "markdown-it-html5-embed"; import markdown_it_sub from "markdown-it-sub"; import markdown_it_sup from "markdown-it-sup"; import Renderer from "markdown-it/lib/renderer"; import Token from "markdown-it/lib/token"; -import { getHttpBase } from "./env"; export let Tribute: any; From c5779cd9b1e669fe6b8220e5c2dff0c125ed04e9 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 14:28:32 -0300 Subject: [PATCH 04/11] Update community link markdown parsing Remove links without remote servers, add kbin support, add user support --- src/shared/markdown.ts | 114 +++++++++++++++++++---------------------- 1 file changed, 53 insertions(+), 61 deletions(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index d35b469e..f56817e6 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -8,7 +8,6 @@ import { CustomEmojiView } from "lemmy-js-client"; import { default as MarkdownIt } from "markdown-it"; import markdown_it_container from "markdown-it-container"; // import markdown_it_emoji from "markdown-it-emoji/bare"; -import { getHttpBase } from "@utils/env"; import markdown_it_footnote from "markdown-it-footnote"; import markdown_it_html5_embed from "markdown-it-html5-embed"; import markdown_it_sub from "markdown-it-sub"; @@ -74,73 +73,66 @@ const html5EmbedConfig = { }; function localCommunityLinkParser(md) { - const pattern = - /(!\b[^@\s]+@[^@\s]+\.[^.\s]+\b)|\/c\/([^@\s]+)(@[^@\s]+\.[^.\s]+\b)?/g; - md.core.ruler.push("replace-text", state => { - const tokens = state.tokens; + /** + * Accepted formats: + * !community@server.com + * /c/community@server.com + * /m/community@server.com + * /u/username@server.com + */ + const pattern = + /(\/[c|m|u]\/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|![a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/g; - for (let i = 0; i < tokens.length; i++) { - if (tokens[i].type === "inline") { - const token = tokens[i]; + for (let i = 0; i < state.tokens.length; i++) { + if (state.tokens[i].type !== "inline") { + continue; + } + const inlineTokens = state.tokens[i].children; + for (let j = inlineTokens.length - 1; j >= 0; j--) { + if ( + inlineTokens[j].type === "text" && + pattern.test(inlineTokens[j].content) + ) { + const textParts = inlineTokens[j].content.split(pattern); + const newTokens: Token[] = []; - const originalContent = token.content; + for (const part of textParts) { + let linkClass = "community-link"; + if (pattern.test(part)) { + // Rewrite !community@server.com and KBin /m/community@server.com to local urls + let href; + if (part.startsWith("!")) { + href = "/c/" + part.substring(1); + } else if (part.startsWith("/m/")) { + href = "/c/" + part.substring(3); + } else { + href = part; + if (part.startsWith("/u/")) { + linkClass = "user-link"; + } + } - let lastIndex = 0; - originalContent.replace( - pattern, - (match, fullDomainMatch, name, domainTld, index) => { - let url; - // ex: !Testing@example.com - if (fullDomainMatch) { - const [name, domain, tld] = fullDomainMatch - .slice(1) - .split("@") - .join(".") - .split("."); - url = `${getHttpBase()}/c/${name}@${domain}.${tld}`; + const linkOpenToken = new state.Token("link_open", "a", 1); + linkOpenToken.attrs = [ + ["href", href], + ["class", linkClass], + ]; + const textToken = new state.Token("text", "", 0); + textToken.content = part; + const linkCloseToken = new state.Token("link_close", "a", -1); + + newTokens.push(linkOpenToken, textToken, linkCloseToken); } else { - // ex: /c/Testing or /c/Testing@example.com - url = `${getHttpBase()}/c/${name}${domainTld || ""}`; + const textToken = new state.Token("text", "", 0); + textToken.content = part; + newTokens.push(textToken); } - - const beforeContent = originalContent.slice(lastIndex, index); - lastIndex = index + match.length; - - const beforeToken = new state.Token("text", "", 0); - beforeToken.content = beforeContent; - - const linkOpenToken = new state.Token("link_open", "a", 1); - linkOpenToken.attrs = [ - ["href", url], - ["class", "community-link"], - ]; - - const textToken = new state.Token("text", "", 0); - textToken.content = match; - - const linkCloseToken = new state.Token("link_close", "a", -1); - - const afterContent = originalContent.slice(lastIndex); - const afterToken = new state.Token("text", "", 0); - afterToken.content = afterContent; - - tokens.splice(i, 1); - - tokens.splice( - i, - 0, - beforeToken, - linkOpenToken, - textToken, - linkCloseToken, - afterToken - ); - - // Update i to skip the newly added tokens - i += 4; } - ); + + // Replace the original token with the new tokens + inlineTokens.splice(j, 1, ...newTokens); + } } } }); From fc0c0634269fb57dad2e75f6b5bf5c1aa06fabf8 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 15:17:34 -0300 Subject: [PATCH 05/11] Move regex pattern to config --- src/shared/config.ts | 10 ++++++++++ src/shared/markdown.ts | 17 ++++------------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/src/shared/config.ts b/src/shared/config.ts index 28e8ce51..6b462244 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -25,4 +25,14 @@ export const fetchLimit = 40; export const relTags = "noopener nofollow"; export const emDash = "\u2014"; +/** + * Accepted formats: + * !community@server.com + * /c/community@server.com + * /m/community@server.com + * /u/username@server.com + */ +export const instanceLinkRegex = + /(\/[c|m|u]\/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|![a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/g; + export const testHost = "0.0.0.0:8536"; diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index f56817e6..d8ed9d50 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -14,6 +14,7 @@ import markdown_it_sub from "markdown-it-sub"; import markdown_it_sup from "markdown-it-sup"; import Renderer from "markdown-it/lib/renderer"; import Token from "markdown-it/lib/token"; +import { instanceLinkRegex } from "./config"; export let Tribute: any; @@ -74,16 +75,6 @@ const html5EmbedConfig = { function localCommunityLinkParser(md) { md.core.ruler.push("replace-text", state => { - /** - * Accepted formats: - * !community@server.com - * /c/community@server.com - * /m/community@server.com - * /u/username@server.com - */ - const pattern = - /(\/[c|m|u]\/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|![a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/g; - for (let i = 0; i < state.tokens.length; i++) { if (state.tokens[i].type !== "inline") { continue; @@ -92,14 +83,14 @@ function localCommunityLinkParser(md) { for (let j = inlineTokens.length - 1; j >= 0; j--) { if ( inlineTokens[j].type === "text" && - pattern.test(inlineTokens[j].content) + instanceLinkRegex.test(inlineTokens[j].content) ) { - const textParts = inlineTokens[j].content.split(pattern); + const textParts = inlineTokens[j].content.split(instanceLinkRegex); const newTokens: Token[] = []; for (const part of textParts) { let linkClass = "community-link"; - if (pattern.test(part)) { + if (instanceLinkRegex.test(part)) { // Rewrite !community@server.com and KBin /m/community@server.com to local urls let href; if (part.startsWith("!")) { From 73147ae37c5ba930a4915fc704a01bfb2744b173 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 15:33:45 -0300 Subject: [PATCH 06/11] Use shorter regex in community link parser --- src/shared/config.ts | 2 +- src/shared/markdown.ts | 69 +++++++++++++++++++++++------------------- 2 files changed, 39 insertions(+), 32 deletions(-) diff --git a/src/shared/config.ts b/src/shared/config.ts index 6b462244..db7e688a 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -33,6 +33,6 @@ export const emDash = "\u2014"; * /u/username@server.com */ export const instanceLinkRegex = - /(\/[c|m|u]\/[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}|![a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,})/g; + /(\/[c|m|u]\/|!)[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g; export const testHost = "0.0.0.0:8536"; diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index d8ed9d50..faab8756 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -73,7 +73,7 @@ const html5EmbedConfig = { }, }; -function localCommunityLinkParser(md) { +function localCommunityLinkParser(md: MarkdownIt) { md.core.ruler.push("replace-text", state => { for (let i = 0; i < state.tokens.length; i++) { if (state.tokens[i].type !== "inline") { @@ -83,42 +83,49 @@ function localCommunityLinkParser(md) { for (let j = inlineTokens.length - 1; j >= 0; j--) { if ( inlineTokens[j].type === "text" && - instanceLinkRegex.test(inlineTokens[j].content) + new RegExp(instanceLinkRegex).test(inlineTokens[j].content) ) { - const textParts = inlineTokens[j].content.split(instanceLinkRegex); + const text = inlineTokens[j].content; + const matches = Array.from(text.matchAll(instanceLinkRegex)); + + let lastIndex = 0; const newTokens: Token[] = []; - for (const part of textParts) { - let linkClass = "community-link"; - if (instanceLinkRegex.test(part)) { - // Rewrite !community@server.com and KBin /m/community@server.com to local urls - let href; - if (part.startsWith("!")) { - href = "/c/" + part.substring(1); - } else if (part.startsWith("/m/")) { - href = "/c/" + part.substring(3); - } else { - href = part; - if (part.startsWith("/u/")) { - linkClass = "user-link"; - } - } - - const linkOpenToken = new state.Token("link_open", "a", 1); - linkOpenToken.attrs = [ - ["href", href], - ["class", linkClass], - ]; + for (const match: RegExpMatchArray of matches) { + // If there is plain text before the match, add it as a separate token + if (match.index !== undefined && match.index > lastIndex) { const textToken = new state.Token("text", "", 0); - textToken.content = part; - const linkCloseToken = new state.Token("link_close", "a", -1); - - newTokens.push(linkOpenToken, textToken, linkCloseToken); - } else { - const textToken = new state.Token("text", "", 0); - textToken.content = part; + textToken.content = text.slice(lastIndex, match.index); newTokens.push(textToken); } + + // Determine the new href + let href; + if (match[0].startsWith("!")) { + href = "/c/" + match[0].substring(1); + } else if (match[0].startsWith("/m/")) { + href = "/c/" + match[0].substring(3); + } else { + href = match[0]; + } + + const linkOpenToken = new state.Token("link_open", "a", 1); + linkOpenToken.attrs = [["href", href]]; + const textToken = new state.Token("text", "", 0); + textToken.content = match[0]; + const linkCloseToken = new state.Token("link_close", "a", -1); + + newTokens.push(linkOpenToken, textToken, linkCloseToken); + + lastIndex = + (match.index !== undefined ? match.index : 0) + match[0].length; + } + + // If there is plain text after the last match, add it as a separate token + if (lastIndex < text.length) { + const textToken = new state.Token("text", "", 0); + textToken.content = text.slice(lastIndex); + newTokens.push(textToken); } // Replace the original token with the new tokens From d6e9b20a6ce29efdb4efe05014581ab5a4142f3b Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 15:40:06 -0300 Subject: [PATCH 07/11] Add missing classes --- src/shared/markdown.ts | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index faab8756..062b150d 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -91,6 +91,8 @@ function localCommunityLinkParser(md: MarkdownIt) { let lastIndex = 0; const newTokens: Token[] = []; + let linkClass = "community-link"; + for (const match: RegExpMatchArray of matches) { // If there is plain text before the match, add it as a separate token if (match.index !== undefined && match.index > lastIndex) { @@ -109,8 +111,15 @@ function localCommunityLinkParser(md: MarkdownIt) { href = match[0]; } + if (match[0].startsWith("/u/")) { + linkClass = "user-link"; + } + const linkOpenToken = new state.Token("link_open", "a", 1); - linkOpenToken.attrs = [["href", href]]; + linkOpenToken.attrs = [ + ["href", href], + ["class", linkClass], + ]; const textToken = new state.Token("text", "", 0); textToken.content = match[0]; const linkCloseToken = new state.Token("link_close", "a", -1); From 773eef5126c8381b42bf1833dc7f040ad7f20b19 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 15:43:01 -0300 Subject: [PATCH 08/11] Remove pipe from community link regex --- src/shared/config.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared/config.ts b/src/shared/config.ts index db7e688a..c56c64b0 100644 --- a/src/shared/config.ts +++ b/src/shared/config.ts @@ -33,6 +33,6 @@ export const emDash = "\u2014"; * /u/username@server.com */ export const instanceLinkRegex = - /(\/[c|m|u]\/|!)[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g; + /(\/[cmu]\/|!)[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\.[a-zA-Z]{2,}/g; export const testHost = "0.0.0.0:8536"; From 110be607c5da59b14d285a8a8b597e202fa42473 Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 15:51:25 -0300 Subject: [PATCH 09/11] Typescript linter fixes --- src/shared/markdown.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index 062b150d..ececf479 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -79,7 +79,7 @@ function localCommunityLinkParser(md: MarkdownIt) { if (state.tokens[i].type !== "inline") { continue; } - const inlineTokens = state.tokens[i].children; + const inlineTokens: Token[] = state.tokens[i].children || []; for (let j = inlineTokens.length - 1; j >= 0; j--) { if ( inlineTokens[j].type === "text" && @@ -93,7 +93,7 @@ function localCommunityLinkParser(md: MarkdownIt) { let linkClass = "community-link"; - for (const match: RegExpMatchArray of matches) { + for (const match of matches) { // If there is plain text before the match, add it as a separate token if (match.index !== undefined && match.index > lastIndex) { const textToken = new state.Token("text", "", 0); From 7bd90da1f84bf1cfccfa192e4aebf93549685e4e Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 16:02:27 -0300 Subject: [PATCH 10/11] Rename function to be more generic, since it parses users --- src/shared/markdown.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index ececf479..adee474f 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -73,7 +73,7 @@ const html5EmbedConfig = { }, }; -function localCommunityLinkParser(md: MarkdownIt) { +function localInstanceLinkParser(md: MarkdownIt) { md.core.ruler.push("replace-text", state => { for (let i = 0; i < state.tokens.length; i++) { if (state.tokens[i].type !== "inline") { From d58fd63113bc2933377ec2d90f8091b34b16e74f Mon Sep 17 00:00:00 2001 From: Zetaphor Date: Thu, 22 Jun 2023 16:05:20 -0300 Subject: [PATCH 11/11] Cleanup, only check for /u/ if /c/ and /m/ checks fail --- src/shared/markdown.ts | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/src/shared/markdown.ts b/src/shared/markdown.ts index adee474f..9f1ec733 100644 --- a/src/shared/markdown.ts +++ b/src/shared/markdown.ts @@ -101,7 +101,6 @@ function localInstanceLinkParser(md: MarkdownIt) { newTokens.push(textToken); } - // Determine the new href let href; if (match[0].startsWith("!")) { href = "/c/" + match[0].substring(1); @@ -109,10 +108,9 @@ function localInstanceLinkParser(md: MarkdownIt) { href = "/c/" + match[0].substring(3); } else { href = match[0]; - } - - if (match[0].startsWith("/u/")) { - linkClass = "user-link"; + if (match[0].startsWith("/u/")) { + linkClass = "user-link"; + } } const linkOpenToken = new state.Token("link_open", "a", 1); @@ -137,7 +135,6 @@ function localInstanceLinkParser(md: MarkdownIt) { newTokens.push(textToken); } - // Replace the original token with the new tokens inlineTokens.splice(j, 1, ...newTokens); } } @@ -162,7 +159,7 @@ export function setupMarkdown() { .use(markdown_it_footnote) .use(markdown_it_html5_embed, html5EmbedConfig) .use(markdown_it_container, "spoiler", spoilerConfig) - .use(localCommunityLinkParser); + .use(localInstanceLinkParser); // .use(markdown_it_emoji, { // defs: emojiDefs, // }); @@ -173,7 +170,7 @@ export function setupMarkdown() { .use(markdown_it_footnote) .use(markdown_it_html5_embed, html5EmbedConfig) .use(markdown_it_container, "spoiler", spoilerConfig) - .use(localCommunityLinkParser) + .use(localInstanceLinkParser) // .use(markdown_it_emoji, { // defs: emojiDefs, // })