Add support for PWA (#1005)

* Add logic for dynamically generating web manifest

* Make PWA icon get autogenerated

* Make service worker work

* Tweak things for PWA

* Handle apple icons and refactor

* Update prod dockerfile

* Remove jimp

* Remove unnecessary option

* Use different function syntax

Dockerfile

@@ -4,9 +4,13 @@ RUN curl -sf https://gobinaries.com/tj/node-prune | sh
 
 WORKDIR /usr/src/app
 
+ENV npm_config_target_arch=x64
+ENV npm_config_target_platform=linux
+ENV npm_config_target_libc=musl
+
 # Cache deps
 COPY package.json yarn.lock ./
-RUN yarn install --production --ignore-scripts --prefer-offline --pure-lockfile
+RUN yarn --production --prefer-offline --pure-lockfile
 
 # Build
 COPY generate_translations.js \
@@ -22,7 +26,7 @@ COPY .git .git
 # Set UI version 
 RUN echo "export const VERSION = '$(git describe --tag)';" > "src/shared/version.ts"
 
-RUN yarn install --production --ignore-scripts --prefer-offline
+RUN yarn --production --prefer-offline
 RUN yarn build:prod
 
 # Prune the image

dev.dockerfile

@@ -3,9 +3,13 @@ RUN apk update && apk add curl yarn python3 build-base gcc wget git --no-cache
 
 WORKDIR /usr/src/app
 
+ENV npm_config_target_arch=x64
+ENV npm_config_target_platform=linux
+ENV npm_config_target_libc=musl
+
 # Cache deps
 COPY package.json yarn.lock ./
-RUN yarn install --ignore-scripts --prefer-offline --pure-lockfile
+RUN yarn --prefer-offline --pure-lockfile
 
 # Build
 COPY generate_translations.js \
@@ -20,7 +24,7 @@ COPY src src
 # Set UI version 
 RUN echo "export const VERSION = 'dev';" > "src/shared/version.ts"
 
-RUN yarn install --ignore-scripts --prefer-offline
+RUN yarn --prefer-offline
 RUN yarn build:dev
 
 FROM node:alpine as runner

lemmy-translations

@@ -1 +1 @@
-Subproject commit 3bb45c26cb54325c3d8d605f4334447b9b78293a
+Subproject commit 007e53683768aeba63e9e4c179c1d240217bcee2

package.json

@@ -44,6 +44,7 @@
     "classnames": "^2.3.1",
     "clean-webpack-plugin": "^4.0.0",
     "copy-webpack-plugin": "^11.0.0",
+    "cross-fetch": "^3.1.5",
     "css-loader": "^6.7.3",
     "emoji-mart": "^5.4.0",
     "emoji-short-name": "^2.0.0",
@@ -76,6 +77,7 @@
     "sass": "^1.62.1",
     "sass-loader": "^13.2.2",
     "serialize-javascript": "^6.0.1",
+    "sharp": "^0.32.1",
     "tippy.js": "^6.3.7",
     "toastify-js": "^1.12.0",
     "tributejs": "^5.1.3",
@@ -113,7 +115,8 @@
     "style-loader": "^3.3.2",
     "terser": "^5.17.3",
     "typescript": "^5.0.4",
-    "webpack-dev-server": "4.15.0"
+    "webpack-dev-server": "4.15.0",
+    "service-worker-webpack": "^1.0.0"
   },
   "engines": {
     "node": ">=8.9.0"

src/assets/manifest.webmanifest

@@ -1,49 +0,0 @@
-{
-  "name": "Lemmy",
-  "description": "A link aggregator for the fediverse",
-  "start_url": "/",
-  "display": "standalone",
-  "background_color": "#222222",
-  "icons": [
-    {
-      "src": "/static/assets/icons/icon-72x72.png",
-      "sizes": "72x72",
-      "type": "image/png"
-    },
-    {
-      "src": "/static/assets/icons/icon-96x96.png",
-      "sizes": "96x96",
-      "type": "image/png"
-    },
-    {
-      "src": "/static/assets/icons/icon-128x128.png",
-      "sizes": "128x128",
-      "type": "image/png"
-    },
-    {
-      "src": "/static/assets/icons/icon-144x144.png",
-      "sizes": "144x144",
-      "type": "image/png"
-    },
-    {
-      "src": "/static/assets/icons/icon-152x152.png",
-      "sizes": "152x152",
-      "type": "image/png"
-    },
-    {
-      "src": "/static/assets/icons/icon-192x192.png",
-      "sizes": "192x192",
-      "type": "image/png"
-    },
-    {
-      "src": "/static/assets/icons/icon-384x384.png",
-      "sizes": "384x384",
-      "type": "image/png"
-    },
-    {
-      "src": "/static/assets/icons/icon-512x512.png",
-      "sizes": "512x512",
-      "type": "image/png"
-    }
-  ]
-}

src/server/index.tsx

@@ -1,23 +1,25 @@
 import express from "express";
-import fs from "fs";
+import { existsSync } from "fs";
+import { readdir, readFile } from "fs/promises";
 import { IncomingHttpHeaders } from "http";
 import { Helmet } from "inferno-helmet";
 import { matchPath, StaticRouter } from "inferno-router";
 import { renderToString } from "inferno-server";
 import IsomorphicCookie from "isomorphic-cookie";
-import { GetSite, GetSiteResponse, LemmyHttp } from "lemmy-js-client";
+import { GetSite, GetSiteResponse, LemmyHttp, Site } from "lemmy-js-client";
 import path from "path";
 import process from "process";
 import serialize from "serialize-javascript";
+import sharp from "sharp";
 import { App } from "../shared/components/app/app";
-import { httpBaseInternal } from "../shared/env";
+import { getHttpBase, getHttpBaseInternal } from "../shared/env";
 import {
   ILemmyConfig,
   InitialFetchRequest,
   IsoData,
 } from "../shared/interfaces";
 import { routes } from "../shared/routes";
-import { initializeSite } from "../shared/utils";
+import { favIconPngUrl, favIconUrl, initializeSite } from "../shared/utils";
 
 const server = express();
 const [hostname, port] = process.env["LEMMY_UI_HOST"]
@@ -54,6 +56,11 @@ Disallow: /password_change
 Disallow: /search/
 `;
 
+server.get("/service-worker.js", async (_req, res) => {
+  res.setHeader("Content-Type", "application/javascript");
+  res.sendFile(path.resolve("./dist/service-worker.js"));
+});
+
 server.get("/robots.txt", async (_req, res) => {
   res.setHeader("content-type", "text/plain; charset=utf-8");
   res.send(robotstxt);
@@ -67,13 +74,13 @@ server.get("/css/themes/:name", async (req, res) => {
   }
 
   const customTheme = path.resolve(`./${extraThemesFolder}/${theme}`);
-  if (fs.existsSync(customTheme)) {
+  if (existsSync(customTheme)) {
     res.sendFile(customTheme);
   } else {
     const internalTheme = path.resolve(`./dist/assets/css/themes/${theme}`);
 
     // If the theme doesn't exist, just send litely
-    if (fs.existsSync(internalTheme)) {
+    if (existsSync(internalTheme)) {
       res.sendFile(internalTheme);
     } else {
       res.sendFile(path.resolve("./dist/assets/css/themes/litely.css"));
@@ -81,11 +88,11 @@ server.get("/css/themes/:name", async (req, res) => {
   }
 });
 
-function buildThemeList(): string[] {
+async function buildThemeList(): Promise<string[]> {
-  let themes = ["darkly", "darkly-red", "litely", "litely-red"];
+  const themes = ["darkly", "darkly-red", "litely", "litely-red"];
-  if (fs.existsSync(extraThemesFolder)) {
+  if (existsSync(extraThemesFolder)) {
-    let dirThemes = fs.readdirSync(extraThemesFolder);
+    const dirThemes = await readdir(extraThemesFolder);
-    let cssThemes = dirThemes
+    const cssThemes = dirThemes
       .filter(d => d.endsWith(".css"))
       .map(d => d.replace(".css", ""));
     themes.push(...cssThemes);
@@ -95,7 +102,7 @@ function buildThemeList(): string[] {
 
 server.get("/css/themelist", async (_req, res) => {
   res.type("json");
-  res.send(JSON.stringify(buildThemeList()));
+  res.send(JSON.stringify(await buildThemeList()));
 });
 
 // server.use(cookieParser());
@@ -110,7 +117,7 @@ server.get("/*", async (req, res) => {
     const promises: Promise<any>[] = [];
 
     const headers = setForwardedHeaders(req.headers);
-    const client = new LemmyHttp(httpBaseInternal, headers);
+    const client = new LemmyHttp(getHttpBaseInternal(), headers);
 
     // Get site data first
     // This bypasses errors, so that the client can hit the error on its own,
@@ -180,6 +187,23 @@ server.get("/*", async (req, res) => {
 
     const config: ILemmyConfig = { wsHost: process.env.LEMMY_UI_LEMMY_WS_HOST };
 
+    const appleTouchIcon = site.site_view.site.icon
+      ? `data:image/png;base64,${sharp(
+          await fetchIconPng(site.site_view.site.icon)
+        )
+          .resize(180, 180)
+          .extend({
+            bottom: 20,
+            top: 20,
+            left: 20,
+            right: 20,
+            background: "#222222",
+          })
+          .png()
+          .toBuffer()
+          .then(buf => buf.toString("base64"))}`
+      : favIconPngUrl;
+
     res.send(`
            <!DOCTYPE html>
            <html ${helmet.htmlAttributes.toString()} lang="en">
@@ -200,9 +224,19 @@ server.get("/*", async (req, res) => {
            <meta name="Description" content="Lemmy">
            <meta charset="utf-8">
            <meta name="viewport" content="width=device-width, initial-scale=1, shrink-to-fit=no">
+           <link
+              id="favicon"
+              rel="shortcut icon"
+              type="image/x-icon"
+              href=${site.site_view.site.icon ?? favIconUrl}
+            />
 
            <!-- Web app manifest -->
-           <link rel="manifest" href="/static/assets/manifest.webmanifest">
+           <link rel="manifest" href="data:application/manifest+json;base64,${await generateManifestBase64(
+             site.site_view.site
+           )}">
+           <link rel="apple-touch-icon" href=${appleTouchIcon} />
+           <link rel="apple-touch-startup-image" href=${appleTouchIcon} />
 
            <!-- Styles -->
            <link rel="stylesheet" type="text/css" href="/static/styles/styles.css" />
@@ -267,3 +301,63 @@ function removeParam(url: string, parameter: string): string {
     .replace(new RegExp("[?&]" + parameter + "=[^&#]*(#.*)?$"), "$1")
     .replace(new RegExp("([?&])" + parameter + "=[^&]*&"), "$1");
 }
+
+const iconSizes = [72, 96, 128, 144, 152, 192, 384, 512];
+const defaultLogoPathDirectory = path.join(
+  process.cwd(),
+  "dist",
+  "assets",
+  "icons"
+);
+
+export async function generateManifestBase64(site: Site) {
+  const url = (
+    process.env.NODE_ENV === "development"
+      ? "http://localhost:1236/"
+      : getHttpBase()
+  ).replace(/\/$/g, "");
+  const icon = site.icon ? await fetchIconPng(site.icon) : null;
+
+  const manifest = {
+    name: site.name,
+    description: site.description ?? "A link aggregator for the fediverse",
+    start_url: url,
+    scope: url,
+    display: "standalone",
+    id: "/",
+    background_color: "#222222",
+    theme_color: "#222222",
+    icons: await Promise.all(
+      iconSizes.map(async size => {
+        let src = await readFile(
+          path.join(defaultLogoPathDirectory, `icon-${size}x${size}.png`)
+        ).then(buf => buf.toString("base64"));
+
+        if (icon) {
+          src = await sharp(icon)
+            .resize(size, size)
+            .png()
+            .toBuffer()
+            .then(buf => buf.toString("base64"));
+        }
+
+        return {
+          sizes: `${size}x${size}`,
+          type: "image/png",
+          src: `data:image/png;base64,${src}`,
+          purpose: "any maskable",
+        };
+      })
+    ),
+  };
+
+  return Buffer.from(JSON.stringify(manifest)).toString("base64");
+}
+
+async function fetchIconPng(iconUrl: string) {
+  return await fetch(
+    iconUrl.replace(/https?:\/\/localhost:\d+/g, getHttpBaseInternal())
+  )
+    .then(res => res.blob())
+    .then(blob => blob.arrayBuffer());
+}

src/service-worker.ts

@@ -1,28 +0,0 @@
-import { register } from "register-service-worker";
-
-register("/service-worker.js", {
-  registrationOptions: { scope: "./" },
-  ready() {
-    console.log("Service worker is active.");
-  },
-  registered() {
-    console.log("Service worker has been registered.");
-  },
-  cached() {
-    console.log("Content has been cached for offline use.");
-  },
-  updatefound() {
-    console.log("New content is downloading.");
-  },
-  updated() {
-    console.log("New content is available; please refresh.");
-  },
-  offline() {
-    console.log(
-      "No internet connection found. App is running in offline mode."
-    );
-  },
-  error(error) {
-    console.error("Error during service worker registration:", error);
-  },
-});

src/shared/components/app/app.tsx

@@ -1,10 +1,9 @@
 import { Component } from "inferno";
-import { Helmet } from "inferno-helmet";
 import { Provider } from "inferno-i18next-dess";
 import { Route, Switch } from "inferno-router";
 import { i18n } from "../../i18next";
 import { routes } from "../../routes";
-import { favIconPngUrl, favIconUrl, setIsoData } from "../../utils";
+import { setIsoData } from "../../utils";
 import { Footer } from "./footer";
 import { Navbar } from "./navbar";
 import { NoMatch } from "./no-match";
@@ -19,24 +18,12 @@ export class App extends Component<any, any> {
   render() {
     let siteRes = this.isoData.site_res;
     let siteView = siteRes.site_view;
-    let icon = siteView.site.icon;
 
     return (
       <>
         <Provider i18next={i18n}>
           <div id="app">
             <Theme defaultTheme={siteView.local_site.default_theme} />
-            {icon && (
-              <Helmet>
-                <link
-                  id="favicon"
-                  rel="shortcut icon"
-                  type="image/x-icon"
-                  href={icon || favIconUrl}
-                />
-                <link rel="apple-touch-icon" href={icon || favIconPngUrl} />
-              </Helmet>
-            )}
             <Navbar siteRes={siteRes} />
             <div className="mt-4 p-0 fl-1">
               <Switch>

src/shared/components/post/post-listing.tsx

@@ -22,7 +22,7 @@ import {
   SavePost,
   TransferCommunity,
 } from "lemmy-js-client";
-import { externalHost } from "../../env";
+import { getExternalHost } from "../../env";
 import { i18n } from "../../i18next";
 import { BanType, PostFormParams, PurgeType } from "../../interfaces";
 import { UserService, WebSocketService } from "../../services";
@@ -350,7 +350,7 @@ export class PostListing extends Component<PostListingProps, PostListingState> {
           )}
         </li>
         <li className="list-inline-item">•</li>
-        {url && !(hostname(url) == externalHost) && (
+        {url && !(hostname(url) === getExternalHost()) && (
           <>
             <li className="list-inline-item">
               <a

src/shared/env.ts

@@ -2,49 +2,69 @@ import { isBrowser } from "./utils";
 
 const testHost = "0.0.0.0:8536";
 
-let internalHost =
+function getInternalHost() {
-  (!isBrowser() && process.env.LEMMY_UI_LEMMY_INTERNAL_HOST) || testHost; // used for local dev
+  return !isBrowser()
-export let externalHost: string;
+    ? process.env.LEMMY_UI_LEMMY_INTERNAL_HOST ?? testHost
-let host: string;
+    : testHost; // used for local dev
-let wsHost: string;
+}
-let secure: string;
 
-if (isBrowser()) {
+export function getExternalHost() {
-  // browser
+  return isBrowser()
-  const lemmyConfig =
+    ? `${window.location.hostname}${
-    typeof window.lemmyConfig !== "undefined" ? window.lemmyConfig : {};
-
-  externalHost = `${window.location.hostname}${
         ["1234", "1235"].includes(window.location.port)
           ? ":8536"
           : window.location.port == ""
           ? ""
           : `:${window.location.port}`
-  }`;
+      }`
-
+    : process.env.LEMMY_UI_LEMMY_EXTERNAL_HOST || testHost;
-  host = externalHost;
-  wsHost = lemmyConfig.wsHost || host;
-  secure = window.location.protocol == "https:" ? "s" : "";
-} else {
-  // server-side
-  externalHost = process.env.LEMMY_UI_LEMMY_EXTERNAL_HOST || testHost;
-  host = internalHost;
-  wsHost = process.env.LEMMY_UI_LEMMY_WS_HOST || externalHost;
-  secure = process.env.LEMMY_UI_HTTPS == "true" ? "s" : "";
 }
 
-export const httpBaseInternal = `http://${host}`; // Don't use secure here
+function getSecure() {
-export const httpBase = `http${secure}://${host}`;
+  return (
-export const wsUriBase = `ws${secure}://${wsHost}`;
+    isBrowser()
-export const wsUri = `${wsUriBase}/api/v3/ws`;
+      ? window.location.protocol.includes("https")
-export const isHttps = secure.endsWith("s");
+      : process.env.LEMMY_UI_HTTPS === "true"
+  )
+    ? "s"
+    : "";
+}
 
-console.log(`httpbase: ${httpBase}`);
+function getHost() {
-console.log(`wsUri: ${wsUri}`);
+  return isBrowser() ? getExternalHost() : getInternalHost();
-console.log(`isHttps: ${isHttps}`);
+}
+
+function getWsHost() {
+  return isBrowser()
+    ? window.lemmyConfig?.wsHost ?? getHost()
+    : process.env.LEMMY_UI_LEMMY_WS_HOST ?? getExternalHost();
+}
+
+function getBaseLocal(s = "") {
+  return `http${s}://${getHost()}`;
+}
+
+export function getHttpBaseInternal() {
+  return getBaseLocal(); // Don't use secure here
+}
+export function getHttpBase() {
+  return getBaseLocal(getSecure());
+}
+export function getWsUri() {
+  return `ws${getSecure()}://${getWsHost()}/api/v3/ws`;
+}
+export function isHttps() {
+  return getSecure() === "s";
+}
+
+console.log(`httpbase: ${getHttpBase()}`);
+console.log(`wsUri: ${getWsUri()}`);
+console.log(`isHttps: ${isHttps()}`);
 
 // This is for html tags, don't include port
-const httpExternalUri = `http${secure}://${externalHost.split(":")[0]}`;
 export function httpExternalPath(path: string) {
-  return `${httpExternalUri}${path}`;
+  return `http${getSecure()}://${getExternalHost().replace(
+    /:\d+/g,
+    ""
+  )}${path}`;
 }

src/shared/services/UserService.ts

@@ -38,7 +38,7 @@ export class UserService {
     expires.setDate(expires.getDate() + 365);
     if (res.jwt) {
       toast(i18n.t("logged_in"));
-      IsomorphicCookie.save("jwt", res.jwt, { expires, secure: isHttps });
+      IsomorphicCookie.save("jwt", res.jwt, { expires, secure: isHttps() });
       this.setJwtInfo();
     }
   }

src/shared/services/WebSocketService.ts

@@ -6,7 +6,7 @@ import {
   Websocket as WS,
   WebsocketBuilder,
 } from "websocket-ts";
-import { wsUri } from "../env";
+import { getWsUri } from "../env";
 import { isBrowser } from "../utils";
 
 export class WebSocketService {
@@ -18,7 +18,7 @@ export class WebSocketService {
     let firstConnect = true;
 
     this.subject = new Observable((obs: any) => {
-      this.ws = new WebsocketBuilder(wsUri)
+      this.ws = new WebsocketBuilder(getWsUri())
         .onMessage((_i, e) => {
           try {
             obs.next(JSON.parse(e.data.toString()));
@@ -27,7 +27,7 @@ export class WebSocketService {
           }
         })
         .onOpen(() => {
-          console.log(`Connected to ${wsUri}`);
+          console.log(`Connected to ${getWsUri()}`);
 
           if (!firstConnect) {
             let res = {

src/shared/utils.ts

@@ -41,12 +41,12 @@ import { Subscription } from "rxjs";
 import { delay, retryWhen, take } from "rxjs/operators";
 import tippy from "tippy.js";
 import Toastify from "toastify-js";
-import { httpBase } from "./env";
+import { getHttpBase } from "./env";
 import { i18n, languages } from "./i18next";
 import { CommentNodeI, DataType, IsoData } from "./interfaces";
 import { UserService, WebSocketService } from "./services";
 
-var Tribute: any;
+let Tribute: any;
 if (isBrowser()) {
   Tribute = require("tributejs");
 }
@@ -344,7 +344,7 @@ export function capitalizeFirstLetter(str: string): string {
 
 export async function getSiteMetadata(url: string) {
   let form: GetSiteMetadata = { url };
-  let client = new LemmyHttp(httpBase);
+  let client = new LemmyHttp(getHttpBase());
   return client.getSiteMetadata(form);
 }
 
@@ -1362,7 +1362,7 @@ export async function fetchCommunities(q: string) {
     limit: fetchLimit,
     auth: myAuth(false),
   };
-  let client = new LemmyHttp(httpBase);
+  let client = new LemmyHttp(getHttpBase());
   return client.search(form);
 }
 
@@ -1376,7 +1376,7 @@ export async function fetchUsers(q: string) {
     limit: fetchLimit,
     auth: myAuth(false),
   };
-  let client = new LemmyHttp(httpBase);
+  let client = new LemmyHttp(getHttpBase());
   return client.search(form);
 }
 
@@ -1540,7 +1540,7 @@ export function selectableLanguages(
 }
 
 export function uploadImage(image: File): Promise<UploadImageResponse> {
-  const client = new LemmyHttp(httpBase);
+  const client = new LemmyHttp(getHttpBase());
 
   return client.uploadImage({ image });
 }

webpack.config.js

@@ -3,8 +3,8 @@ const MiniCssExtractPlugin = require("mini-css-extract-plugin");
 const nodeExternals = require("webpack-node-externals");
 const CopyPlugin = require("copy-webpack-plugin");
 const RunNodeWebpackPlugin = require("run-node-webpack-plugin");
-const { merge } = require("lodash");
+const merge = require("lodash/merge");
-
+const { ServiceWorkerPlugin } = require("service-worker-webpack");
 const banner = `
   hash:[contentHash], chunkhash:[chunkhash], name:[name], filebase:[base], query:[query], file:[file]
   Source code: https://github.com/LemmyNet/lemmy-ui
@@ -83,6 +83,7 @@ const createServerConfig = (_env, mode) => {
 
   return config;
 };
+
 const createClientConfig = (_env, mode) => {
   const config = merge({}, base, {
     mode,
@@ -90,6 +91,58 @@ const createClientConfig = (_env, mode) => {
     output: {
       filename: "js/client.js",
     },
+    plugins: [
+      ...base.plugins,
+      new ServiceWorkerPlugin({
+        enableInDevelopment: true,
+        workbox: {
+          modifyURLPrefix: {
+            "/": "/static/",
+          },
+          cacheId: "lemmy",
+          include: [/(assets|styles)\/.+\..+|client\.js$/g],
+          inlineWorkboxRuntime: true,
+          runtimeCaching: [
+            {
+              urlPattern: ({
+                sameOrigin,
+                url: { pathname, host },
+                request: { method },
+              }) =>
+                (sameOrigin || host.includes("localhost")) &&
+                (!(
+                  pathname.includes("pictrs") || pathname.includes("static")
+                ) ||
+                  method === "POST"),
+              handler: "NetworkFirst",
+              options: {
+                cacheName: "instance-cache",
+              },
+            },
+            {
+              urlPattern: ({ url: { pathname, host }, sameOrigin }) =>
+                (sameOrigin || host.includes("localhost")) &&
+                pathname.includes("static"),
+              handler: "CacheFirst",
+              options: {
+                cacheName: "static-cache",
+              },
+            },
+            {
+              urlPattern: ({ url: { pathname }, request: { method } }) =>
+                pathname.includes("pictrs") && method === "GET",
+              handler: "StaleWhileRevalidate",
+              options: {
+                cacheName: "image-cache",
+                expiration: {
+                  maxAgeSeconds: 60 * 60 * 24,
+                },
+              },
+            },
+          ],
+        },
+      }),
+    ],
   });
 
   if (mode === "development") {