From 55f91ac5dc1816463fb99d6974f89acd46de3444 Mon Sep 17 00:00:00 2001 From: Dessalines Date: Mon, 17 Feb 2020 11:18:01 -0500 Subject: [PATCH] First pass at adding oembeds / iframely. --- ansible/lemmy.yml | 1 + ansible/lemmy_dev.yml | 1 + ansible/templates/docker-compose.yml | 9 + ansible/templates/nginx.conf | 7 + docker/dev/docker-compose.yml | 8 + docker/iframely.config.local.js | 283 ++++++++++++++++++++++ docker/prod/docker-compose.yml | 8 + docs/src/administration_install_docker.md | 1 + ui/src/components/iframely-card.tsx | 100 ++++++++ ui/src/components/post-listing.tsx | 53 +++- ui/src/interfaces.ts | 15 ++ 11 files changed, 482 insertions(+), 4 deletions(-) create mode 100644 docker/iframely.config.local.js create mode 100644 ui/src/components/iframely-card.tsx diff --git a/ansible/lemmy.yml b/ansible/lemmy.yml index c415abef5..8d5e22641 100644 --- a/ansible/lemmy.yml +++ b/ansible/lemmy.yml @@ -35,6 +35,7 @@ with_items: - { src: 'templates/docker-compose.yml', dest: '/lemmy/docker-compose.yml', mode: '0600' } - { src: 'templates/nginx.conf', dest: '/etc/nginx/sites-enabled/lemmy.conf', mode: '0644' } + - { src: '../docker/iframely.config.local.js', dest: '/lemmy/iframely.config.local.js', mode: '0600' } - name: add config file (only during initial setup) template: src='templates/config.hjson' dest='/lemmy/lemmy.hjson' mode='0600' force='no' owner='1000' group='1000' diff --git a/ansible/lemmy_dev.yml b/ansible/lemmy_dev.yml index c150714ca..e9b8364f3 100644 --- a/ansible/lemmy_dev.yml +++ b/ansible/lemmy_dev.yml @@ -37,6 +37,7 @@ with_items: - { src: 'templates/docker-compose.yml', dest: '/lemmy/docker-compose.yml', mode: '0600' } - { src: 'templates/nginx.conf', dest: '/etc/nginx/sites-enabled/lemmy.conf', mode: '0644' } + - { src: '../docker/iframely.config.local.js', dest: '/lemmy/iframely.config.local.js', mode: '0600' } - name: add config file (only during initial setup) template: src='templates/config.hjson' dest='/lemmy/lemmy.hjson' mode='0600' force='no' owner='1000' group='1000' diff --git a/ansible/templates/docker-compose.yml b/ansible/templates/docker-compose.yml index 2693d7ad2..bf9aeeb5a 100644 --- a/ansible/templates/docker-compose.yml +++ b/ansible/templates/docker-compose.yml @@ -30,6 +30,14 @@ services: - lemmy_pictshare:/usr/share/nginx/html/data restart: always + lemmy_iframely: + image: dogbin/iframely:latest + ports: + - "127.0.0.1:8061:8061" + volumes: + - ./iframely.config.local.js:/iframely/config.local.js:ro + restart: always + postfix: image: mwader/postfix-relay environment: @@ -38,3 +46,4 @@ services: volumes: lemmy_db: lemmy_pictshare: + lemmy_iframely: diff --git a/ansible/templates/nginx.conf b/ansible/templates/nginx.conf index 9f31140b2..04e5a6436 100644 --- a/ansible/templates/nginx.conf +++ b/ansible/templates/nginx.conf @@ -80,6 +80,13 @@ server { add_header Cache-Control "public, max-age=31536000, immutable"; } } + + location /iframely/ { + proxy_pass http://0.0.0.0:8061/; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + } } # Anonymize IP addresses diff --git a/docker/dev/docker-compose.yml b/docker/dev/docker-compose.yml index eabd334d5..987be4d5b 100644 --- a/docker/dev/docker-compose.yml +++ b/docker/dev/docker-compose.yml @@ -28,6 +28,14 @@ services: volumes: - lemmy_pictshare:/usr/share/nginx/html/data restart: always + lemmy_iframely: + image: dogbin/iframely:latest + ports: + - "127.0.0.1:8061:8061" + volumes: + - ../iframely.config.local.js:/iframely/config.local.js:ro + restart: always volumes: lemmy_db: lemmy_pictshare: + lemmy_iframely: diff --git a/docker/iframely.config.local.js b/docker/iframely.config.local.js new file mode 100644 index 000000000..5c00cb143 --- /dev/null +++ b/docker/iframely.config.local.js @@ -0,0 +1,283 @@ +(function() { + var config = { + + // Specify a path for custom plugins. Custom plugins will override core plugins. + // CUSTOM_PLUGINS_PATH: __dirname + '/yourcustom-plugin-folder', + + DEBUG: false, + RICH_LOG_ENABLED: false, + + // For embeds that require render, baseAppUrl will be used as the host. + baseAppUrl: "http://yourdomain.com", + relativeStaticUrl: "/r", + + // Or just skip built-in renders altogether + SKIP_IFRAMELY_RENDERS: true, + + // For legacy reasons the response format of Iframely open-source is + // different by default as it does not group the links array by rel. + // In order to get the same grouped response as in Cloud API, + // add `&group=true` to your request to change response per request + // or set `GROUP_LINKS` in your config to `true` for a global change. + GROUP_LINKS: true, + + // Number of maximum redirects to follow before aborting the page + // request with `redirect loop` error. + MAX_REDIRECTS: 4, + + SKIP_OEMBED_RE_LIST: [ + // /^https?:\/\/yourdomain\.com\//, + ], + + /* + // Used to pass parameters to the generate functions when creating HTML elements + // disableSizeWrapper: Don't wrap element (iframe, video, etc) in a positioned div + GENERATE_LINK_PARAMS: { + disableSizeWrapper: true + }, + */ + + port: 8061, //can be overridden by PORT env var + host: '0.0.0.0', // Dockers beware. See https://github.com/itteco/iframely/issues/132#issuecomment-242991246 + //can be overridden by HOST env var + + // Optional SSL cert, if you serve under HTTPS. + /* + ssl: { + key: require('fs').readFileSync(__dirname + '/key.pem'), + cert: require('fs').readFileSync(__dirname + '/cert.pem'), + port: 443 + }, + */ + + /* + Supported cache engines: + - no-cache - no caching will be used. + - node-cache - good for debug, node memory will be used (https://github.com/tcs-de/nodecache). + - redis - https://github.com/mranney/node_redis. + - memcached - https://github.com/3rd-Eden/node-memcached + */ + CACHE_ENGINE: 'node-cache', + CACHE_TTL: 0, // In seconds. + // 0 = 'never expire' for memcached & node-cache to let cache engine decide itself when to evict the record + // 0 = 'no cache' for redis. Use high enough (e.g. 365*24*60*60*1000) ttl for similar 'never expire' approach instead + + /* + // Redis cache options. + REDIS_OPTIONS: { + host: '127.0.0.1', + port: 6379 + }, + */ + + /* + // Memcached options. See https://github.com/3rd-Eden/node-memcached#server-locations + MEMCACHED_OPTIONS: { + locations: "127.0.0.1:11211" + } + */ + + /* + // Access-Control-Allow-Origin list. + allowedOrigins: [ + "*", + "http://another_domain.com" + ], + */ + + /* + // Uncomment to enable plugin testing framework. + tests: { + mongodb: 'mongodb://localhost:27017/iframely-tests', + single_test_timeout: 10 * 1000, + plugin_test_period: 2 * 60 * 60 * 1000, + relaunch_script_period: 5 * 60 * 1000 + }, + */ + + // If there's no response from remote server, the timeout will occur after + RESPONSE_TIMEOUT: 5 * 1000, //ms + + /* From v1.4.0, Iframely supports HTTP/2 by default. Disable it, if you'd rather not. + Alternatively, you can also disable per origin. See `proxy` option below. + */ + // DISABLE_HTTP2: true, + + // Customize API calls to oembed endpoints. + ADD_OEMBED_PARAMS: [{ + // Endpoint url regexp array. + re: [/^http:\/\/api\.instagram\.com\/oembed/], + // Custom get params object. + params: { + hidecaption: true + } + }, { + re: [/^https:\/\/www\.facebook\.com\/plugins\/page\/oembed\.json/i], + params: { + show_posts: 0, + show_facepile: 0, + maxwidth: 600 + } + }, { + // match i=user or i=moment or i=timeline to configure these types invidually + // see params spec at https://dev.twitter.com/web/embedded-timelines/oembed + re: [/^https?:\/\/publish\.twitter\.com\/oembed\?i=user/i], + params: { + limit: 1, + maxwidth: 600 + } + /* + }, { + // Facebook https://developers.facebook.com/docs/plugins/oembed-endpoints + re: [/^https:\/\/www\.facebook\.com\/plugins\/\w+\/oembed\.json/i], + params: { + // Skip script tag and fb-root div. + omitscript: true + } + */ + }], + + /* + // Configure use of HTTP proxies as needed. + // You don't have to specify all options per regex - just what you need to override + PROXY: [{ + re: [/^https?:\/\/www\.domain\.com/], + proxy_server: 'http://1.2.3.4:8080', + user_agent: 'CHANGE YOUR AGENT', + headers: { + // HTTP headers + // Overrides previous params if overlapped. + }, + request_options: { + // Refer to: https://github.com/request/request + // Overrides previous params if overlapped. + }, + disable_http2: true + }], + */ + + // Customize API calls to 3rd parties. At the very least - configure required keys. + providerOptions: { + locale: "en_US", // ISO 639-1 two-letter language code, e.g. en_CA or fr_CH. + // Will be added as highest priotity in accept-language header with each request. + // Plus is used in FB, YouTube and perhaps other plugins + "twitter": { + "max-width": 550, + "min-width": 250, + hide_media: false, + hide_thread: false, + omit_script: false, + center: false, + // dnt: true, + cache_ttl: 100 * 365 * 24 * 3600 // 100 Years. + }, + readability: { + enabled: false + // allowPTagDescription: true // to enable description fallback to first paragraph + }, + images: { + loadSize: false, // if true, will try an load first bytes of all images to get/confirm the sizes + checkFavicon: false // if true, will verify all favicons + }, + tumblr: { + consumer_key: "INSERT YOUR VALUE" + // media_only: true // disables status embeds for images and videos - will return plain media + }, + google: { + // https://developers.google.com/maps/documentation/embed/guide#api_key + maps_key: "INSERT YOUR VALUE" + }, + + /* + // Optional Camo Proxy to wrap all images: https://github.com/atmos/camo + camoProxy: { + camo_proxy_key: "INSERT YOUR VALUE", + camo_proxy_host: "INSERT YOUR VALUE" + // ssl_only: true // will only proxy non-ssl images + }, + */ + + // List of query parameters to add to YouTube and Vimeo frames + // Start it with leading "?". Or omit alltogether for default values + // API key is optional, youtube will work without it too. + // It is probably the same API key you use for Google Maps. + youtube: { + // api_key: "INSERT YOUR VALUE", + get_params: "?rel=0&showinfo=1" // https://developers.google.com/youtube/player_parameters + }, + vimeo: { + get_params: "?byline=0&badge=0" // https://developer.vimeo.com/player/embedding + }, + + /* + soundcloud: { + old_player: true // enables classic player + }, + giphy: { + media_only: true // disables branded player for gifs and returns just the image + } + */ + /* + bandcamp: { + get_params: '/size=large/bgcol=333333/linkcol=ffffff/artwork=small/transparent=true/', + media: { + album: { + height: 472, + 'max-width': 700 + }, + track: { + height: 120, + 'max-width': 700 + } + } + } + */ + }, + + // WHITELIST_WILDCARD, if present, will be added to whitelist as record for top level domain: "*" + // with it, you can define what parsers do when they run accross unknown publisher. + // If absent or empty, all generic media parsers will be disabled except for known domains + // More about format: https://iframely.com/docs/qa-format + + /* + WHITELIST_WILDCARD: { + "twitter": { + "player": "allow", + "photo": "deny" + }, + "oembed": { + "video": "allow", + "photo": "allow", + "rich": "deny", + "link": "deny" + }, + "og": { + "video": ["allow", "ssl", "responsive"] + }, + "iframely": { + "survey": "allow", + "reader": "allow", + "player": "allow", + "image": "allow" + }, + "html-meta": { + "video": ["allow", "responsive"], + "promo": "allow" + } + } + */ + + // Black-list any of the inappropriate domains. Iframely will return 417 + // At minimum, keep your localhosts blacklisted to avoid SSRF + BLACKLIST_DOMAINS_RE: [ + /^https?:\/\/127\.0\.0\.1/i, + /^https?:\/\/localhost/i, + + // And this is AWS metadata service + // https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-instance-metadata.html + /^https?:\/\/169\.254\.169\.254/ + ] + }; + + module.exports = config; +})(); diff --git a/docker/prod/docker-compose.yml b/docker/prod/docker-compose.yml index 3472be5d6..8469a1e7b 100644 --- a/docker/prod/docker-compose.yml +++ b/docker/prod/docker-compose.yml @@ -26,6 +26,14 @@ services: volumes: - lemmy_pictshare:/usr/share/nginx/html/data restart: always + lemmy_iframely: + image: dogbin/iframely:latest + ports: + - "127.0.0.1:8061:8061" + volumes: + - ./iframely.config.local.js:/iframely/config.local.js:ro + restart: always volumes: lemmy_db: lemmy_pictshare: + lemmy_iframely: diff --git a/docs/src/administration_install_docker.md b/docs/src/administration_install_docker.md index f92cbd5be..992049839 100644 --- a/docs/src/administration_install_docker.md +++ b/docs/src/administration_install_docker.md @@ -7,6 +7,7 @@ mkdir lemmy/ cd lemmy/ wget https://raw.githubusercontent.com/dessalines/lemmy/master/docker/prod/docker-compose.yml wget https://raw.githubusercontent.com/dessalines/lemmy/master/docker/lemmy.hjson +wget https://raw.githubusercontent.com/dessalines/lemmy/master/docker/iframely.config.local.js # Edit lemmy.hjson, and docker-compose.yml to do more configuration (like adding a custom password) docker-compose up -d ``` diff --git a/ui/src/components/iframely-card.tsx b/ui/src/components/iframely-card.tsx new file mode 100644 index 000000000..73f3cef72 --- /dev/null +++ b/ui/src/components/iframely-card.tsx @@ -0,0 +1,100 @@ +import { Component, linkEvent } from 'inferno'; +import { FramelyData } from '../interfaces'; +import { mdToHtml } from '../utils'; + +interface FramelyCardProps { + iframely: FramelyData; +} + +interface FramelyCardState { + expanded: boolean; +} + +export class IFramelyCard extends Component< + FramelyCardProps, + FramelyCardState +> { + private emptyState: FramelyCardState = { + expanded: false, + }; + + constructor(props: any, context: any) { + super(props, context); + this.state = this.emptyState; + } + + render() { + let iframely = this.props.iframely; + return ( + <> +
+
+ {iframely.thumbnail_url && ( +
+ {iframely.html ? ( + + + + ) : ( + + )} +
+ )} +
+
+
+ + + {iframely.title} + + +
+ + + {new URL(iframely.url).hostname} + + + + + {iframely.html && ( + + {this.state.expanded ? '[-]' : '[+]'} + + )} + + {iframely.description && ( +
+ )} +
+
+
+
+ {this.state.expanded && ( +
+
+
+ )} + + ); + } + + handleIframeExpand(i: IFramelyCard) { + i.state.expanded = !i.state.expanded; + i.setState(i.state); + } +} diff --git a/ui/src/components/post-listing.tsx b/ui/src/components/post-listing.tsx index d37725440..5cc632517 100644 --- a/ui/src/components/post-listing.tsx +++ b/ui/src/components/post-listing.tsx @@ -15,9 +15,11 @@ import { AddAdminForm, TransferSiteForm, TransferCommunityForm, + FramelyData, } from '../interfaces'; import { MomentTime } from './moment-time'; import { PostForm } from './post-form'; +import { IFramelyCard } from './iframely-card'; import { mdToHtml, canMod, @@ -47,6 +49,7 @@ interface PostListingState { score: number; upvotes: number; downvotes: number; + iframely: FramelyData; } interface PostListingProps { @@ -74,6 +77,7 @@ export class PostListing extends Component { score: this.props.post.score, upvotes: this.props.post.upvotes, downvotes: this.props.post.downvotes, + iframely: null, }; constructor(props: any, context: any) { @@ -84,6 +88,10 @@ export class PostListing extends Component { this.handlePostDisLike = this.handlePostDisLike.bind(this); this.handleEditPost = this.handleEditPost.bind(this); this.handleEditCancel = this.handleEditCancel.bind(this); + + if (this.props.post.url) { + this.fetchIframely(); + } } componentWillReceiveProps(nextProps: PostListingProps) { @@ -141,7 +149,7 @@ export class PostListing extends Component { )}
- {post.url && isImage(post.url) && !this.state.imageExpanded && ( + {this.hasImage() && !this.state.imageExpanded && ( { className={`mx-2 mt-1 float-left img-fluid thumbnail rounded ${(post.nsfw || post.community_nsfw) && 'img-blur'}`} - src={imageThumbnailer(post.url)} + src={imageThumbnailer(this.getImage())} /> )} @@ -205,7 +213,7 @@ export class PostListing extends Component { )} - {post.url && isImage(post.url) && ( + {this.hasImage() && ( <> {!this.state.imageExpanded ? ( { class="pointer" onClick={linkEvent(this, this.handleImageExpandClick)} > - +
@@ -587,6 +598,9 @@ export class PostListing extends Component { )} + {post.url && this.props.showBody && this.state.iframely && ( + + )} {this.state.showRemoveDialog && (
{ ); } + fetchIframely() { + fetch(`/iframely/oembed?url=${this.props.post.url}`) + .then(res => res.json()) + .then(res => { + this.state.iframely = res; + this.setState(this.state); + }) + .catch(error => { + console.error(`Iframely service not set up properly. ${error}`); + }); + } + + hasImage(): boolean { + return ( + (this.props.post.url && isImage(this.props.post.url)) || + (this.state.iframely && this.state.iframely.thumbnail_url !== undefined) + ); + } + + getImage(): string { + let simpleImg = isImage(this.props.post.url); + if (simpleImg) { + return this.props.post.url; + } else if (this.state.iframely) { + let iframelyThumbnail = this.state.iframely.thumbnail_url; + if (iframelyThumbnail) { + return iframelyThumbnail; + } + } + } + handlePostLike(i: PostListing) { let new_vote = i.state.my_vote == 1 ? 0 : 1; diff --git a/ui/src/interfaces.ts b/ui/src/interfaces.ts index 5846b548c..5baadb170 100644 --- a/ui/src/interfaces.ts +++ b/ui/src/interfaces.ts @@ -876,3 +876,18 @@ export interface WebSocketJsonResponse { error?: string; reconnect?: boolean; } + +export interface FramelyData { + url: string; + type: string; + version?: string; + title: string; + author?: string; + author_url?: string; + provider_name?: string; + thumbnail_url?: string; + thumbnail_width?: number; + thumbnail_height?: number; + description?: string; + html?: string; +}