Add screenshots for nsites

.changeset/stupid-poems-approve.md

@@ -0,0 +1,5 @@
+---
+"nsite-ts": minor
+---
+
+Add screenshots for nsites

.gitignore

@@ -3,3 +3,4 @@ build
 .env
 data
 .netrc
+screenshots

.vscode/launch.json

@@ -26,8 +26,7 @@
       "internalConsoleOptions": "openOnSessionStart",
       "outputCapture": "std",
       "env": {
-        "DEBUG": "nsite,nsite:*",
-        "ALL_PROXY": "pac+file://proxy.pac"
+        "DEBUG": "nsite,nsite:*"
       }
     }
   ]

Dockerfile

@@ -39,10 +39,12 @@ COPY ./public ./public
 COPY tor-and-i2p.pac proxy.pac
 
 VOLUME [ "/var/cache/nginx" ]
+VOLUME [ "/screenshots" ]
 
 EXPOSE 80 3000
 ENV NSITE_PORT="3000"
 ENV NGINX_CACHE_DIR="/var/cache/nginx"
+ENV SCREENSHOTS_DIR="/screenshots"
 
 COPY docker-entrypoint.sh /
 RUN chmod +x /docker-entrypoint.sh

docker-compose.yml

@@ -1,5 +1,6 @@
 version: "3.7"
 
+
 services:
   nsite:
     build: .
@@ -8,6 +9,10 @@ services:
       LOOKUP_RELAYS: wss://user.kindpag.es,wss://purplepag.es
       SUBSCRIPTION_RELAYS: wss://nostrue.com/,wss://nos.lol/,wss://relay.damus.io/,wss://purplerelay.com/
     volumes:
+      - type: tmpfs
+        target: /screenshots
+        tmpfs:
+          size: 100M
       - type: tmpfs
         target: /var/cache/nginx
         tmpfs:

package.json

@@ -27,11 +27,13 @@
     "keyv": "^5.0.1",
     "koa": "^2.15.3",
     "koa-morgan": "^1.0.1",
+    "koa-send": "^5.0.1",
     "koa-static": "^5.0.0",
     "mime": "^4.0.4",
     "nostr-tools": "^2.7.2",
     "pac-proxy-agent": "^7.0.2",
     "proxy-agent": "^6.4.0",
+    "puppeteer": "^23.5.0",
     "websocket-polyfill": "^1.0.0",
     "ws": "^8.18.0",
     "xbytes": "^1.9.1"
@@ -44,6 +46,7 @@
     "@types/follow-redirects": "^1.14.4",
     "@types/koa": "^2.14.0",
     "@types/koa-morgan": "^1.0.8",
+    "@types/koa-send": "^4.1.6",
     "@types/koa-static": "^4.0.4",
     "@types/koa__cors": "^5.0.0",
     "@types/koa__router": "^12.0.4",

public/components/nsite-card.js

@@ -0,0 +1,100 @@
+import { html, css, LitElement } from "lit";
+import { nip19 } from "nostr-tools";
+import { pool, relays } from "../pool.js";
+
+export class NsiteCard extends LitElement {
+  static styles = css`
+    :host {
+      min-width: 3in;
+      max-width: 4in;
+      border: 1px solid lightslategray;
+      display: flex;
+      flex-direction: column;
+      padding: 0.5em;
+      gap: 0.3em;
+      border-radius: 0.5em;
+    }
+
+    .title {
+      display: flex;
+      gap: 0.5em;
+      align-items: center;
+      color: initial;
+      text-decoration: none;
+    }
+    .title h3 {
+      margin: 0;
+    }
+    .avatar {
+      width: 3rem;
+      height: 3rem;
+      border: none;
+      outline: none;
+      border-radius: 50%;
+    }
+
+    .thumb {
+      display: flex;
+      overflow: hidden;
+    }
+
+    .thumb > img {
+      width: 100%;
+      border-radius: 0.5em;
+    }
+
+    .about {
+      display: -webkit-box;
+      -webkit-line-clamp: 3;
+      -webkit-box-orient: vertical;
+      overflow: hidden;
+    }
+
+    time {
+      margin-top: auto;
+    }
+  `;
+
+  static properties = {
+    nsite: { type: Object },
+    profile: { state: true, type: Object },
+  };
+
+  constructor() {
+    super();
+  }
+
+  connectedCallback() {
+    super.connectedCallback();
+
+    pool.get(relays, { kinds: [0], authors: [this.nsite.pubkey] }).then((event) => {
+      if (event) this.profile = JSON.parse(event.content);
+    });
+  }
+
+  render() {
+    const npub = nip19.npubEncode(this.nsite.pubkey);
+    const url = new URL("/", `${location.protocol}//${npub}.${location.host}`);
+
+    return html`
+      <a class="thumb" href="${url}" target="_blank">
+        <img src="/screenshot/${this.nsite.pubkey}.png" />
+      </a>
+      <a class="title" href="${url}" target="_blank">
+        ${this.profile && html`<img src="${this.profile.image || this.profile.picture}" class="avatar" />`}
+        <div>
+          ${this.profile
+            ? html`
+                <h3>${this.profile.display_name || this.profile.name}</h3>
+                <small>${this.profile.nip05}</small>
+              `
+            : html`<h3>${npub.slice(0, 8)}</h3>`}
+        </div>
+      </a>
+      ${this.profile && html`<p class="about">${this.profile.about}</p>`}
+      <time>${new Date(this.nsite.created_at * 1000).toDateString()}</time>
+    `;
+  }
+}
+
+customElements.define("nsite-card", NsiteCard);

public/index.html

@@ -5,16 +5,13 @@
     <meta name="viewport" content="width=device-width, initial-scale=1.0" />
     <title>nsite</title>
 
-    <link rel="stylesheet" href="/lib/normalize.css" />
-    <link rel="stylesheet" href="/lib/milligram.css" />
-    <script src="/lib/nostr-name.js"></script>
-    <script src="/lib/nostr-picture.js"></script>
-
     <script type="importmap">
       {
         "imports": {
           "blossom-client-sdk": "https://esm.run/blossom-client-sdk",
-          "nostr-tools": "https://esm.run/nostr-tools"
+          "nostr-tools": "https://esm.run/nostr-tools",
+          "lit": "https://esm.run/lit",
+          "lit/directives/repeat.js": "https://esm.run/lit/directives/repeat.js"
         }
       }
     </script>
@@ -24,25 +21,9 @@
         width: 100%;
       }
     </style>
+    <script type="module" src="./main.js"></script>
   </head>
   <body>
-    <div class="container">
-      <img src="/logo.jpg" style="max-height: 2in" />
-      <h1>nsite</h1>
-      <a class="navbar-item" href="https://github.com/hzrd149/nsite-ts" target="_blank">Source Code</a>
-
-      <h2 class="subtitle is-2">Latest nsites:</h2>
-      <div id="sites" style="display: flex; flex-direction: column; gap: 0.5em"></div>
-    </div>
-
-    <template id="site">
-      <a class="card nsite-link" style="display: flex; flex-direction: column">
-        <nostr-picture style="max-width: 48px; max-height: 48px; overflow: hidden"></nostr-picture>
-        <nostr-name class="title is-4"></nostr-name>
-        <time datetime="2016-1-1">11:09 PM - 1 Jan 2016</time>
-      </a>
-    </template>
-
-    <script type="module" src="./main.js"></script>
+    <nsite-app></nsite-app>
   </body>
 </html>

public/main.js

@@ -1,35 +1,58 @@
-import { nip19, SimplePool } from "nostr-tools";
+import { LitElement, html, css } from "lit";
+import { repeat } from "lit/directives/repeat.js";
+import "./components/nsite-card.js";
+import { pool, relays } from "./pool.js";
 
-const relays = ["wss://relay.damus.io", "wss://nos.lol", "wss://nostr.wine"];
-const pool = new SimplePool();
+export class NsiteApp extends LitElement {
+  static properties = {
+    selected: { state: true },
+    status: { state: true, type: String },
+    sites: { state: true, type: Array },
+  };
 
-const seen = new Set();
-function addSite(event) {
-  if (seen.has(event.pubkey)) return;
-  seen.add(event.pubkey);
+  static styles = css`
+    .sites {
+      display: flex;
+      gap: 0.5em;
+      flex-wrap: wrap;
+    }
+  `;
 
-  try {
-    const template = document.getElementById("site");
-    const site = template.content.cloneNode(true);
-    const npub = nip19.npubEncode(event.pubkey);
+  seen = new Set();
+  constructor() {
+    super();
+    this.sites = [];
+  }
 
-    site.querySelector("nostr-name").setAttribute("pubkey", event.pubkey);
-    site.querySelector("nostr-name").textContent = npub.slice(0, 8);
-    site.querySelector("nostr-picture").setAttribute("pubkey", event.pubkey);
+  connectedCallback() {
+    super.connectedCallback();
 
-    site
-      .querySelector(".nsite-link")
-      ?.setAttribute("href", new URL("/", `${location.protocol}//${npub}.${location.host}`).toString());
-    site.querySelector("time").textContent = new Date(event.created_at * 1000).toDateString();
+    pool.subscribeMany(relays, [{ kinds: [34128], "#d": ["/index.html"] }], {
+      onevent: (event) => {
+        if (this.seen.has(event.pubkey)) return;
+        this.seen.add(event.pubkey);
 
-    document.getElementById("sites").appendChild(site);
-  } catch (error) {
-    console.log("Failed to add site", event);
-    console.log(error);
+        this.sites = [...this.sites, event].sort((a, b) => b.created_at - a.created_at);
+      },
+    });
+  }
+
+  render() {
+    return html`<div class="container">
+      <img src="/logo.jpg" style="max-height: 2in" />
+      <h1>nsite</h1>
+      <a class="navbar-item" href="https://github.com/hzrd149/nsite-ts" target="_blank">Source Code</a>
+
+      <h2 class="subtitle is-2">Latest nsites:</h2>
+      <div class="sites">
+        ${repeat(
+          this.sites,
+          (nsite) => nsite.pubkey,
+          (nsite) => html`<nsite-card .nsite="${nsite}"></nsite-card>`,
+        )}
+      </div>
+    </div>`;
   }
 }
 
-console.log("Loading sites");
-pool.subscribeMany(relays, [{ kinds: [34128], "#d": ["/index.html"] }], {
-  onevent: addSite,
-});
+customElements.define("nsite-app", NsiteApp);

public/pool.js

@@ -0,0 +1,4 @@
+import { SimplePool } from "nostr-tools";
+
+export const relays = ["wss://relay.damus.io", "wss://nos.lol", "wss://nostr.wine"];
+export const pool = new SimplePool();

src/env.ts

@@ -5,7 +5,10 @@ const LOOKUP_RELAYS = process.env.LOOKUP_RELAYS?.split(",").map((u) => u.trim())
   "wss://user.kindpag.es/",
   "wss://purplepag.es/",
 ];
-const SUBSCRIPTION_RELAYS = process.env.SUBSCRIPTION_RELAYS?.split(",").map((u) => u.trim()) ?? [];
+const SUBSCRIPTION_RELAYS = process.env.SUBSCRIPTION_RELAYS?.split(",").map((u) => u.trim()) ?? [
+  "wss://nos.lol",
+  "wss://relay.damus.io",
+];
 const BLOSSOM_SERVERS = process.env.BLOSSOM_SERVERS?.split(",").map((u) => u.trim()) ?? [];
 
 const MAX_FILE_SIZE = process.env.MAX_FILE_SIZE ? xbytes.parseSize(process.env.MAX_FILE_SIZE) : Infinity;
@@ -17,6 +20,12 @@ const PAC_PROXY = process.env.PAC_PROXY;
 const TOR_PROXY = process.env.TOR_PROXY;
 const I2P_PROXY = process.env.I2P_PROXY;
 
+const NSITE_HOST = process.env.NSITE_HOST || "0.0.0.0";
+const NSITE_PORT = process.env.NSITE_PORT ? parseInt(process.env.NSITE_PORT) : 3000;
+const HOST = `${NSITE_HOST}:${NSITE_PORT}`;
+
+const SCREENSHOTS_DIR = process.env.SCREENSHOTS_DIR || "./screenshots";
+
 export {
   SUBSCRIPTION_RELAYS,
   LOOKUP_RELAYS,
@@ -27,4 +36,8 @@ export {
   PAC_PROXY,
   TOR_PROXY,
   I2P_PROXY,
+  NSITE_HOST,
+  NSITE_PORT,
+  HOST,
+  SCREENSHOTS_DIR,
 };

src/index.ts

@@ -2,21 +2,22 @@
 import "./polyfill.js";
 import Koa from "koa";
 import serve from "koa-static";
-import path from "node:path";
+import path, { basename } from "node:path";
 import cors from "@koa/cors";
 import fs from "node:fs";
 import { fileURLToPath } from "node:url";
 import mime from "mime";
 import morgan from "koa-morgan";
+import send from "koa-send";
 
 import { resolveNpubFromHostname } from "./helpers/dns.js";
 import { getNsiteBlobs, parseNsiteEvent } from "./events.js";
 import { downloadFile, getUserBlossomServers } from "./blossom.js";
-import { BLOSSOM_SERVERS, NGINX_CACHE_DIR, SUBSCRIPTION_RELAYS } from "./env.js";
+import { BLOSSOM_SERVERS, HOST, NGINX_CACHE_DIR, NSITE_HOST, NSITE_PORT, SUBSCRIPTION_RELAYS } from "./env.js";
 import { userDomains, userRelays, userServers } from "./cache.js";
-import { NSITE_KIND } from "./const.js";
 import { invalidatePubkeyPath } from "./nginx.js";
 import pool, { getUserOutboxes, subscribeForEvents } from "./nostr.js";
+import { getScreenshotPath, hasScreenshot, removeScreenshot, takeScreenshot } from "./screenshots.js";
 
 const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
@@ -43,7 +44,7 @@ app.use(async (ctx, next) => {
   } catch (err) {
     console.log(err);
     ctx.status = 500;
-    ctx.body = { message: "Something went wrong" };
+    if (err instanceof Error) ctx.body = { message: err.message };
   }
 });
 
@@ -84,6 +85,10 @@ app.use(async (ctx, next) => {
       }
     }
 
+    relays.push(...SUBSCRIPTION_RELAYS);
+
+    if (relays.length === 0) throw new Error("No nostr relays");
+
     console.log(`${pubkey}: Searching for ${ctx.path}`);
     const blobs = await getNsiteBlobs(pubkey, ctx.path, relays);
 
@@ -145,27 +150,40 @@ try {
   app.use(serve(www));
 }
 
-app.listen(
-  {
-    port: process.env.NSITE_PORT || 3000,
-    host: process.env.NSITE_HOST || "0.0.0.0",
-  },
-  () => {
-    console.log("Started on port", process.env.PORT || 3000);
-  },
-);
+// get screenshots for websites
+app.use(async (ctx, next) => {
+  if (ctx.method === "GET" && ctx.path.startsWith("/screenshot")) {
+    const [pubkey, etx] = basename(ctx.path).split(".");
 
-// invalidate nginx cache on new events
-if (NGINX_CACHE_DIR && SUBSCRIPTION_RELAYS.length > 0) {
+    if (pubkey) {
+      if (!(await hasScreenshot(pubkey))) await takeScreenshot(pubkey);
+
+      await send(ctx, getScreenshotPath(pubkey));
+    } else throw Error("Missing pubkey");
+  } else return next();
+});
+
+app.listen({ host: NSITE_HOST, port: NSITE_PORT }, () => {
+  console.log("Started on port", HOST);
+});
+
+// invalidate nginx cache and screenshots on new events
+if (SUBSCRIPTION_RELAYS.length > 0) {
   console.log(`Listening for new nsite events`);
-
   subscribeForEvents(SUBSCRIPTION_RELAYS, async (event) => {
     try {
       const nsite = parseNsiteEvent(event);
       if (nsite) {
+        if (NGINX_CACHE_DIR) {
           console.log(`${nsite.pubkey}: Invalidating ${nsite.path}`);
           await invalidatePubkeyPath(nsite.pubkey, nsite.path);
         }
+
+        // invalidate screenshot for nsite
+        if (nsite.path === "/" || nsite.path === "/index.html") {
+          await removeScreenshot(nsite.pubkey);
+        }
+      }
     } catch (error) {
       console.log(`Failed to invalidate ${event.id}`);
     }

src/screenshots.ts

@@ -0,0 +1,40 @@
+import { nip19 } from "nostr-tools";
+import puppeteer from "puppeteer";
+import { join } from "path";
+import pfs from "fs/promises";
+
+import { NSITE_PORT, SCREENSHOTS_DIR } from "./env.js";
+
+try {
+  await pfs.mkdir(SCREENSHOTS_DIR, { recursive: true });
+} catch (error) {}
+
+export function getScreenshotPath(pubkey: string) {
+  return join(SCREENSHOTS_DIR, pubkey + ".png");
+}
+
+export async function hasScreenshot(pubkey: string) {
+  try {
+    await pfs.stat(getScreenshotPath(pubkey));
+    return true;
+  } catch (error) {
+    return false;
+  }
+}
+
+export async function takeScreenshot(pubkey: string) {
+  console.log(`${pubkey}: Generating screenshot`);
+
+  const browser = await puppeteer.launch();
+  const page = await browser.newPage();
+  const url = new URL(`http://${nip19.npubEncode(pubkey)}.localhost:${NSITE_PORT}`);
+  await page.goto(url.toString());
+  await page.screenshot({ path: getScreenshotPath(pubkey) });
+  await browser.close();
+}
+
+export async function removeScreenshot(pubkey: string) {
+  try {
+    await pfs.rm(getScreenshotPath(pubkey));
+  } catch (error) {}
+}