diff --git a/.env b/.env index 98e4615..bafcc40 100644 --- a/.env +++ b/.env @@ -1,4 +1,6 @@ +CACHE_PATH="in-memory" + NOSTR_RELAYS=wss://nos.lol,wss://relay.damus.io BLOSSOM_SERVERS=https://cdn.hzrd149.com -MAX_FILE_SIZE='2 MB' +NGINX_HOST='nginx' diff --git a/.env.example b/.env.example index 12008f3..4e33d8b 100644 --- a/.env.example +++ b/.env.example @@ -1,3 +1,7 @@ +# where to cache nostr events +# can be in-memory, redis:// or sqlite:// +CACHE_PATH="in-memory" + # A list of nostr relays to search NOSTR_RELAYS=wss://nos.lol,wss://relay.damus.io @@ -6,3 +10,6 @@ BLOSSOM_SERVERS=https://cdn.satellite.earth # The max file size to serve MAX_FILE_SIZE='2 MB' + +# the hostname or ip of the upstream nginx proxy cache +NGINX_HOST='nginx' diff --git a/.github/workflows/docker-image.yml b/.github/workflows/docker-image.yml new file mode 100644 index 0000000..7a12791 --- /dev/null +++ b/.github/workflows/docker-image.yml @@ -0,0 +1,60 @@ +name: Docker image + +on: + push: + branches: + - "**" + tags: + - "v*.*.*" + pull_request: + branches: + - "master" + +env: + REGISTRY: ghcr.io + IMAGE_NAME: ${{ github.repository }} + +jobs: + build-and-push-image: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Log in to the Container registry + uses: docker/login-action@v3 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract metadata (tags, labels) for Docker + id: meta + uses: docker/metadata-action@v5 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + tags: | + type=semver,pattern={{version}} + type=semver,pattern={{major}}.{{minor}} + type=semver,pattern={{major}} + type=ref,event=branch + type=ref,event=pr + + - name: Build and push Docker image + uses: docker/build-push-action@v5 + with: + context: . + push: true + platforms: linux/amd64,linux/arm64 + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} diff --git a/.vscode/launch.json b/.vscode/launch.json index 7f7b360..a9b8b50 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -1,11 +1,8 @@ { - // Use IntelliSense to learn about possible attributes. - // Hover to view descriptions of existing attributes. - // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 "version": "0.2.0", "configurations": [ { - "name": "node", + "name": "dev", "type": "node", "request": "launch", "args": ["./src/index.ts"], diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..5e41345 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,30 @@ +# syntax=docker/dockerfile:1 +FROM node:20-alpine AS base + +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +RUN corepack enable + +WORKDIR /app +COPY package.json . +COPY pnpm-lock.yaml . + +FROM base AS prod-deps +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --prod --frozen-lockfile + +FROM base AS build +RUN --mount=type=cache,id=pnpm,target=/pnpm/store pnpm install --frozen-lockfile +COPY tsconfig.json . +COPY src ./src +RUN pnpm build + +FROM base AS main +COPY --from=prod-deps /app/node_modules /app/node_modules +COPY --from=build ./app/build ./build + +COPY ./public ./public + +EXPOSE 80 +ENV PORT="80" + +ENTRYPOINT [ "node", "." ] diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..6dd837c --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,21 @@ +version: "3.7" + +volumes: + cache: {} + +services: + nginx: + image: nginx:alpine + ports: + - 8080:80 + volumes: + - cache:/var/cache/nginx + - ./nginx/nginx.conf:/etc/nginx/nginx.conf + - ./nginx/default.conf:/etc/nginx/conf.d/default.conf + + nsite: + build: . + environment: + NOSTR_RELAYS: wss://nos.lol,wss://relay.damus.io + ports: + - 3000:80 diff --git a/nginx/default.conf b/nginx/default.conf new file mode 100644 index 0000000..8ff7ac4 --- /dev/null +++ b/nginx/default.conf @@ -0,0 +1,28 @@ +server { + listen 80; + server_name nsite-proxy; + + location / { + proxy_cache request_cache; + proxy_cache_valid 200 60m; + proxy_cache_valid 404 10m; + proxy_cache_key $scheme$proxy_host$request_uri; + proxy_cache_use_stale error timeout updating http_500 http_502 http_503 http_504; + + add_header X-Cache $upstream_cache_status; + add_header X-Cache-Status $upstream_status; + + expires 30d; + add_header Cache-Control "public, no-transform"; + + proxy_set_header Host $host; + proxy_pass http://nsite; + } + + # Manual cache invalidation ( cant use proxy_cache_purge ) + # location ~ /purge(/.*) { + # allow 127.0.0.1; + # deny all; + # proxy_cache_purge request_cache $scheme$proxy_host$1; + # } +} diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..e3236e3 --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,33 @@ +user nginx; +worker_processes auto; + +error_log /var/log/nginx/error.log notice; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + # add custom cache + proxy_cache_path /var/cache/nginx levels=1:2 keys_zone=request_cache:10m max_size=10g inactive=60m use_temp_path=off; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent" "$http_x_forwarded_for"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + #tcp_nopush on; + + keepalive_timeout 65; + + gzip on; + + include /etc/nginx/conf.d/*.conf; +} + diff --git a/package.json b/package.json index b8e182a..2803b96 100644 --- a/package.json +++ b/package.json @@ -18,23 +18,19 @@ "public" ], "dependencies": { + "@keyv/redis": "^3.0.1", "@keyv/sqlite": "^4.0.1", "@koa/cors": "^5.0.0", - "@koa/router": "^12.0.1", "@nostr-dev-kit/ndk": "^2.10.0", "blossom-client-sdk": "^1.1.1", - "debug": "^4.3.5", "dotenv": "^16.4.5", "follow-redirects": "^1.15.6", - "http-errors": "1", "keyv": "^5.0.1", "koa": "^2.15.3", - "koa-mount": "^4.0.0", - "koa-send": "^5.0.1", + "koa-morgan": "^1.0.1", "koa-static": "^5.0.0", "mime": "^4.0.4", "nostr-tools": "^2.7.2", - "socks-proxy-agent": "^8.0.4", "websocket-polyfill": "^1.0.0", "ws": "^8.18.0", "xbytes": "^1.9.1" @@ -44,13 +40,9 @@ "@swc-node/register": "^1.9.0", "@swc/core": "^1.5.0", "@types/better-sqlite3": "^7.6.9", - "@types/debug": "^4.1.12", "@types/follow-redirects": "^1.14.4", - "@types/http-errors": "^2.0.4", "@types/koa": "^2.14.0", - "@types/koa-basic-auth": "^2.0.6", - "@types/koa-mount": "^4.0.5", - "@types/koa-send": "^4.1.6", + "@types/koa-morgan": "^1.0.8", "@types/koa-static": "^4.0.4", "@types/koa__cors": "^5.0.0", "@types/koa__router": "^12.0.4", @@ -62,5 +54,6 @@ }, "resolutions": { "websocket-polyfill": "1.0.0" - } + }, + "packageManager": "pnpm@9.6.0" } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 22efa6b..bb1127b 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -11,45 +11,36 @@ importers: .: dependencies: + '@keyv/redis': + specifier: ^3.0.1 + version: 3.0.1 '@keyv/sqlite': specifier: ^4.0.1 version: 4.0.1 '@koa/cors': specifier: ^5.0.0 version: 5.0.0 - '@koa/router': - specifier: ^12.0.1 - version: 12.0.2 '@nostr-dev-kit/ndk': specifier: ^2.10.0 version: 2.10.0(typescript@5.6.2) blossom-client-sdk: specifier: ^1.1.1 version: 1.1.1 - debug: - specifier: ^4.3.5 - version: 4.3.7(supports-color@5.5.0) dotenv: specifier: ^16.4.5 version: 16.4.5 follow-redirects: specifier: ^1.15.6 - version: 1.15.9(debug@4.3.7) - http-errors: - specifier: '1' - version: 1.8.1 + version: 1.15.9 keyv: specifier: ^5.0.1 version: 5.0.1 koa: specifier: ^2.15.3 version: 2.15.3 - koa-mount: - specifier: ^4.0.0 - version: 4.0.0 - koa-send: - specifier: ^5.0.1 - version: 5.0.1 + koa-morgan: + specifier: ^1.0.1 + version: 1.0.1 koa-static: specifier: ^5.0.0 version: 5.0.0 @@ -59,9 +50,6 @@ importers: nostr-tools: specifier: ^2.7.2 version: 2.7.2(typescript@5.6.2) - socks-proxy-agent: - specifier: ^8.0.4 - version: 8.0.4 websocket-polyfill: specifier: 1.0.0 version: 1.0.0 @@ -84,27 +72,15 @@ importers: '@types/better-sqlite3': specifier: ^7.6.9 version: 7.6.11 - '@types/debug': - specifier: ^4.1.12 - version: 4.1.12 '@types/follow-redirects': specifier: ^1.14.4 version: 1.14.4 - '@types/http-errors': - specifier: ^2.0.4 - version: 2.0.4 '@types/koa': specifier: ^2.14.0 version: 2.15.0 - '@types/koa-basic-auth': - specifier: ^2.0.6 - version: 2.0.6 - '@types/koa-mount': - specifier: ^4.0.5 - version: 4.0.5 - '@types/koa-send': - specifier: ^4.1.6 - version: 4.1.6 + '@types/koa-morgan': + specifier: ^1.0.8 + version: 1.0.8 '@types/koa-static': specifier: ^4.0.4 version: 4.0.4 @@ -203,6 +179,13 @@ packages: '@gar/promisify@1.1.3': resolution: {integrity: sha512-k2Ty1JcVojjJFwrg/ThKi2ujJ7XNLYaFGNB/bWT9wGR+oSMJHMa5w+CUq6p/pVrKeNNgA7pCqEcjSnHVoqJQFw==} + '@ioredis/commands@1.2.0': + resolution: {integrity: sha512-Sx1pU8EM64o2BrqNpEO1CNLtKQwyhuXuqyfH7oGKCk+1a33d2r5saW8zNwm3j6BTExtjrv2BxTgzzkMwts6vGg==} + + '@keyv/redis@3.0.1': + resolution: {integrity: sha512-eyqzomQC76LjUOEkPP8rdR2Fk4eZBSS0Ma47i7CNiQuv8NCw3trZvghx8L5Xruk7XPEj/eRAMrAxP//xQFOPdQ==} + engines: {node: '>= 18'} + '@keyv/serialize@1.0.1': resolution: {integrity: sha512-kKXeynfORDGPUEEl2PvTExM2zs+IldC6ZD8jPcfvI351MDNtfMlw9V9s4XZXuJNDK2qR5gbEKxRyoYx3quHUVQ==} @@ -214,10 +197,6 @@ packages: resolution: {integrity: sha512-x/iUDjcS90W69PryLDIMgFyV21YLTnG9zOpPXS7Bkt2b8AsY3zZsIpOLBkYr9fBcF3HbkKaER5hOBZLfpLgYNw==} engines: {node: '>= 14.0.0'} - '@koa/router@12.0.2': - resolution: {integrity: sha512-sYcHglGKTxGF+hQ6x67xDfkE9o+NhVlRHBqq6gLywaMc6CojK/5vFZByphdonKinYlMLkEkacm+HEse9HzwgTA==} - engines: {node: '>= 12'} - '@manypkg/find-root@1.1.0': resolution: {integrity: sha512-mki5uBvhHzO8kYYix/WRy2WX8S3B5wdVSc9D6KcU5lQNglP2yt58/VfLuAK49glRXChosY8ap2oJ1qgma3GUVA==} @@ -462,9 +441,6 @@ packages: '@types/cookies@0.9.0': resolution: {integrity: sha512-40Zk8qR147RABiQ7NQnBzWzDcjKzNrntB5BAmeGCb2p/MIyOE+4BVvc17wumsUqUw00bJYqoXFHYygQnEFh4/Q==} - '@types/debug@4.1.12': - resolution: {integrity: sha512-vIChWdVG3LG1SMxEvI/AK+FWJthlrqlTu7fbrlywTkkaONwk/UAGaULXRlf8vkzFBLVm0zkMdCquhL5aOjhXPQ==} - '@types/express-serve-static-core@4.19.5': resolution: {integrity: sha512-y6W03tvrACO72aijJ5uF02FRq5cgDR9lUxddQ8vyF+GvmjJQqbzDcJngEjURc+ZsG31VI3hODNZJ2URj86pzmg==} @@ -483,14 +459,11 @@ packages: '@types/keygrip@1.0.6': resolution: {integrity: sha512-lZuNAY9xeJt7Bx4t4dx0rYCDqGPW8RXhQZK1td7d4H6E9zYbLoOtjBvfwdTKpsyxQI/2jv+armjX/RW+ZNpXOQ==} - '@types/koa-basic-auth@2.0.6': - resolution: {integrity: sha512-1/FdT3KiHIkVf+TxYiPPey0wnPBzuts6lz/Obskgo9ZY485J02+uI6STnD114L2iG+Wi5MBqU7EYNphKdKwZWQ==} - '@types/koa-compose@3.2.8': resolution: {integrity: sha512-4Olc63RY+MKvxMwVknCUDhRQX1pFQoBZ/lXcRLP69PQkEpze/0cr8LNqJQe5NFb/b19DWi2a5bTi2VAlQzhJuA==} - '@types/koa-mount@4.0.5': - resolution: {integrity: sha512-pV1njJ7r94iqAFzT9D5sGSYKUHFGudCLAnmr4WFli7V5tJf5MAgRQK9leTPJ4gjvgr+hnTf86fZsKoFN358c7w==} + '@types/koa-morgan@1.0.8': + resolution: {integrity: sha512-2GredUi+iA3V0XrbzdsOAYgwj4F6+FnN+f5YjoKjessIE2lrMkqnc06YQQnzbMG75hRsXjyD+p6d5vlI70s1vg==} '@types/koa-send@4.1.6': resolution: {integrity: sha512-vgnNGoOJkx7FrF0Jl6rbK1f8bBecqAchKpXtKuXzqIEdXTDO6dsSTjr+eZ5m7ltSjH4K/E7auNJEQCAd0McUPA==} @@ -510,8 +483,8 @@ packages: '@types/mime@1.3.5': resolution: {integrity: sha512-/pyBZWSLD2n0dcHE3hq8s8ZvcETHtEuF+3E7XVt0Ig2nvsVQXdghHVcEkIWjy9A0wKfTn97a/PSDYohKIlnP/w==} - '@types/ms@0.7.34': - resolution: {integrity: sha512-nG96G3Wp6acyAgJqGasjODb+acrI7KltPiRxzHPXnP3NgI28bpQDRv53olbqGXbfcgF5aiiHmO3xpwEpS5Ld9g==} + '@types/morgan@1.9.9': + resolution: {integrity: sha512-iRYSDKVaC6FkGSpEVVIvrRGw0DfJMiQzIn3qr2G5B3C//AWkulhXgaBd7tS9/J79GWSYMTHGs7PfI5b3Y8m+RQ==} '@types/node@12.20.55': resolution: {integrity: sha512-J8xLz7q2OFulZ2cyGTLE1TbbZcjpno7FaN6zdJNrgAdrJ+DZzh/uFR6YrTb4C+nXakvud8Q4+rbhoIWlYQbUFQ==} @@ -548,10 +521,6 @@ packages: resolution: {integrity: sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==} engines: {node: '>= 6.0.0'} - agent-base@7.1.1: - resolution: {integrity: sha512-H0TSyFNDMomMNJQBn8wFV5YC/2eJ+VXECwOadZJT554xP6cODZHPX3H9QMQECxvrgiSOP1pHjy1sMWQVYJOUOA==} - engines: {node: '>= 14'} - agentkeepalive@4.5.0: resolution: {integrity: sha512-5GG/5IbQQpC9FpkRGsSvZI5QYeSCzlJHdpBQntCsuTOxhKD8lqKhrleg2Yi7yvMIf82Ycmmqln9U8V9qwEiJew==} engines: {node: '>= 8.0.0'} @@ -593,6 +562,10 @@ packages: base64-js@1.5.1: resolution: {integrity: sha512-AKpaYlHn8t4SVbOHCy+b5+KKgvR4vrsD8vbvrbiQJps7fKDTkjkDry6ji0rUJjC0kzbNePLwzxq8iypo41qeWA==} + basic-auth@2.0.1: + resolution: {integrity: sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==} + engines: {node: '>= 0.8'} + better-path-resolve@1.0.0: resolution: {integrity: sha512-pbnl5XzGBdrFU/wT4jqmJVPn2B6UHPBOhzMQkY/SPUPB6QtUXtmBHBIwCbXJol93mOpGMnQyP/+BB19q04xj7g==} engines: {node: '>=4'} @@ -657,6 +630,10 @@ packages: resolution: {integrity: sha512-4diC9HaTE+KRAMWhDhrGOECgWZxoevMc5TlkObMqNSsVU62PYzXZ/SMTjzyGAFF1YusgxGcSWTEXBhp0CPwQ1A==} engines: {node: '>=6'} + cluster-key-slot@1.1.2: + resolution: {integrity: sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA==} + engines: {node: '>=0.10.0'} + co@4.6.0: resolution: {integrity: sha512-QVb0dM5HvG+uaxitm8wONl7jltx8dqhfU33DcqtOZcLSVIKSDDLDi7+0LbAKiyI8hD9u42m2YxXSkMGWThaecQ==} engines: {iojs: '>= 1.0.0', node: '>= 0.12.0'} @@ -693,6 +670,14 @@ packages: resolution: {integrity: sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==} engines: {node: '>= 12'} + debug@2.6.9: + resolution: {integrity: sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==} + peerDependencies: + supports-color: '*' + peerDependenciesMeta: + supports-color: + optional: true + debug@3.2.7: resolution: {integrity: sha512-CFjzYYAi4ThfiQvizrFQevTTXHtnCqWfe7x1AhgEscTz6ZbLbfoLRLPugTQyBth6f8ZERVUSyWHFD/7Wu4t1XQ==} peerDependencies: @@ -724,6 +709,10 @@ packages: delegates@1.0.0: resolution: {integrity: sha512-bd2L678uiWATM6m5Z1VzNCErI3jiGzt6HGY8OVICs40JQq/HALfbyNJmp0UDakEY4pMMaN0Ly5om/B1VI/+xfQ==} + denque@2.1.0: + resolution: {integrity: sha512-HVQE3AAb/pxF8fQAoiqpvg9i3evqug3hoiwakOyZAwJm+6vZehbkYXZ0l4JxS+I3QxM97v5aaRNhj8v5oBhekw==} + engines: {node: '>=0.10'} + depd@1.1.2: resolution: {integrity: sha512-7emPTl6Dpo6JRXOXjLRxck+FlLRX5847cLKEn00PLAgc3g2hTZZgr+e4c2v6QpSmLeFP3n5yUo7ft6avBK/5jQ==} engines: {node: '>= 0.6'} @@ -913,10 +902,6 @@ packages: resolution: {integrity: sha512-Kpk9Sm7NmI+RHhnj6OIWDI1d6fIoFAtFt9RLaTMRlg/8w49juAStsrBgp0Dp4OdxdVbRIeKhtCUvoi/RuAhO4g==} engines: {node: '>= 0.6'} - http-errors@2.0.0: - resolution: {integrity: sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==} - engines: {node: '>= 0.8'} - http-proxy-agent@4.0.1: resolution: {integrity: sha512-k0zdNgqWTGA6aeIRVpvfVob4fL52dTfaehylg0Y4UvSySvOq/Y+BOyPrgpUrA7HylqvU8vIZGsRuXmspskV0Tg==} engines: {node: '>= 6'} @@ -976,6 +961,10 @@ packages: ini@1.3.8: resolution: {integrity: sha512-JV/yugV2uzW5iMRSiZAyDtQd+nxtUnjeLt0acNdw98kKLrvuRVyB80tsREOE7yvGVgalhZ6RNXCmEHkUKBKxew==} + ioredis@5.4.1: + resolution: {integrity: sha512-2YZsvl7jopIa1gaePkeMtd9rAcSjOOjPtpcLlOeusyO+XH2SK5ZcT+UCrElPP+WVIInh2TzeI4XW9ENaSLVVHA==} + engines: {node: '>=12.22.0'} + ip-address@9.0.5: resolution: {integrity: sha512-zHtQzGojZXTwZTHQqra+ETKd4Sn3vgi7uBmlPoXVWZqYvuKmtI0l/VZTjqGmJY9x88GGOaZ9+G9ES8hC4T4X8g==} engines: {node: '>= 12'} @@ -1042,9 +1031,8 @@ packages: resolution: {integrity: sha512-asOvN6bFlSnxewce2e/DK3p4tltyfC4VM7ZwuTuepI7dEQVcvpyFuBcEARu1+Hxg8DIwytce2n7jrZtRlPrARA==} engines: {node: '>= 10'} - koa-mount@4.0.0: - resolution: {integrity: sha512-rm71jaA/P+6HeCpoRhmCv8KVBIi0tfGuO/dMKicbQnQW/YJntJ6MnnspkodoA4QstMVEZArsCphmd0bJEtoMjQ==} - engines: {node: '>= 7.6.0'} + koa-morgan@1.0.1: + resolution: {integrity: sha512-JOUdCNlc21G50afBXfErUrr1RKymbgzlrO5KURY+wmDG1Uvd2jmxUJcHgylb/mYXy2SjiNZyYim/ptUBGsIi3A==} koa-send@5.0.1: resolution: {integrity: sha512-tmcyQ/wXXuxpDxyNXv5yNNkdAMdFRqwtegBXUaowiQzUKqJehttS0x2j0eOZDQAyloAth5w6wwBImnFzkUz3pQ==} @@ -1065,6 +1053,12 @@ packages: resolution: {integrity: sha512-t7hw9pI+WvuwNJXwk5zVHpyhIqzg2qTlklJOf0mVxGSbe3Fp2VieZcduNYjaLDoy6p9uGpQEGWG87WpMKlNq8g==} engines: {node: '>=8'} + lodash.defaults@4.2.0: + resolution: {integrity: sha512-qjxPLHd3r5DnsdGacqOMU6pb/avJzdh9tFX2ymgoZE27BmjXrNy/y4LoaiTeAb+O3gL8AfpJGtqfX/ae2leYYQ==} + + lodash.isarguments@3.1.0: + resolution: {integrity: sha512-chi4NHZlZqZD18a0imDHnZPrDeBbTtVN7GXMwuGdRH9qotxAjYs3aVLKc7zNOG9eddR5Ksd8rvFEBc9SsggPpg==} + lodash.startcase@4.4.0: resolution: {integrity: sha512-+WKqsK294HMSc2jEbNgpHpd0JfIBhp7rEV4aqXWqFr6AlXov+SlcgB1Fv01y2kGe3Gc8nMW7VA0SrGuSkRfIEg==} @@ -1087,10 +1081,6 @@ packages: resolution: {integrity: sha512-8q7VEgMJW4J8tcfVPy8g09NcQwZdbwFEqhe/WZkoIzjn/3TGDwtOCYtXGxA3O8tPzpczCCDgv+P2P5y00ZJOOg==} engines: {node: '>= 8'} - methods@1.1.2: - resolution: {integrity: sha512-iclAHeNqNm68zFtnZ0e+1L2yUIdvzNoauKU4WBA3VvH/vPFieF7qfRlwUZU+DA9P9bPXIS90ulxoUoCH23sV2w==} - engines: {node: '>= 0.6'} - micromatch@4.0.8: resolution: {integrity: sha512-PXwfBhYu0hBCPw8Dn0E+WDYb7af3dSLVWKi3HGv84IdF4TyFoC0ysxFd0Goxw7nSv4T/PzEJQxsYsEiFCKo2BA==} engines: {node: '>=8.6'} @@ -1158,10 +1148,17 @@ packages: engines: {node: '>=10'} hasBin: true + morgan@1.10.0: + resolution: {integrity: sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==} + engines: {node: '>= 0.8.0'} + mri@1.2.0: resolution: {integrity: sha512-tzzskb3bG8LvYGFF/mDTpq3jpI6Q9wc3LEmBaghu+DdCssd1FakN7Bc0hVNmEyGq1bq3RgfkCb3cmQLpNPOroA==} engines: {node: '>=4'} + ms@2.0.0: + resolution: {integrity: sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==} + ms@2.1.3: resolution: {integrity: sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==} @@ -1222,10 +1219,18 @@ packages: engines: {node: ^12.13.0 || ^14.15.0 || >=16.0.0} deprecated: This package is no longer supported. + on-finished@2.3.0: + resolution: {integrity: sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==} + engines: {node: '>= 0.8'} + on-finished@2.4.1: resolution: {integrity: sha512-oVlzkg3ENAhCk2zdv7IJwd/QUD4z2RxRwpkcGY8psCVcCYZNq4wYnVWALHM+brtuJjePWiYF/ClmuDr8Ch5+kg==} engines: {node: '>= 0.8'} + on-headers@1.0.2: + resolution: {integrity: sha512-pZAE+FJLoyITytdqK0U5s+FIpjN0JP3OzFi/u8Rx+EV5/W+JTWGXG8xFzevE7AjBfDqHv/8vL8qQsIhHnqRkrA==} + engines: {node: '>= 0.8'} + once@1.4.0: resolution: {integrity: sha512-lNaJgI+2Q5URQBkccEKHTQOPaXdUxnZZElQTZY0MFUAuaEqe1E+Nyvgdz/aIyNi6Z9MzO5dv1H8n58/GELp3+w==} @@ -1281,9 +1286,6 @@ packages: resolution: {integrity: sha512-AVbw3UJ2e9bq64vSaS9Am0fje1Pa8pbGqTTsmXfaIiMpnr5DlDhfJOuLj9Sf95ZPVDAUerDfEk88MPmPe7UCQg==} engines: {node: '>=0.10.0'} - path-to-regexp@6.3.0: - resolution: {integrity: sha512-Yhpw4T9C6hPpgPeA28us07OJeqZ5EzQTkbfwuhsUg0c237RomFoETJgmp2sa3F/41gfLE6G5cqcYwznmeEeOlQ==} - path-type@4.0.0: resolution: {integrity: sha512-gDKb8aZMDeD/tZWs9P6+q0J9Mwkdl6xMV8TjnGP3qJVJ06bdMgkbBlLU8IdfOsIsFz2BW1rNVT3XuNEl8zPAvw==} engines: {node: '>=8'} @@ -1358,6 +1360,14 @@ packages: resolution: {integrity: sha512-hOS089on8RduqdbhvQ5Z37A0ESjsqz6qnRcffsMU3495FuTdqSm+7bhJ29JvIOsBDEEnan5DPu9t3To9VRlMzA==} engines: {node: '>=8.10.0'} + redis-errors@1.2.0: + resolution: {integrity: sha512-1qny3OExCf0UvUV/5wpYKf2YwPcOqXzkwKKSmKHiE6ZMQs5heeE/c8eXK+PNllPvmjgAbfnsbpkGZWy8cBpn9w==} + engines: {node: '>=4'} + + redis-parser@3.0.0: + resolution: {integrity: sha512-DJnGAeenTdpMEH6uAJRK/uiyEIH9WVsUmoLwzudwGJUwZPp80PDBWPHXSAGNPwNvIXAbe7MSUB1zQFugFml66A==} + engines: {node: '>=4'} + regenerator-runtime@0.14.1: resolution: {integrity: sha512-dYnhHh0nJoMfnkZs6GmmhFknAGRrLznOu5nc9ML+EJxGvrx6H7teuevqVqCuPcPK//3eDrrjQhehXVx9cnkGdw==} @@ -1385,6 +1395,9 @@ packages: run-parallel@1.2.0: resolution: {integrity: sha512-5l4VyZR86LZ/lDxZTR6jqL8AFE2S0IFLMP26AbjsLVADxHdhB/c0GUsH+y39UfCi3dzz8OlQuPmnaJOMoDHQBA==} + safe-buffer@5.1.2: + resolution: {integrity: sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==} + safe-buffer@5.2.1: resolution: {integrity: sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==} @@ -1438,10 +1451,6 @@ packages: resolution: {integrity: sha512-a6KW9G+6B3nWZ1yB8G7pJwL3ggLy1uTzKAgCb7ttblwqdz9fMGJUuTy3uFzEP48FAs9FLILlmzDlE2JJhVQaXQ==} engines: {node: '>= 10'} - socks-proxy-agent@8.0.4: - resolution: {integrity: sha512-GNAq/eg8Udq2x0eNiFkr9gRg5bA7PXEWagQdeRX4cPSG+X/8V38v637gim9bjFptMk1QWsCTr0ttrJEiXbNnRw==} - engines: {node: '>= 14'} - socks@2.8.3: resolution: {integrity: sha512-l5x7VUUWbjVFbafGLxPWkYsHIhEvmF85tbIeFZWc8ZPtoMyybuEhL7Jye/ooC4/d48FgOjSJXgsF/AJPYCW8Zw==} engines: {node: '>= 10.0.0', npm: '>= 3.0.0'} @@ -1469,14 +1478,13 @@ packages: resolution: {integrity: sha512-97qShzy1AiyxvPNIkLWoGua7xoQzzPjQ0HAH4B0rWKo7SZ6USuPcrUiAFrws0UH8RrbWmgq3LMTObhPIHbbBeQ==} engines: {node: '>= 8'} + standard-as-callback@2.1.0: + resolution: {integrity: sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A==} + statuses@1.5.0: resolution: {integrity: sha512-OpZ3zP+jT1PI7I8nemJX4AKmAX070ZkYPVWV/AaKTJl+tXCTGyVdC1a4SL8RUQYEwk/f34ZX8UTykN68FwrqAA==} engines: {node: '>= 0.6'} - statuses@2.0.1: - resolution: {integrity: sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==} - engines: {node: '>= 0.8'} - string-width@4.2.3: resolution: {integrity: sha512-wKyQRQpjJ0sIp62ErSZdGsjMJWsap5oRNihHhu6G7JVO/9jIB6UyevL+tXuOqrng8j/cxKTWyWUwvSTriiZz/g==} engines: {node: '>=8'} @@ -1803,6 +1811,14 @@ snapshots: '@gar/promisify@1.1.3': optional: true + '@ioredis/commands@1.2.0': {} + + '@keyv/redis@3.0.1': + dependencies: + ioredis: 5.4.1 + transitivePeerDependencies: + - supports-color + '@keyv/serialize@1.0.1': dependencies: buffer: 6.0.3 @@ -1818,16 +1834,6 @@ snapshots: dependencies: vary: 1.1.2 - '@koa/router@12.0.2': - dependencies: - debug: 4.3.7(supports-color@5.5.0) - http-errors: 2.0.0 - koa-compose: 4.1.0 - methods: 1.1.2 - path-to-regexp: 6.3.0 - transitivePeerDependencies: - - supports-color - '@manypkg/find-root@1.1.0': dependencies: '@babel/runtime': 7.25.6 @@ -2078,10 +2084,6 @@ snapshots: '@types/keygrip': 1.0.6 '@types/node': 20.16.5 - '@types/debug@4.1.12': - dependencies: - '@types/ms': 0.7.34 - '@types/express-serve-static-core@4.19.5': dependencies: '@types/node': 20.16.5 @@ -2106,17 +2108,14 @@ snapshots: '@types/keygrip@1.0.6': {} - '@types/koa-basic-auth@2.0.6': - dependencies: - '@types/koa': 2.15.0 - '@types/koa-compose@3.2.8': dependencies: '@types/koa': 2.15.0 - '@types/koa-mount@4.0.5': + '@types/koa-morgan@1.0.8': dependencies: '@types/koa': 2.15.0 + '@types/morgan': 1.9.9 '@types/koa-send@4.1.6': dependencies: @@ -2148,7 +2147,9 @@ snapshots: '@types/mime@1.3.5': {} - '@types/ms@0.7.34': {} + '@types/morgan@1.9.9': + dependencies: + '@types/node': 20.16.5 '@types/node@12.20.55': {} @@ -2192,12 +2193,6 @@ snapshots: - supports-color optional: true - agent-base@7.1.1: - dependencies: - debug: 4.3.7(supports-color@5.5.0) - transitivePeerDependencies: - - supports-color - agentkeepalive@4.5.0: dependencies: humanize-ms: 1.2.1 @@ -2237,6 +2232,10 @@ snapshots: base64-js@1.5.1: {} + basic-auth@2.0.1: + dependencies: + safe-buffer: 5.1.2 + better-path-resolve@1.0.0: dependencies: is-windows: 1.0.2 @@ -2330,6 +2329,8 @@ snapshots: clean-stack@2.2.0: optional: true + cluster-key-slot@1.1.2: {} + co@4.6.0: {} color-support@1.1.3: @@ -2361,6 +2362,10 @@ snapshots: data-uri-to-buffer@4.0.1: {} + debug@2.6.9: + dependencies: + ms: 2.0.0 + debug@3.2.7: dependencies: ms: 2.1.3 @@ -2381,6 +2386,8 @@ snapshots: delegates@1.0.0: {} + denque@2.1.0: {} + depd@1.1.2: {} depd@2.0.0: {} @@ -2466,9 +2473,7 @@ snapshots: locate-path: 5.0.0 path-exists: 4.0.0 - follow-redirects@1.15.9(debug@4.3.7): - optionalDependencies: - debug: 4.3.7(supports-color@5.5.0) + follow-redirects@1.15.9: {} formdata-polyfill@4.0.10: dependencies: @@ -2573,14 +2578,6 @@ snapshots: statuses: 1.5.0 toidentifier: 1.0.1 - http-errors@2.0.0: - dependencies: - depd: 2.0.0 - inherits: 2.0.4 - setprototypeof: 1.2.0 - statuses: 2.0.1 - toidentifier: 1.0.1 - http-proxy-agent@4.0.1: dependencies: '@tootallnate/once': 1.1.2 @@ -2643,10 +2640,25 @@ snapshots: ini@1.3.8: {} + ioredis@5.4.1: + dependencies: + '@ioredis/commands': 1.2.0 + cluster-key-slot: 1.1.2 + debug: 4.3.7(supports-color@5.5.0) + denque: 2.1.0 + lodash.defaults: 4.2.0 + lodash.isarguments: 3.1.0 + redis-errors: 1.2.0 + redis-parser: 3.0.0 + standard-as-callback: 2.1.0 + transitivePeerDependencies: + - supports-color + ip-address@9.0.5: dependencies: jsbn: 1.1.0 sprintf-js: 1.1.3 + optional: true is-binary-path@2.1.0: dependencies: @@ -2683,7 +2695,8 @@ snapshots: argparse: 1.0.10 esprima: 4.0.1 - jsbn@1.1.0: {} + jsbn@1.1.0: + optional: true jsonfile@4.0.0: optionalDependencies: @@ -2704,10 +2717,9 @@ snapshots: co: 4.6.0 koa-compose: 4.1.0 - koa-mount@4.0.0: + koa-morgan@1.0.1: dependencies: - debug: 4.3.7(supports-color@5.5.0) - koa-compose: 4.1.0 + morgan: 1.10.0 transitivePeerDependencies: - supports-color @@ -2762,6 +2774,10 @@ snapshots: dependencies: p-locate: 4.1.0 + lodash.defaults@4.2.0: {} + + lodash.isarguments@3.1.0: {} + lodash.startcase@4.4.0: {} lru-cache@4.1.5: @@ -2801,8 +2817,6 @@ snapshots: merge2@1.4.1: {} - methods@1.1.2: {} - micromatch@4.0.8: dependencies: braces: 3.0.3 @@ -2868,8 +2882,20 @@ snapshots: mkdirp@1.0.4: {} + morgan@1.10.0: + dependencies: + basic-auth: 2.0.1 + debug: 2.6.9 + depd: 2.0.0 + on-finished: 2.3.0 + on-headers: 1.0.2 + transitivePeerDependencies: + - supports-color + mri@1.2.0: {} + ms@2.0.0: {} + ms@2.1.3: {} napi-build-utils@1.0.2: {} @@ -2950,10 +2976,16 @@ snapshots: set-blocking: 2.0.0 optional: true + on-finished@2.3.0: + dependencies: + ee-first: 1.1.1 + on-finished@2.4.1: dependencies: ee-first: 1.1.1 + on-headers@1.0.2: {} + once@1.4.0: dependencies: wrappy: 1.0.2 @@ -3007,8 +3039,6 @@ snapshots: path-is-absolute@1.0.1: {} - path-to-regexp@6.3.0: {} - path-type@4.0.0: {} picocolors@1.1.0: {} @@ -3082,6 +3112,12 @@ snapshots: dependencies: picomatch: 2.3.1 + redis-errors@1.2.0: {} + + redis-parser@3.0.0: + dependencies: + redis-errors: 1.2.0 + regenerator-runtime@0.14.1: {} resolve-from@5.0.0: {} @@ -3105,6 +3141,8 @@ snapshots: dependencies: queue-microtask: 1.2.3 + safe-buffer@5.1.2: {} + safe-buffer@5.2.1: {} safer-buffer@2.1.2: {} @@ -3140,7 +3178,8 @@ snapshots: slash@3.0.0: {} - smart-buffer@4.2.0: {} + smart-buffer@4.2.0: + optional: true socks-proxy-agent@6.2.1: dependencies: @@ -3151,18 +3190,11 @@ snapshots: - supports-color optional: true - socks-proxy-agent@8.0.4: - dependencies: - agent-base: 7.1.1 - debug: 4.3.7(supports-color@5.5.0) - socks: 2.8.3 - transitivePeerDependencies: - - supports-color - socks@2.8.3: dependencies: ip-address: 9.0.5 smart-buffer: 4.2.0 + optional: true source-map-support@0.5.21: dependencies: @@ -3178,7 +3210,8 @@ snapshots: sprintf-js@1.0.3: {} - sprintf-js@1.1.3: {} + sprintf-js@1.1.3: + optional: true sqlite3@5.1.7: dependencies: @@ -3197,9 +3230,9 @@ snapshots: minipass: 3.3.6 optional: true - statuses@1.5.0: {} + standard-as-callback@2.1.0: {} - statuses@2.0.1: {} + statuses@1.5.0: {} string-width@4.2.3: dependencies: diff --git a/src/blossom.ts b/src/blossom.ts new file mode 100644 index 0000000..30081fd --- /dev/null +++ b/src/blossom.ts @@ -0,0 +1,35 @@ +import { getServersFromServerListEvent, USER_BLOSSOM_SERVER_LIST_KIND } from "blossom-client-sdk"; + +import ndk from "./ndk.js"; +import { BLOSSOM_SERVERS, MAX_FILE_SIZE } from "./env.js"; +import { makeRequestWithAbort } from "./helpers/http.js"; + +export async function getUserBlossomServers(pubkey: string) { + const blossomServersEvent = await ndk.fetchEvent([{ kinds: [USER_BLOSSOM_SERVER_LIST_KIND], authors: [pubkey] }]); + + return blossomServersEvent && getServersFromServerListEvent(blossomServersEvent).map((u) => u.toString()); +} + +// TODO: download the file to /tmp and verify it +export async function downloadFile(sha256: string, servers = BLOSSOM_SERVERS) { + for (const server of servers) { + try { + const { response } = await makeRequestWithAbort(new URL(sha256, server)); + if (!response.statusCode) throw new Error("Missing headers or status code"); + + const size = response.headers["content-length"]; + if (size && parseInt(size) > MAX_FILE_SIZE) { + throw new Error("File too large"); + } + + if (response.statusCode >= 200 && response.statusCode < 300) { + return response; + } else { + // Consume response data to free up memory + response.resume(); + } + } catch (error) { + // ignore error, try next server + } + } +} diff --git a/src/cache.ts b/src/cache.ts index 35b6852..b8c6b13 100644 --- a/src/cache.ts +++ b/src/cache.ts @@ -1,16 +1,43 @@ import Keyv from "keyv"; -import KeyvSqlite from "@keyv/sqlite"; import pfs from "fs/promises"; +import { CACHE_PATH } from "./env.js"; try { await pfs.mkdir("data"); } catch (error) {} -const keyvSqlite = new KeyvSqlite({ dialect: "sqlite", uri: "./data/cache.db" }); -keyvSqlite.on("error", (err) => { +async function createStore() { + if (!CACHE_PATH || CACHE_PATH === "in-memory") return undefined; + else if (CACHE_PATH.startsWith("redis://")) { + const { default: KeyvRedis } = await import("@keyv/redis"); + return new KeyvRedis(CACHE_PATH); + } else if (CACHE_PATH.startsWith("sqlite://")) { + const { default: KeyvSqlite } = await import("@keyv/sqlite"); + return new KeyvSqlite(CACHE_PATH); + } +} + +const store = await createStore(); + +store?.on("error", (err) => { console.log("Connection Error", err); process.exit(1); }); -export const files = new Keyv({ store: keyvSqlite, namespace: "files" }); -export const downloaded = new Keyv({ store: keyvSqlite, ttl: 1000 * 60 * 5, namespace: "downloaded" }); +const opts = store ? { store } : {}; + +/** pubkey -> blossom servers */ +export const userServers = new Keyv({ + ...opts, + namespace: "servers", + // cache servers for an hour + ttl: 60 * 60 * 1000, +}); + +/** pubkey -> relays */ +export const userRelays = new Keyv({ + ...opts, + namespace: "relays", + // cache relays for an hour + ttl: 60 * 60 * 1000, +}); diff --git a/src/downloader.ts b/src/downloader.ts deleted file mode 100644 index d974f2b..0000000 --- a/src/downloader.ts +++ /dev/null @@ -1,77 +0,0 @@ -import fs from "fs"; -import pfs from "fs/promises"; - -import { NSITE_KIND } from "./const.js"; -import ndk from "./ndk.js"; -import { BLOSSOM_SERVERS, MAX_FILE_SIZE } from "./env.js"; -import { makeRequestWithAbort } from "./helpers/http.js"; -import { dirname, join } from "path"; -import { downloaded, files } from "./cache.js"; -import { getServersFromServerListEvent, USER_BLOSSOM_SERVER_LIST_KIND } from "blossom-client-sdk"; - -// TODO: download the file to /tmp and verify it -async function downloadFile(sha256: string, servers = BLOSSOM_SERVERS) { - for (const server of servers) { - try { - const { response } = await makeRequestWithAbort(new URL(sha256, server)); - if (!response.statusCode) throw new Error("Missing headers or status code"); - - const size = response.headers["content-length"]; - if (size && parseInt(size) > MAX_FILE_SIZE) { - throw new Error("File too large"); - } - - if (response.statusCode >= 200 && response.statusCode < 300) { - return response; - } else { - // Consume response data to free up memory - response.resume(); - } - } catch (error) { - // ignore error, try next server - } - } - - throw new Error("No server found"); -} - -export async function downloadSite(pubkey: string) { - const user = await ndk.getUser({ pubkey }); - - const blossomServers = await ndk.fetchEvent([{ kinds: [USER_BLOSSOM_SERVER_LIST_KIND], authors: [pubkey] }]); - const servers = blossomServers ? getServersFromServerListEvent(blossomServers).map((u) => u.toString()) : []; - - const nsiteEvents = await ndk.fetchEvents([{ kinds: [NSITE_KIND], authors: [pubkey] }]); - - servers.push(...BLOSSOM_SERVERS); - - console.log(`Found ${nsiteEvents.size} events for ${pubkey}`); - - for (const event of nsiteEvents) { - const path = event.dTag; - const sha256 = event.tagValue("x") || event.tagValue("sha256"); - - if (!path || !sha256) continue; - - const current = await files.get(join(pubkey, path)); - if (sha256 === current) continue; - - try { - await pfs.mkdir(dirname(join("data/sites", pubkey, path)), { recursive: true }); - } catch (error) {} - - try { - const res = await downloadFile(sha256, servers); - - console.log(`Downloading ${join(pubkey, path)}`); - res.pipe(fs.createWriteStream(join("data/sites", pubkey, path))); - - await files.set(join(pubkey, path), sha256); - } catch (error) { - console.log(`Failed to download ${join(pubkey, path)}`, error); - } - } - - console.log(`Finished downloading ${pubkey}`); - await downloaded.set(pubkey, true); -} diff --git a/src/env.ts b/src/env.ts index 1f55e18..d31b1be 100644 --- a/src/env.ts +++ b/src/env.ts @@ -6,6 +6,9 @@ const BLOSSOM_SERVERS = process.env.BLOSSOM_SERVERS?.split(",") ?? []; const MAX_FILE_SIZE = process.env.MAX_FILE_SIZE ? xbytes.parseSize(process.env.MAX_FILE_SIZE) : Infinity; +const NGINX_HOST = process.env.NGINX_HOST; +const CACHE_PATH = process.env.CACHE_PATH; + if (NOSTR_RELAYS.length === 0) throw new Error("Requires at least one relay in NOSTR_RELAYS"); -export { NOSTR_RELAYS, BLOSSOM_SERVERS, MAX_FILE_SIZE }; +export { NOSTR_RELAYS, BLOSSOM_SERVERS, MAX_FILE_SIZE, NGINX_HOST, CACHE_PATH }; diff --git a/src/events.ts b/src/events.ts new file mode 100644 index 0000000..584de88 --- /dev/null +++ b/src/events.ts @@ -0,0 +1,39 @@ +import { extname, isAbsolute, join } from "path"; +import { NSITE_KIND } from "./const.js"; +import ndk from "./ndk.js"; + +export function getSearchPaths(path: string) { + const paths = [path]; + + // if the path does not have an extension, also look for index.html + if (extname(path) === "") paths.push(join(path, "index.html")); + + // also look for relative paths + for (const p of Array.from(paths)) { + if (isAbsolute(p)) paths.push(path.replace(/^\//, "")); + } + + return paths; +} + +export function parseNsiteEvent(event: { pubkey: string; tags: string[][] }) { + const path = event.tags.find((t) => t[0] === "d" && t[1])?.[1]; + const sha256 = event.tags.find((t) => t[0] === "x" && t[1])?.[1]; + + if (path && sha256) + return { + pubkey: event.pubkey, + path: join("/", path), + sha256, + }; +} + +export async function getNsiteBlobs(pubkey: string, path: string) { + const paths = getSearchPaths(path); + const events = await ndk.fetchEvents({ kinds: [NSITE_KIND], "#d": paths, authors: [pubkey] }); + + return Array.from(events) + .map(parseNsiteEvent) + .filter((e) => !!e) + .sort((a, b) => paths.indexOf(a.path) - paths.indexOf(b.path)); +} diff --git a/src/helpers/error.ts b/src/helpers/error.ts deleted file mode 100644 index 7ba31d8..0000000 --- a/src/helpers/error.ts +++ /dev/null @@ -1,7 +0,0 @@ -import HttpErrors from "http-errors"; - -export function isHttpError(error: unknown): error is HttpErrors.HttpError { - if (!error) return false; - // @ts-expect-error - return error instanceof HttpErrors.HttpError || !!error.status || !!error.headers; -} diff --git a/src/index.ts b/src/index.ts index 61f1776..43d9976 100644 --- a/src/index.ts +++ b/src/index.ts @@ -2,22 +2,27 @@ import "./polyfill.js"; import Koa from "koa"; import serve from "koa-static"; -import path, { join } from "node:path"; +import path from "node:path"; import cors from "@koa/cors"; import fs from "node:fs"; import { fileURLToPath } from "node:url"; -import send from "koa-send"; +import mime from "mime"; +import morgan from "koa-morgan"; -import logger from "./logger.js"; -import { isHttpError } from "./helpers/error.js"; import { resolveNpubFromHostname } from "./helpers/dns.js"; -import { downloadSite } from "./downloader.js"; -import { downloaded } from "./cache.js"; +import { getNsiteBlobs } from "./events.js"; +import { downloadFile, getUserBlossomServers } from "./blossom.js"; +import { BLOSSOM_SERVERS } from "./env.js"; +import { userServers } from "./cache.js"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const app = new Koa(); +morgan.token("host", (req) => req.headers.host ?? ""); + +app.use(morgan(":method :host:url :status :response-time ms - :res[content-length]")); + // set CORS headers app.use( cors({ @@ -33,15 +38,9 @@ app.use(async (ctx, next) => { try { await next(); } catch (err) { - if (isHttpError(err)) { - const status = (ctx.status = err.status || 500); - if (status >= 500) console.error(err.stack); - ctx.body = status > 500 ? { message: "Something went wrong" } : { message: err.message }; - } else { - console.log(err); - ctx.status = 500; - ctx.body = { message: "Something went wrong" }; - } + console.log(err); + ctx.status = 500; + ctx.body = { message: "Something went wrong" }; } }); @@ -50,20 +49,45 @@ app.use(async (ctx, next) => { const pubkey = (ctx.state.pubkey = await resolveNpubFromHostname(ctx.hostname)); if (pubkey) { - if (!(await downloaded.get(pubkey))) { - // don't wait for download - downloadSite(pubkey); + console.log(`${pubkey}: Searching for ${ctx.path}`); + const blobs = await getNsiteBlobs(pubkey, ctx.path); - await downloaded.set(pubkey, true); + if (blobs.length === 0) { + ctx.status = 404; + ctx.body = "Not Found"; + return; } - await send(ctx, join(pubkey, ctx.path), { root: "data/sites", index: "index.html" }); + let servers = await userServers.get(pubkey); + if (!servers) { + console.log(`${pubkey}: Searching for blossom servers`); + servers = (await getUserBlossomServers(pubkey)) ?? []; + + await userServers.set(pubkey, servers); + } + servers.push(...BLOSSOM_SERVERS); + + for (const blob of blobs) { + const res = await downloadFile(blob.sha256, servers); + + if (res) { + const type = mime.getType(blob.path); + if (type) ctx.set("Content-Type", type); + else if (res.headers["content-type"]) ctx.set("content-type", res.headers["content-type"]); + + // pass headers along + if (res.headers["content-length"]) ctx.set("content-length", res.headers["content-length"]); + + ctx.body = res; + return; + } + } + + ctx.status = 500; + ctx.body = "Failed to download blob"; } else await next(); }); -// serve static sites -app.use(serve("sites")); - // serve static files from public try { const www = path.resolve(process.cwd(), "public"); @@ -74,11 +98,12 @@ try { app.use(serve(www)); } -app.listen(process.env.PORT || 3000); -logger("Started on port", process.env.PORT || 3000); +app.listen(process.env.PORT || 3000, () => { + console.log("Started on port", process.env.PORT || 3000); +}); async function shutdown() { - logger("Shutting down..."); + console.log("Shutting down..."); process.exit(0); } diff --git a/src/logger.ts b/src/logger.ts deleted file mode 100644 index 1ce81b3..0000000 --- a/src/logger.ts +++ /dev/null @@ -1,7 +0,0 @@ -import debug from "debug"; - -if (!process.env.DEBUG) debug.enable("nsite, nsite:*"); - -const logger = debug("nsite"); - -export default logger; diff --git a/src/nginx.ts b/src/nginx.ts new file mode 100644 index 0000000..d3dc05f --- /dev/null +++ b/src/nginx.ts @@ -0,0 +1,26 @@ +import http from "node:http"; +import { NGINX_HOST } from "./env.js"; + +export function invalidateCache(host: string, path: string) { + if (!NGINX_HOST) return Promise.resolve(false); + + return new Promise((resolve, reject) => { + const req = http.request( + { + hostname: NGINX_HOST, + method: "GET", + port: 80, + path, + headers: { + Host: host, + }, + }, + (res) => { + if (res.statusCode && res.statusCode >= 200 && res.statusCode < 300) resolve(true); + else reject(new Error("Failed to invalidate")); + }, + ); + + req.end(); + }); +}