Merge pull request #3 from vitorpamplona/main

Moves to Backup everything from PubKey or about PubKey from all available relays.

CNAME

@@ -1 +1 @@
-nostryfied.online
+nostryfied.amethyst.social

index.html

@@ -108,11 +108,17 @@
               id="fetching-progress"
               name="fetching-progress"
               min="0"
-              max="20"
+              max="180"
               value="0"
               style="visibility: hidden" />
           </p>
         </div>
+        <div class="box-content" id="checking-relays-header-box">
+          <p id="checking-relays-header"></p>
+        </div>
+        <div class="box-content" id="checking-relays-box">
+          <p id="checking-relays"></p>
+        </div>
         <div class="box-content">
           <p id="file-download"></p>
           <p id="events-found"></p>
@@ -125,7 +131,7 @@
               id="broadcasting-progress"
               name="broadcasting-progress"
               min="0"
-              max="20"
+              max="180"
               value="0"
               style="visibility: hidden" />
           </p>
@@ -191,7 +197,7 @@
     <script src="https://bundle.run/buffer@6.0.3"></script>
     <script src="https://bundle.run/bech32@2.0.0"></script>
     <script src="https://nostr-utils.pages.dev/js/jquery-3.6.2.min.js"></script>
-    <script src="https://nostr-utils.pages.dev/js/nostr-utils.js"></script>
+    <script src="js/nostr-utils.js"></script>
     <script src="js/relays.js"></script>
     <script src="js/nostr-broadcast.js"></script>
   </body>

js/nostr-broadcast.js

@@ -16,6 +16,7 @@ const fetchAndBroadcast = async () => {
     fetching: 'Fetching from relays... ',
     download: `Downloading Backup file... ${checkMark}`,
   }
+  $('#checking-relays-header').text("Waiting for Relays: ")
   // parse pubkey ('npub' or hexa)
   const pubkey = parsePubkey($('#pubkey').val())
   if (!pubkey) return
@@ -25,18 +26,25 @@ const fetchAndBroadcast = async () => {
   $('#fetching-status').text(txt.fetching)
   // show and update fetching progress bar
   $('#fetching-progress').css('visibility', 'visible')
-  const fetchInterval = setInterval(() => {
-    // update fetching progress bar
-    const currValue = parseInt($('#fetching-progress').val())
-    $('#fetching-progress').val(currValue + 1)
-  }, 1000)
+  $('#fetching-progress').prop('max', relays.length)
+
+  $('#checking-relays-header-box').css('display', 'flex')
+  $('#checking-relays-box').css('display', 'flex')
+  $('#checking-relays-header').text("Waiting for Relays:")
+
   // get all events from relays
-  const filter = { authors: [pubkey] }
-  const data = await getEvents(filter)
+  const filters =[{ authors: [pubkey] }, { "#p": [pubkey] }] 
+  const data = (await getEvents(filters, pubkey)).sort((a, b) => b.created_at - a.created_at)
+  const latestKind3 = data.filter((it) => it.kind == 3 && it.pubkey === pubkey)[0]  
+  const myRelaySet = JSON.parse(latestKind3.content)
+  relays = Object.keys(myRelaySet).filter(url => myRelaySet[url].write).map(url => url)
+
+  $('#checking-relays-header-box').css('display', 'none')
+  $('#checking-relays-box').css('display', 'none')
+
   // inform user fetching is done
   $('#fetching-status').html(txt.fetching + checkMark)
-  clearInterval(fetchInterval)
-  $('#fetching-progress').val(20)
+  $('#fetching-progress').val(relays.length)
   // inform user that backup file (js format) is being downloaded
   $('#file-download').html(txt.download)
   downloadFile(data, 'nostr-backup.js')
@@ -44,16 +52,17 @@ const fetchAndBroadcast = async () => {
   $('#broadcasting-status').html(txt.broadcasting)
   // show and update broadcasting progress bar
   $('#broadcasting-progress').css('visibility', 'visible')
-  const broadcastInterval = setInterval(() => {
-    // update fetching progress bar
-    const currValue = parseInt($('#broadcasting-progress').val())
-    $('#broadcasting-progress').val(currValue + 1)
-  }, 1000)
+  $('#broadcasting-progress').prop('max', relays.length)
+  
+  $('#checking-relays-header-box').css('display', 'flex')
+  $('#checking-relays-box').css('display', 'flex')
+  $('#checking-relays-header').text("Broadcasting to Relays:")
+
   await broadcastEvents(data)
+
   // inform user that broadcasting is done
   $('#broadcasting-status').html(txt.broadcasting + checkMark)
-  clearInterval(broadcastInterval)
-  $('#broadcasting-progress').val(20)
+  $('#broadcasting-progress').val(relays.length)
   // re-enable broadcast button
   $('#fetch-and-broadcast').prop('disabled', false)
 }

js/nostr-utils.js

@@ -0,0 +1,236 @@
+// from https://github.com/paulmillr/noble-secp256k1/blob/main/index.ts#L803
+function hexToBytes(hex) {
+    if (typeof hex !== 'string') {
+      throw new TypeError('hexToBytes: expected string, got ' + typeof hex)
+    }
+    if (hex.length % 2)
+      throw new Error('hexToBytes: received invalid unpadded hex' + hex.length)
+    const array = new Uint8Array(hex.length / 2)
+    for (let i = 0; i < array.length; i++) {
+      const j = i * 2
+      const hexByte = hex.slice(j, j + 2)
+      const byte = Number.parseInt(hexByte, 16)
+      if (Number.isNaN(byte) || byte < 0) throw new Error('Invalid byte sequence')
+      array[i] = byte
+    }
+    return array
+  }
+  
+  // decode nip19 ('npub') to hex
+  const npub2hexa = (npub) => {
+    let { prefix, words } = bech32.bech32.decode(npub, 90)
+    if (prefix === 'npub') {
+      let data = new Uint8Array(bech32.bech32.fromWords(words))
+      return buffer.Buffer.from(data).toString('hex')
+    }
+  }
+  
+  // encode hex to nip19 ('npub')
+  const hexa2npub = (hex) => {
+    const data = hexToBytes(hex)
+    const words = bech32.bech32.toWords(data)
+    const prefix = 'npub'
+    return bech32.bech32.encode(prefix, words, 90)
+  }
+  
+  // parse inserted pubkey
+  const parsePubkey = (pubkey) =>
+    pubkey.match('npub1') ? npub2hexa(pubkey) : pubkey
+  
+  // download js file
+  const downloadFile = (data, fileName) => {
+    const prettyJs = 'const data = ' + JSON.stringify(data, null, 2)
+    const tempLink = document.createElement('a')
+    const taBlob = new Blob([prettyJs], { type: 'text/javascript' })
+    tempLink.setAttribute('href', URL.createObjectURL(taBlob))
+    tempLink.setAttribute('download', fileName)
+    tempLink.click()
+  }
+
+  const updateRelayStatus = (relayStatus) => {
+    if (Object.keys(relayStatus).length > 0) {
+      let newText = Object.keys(relayStatus).map(
+        it => it.replace("wss://", "").replace("ws://", "") + ": " + relayStatus[it]
+      ).join("<br />")
+      $('#checking-relays').html(newText)
+    } else {
+      $('#checking-relays-header').html("")
+      $('#checking-relays').html("")
+    }
+  }
+
+  // fetch events from relay, returns a promise
+  const fetchFromRelay = async (relay, filters, pubkey, events, relayStatus) =>
+    new Promise((resolve, reject) => {
+      try {
+        relayStatus[relay] = "Starting"
+        updateRelayStatus(relayStatus)
+        // open websocket
+        const ws = new WebSocket(relay)
+
+        // prevent hanging forever
+        let myTimeout = setTimeout(() => {
+          ws.close()
+          reject('timeout')
+        }, 10_000)
+
+
+        // subscription id
+        const subsId = 'my-sub'
+        // subscribe to events filtered by author
+        ws.onopen = () => {
+          clearTimeout(myTimeout)
+          myTimeout = setTimeout(() => {
+            ws.close()
+            reject('timeout')
+          }, 10_000)
+          relayStatus[relay] = "Downloading"
+          updateRelayStatus(relayStatus)
+          ws.send(JSON.stringify(['REQ', subsId].concat(filters)))
+        }
+  
+        // Listen for messages
+        ws.onmessage = (event) => {
+          const [msgType, subscriptionId, data] = JSON.parse(event.data)
+          // event messages
+          if (msgType === 'EVENT' && subscriptionId === subsId) {
+            clearTimeout(myTimeout)
+            myTimeout = setTimeout(() => {
+              ws.close()
+              reject('timeout')
+            }, 10_000)
+
+            const { id } = data
+
+            // don't save/reboradcast kind 3s that are not from the author. 
+            // their are too big. 
+            if (data.kind == 3 && data.pubkey != pubkey) {
+              return
+            }
+
+            // prevent duplicated events
+            if (events[id]) return
+            else events[id] = data
+            // show how many events were found until this moment
+            $('#events-found').text(`${Object.keys(events).length} events found`)
+          }
+          // end of subscription messages
+          if (msgType === 'EOSE' && subscriptionId === subsId) {
+            relayStatus[relay] = "Done"
+            updateRelayStatus(relayStatus)
+            ws.close()
+            resolve()
+          }
+        }
+        ws.onerror = (err) => {
+          relayStatus[relay] = "Done"
+          updateRelayStatus(relayStatus)
+          ws.close()
+          reject(err)
+        }
+        ws.onclose = (socket, event) => {
+          relayStatus[relay] = "Done"
+          updateRelayStatus(relayStatus)
+          resolve()
+        }
+      } catch (exception) {
+        console.log(exception)
+        relayStatus[relay] = "Error"
+        updateRelayStatus(relayStatus)
+        try {
+          ws.close()
+        } catch (exception) {
+        }
+        
+        reject(exception)
+      }
+    })
+  
+  // query relays for events published by this pubkey
+  const getEvents = async (filters, pubkey) => {
+    // events hash
+    const events = {}
+
+    // batch processing of 10 relays
+    let fetchFunctions = [...relays]
+    while (fetchFunctions.length) {
+      let relaysForThisRound = fetchFunctions.splice(0, 10)
+      let relayStatus = {}
+      $('#fetching-progress').val(relays.length - fetchFunctions.length)
+      await Promise.allSettled( relaysForThisRound.map((relay) => fetchFromRelay(relay, filters, pubkey, events, relayStatus)) )
+    }
+    updateRelayStatus({})
+
+    // return data as an array of events
+    return Object.keys(events).map((id) => events[id])
+  }
+  
+  // send events to a relay, returns a promisse
+  const sendToRelay = async (relay, data, relayStatus) =>
+    new Promise((resolve, reject) => {
+      try {
+        const ws = new WebSocket(relay)
+
+        relayStatus[relay] = "Starting"
+        updateRelayStatus(relayStatus)
+
+        // prevent hanging forever
+        let myTimeout = setTimeout(() => {
+          ws.close()
+          reject('timeout')
+        }, 10_000)
+
+        // fetch events from relay
+        ws.onopen = () => {
+          relayStatus[relay] = "Sending"
+          updateRelayStatus(relayStatus)
+          for (evnt of data) {
+            clearTimeout(myTimeout)
+            myTimeout = setTimeout(() => {
+              ws.close()
+              reject('timeout')
+            }, 5_000)
+
+            ws.send(JSON.stringify(['EVENT', evnt]))
+          }
+          relayStatus[relay] = "Done"
+          updateRelayStatus(relayStatus)
+          ws.close()
+          resolve(`done for ${relay}`)
+        }
+        ws.onerror = (err) => {
+          relayStatus[relay] = "Error"
+          updateRelayStatus(relayStatus)
+          console.log("Error", err)
+          ws.close()
+          reject(err)
+        }
+        ws.onclose = (socket, event) => {
+          relayStatus[relay] = "Done"
+          updateRelayStatus(relayStatus)
+          resolve()
+        }
+      } catch (exception) {
+        relayStatus[relay] = "Error"
+        updateRelayStatus(relayStatus)
+        try {
+          ws.close()
+        } catch (exception) {
+        }
+        reject(exception)
+      }
+    })
+  
+  // broadcast events to list of relays
+  const broadcastEvents = async (data) => {
+    // batch processing of 10 relays
+    let broadcastFunctions = [...relays]
+    let relayStatus = {}
+    while (broadcastFunctions.length) {
+      let relaysForThisRound = broadcastFunctions.splice(0, 10)
+      $('#broadcasting-progress').val(relays.length - broadcastFunctions.length)
+      await Promise.allSettled( relaysForThisRound.map((relay) => sendToRelay(relay, data, relayStatus)) )
+    }
+
+    updateRelayStatus(relayStatus)
+  }

js/relays.js

@@ -1,7 +1,8 @@
 // list of paid relays from:
 // https://thebitcoinmanual.com/articles/paid-nostr-relay/
 
-const relays = [
+
+const fixedRelays = [
   'wss://atlas.nostr.land', // paid relay	15000	npub12262qa4uhw7u8gdwlgmntqtv7aye8vdcmvszkqwgs0zchel6mz7s6cgrkj
   'wss://bitcoiner.social', // paid relay	1000	npub1dxs2pygtfxsah77yuncsmu3ttqr274qr5g5zva3c7t5s3jtgy2xszsn4st
   'wss://brb.io',
@@ -9,6 +10,7 @@ const relays = [
   'wss://expensive-relay.fiatjaf.com',
   'wss://freedom-relay.herokuapp.com',
   'wss://nos.lol',
+  'wss://a.nos.lol',
   'wss://nostr-2.zebedee.cloud',
   'wss://nostr-pub.wellorder.net',
   'wss://nostr-relay.alekberg.net',
@@ -43,4 +45,15 @@ const relays = [
   'wss://relay.snort.social',
   'wss://relayer.fiatjaf.com',
   'wss://rsslay.fiatjaf.com',
+  'wss://no.str.cr',
+  'wss://nostr.oxtr.dev',
+  'wss://nostr.mom',
+  'wss://relay.nostr.ch',
+  'wss://relay.nostr.band'
 ]
+
+var relays = []
+
+fetch("https://api.nostr.watch/v1/online")
+     .then(response => response.json())
+     .then(json => relays = fixedRelays.concat(json));