From da1bbfc8e3085325dc91aa99764447d49502923d Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 4 Nov 2023 13:52:07 -0500
Subject: [PATCH] readall for crawler

---
 scripts/crawl.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/scripts/crawl.py b/scripts/crawl.py
index 12e6425..01e462f 100644
--- a/scripts/crawl.py
+++ b/scripts/crawl.py
@@ -15,6 +15,17 @@ else:
   socktype = socket.AF_UNIX
   sockaddr = "/var/run/yggdrasil/yggdrasil.sock"
 
+def readall(sock):
+  data = []
+  sock.settimeout(1)
+  while True:
+    try:
+      frag = sock.recv(4096)
+      data.append(frag)
+    except:
+      break
+  return "".join(data)
+
 def getPeersRequest(key):
   return '{{"keepalive":true, "request":"debug_remoteGetPeers", "arguments": {{"key":"{}"}}}}'.format(key)
 
@@ -23,7 +34,7 @@ def doRequest(req):
     ygg = socket.socket(socktype, socket.SOCK_STREAM)
     ygg.connect(sockaddr)
     ygg.send(req)
-    data = json.loads(ygg.recv(1048576))
+    data = json.loads(readall(ygg))
     return data
   except:
     return None