From e1638133b64e9686e7ff7ce60c2f28e1b754fbe7 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Wed, 25 Dec 2019 17:27:34 -0600 Subject: [PATCH 01/11] update crawler --- scripts/crawl-dht.py | 28 +++++++++++++++++++++++++--- web/updateGraph.py | 3 ++- 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/scripts/crawl-dht.py b/scripts/crawl-dht.py index b5e9306..fe30b39 100644 --- a/scripts/crawl-dht.py +++ b/scripts/crawl-dht.py @@ -26,6 +26,17 @@ def doRequest(req): except: return None +def getNodeInfo(key, coords): + try: + req = '{{"keepalive":true, "request":"getNodeInfo", "box_pub_key":"{}", "coords":"{}"}}'.format(key, coords) + ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ygg.connect(host_port) + ygg.send(req) + data = json.loads(ygg.recv(1024*15)) + return data + except: + return None + visited = dict() # Add nodes after a successful lookup response rumored = dict() # Add rumors about nodes to ping timedout = dict() @@ -45,8 +56,19 @@ def handleResponse(address, info, data): now = time.time() visited[str(address)] = {'box_pub_key':str(info['box_pub_key']), 'coords':str(info['coords']), 'time':now} if address in timedout: del timedout[address] + nodeinfo = getNodeInfo(str(info['box_pub_key']), str(info['coords'])) + #print "\nDEBUG:", info, nodeinfo if len(visited) > 1: sys.stdout.write(",\n") - sys.stdout.write('"{}": ["{}", {}]'.format(address, info['coords'], int(now))) + nodename = None + try: + if nodeinfo and 'response' in nodeinfo and 'nodeinfo' in nodeinfo['response'] and 'name' in nodeinfo['response']['nodeinfo']: + nodename = '"' + str(nodeinfo['response']['nodeinfo']['name']) + '"' + except: + pass + if nodename: + sys.stdout.write('"{}": ["{}", {}, {}]'.format(address, info['coords'], int(now), nodename)) + else: + sys.stdout.write('"{}": ["{}", {}]'.format(address, info['coords'], int(now))) sys.stdout.flush() # End handleResponse @@ -62,8 +84,8 @@ while len(rumored) > 0: for k,v in rumored.iteritems(): handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords']))) # These next two are imperfect workarounds to deal with old kad nodes - handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'], '0'*128))) - handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'], 'f'*128))) + #handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'], '0'*128))) + #handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'], 'f'*128))) break del rumored[k] print '\n}}' diff --git a/web/updateGraph.py b/web/updateGraph.py index 6d3cb97..660214c 100755 --- a/web/updateGraph.py +++ b/web/updateGraph.py @@ -5,7 +5,8 @@ import graphPlotter import cgi import urllib, json -url = "http://y.yakamo.org:3000/current" +#url = "http://y.yakamo.org:3000/current" +url = "current" # nodes indexed by coords class NodeInfo: From b47b1fffd445a652344903aa7515408b00a62391 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sat, 22 May 2021 22:47:16 -0500 Subject: [PATCH 02/11] WIP update crawler for ygg future branch --- scripts/crawl-dht.py | 97 -------------------------------------------- scripts/crawl.py | 93 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 93 insertions(+), 97 deletions(-) delete mode 100644 scripts/crawl-dht.py create mode 100644 scripts/crawl.py diff --git a/scripts/crawl-dht.py b/scripts/crawl-dht.py deleted file mode 100644 index fe30b39..0000000 --- a/scripts/crawl-dht.py +++ /dev/null @@ -1,97 +0,0 @@ -import json -import socket -import sys -import time - -#gives the option to get data from an external server instead and send that -#if no options given it will default to localhost instead -if len(sys.argv) == 3: - host_port = (sys.argv[1], int(sys.argv[2])) -else: - host_port = ('localhost', 9001) - -def getDHTPingRequest(key, coords, target=None): - if target: - return '{{"keepalive":true, "request":"dhtPing", "box_pub_key":"{}", "coords":"{}", "target":"{}"}}'.format(key, coords, target) - else: - return '{{"keepalive":true, "request":"dhtPing", "box_pub_key":"{}", "coords":"{}"}}'.format(key, coords) - -def doRequest(req): - try: - ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ygg.connect(host_port) - ygg.send(req) - data = json.loads(ygg.recv(1024*15)) - return data - except: - return None - -def getNodeInfo(key, coords): - try: - req = '{{"keepalive":true, "request":"getNodeInfo", "box_pub_key":"{}", "coords":"{}"}}'.format(key, coords) - ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ygg.connect(host_port) - ygg.send(req) - data = json.loads(ygg.recv(1024*15)) - return data - except: - return None - -visited = dict() # Add nodes after a successful lookup response -rumored = dict() # Add rumors about nodes to ping -timedout = dict() -def handleResponse(address, info, data): - global visited - global rumored - global timedout - timedout[str(address)] = {'box_pub_key':str(info['box_pub_key']), 'coords':str(info['coords'])} - if not data: return - if 'response' not in data: return - if 'nodes' not in data['response']: return - for addr,rumor in data['response']['nodes'].iteritems(): - if addr in visited: continue - rumored[addr] = rumor - if address not in visited: - # TODO? remove this, it's debug output that happens to be in the same format as yakamo's "current" json file - now = time.time() - visited[str(address)] = {'box_pub_key':str(info['box_pub_key']), 'coords':str(info['coords']), 'time':now} - if address in timedout: del timedout[address] - nodeinfo = getNodeInfo(str(info['box_pub_key']), str(info['coords'])) - #print "\nDEBUG:", info, nodeinfo - if len(visited) > 1: sys.stdout.write(",\n") - nodename = None - try: - if nodeinfo and 'response' in nodeinfo and 'nodeinfo' in nodeinfo['response'] and 'name' in nodeinfo['response']['nodeinfo']: - nodename = '"' + str(nodeinfo['response']['nodeinfo']['name']) + '"' - except: - pass - if nodename: - sys.stdout.write('"{}": ["{}", {}, {}]'.format(address, info['coords'], int(now), nodename)) - else: - sys.stdout.write('"{}": ["{}", {}]'.format(address, info['coords'], int(now))) - sys.stdout.flush() -# End handleResponse - -# Get self info -selfInfo = doRequest('{"keepalive":true, "request":"getSelf"}') - -# Initialize dicts of visited/rumored nodes -for k,v in selfInfo['response']['self'].iteritems(): rumored[k] = v - -# Loop over rumored nodes and ping them, adding to visited if they respond -print '{"yggnodes": {' -while len(rumored) > 0: - for k,v in rumored.iteritems(): - handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords']))) - # These next two are imperfect workarounds to deal with old kad nodes - #handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'], '0'*128))) - #handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'], 'f'*128))) - break - del rumored[k] -print '\n}}' -#End - -# TODO do something with the results - -#print visited -#print timedout diff --git a/scripts/crawl.py b/scripts/crawl.py new file mode 100644 index 0000000..6bad223 --- /dev/null +++ b/scripts/crawl.py @@ -0,0 +1,93 @@ +import json +import socket +import sys +import time + +#gives the option to get data from an external server instead and send that +#if no options given it will default to localhost instead +if len(sys.argv) == 3: + socktype = socket.AF_INET + sockaddr = (sys.argv[1], int(sys.argv[2])) +elif len(sys.argv) == 2: + socktype = socket.AF_UNIX + sockaddr = sys.argv[1] +else: + socktype = socket.AF_UNIX + sockaddr = "/var/run/yggdrasil.sock" + +def getPeersRequest(key): + return '{{"keepalive":true, "request":"debugGetPeers", "key":"{}"}}'.format(key) + +def doRequest(req): + try: + ygg = socket.socket(socktype, socket.SOCK_STREAM) + ygg.connect(sockaddr) + ygg.send(req) + data = json.loads(ygg.recv(1024*15)) + return data + except: + return None + +visited = set() # Add nodes after a successful lookup response +rumored = set() # Add rumors about nodes to ping +timedout = set() +def handleResponse(address, data): + global visited + global rumored + global timedout + if address in visited: return + if not data: return + if 'response' not in data: return + for k,v in data['response'].iteritems(): + if 'keys' not in v: continue + keys = v['keys'] + for key in keys: + if key in visited: continue + if key in timedout: continue + rumored.add(key) + selfInfo = doRequest('{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(address)) + if 'response' not in selfInfo: return + coords = None + for _,v in selfInfo['response'].iteritems(): + if 'Coords' not in v: continue + coords = str(v['Coords']) + break + if coords == None: return + nodename = None + nodeinfo = doRequest('{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(address)) + try: + if nodeinfo and 'response' in nodeinfo and 'nodeinfo' in nodeinfo['response'] and 'name' in nodeinfo['response']['nodeinfo']: + nodename = '"' + str(nodeinfo['response']['nodeinfo']['name']) + '"' + except: + pass + now = time.time() + if len(visited) > 0: sys.stdout.write(",\n") + if nodename: + sys.stdout.write('"{}": ["{}", {}, {}]'.format(address, coords, int(now), nodename)) + else: + sys.stdout.write('"{}": ["{}", {}]'.format(address, coords, int(now))) + sys.stdout.flush() + visited.add(address) +# End handleResponse + +# Get self info +selfInfo = doRequest('{"keepalive":true, "request":"getSelf"}') +for k,v in selfInfo['response']['self'].iteritems(): rumored.add(v['key']) + +# Initialize dicts of visited/rumored nodes +#for k,v in selfInfo['response']['self'].iteritems(): rumored[k] = v + +# Loop over rumored nodes and ping them, adding to visited if they respond +print '{"yggnodes": {' +while len(rumored) > 0: + for k in rumored: + handleResponse(k, doRequest(getPeersRequest(v['key']))) + break + rumored.remove(k) +print '\n}}' +#End + +# TODO do something with the results + +#print visited +#print timedout From 35b2d87911ff21c05bbec53d59ad000af7d0559a Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 23 May 2021 12:41:27 -0500 Subject: [PATCH 03/11] update crawler --- scripts/crawl.py | 50 ++++++++++++++++++++++++------------------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/scripts/crawl.py b/scripts/crawl.py index 6bad223..77bd68e 100644 --- a/scripts/crawl.py +++ b/scripts/crawl.py @@ -31,43 +31,43 @@ def doRequest(req): visited = set() # Add nodes after a successful lookup response rumored = set() # Add rumors about nodes to ping timedout = set() -def handleResponse(address, data): +def handleResponse(publicKey, data): global visited global rumored global timedout - if address in visited: return + if publicKey in visited: return if not data: return if 'response' not in data: return - for k,v in data['response'].iteritems(): + out = dict() + for addr,v in data['response'].iteritems(): if 'keys' not in v: continue - keys = v['keys'] - for key in keys: + peers = v['keys'] + for key in peers: if key in visited: continue if key in timedout: continue rumored.add(key) - selfInfo = doRequest('{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(address)) - if 'response' not in selfInfo: return - coords = None - for _,v in selfInfo['response'].iteritems(): - if 'Coords' not in v: continue - coords = str(v['Coords']) + out['address'] = addr + out['peers'] = peers break - if coords == None: return - nodename = None - nodeinfo = doRequest('{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(address)) - try: - if nodeinfo and 'response' in nodeinfo and 'nodeinfo' in nodeinfo['response'] and 'name' in nodeinfo['response']['nodeinfo']: - nodename = '"' + str(nodeinfo['response']['nodeinfo']['name']) + '"' - except: - pass - now = time.time() + selfInfo = doRequest('{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(publicKey)) + if 'response' in selfInfo: + for _,v in selfInfo['response'].iteritems(): + if 'coords' in v: + out['coords'] = v['coords'] + dhtInfo = doRequest('{{"keepalive":true, "request":"debugGetDHT", "key":"{}"}}'.format(key)) + if 'response' in dhtInfo: + for _,v in dhtInfo['response'].iteritems(): + if 'keys' in v: + out['dht'] = v['keys'] + nodeInfo = doRequest('{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(publicKey)) + if 'response' in nodeInfo: + for _,v in nodeInfo['response'].iteritems(): + out['nodeinfo'] = v + out['time'] = time.time() if len(visited) > 0: sys.stdout.write(",\n") - if nodename: - sys.stdout.write('"{}": ["{}", {}, {}]'.format(address, coords, int(now), nodename)) - else: - sys.stdout.write('"{}": ["{}", {}]'.format(address, coords, int(now))) + sys.stdout.write('"{}": {}'.format(publicKey, json.dumps(out))) sys.stdout.flush() - visited.add(address) + visited.add(publicKey) # End handleResponse # Get self info From ee6b45de168dfb9807c526ffb8426bf3b452425f Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 23 May 2021 12:44:57 -0500 Subject: [PATCH 04/11] add rumors from dht --- scripts/crawl.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/scripts/crawl.py b/scripts/crawl.py index 77bd68e..0c92c2b 100644 --- a/scripts/crawl.py +++ b/scripts/crawl.py @@ -58,6 +58,11 @@ def handleResponse(publicKey, data): if 'response' in dhtInfo: for _,v in dhtInfo['response'].iteritems(): if 'keys' in v: + dht = v['keys'] + for key in dht: + if key in visited: continue + if key in timedout: continue + rumored.add(key) out['dht'] = v['keys'] nodeInfo = doRequest('{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(publicKey)) if 'response' in nodeInfo: From de7cdee06488a8dc38920f49db07ab11d5508a7b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 23 May 2021 12:52:59 -0500 Subject: [PATCH 05/11] cleanup --- scripts/crawl.py | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/scripts/crawl.py b/scripts/crawl.py index 0c92c2b..03ec1ef 100644 --- a/scripts/crawl.py +++ b/scripts/crawl.py @@ -15,9 +15,18 @@ else: socktype = socket.AF_UNIX sockaddr = "/var/run/yggdrasil.sock" +def getNodeInfoRequest(key): + return '{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(key) + +def getSelfRequest(key): + return '{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(key) + def getPeersRequest(key): return '{{"keepalive":true, "request":"debugGetPeers", "key":"{}"}}'.format(key) +def getDHTRequest(key): + return '{{"keepalive":true, "request":"debugGetDHT", "key":"{}"}}'.format(key) + def doRequest(req): try: ygg = socket.socket(socktype, socket.SOCK_STREAM) @@ -31,7 +40,7 @@ def doRequest(req): visited = set() # Add nodes after a successful lookup response rumored = set() # Add rumors about nodes to ping timedout = set() -def handleResponse(publicKey, data): +def handleNodeInfoResponse(publicKey, data): global visited global rumored global timedout @@ -40,21 +49,24 @@ def handleResponse(publicKey, data): if 'response' not in data: return out = dict() for addr,v in data['response'].iteritems(): - if 'keys' not in v: continue - peers = v['keys'] - for key in peers: - if key in visited: continue - if key in timedout: continue - rumored.add(key) out['address'] = addr - out['peers'] = peers - break - selfInfo = doRequest('{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(publicKey)) + out['nodeinfo'] = v + selfInfo = doRequest(getSelfRequest(publicKey)) if 'response' in selfInfo: for _,v in selfInfo['response'].iteritems(): if 'coords' in v: out['coords'] = v['coords'] - dhtInfo = doRequest('{{"keepalive":true, "request":"debugGetDHT", "key":"{}"}}'.format(key)) + peerInfo = doRequest(getPeersRequest(publicKey)) + if 'response' in peerInfo: + for _,v in peerInfo['response'].iteritems(): + if 'keys' not in v: continue + peers = v['keys'] + for key in peers: + if key in visited: continue + if key in timedout: continue + rumored.add(key) + out['peers'] = peers + dhtInfo = doRequest(getDHTRequest(publicKey)) if 'response' in dhtInfo: for _,v in dhtInfo['response'].iteritems(): if 'keys' in v: @@ -63,11 +75,7 @@ def handleResponse(publicKey, data): if key in visited: continue if key in timedout: continue rumored.add(key) - out['dht'] = v['keys'] - nodeInfo = doRequest('{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(publicKey)) - if 'response' in nodeInfo: - for _,v in nodeInfo['response'].iteritems(): - out['nodeinfo'] = v + out['dht'] = dht out['time'] = time.time() if len(visited) > 0: sys.stdout.write(",\n") sys.stdout.write('"{}": {}'.format(publicKey, json.dumps(out))) @@ -86,7 +94,7 @@ for k,v in selfInfo['response']['self'].iteritems(): rumored.add(v['key']) print '{"yggnodes": {' while len(rumored) > 0: for k in rumored: - handleResponse(k, doRequest(getPeersRequest(v['key']))) + handleNodeInfoResponse(k, doRequest(getNodeInfoRequest(v['key']))) break rumored.remove(k) print '\n}}' From 74770f51b5813fbc226b330dbdb86da85426eab4 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 23 May 2021 13:39:45 -0500 Subject: [PATCH 06/11] update for admin debug rename --- scripts/crawl.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/scripts/crawl.py b/scripts/crawl.py index 03ec1ef..730e5b2 100644 --- a/scripts/crawl.py +++ b/scripts/crawl.py @@ -19,13 +19,13 @@ def getNodeInfoRequest(key): return '{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(key) def getSelfRequest(key): - return '{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(key) + return '{{"keepalive":true, "request":"debug_remoteGetSelf", "key":"{}"}}'.format(key) def getPeersRequest(key): - return '{{"keepalive":true, "request":"debugGetPeers", "key":"{}"}}'.format(key) + return '{{"keepalive":true, "request":"debug_remoteGetPeers", "key":"{}"}}'.format(key) def getDHTRequest(key): - return '{{"keepalive":true, "request":"debugGetDHT", "key":"{}"}}'.format(key) + return '{{"keepalive":true, "request":"debug_remoteGetDHT", "key":"{}"}}'.format(key) def doRequest(req): try: From 3e913d2aa23db6c4bb35723629dc36102ecdedd8 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 23 May 2021 14:16:52 -0500 Subject: [PATCH 07/11] make updateGraph use new crawler result format --- scripts/crawl.py | 2 +- web/updateGraph.py | 26 ++++++++++---------------- 2 files changed, 11 insertions(+), 17 deletions(-) diff --git a/scripts/crawl.py b/scripts/crawl.py index 730e5b2..0800af7 100644 --- a/scripts/crawl.py +++ b/scripts/crawl.py @@ -94,7 +94,7 @@ for k,v in selfInfo['response']['self'].iteritems(): rumored.add(v['key']) print '{"yggnodes": {' while len(rumored) > 0: for k in rumored: - handleNodeInfoResponse(k, doRequest(getNodeInfoRequest(v['key']))) + handleNodeInfoResponse(k, doRequest(getNodeInfoRequest(k))) break rumored.remove(k) print '\n}}' diff --git a/web/updateGraph.py b/web/updateGraph.py index 660214c..e52a448 100755 --- a/web/updateGraph.py +++ b/web/updateGraph.py @@ -1,6 +1,4 @@ #!/usr/bin/env python -from flask import Config -from database import NodeDB import graphPlotter import cgi @@ -35,9 +33,16 @@ def generate_graph(time_limit=60*60*3): data = json.loads(response.read())["yggnodes"] toAdd = [] - for ip in data: - info = NodeInfo(ip, data[ip][0]) - if len(data[ip]) >= 3: info.label = data[ip][2] + for key in data: + if 'address' not in data[key] or 'coords' not in data[key]: continue + ip = data[key]['address'] + coords = data[key]['coords'] + info = NodeInfo(ip, coords) + if 'nodeinfo' in data[key]: + if 'name' in data[key]['nodeinfo']: + label = data[key]['nodeinfo']['name'] + if type(label) == str and len(label) <= 32: + info.label = label info.label = cgi.escape(info.label) toAdd.append(info) @@ -68,16 +73,5 @@ def generate_graph(time_limit=60*60*3): with open('static/graph.json', 'w') as f: f.write(js) - -def load_graph_from_db(time_limit): - config = Config('./') - config.from_pyfile('web_config.cfg') - - with NodeDB(config) as db: - nodes = db.get_nodes(time_limit) - edges = db.get_edges(nodes, 60*60*24*7) - return (nodes, edges) - - if __name__ == '__main__': generate_graph() From a94e7c070f75bb5b1d6c127476cdd841a4ed0d3b Mon Sep 17 00:00:00 2001 From: Arceliar Date: Sun, 20 Jun 2021 18:12:02 -0500 Subject: [PATCH 08/11] add crawler.go --- scripts/crawler.go | 167 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 167 insertions(+) create mode 100644 scripts/crawler.go diff --git a/scripts/crawler.go b/scripts/crawler.go new file mode 100644 index 0000000..77c9fb8 --- /dev/null +++ b/scripts/crawler.go @@ -0,0 +1,167 @@ +package main + +import ( + "encoding/json" + "fmt" + "net" + "sync" + "time" +) + +var waitgroup sync.WaitGroup +var visited sync.Map +var rumored sync.Map + +func dial() (net.Conn, error) { + return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second) +} + +func getRequest(key, request string) map[string]interface{} { + return map[string]interface{}{ + "keepalive": true, + "request": request, + "key": key, + } +} + +func doRequest(request map[string]interface{}) map[string]interface{} { + req, err := json.Marshal(request) + if err != nil { + panic(err) + } + sock, err := dial() + if err != nil { + panic(err) + } + if _,err = sock.Write(req); err != nil { + panic(err) + } + bs := make([]byte, 65535) + n, err := sock.Read(bs) + if err != nil { + panic(bs) + } + bs = bs[:n] + var res map[string]interface{} + if err = json.Unmarshal(bs, &res); err != nil { + panic(err) + } + return res +} + +func getNodeInfo(key string) map[string]interface{} { + return doRequest(getRequest(key, "getNodeInfo")) +} + +func getSelf(key string) map[string]interface{} { + return doRequest(getRequest(key, "debug_remoteGetSelf")) +} + +func getPeers(key string) map[string]interface{} { + return doRequest(getRequest(key, "debug_remoteGetPeers")) +} + +func getDHT(key string) map[string]interface{} { + return doRequest(getRequest(key, "debug_remoteGetDHT")) +} + +type rumorResult struct { + key string + res map[string]interface{} +} + +func doRumor(key string, out chan rumorResult) { + waitgroup.Add(1) + go func() { + defer waitgroup.Done() + if _, known := rumored.LoadOrStore(key, true); known { + return + } + defer rumored.Delete(key) + if _, known := visited.Load(key); known { + return + } + results := make(map[string]interface{}) + if res, ok := getNodeInfo(key)["response"]; ok { + for addr,v := range res.(map[string]interface{}) { + results["address"] = addr + results["nodeinfo"] = v + } + } + if res, ok := getSelf(key)["response"]; ok { + for _,v := range res.(map[string]interface{}) { + vm := v.(map[string]interface{}) + if coords, ok := vm["coords"]; ok { + results["coords"] = coords + } + } + } + if res, ok := getPeers(key)["response"]; ok { + for _,v := range res.(map[string]interface{}) { + vm := v.(map[string]interface{}) + if keys, ok := vm["keys"]; ok { + results["peers"] = keys + for _,key := range keys.([]interface{}) { + doRumor(key.(string), out) + } + } + } + } + if res, ok := getDHT(key)["response"]; ok { + for _,v := range res.(map[string]interface{}) { + vm := v.(map[string]interface{}) + if keys, ok := vm["keys"]; ok { + results["dht"] = keys + for _,key := range keys.([]interface{}) { + doRumor(key.(string), out) + } + } + } + } + if len(results) > 0 { + if _, known := visited.LoadOrStore(key, true); known { + return + } + results["time"] = time.Now().Unix() + out<-rumorResult{key, results} + } + }() +} + +func doPrinter() (chan rumorResult, chan struct{}) { + results := make(chan rumorResult) + done := make(chan struct{}) + go func() { + defer close(done) + fmt.Println("{\"yggnodes\": {") + var notFirst bool + for result := range results { + // TODO correct output + res, err := json.Marshal(result.res) + if err != nil { + panic(err) + } + if notFirst { + fmt.Println(",") + } + fmt.Printf("\"%s\": %s", result.key, res) + notFirst = true + } + fmt.Println("\n}}") + }() + return results, done +} + +func main() { + self := doRequest(map[string]interface{}{"keepalive": true, "request": "getSelf"}) + res := self["response"].(map[string]interface{})["self"].(map[string]interface{}) + var key string + for _,v := range res { + key = v.(map[string]interface{})["key"].(string) + } + results, done := doPrinter() + doRumor(key, results) + waitgroup.Wait() + close(results) + <-done +} From f094991677ff165e2fdf8522fe610e71e2da9d10 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Fri, 2 Jul 2021 20:37:10 -0500 Subject: [PATCH 09/11] fix bug in name parsing, don't print info about time-out nodes, and gofmt --- scripts/crawler.go | 263 ++++++++++++++++++++++++--------------------- web/updateGraph.py | 4 +- 2 files changed, 140 insertions(+), 127 deletions(-) diff --git a/scripts/crawler.go b/scripts/crawler.go index 77c9fb8..d4d06e2 100644 --- a/scripts/crawler.go +++ b/scripts/crawler.go @@ -1,11 +1,11 @@ package main import ( - "encoding/json" - "fmt" - "net" - "sync" - "time" + "encoding/json" + "fmt" + "net" + "sync" + "time" ) var waitgroup sync.WaitGroup @@ -13,155 +13,168 @@ var visited sync.Map var rumored sync.Map func dial() (net.Conn, error) { - return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second) + return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second) } func getRequest(key, request string) map[string]interface{} { - return map[string]interface{}{ - "keepalive": true, - "request": request, - "key": key, - } + return map[string]interface{}{ + "keepalive": true, + "request": request, + "key": key, + } } func doRequest(request map[string]interface{}) map[string]interface{} { - req, err := json.Marshal(request) - if err != nil { - panic(err) - } - sock, err := dial() - if err != nil { - panic(err) - } - if _,err = sock.Write(req); err != nil { - panic(err) - } - bs := make([]byte, 65535) - n, err := sock.Read(bs) - if err != nil { - panic(bs) - } - bs = bs[:n] - var res map[string]interface{} - if err = json.Unmarshal(bs, &res); err != nil { - panic(err) - } - return res + req, err := json.Marshal(request) + if err != nil { + panic(err) + } + sock, err := dial() + if err != nil { + panic(err) + } + if _, err = sock.Write(req); err != nil { + panic(err) + } + bs := make([]byte, 65535) + n, err := sock.Read(bs) + if err != nil { + panic(bs) + } + bs = bs[:n] + var res map[string]interface{} + if err = json.Unmarshal(bs, &res); err != nil { + panic(err) + } + return res } func getNodeInfo(key string) map[string]interface{} { - return doRequest(getRequest(key, "getNodeInfo")) + return doRequest(getRequest(key, "getNodeInfo")) } func getSelf(key string) map[string]interface{} { - return doRequest(getRequest(key, "debug_remoteGetSelf")) + return doRequest(getRequest(key, "debug_remoteGetSelf")) } func getPeers(key string) map[string]interface{} { - return doRequest(getRequest(key, "debug_remoteGetPeers")) + return doRequest(getRequest(key, "debug_remoteGetPeers")) } func getDHT(key string) map[string]interface{} { - return doRequest(getRequest(key, "debug_remoteGetDHT")) + return doRequest(getRequest(key, "debug_remoteGetDHT")) } type rumorResult struct { - key string - res map[string]interface{} + key string + res map[string]interface{} } func doRumor(key string, out chan rumorResult) { - waitgroup.Add(1) - go func() { - defer waitgroup.Done() - if _, known := rumored.LoadOrStore(key, true); known { - return - } - defer rumored.Delete(key) - if _, known := visited.Load(key); known { - return - } - results := make(map[string]interface{}) - if res, ok := getNodeInfo(key)["response"]; ok { - for addr,v := range res.(map[string]interface{}) { - results["address"] = addr - results["nodeinfo"] = v - } - } - if res, ok := getSelf(key)["response"]; ok { - for _,v := range res.(map[string]interface{}) { - vm := v.(map[string]interface{}) - if coords, ok := vm["coords"]; ok { - results["coords"] = coords + waitgroup.Add(1) + go func() { + defer waitgroup.Done() + if _, known := rumored.LoadOrStore(key, true); known { + return + } + defer rumored.Delete(key) + if _, known := visited.Load(key); known { + return + } + results := make(map[string]interface{}) + if res, ok := getNodeInfo(key)["response"]; ok { + for addr, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return } - } - } - if res, ok := getPeers(key)["response"]; ok { - for _,v := range res.(map[string]interface{}) { - vm := v.(map[string]interface{}) - if keys, ok := vm["keys"]; ok { - results["peers"] = keys - for _,key := range keys.([]interface{}) { - doRumor(key.(string), out) - } - } - } - } - if res, ok := getDHT(key)["response"]; ok { - for _,v := range res.(map[string]interface{}) { - vm := v.(map[string]interface{}) - if keys, ok := vm["keys"]; ok { - results["dht"] = keys - for _,key := range keys.([]interface{}) { - doRumor(key.(string), out) - } - } - } - } - if len(results) > 0 { - if _, known := visited.LoadOrStore(key, true); known { - return - } - results["time"] = time.Now().Unix() - out<-rumorResult{key, results} - } - }() + results["address"] = addr + results["nodeinfo"] = vm + } + } + if res, ok := getSelf(key)["response"]; ok { + for _, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return + } + if coords, ok := vm["coords"]; ok { + results["coords"] = coords + } + } + } + if res, ok := getPeers(key)["response"]; ok { + for _, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return + } + if keys, ok := vm["keys"]; ok { + results["peers"] = keys + for _, key := range keys.([]interface{}) { + doRumor(key.(string), out) + } + } + } + } + if res, ok := getDHT(key)["response"]; ok { + for _, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return + } + if keys, ok := vm["keys"]; ok { + results["dht"] = keys + for _, key := range keys.([]interface{}) { + doRumor(key.(string), out) + } + } + } + } + if len(results) > 0 { + if _, known := visited.LoadOrStore(key, true); known { + return + } + results["time"] = time.Now().Unix() + out <- rumorResult{key, results} + } + }() } func doPrinter() (chan rumorResult, chan struct{}) { - results := make(chan rumorResult) - done := make(chan struct{}) - go func() { - defer close(done) - fmt.Println("{\"yggnodes\": {") - var notFirst bool - for result := range results { - // TODO correct output - res, err := json.Marshal(result.res) - if err != nil { - panic(err) - } - if notFirst { - fmt.Println(",") - } - fmt.Printf("\"%s\": %s", result.key, res) - notFirst = true - } - fmt.Println("\n}}") - }() - return results, done + results := make(chan rumorResult) + done := make(chan struct{}) + go func() { + defer close(done) + fmt.Println("{\"yggnodes\": {") + var notFirst bool + for result := range results { + // TODO correct output + res, err := json.Marshal(result.res) + if err != nil { + panic(err) + } + if notFirst { + fmt.Println(",") + } + fmt.Printf("\"%s\": %s", result.key, res) + notFirst = true + } + fmt.Println("\n}}") + }() + return results, done } func main() { - self := doRequest(map[string]interface{}{"keepalive": true, "request": "getSelf"}) - res := self["response"].(map[string]interface{})["self"].(map[string]interface{}) - var key string - for _,v := range res { - key = v.(map[string]interface{})["key"].(string) - } - results, done := doPrinter() - doRumor(key, results) - waitgroup.Wait() - close(results) - <-done + self := doRequest(map[string]interface{}{"keepalive": true, "request": "getSelf"}) + res := self["response"].(map[string]interface{})["self"].(map[string]interface{}) + var key string + for _, v := range res { + key = v.(map[string]interface{})["key"].(string) + } + results, done := doPrinter() + doRumor(key, results) + waitgroup.Wait() + close(results) + <-done } diff --git a/web/updateGraph.py b/web/updateGraph.py index e52a448..54f978a 100755 --- a/web/updateGraph.py +++ b/web/updateGraph.py @@ -40,8 +40,8 @@ def generate_graph(time_limit=60*60*3): info = NodeInfo(ip, coords) if 'nodeinfo' in data[key]: if 'name' in data[key]['nodeinfo']: - label = data[key]['nodeinfo']['name'] - if type(label) == str and len(label) <= 32: + label = str(data[key]['nodeinfo']['name']) + if len(label) <= 32: info.label = label info.label = cgi.escape(info.label) toAdd.append(info) From b744a6ab1a40fccb8bd9f51cae394b67dc954fa5 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 6 Jul 2021 18:48:25 -0500 Subject: [PATCH 10/11] gofmt and minor parsing fix --- scripts/crawler.go | 8 ++++---- web/updateGraph.py | 12 +++++++----- 2 files changed, 11 insertions(+), 9 deletions(-) diff --git a/scripts/crawler.go b/scripts/crawler.go index d4d06e2..0f46391 100644 --- a/scripts/crawler.go +++ b/scripts/crawler.go @@ -84,10 +84,10 @@ func doRumor(key string, out chan rumorResult) { results := make(map[string]interface{}) if res, ok := getNodeInfo(key)["response"]; ok { for addr, v := range res.(map[string]interface{}) { - vm, ok := v.(map[string]interface{}) - if !ok { - return - } + vm, ok := v.(map[string]interface{}) + if !ok { + return + } results["address"] = addr results["nodeinfo"] = vm } diff --git a/web/updateGraph.py b/web/updateGraph.py index 54f978a..a7210ad 100755 --- a/web/updateGraph.py +++ b/web/updateGraph.py @@ -38,11 +38,13 @@ def generate_graph(time_limit=60*60*3): ip = data[key]['address'] coords = data[key]['coords'] info = NodeInfo(ip, coords) - if 'nodeinfo' in data[key]: - if 'name' in data[key]['nodeinfo']: - label = str(data[key]['nodeinfo']['name']) - if len(label) <= 32: - info.label = label + try: + if 'nodeinfo' in data[key]: + if 'name' in data[key]['nodeinfo']: + label = str(data[key]['nodeinfo']['name']) + if len(label) <= 32: + info.label = label + except: pass info.label = cgi.escape(info.label) toAdd.append(info) From 7b8b0e0ba6dc7507517feb36a3df674af4034597 Mon Sep 17 00:00:00 2001 From: Arceliar Date: Tue, 6 Jul 2021 19:12:06 -0500 Subject: [PATCH 11/11] retry requests a few times before failing --- scripts/crawler.go | 41 ++++++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 15 deletions(-) diff --git a/scripts/crawler.go b/scripts/crawler.go index 0f46391..d66cc9e 100644 --- a/scripts/crawler.go +++ b/scripts/crawler.go @@ -12,6 +12,8 @@ var waitgroup sync.WaitGroup var visited sync.Map var rumored sync.Map +const MAX_RETRY = 3 + func dial() (net.Conn, error) { return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second) } @@ -29,22 +31,31 @@ func doRequest(request map[string]interface{}) map[string]interface{} { if err != nil { panic(err) } - sock, err := dial() - if err != nil { - panic(err) - } - if _, err = sock.Write(req); err != nil { - panic(err) - } - bs := make([]byte, 65535) - n, err := sock.Read(bs) - if err != nil { - panic(bs) - } - bs = bs[:n] var res map[string]interface{} - if err = json.Unmarshal(bs, &res); err != nil { - panic(err) + for idx := 0; idx < MAX_RETRY; idx++ { + sock, err := dial() + if err != nil { + panic(err) + } + if _, err = sock.Write(req); err != nil { + panic(err) + } + bs := make([]byte, 65535) + n, err := sock.Read(bs) + if err != nil { + panic(bs) + } + bs = bs[:n] + if err = json.Unmarshal(bs, &res); err != nil { + panic(err) + } + // TODO parse res, check if there's an error + if res, ok := res["response"]; ok { + if _, isIn := res.(map[string]interface{})["error"]; isIn { + continue + } + } + break } return res }