cwinfo/yggdrasil-map
5
0
Fork 0
mirror of https://github.com/cwinfo/yggdrasil-map synced 2024-11-24 15:41:35 +00:00
Code Releases Activity

update crawler

Browse Source
This commit is contained in:
Arceliar 2021-05-23 12:41:27 -05:00
parent b47b1fffd4
commit 35b2d87911
1 changed files with 25 additions and 25 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

50
scripts/crawl.py
View File

@ -31,43 +31,43 @@ def doRequest(req):
visited = set() # Add nodes after a successful lookup response visited = set() # Add nodes after a successful lookup response
rumored = set() # Add rumors about nodes to ping rumored = set() # Add rumors about nodes to ping
timedout = set() timedout = set()
def handleResponse(address, data): def handleResponse(publicKey, data):
global visited global visited
global rumored global rumored
global timedout global timedout
if address in visited: return if publicKey in visited: return
if not data: return if not data: return
if 'response' not in data: return if 'response' not in data: return
for k,v in data['response'].iteritems(): out = dict()
for addr,v in data['response'].iteritems():
if 'keys' not in v: continue if 'keys' not in v: continue
keys = v['keys'] peers = v['keys']
for key in keys: for key in peers:
if key in visited: continue if key in visited: continue
if key in timedout: continue if key in timedout: continue
rumored.add(key) rumored.add(key)
selfInfo = doRequest('{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(address)) out['address'] = addr
if 'response' not in selfInfo: return out['peers'] = peers
coords = None
for _,v in selfInfo['response'].iteritems():
if 'Coords' not in v: continue
coords = str(v['Coords'])
break break
if coords == None: return selfInfo = doRequest('{{"keepalive":true, "request":"debugGetSelf", "key":"{}"}}'.format(publicKey))
nodename = None if 'response' in selfInfo:
nodeinfo = doRequest('{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(address)) for _,v in selfInfo['response'].iteritems():
try: if 'coords' in v:
if nodeinfo and 'response' in nodeinfo and 'nodeinfo' in nodeinfo['response'] and 'name' in nodeinfo['response']['nodeinfo']: out['coords'] = v['coords']
nodename = '"' + str(nodeinfo['response']['nodeinfo']['name']) + '"' dhtInfo = doRequest('{{"keepalive":true, "request":"debugGetDHT", "key":"{}"}}'.format(key))
except: if 'response' in dhtInfo:
pass for _,v in dhtInfo['response'].iteritems():
now = time.time() if 'keys' in v:
out['dht'] = v['keys']
nodeInfo = doRequest('{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(publicKey))
if 'response' in nodeInfo:
for _,v in nodeInfo['response'].iteritems():
out['nodeinfo'] = v
out['time'] = time.time()
if len(visited) > 0: sys.stdout.write(",\n") if len(visited) > 0: sys.stdout.write(",\n")
if nodename: sys.stdout.write('"{}": {}'.format(publicKey, json.dumps(out)))
sys.stdout.write('"{}": ["{}", {}, {}]'.format(address, coords, int(now), nodename))
else:
sys.stdout.write('"{}": ["{}", {}]'.format(address, coords, int(now)))
sys.stdout.flush() sys.stdout.flush()
visited.add(address) visited.add(publicKey)
# End handleResponse # End handleResponse
# Get self info # Get self info