update crawlers for v0.5

scripts/crawl.py

@@ -15,18 +15,9 @@ else:
   socktype = socket.AF_UNIX
   sockaddr = "/var/run/yggdrasil.sock"
 
-def getNodeInfoRequest(key):
-  return '{{"keepalive":true, "request":"getNodeInfo", "arguments": {{"key":"{}"}}}}'.format(key)
-
-def getSelfRequest(key):
-  return '{{"keepalive":true, "request":"debug_remoteGetSelf", "arguments": {{"key":"{}"}}}}'.format(key)
-
 def getPeersRequest(key):
   return '{{"keepalive":true, "request":"debug_remoteGetPeers", "arguments": {{"key":"{}"}}}}'.format(key)
 
-def getDHTRequest(key):
-  return '{{"keepalive":true, "request":"debug_remoteGetDHT", "arguments": {{"key":"{}"}}}}'.format(key)
-
 def doRequest(req):
   try:
     ygg = socket.socket(socktype, socket.SOCK_STREAM)
@@ -40,64 +31,44 @@ def doRequest(req):
 visited = set() # Add nodes after a successful lookup response
 rumored = set() # Add rumors about nodes to ping
 timedout = set()
-def handleNodeInfoResponse(publicKey, data):
-  global visited
+
+def handleGetPeersResponse(publicKey, data):
+  global vistied
   global rumored
   global timedout
   if publicKey in visited: return
-  if not data: return
-  if 'response' not in data: return
-  out = dict()
-  for addr,v in data['response'].iteritems():
-    out['address'] = addr
-    out['nodeinfo'] = v
-  selfInfo = doRequest(getSelfRequest(publicKey))
-  if 'response' in selfInfo:
-    for _,v in selfInfo['response'].iteritems():
-      if 'coords' in v:
-        out['coords'] = v['coords']
-  peerInfo = doRequest(getPeersRequest(publicKey))
-  if 'response' in peerInfo:
-    for _,v in peerInfo['response'].iteritems():
-      if 'keys' not in v: continue
-      peers = v['keys']
-      for key in peers:
-        if key in visited: continue
-        if key in timedout: continue
-        rumored.add(key)
-      out['peers'] = peers
-  dhtInfo = doRequest(getDHTRequest(publicKey))
-  if 'response' in dhtInfo:
-    for _,v in dhtInfo['response'].iteritems():
-      if 'keys' in v:
-        dht = v['keys']
-        for key in dht:
-          if key in visited: continue
-          if key in timedout: continue
-          rumored.add(key)
-        out['dht'] = dht
-  out['time'] = time.time()
-  if len(visited) > 0: sys.stdout.write(",\n")
-  sys.stdout.write('"{}": {}'.format(publicKey, json.dumps(out)))
-  sys.stdout.flush()
-  visited.add(publicKey)
-# End handleResponse
+  #visited.add(publicKey)
+  try:
+    ks = data['response'].values()[0]['keys']
+    for k in ks:
+      if k in visited: continue
+      if k in timedout: continue
+      rumored.add(k)
+    visited.add(publicKey)
+  except:
+    pass
 
 # Get self info
 selfInfo = doRequest('{"keepalive":true, "request":"getSelf"}')
-rumored.add(selfInfo['response']['key'])
+#rumored.add(selfInfo['response']['key'])
+visited.add(selfInfo['response']['key'])
+try:
+  peers = doRequest('{"keepalive":true, "request":"getPeers"}')
+  for p in peers['response']['peers']:
+    rumored.add(p['key'])
+except:
+  pass
 
 # Initialize dicts of visited/rumored nodes
 #for k,v in selfInfo['response']['self'].iteritems(): rumored[k] = v
 
 # Loop over rumored nodes and ping them, adding to visited if they respond
-print '{"yggnodes": {'
 while len(rumored) > 0:
   for k in rumored:
-    handleNodeInfoResponse(k, doRequest(getNodeInfoRequest(k)))
+    print "DEBUG tested:", len(visited), "remaining:", len(rumored)
+    handleGetPeersResponse(k, doRequest(getPeersRequest(k)))
     break
   rumored.remove(k)
-print '\n}}'
 #End
 
 # TODO do something with the results

scripts/crawler.go

@@ -91,6 +91,14 @@ func getDHT(key string) map[string]interface{} {
 	return doRequest(getRequest(key, "debug_remoteGetDHT"))
 }
 
+func getPaths() map[string]interface{} {
+	req := map[string]interface{}{
+		"keepalive": true,
+		"request":   "getPaths",
+	}
+	return doRequest(req)
+}
+
 type rumorResult struct {
 	key string
 	res map[string]interface{}
@@ -116,6 +124,39 @@ func doRumor(key string, out chan rumorResult) {
 				results["nodeinfo"] = vm
 			}
 		}
+		
+		if res, ok := getPaths()["response"]; ok {
+			if _, ok := res.(map[string]interface{}); !ok {
+				return
+			}
+			psi := res.(map[string]interface{})["paths"]
+			if _, ok := psi.([]interface{}); !ok {
+				return
+			}
+			ps := psi.([]interface{})
+			for _, pi := range ps {
+				if _, ok := pi.(map[string]interface{}); !ok {
+					return
+				}
+				p := pi.(map[string]interface{})
+				if _, ok := p["key"]; !ok {
+					return
+				}
+				ki := p["key"]
+				if _, ok := ki.(string); !ok {
+					return
+				}
+				k := ki.(string)
+				if k != key {
+					continue
+				}
+				if _, ok := p["path"]; !ok {
+					return
+				}
+				results["coords"] = fmt.Sprintf("%v", p["path"])
+			}
+		}
+		/*
 		if res, ok := getSelf(key)["response"]; ok {
 			if _, ok := res.(map[string]interface{}); !ok {
 				return
@@ -130,6 +171,7 @@ func doRumor(key string, out chan rumorResult) {
 				}
 			}
 		}
+		*/
 		if res, ok := getPeers(key)["response"]; ok {
 			if _, ok := res.(map[string]interface{}); !ok {
 				return
@@ -144,6 +186,7 @@ func doRumor(key string, out chan rumorResult) {
 				}
 			}
 		}
+		/*
 		if res, ok := getDHT(key)["response"]; ok {
 			if _, ok := res.(map[string]interface{}); !ok {
 				return
@@ -158,6 +201,7 @@ func doRumor(key string, out chan rumorResult) {
 				}
 			}
 		}
+		*/
 		if len(results) > 0 {
 			results["time"] = time.Now().Unix()
 			out <- rumorResult{key, results}

scripts/getcurrent.py

@@ -0,0 +1,45 @@
+import json
+import socket
+import sys
+import time
+
+#gives the option to get data from an external server instead and send that
+#if no options given it will default to localhost instead
+if len(sys.argv) == 3:
+  socktype = socket.AF_INET
+  sockaddr = (sys.argv[1], int(sys.argv[2]))
+elif len(sys.argv) == 2:
+  socktype = socket.AF_UNIX
+  sockaddr = sys.argv[1]
+else:
+  socktype = socket.AF_UNIX
+  sockaddr = "/var/run/yggdrasil.sock"
+
+def doRequest(req):
+  try:
+    ygg = socket.socket(socktype, socket.SOCK_STREAM)
+    ygg.connect(sockaddr)
+    ygg.send(req)
+    data = json.loads(ygg.recv(1048576))
+    return data
+  except:
+    return None
+
+known = doRequest('{"keepalive":true, "request":"lookups"}')
+
+infos = dict()
+for node in known['response']['infos']:
+  coords = json.dumps(node["path"]).replace(",", " ")
+  v = {"address": node["addr"], "coords": coords, "time": node["time"]}
+  infos[node["key"]] = v
+
+# Initialize dicts of visited/rumored nodes
+#for k,v in selfInfo['response']['self'].iteritems(): rumored[k] = v
+
+# Loop over rumored nodes and ping them, adding to visited if they respond
+print json.dumps({"yggnodes": infos})
+
+# TODO do something with the results
+
+#print visited
+#print timedout

web/graphData.py

@@ -1,5 +1,5 @@
 import json
-from database import NodeDB
+#from database import NodeDB
 from graph import Node, Edge
 import traceback
 import time