add histogramming code, update crawler, allow longer names

scripts/crawler.go

@@ -13,6 +13,13 @@ var visited sync.Map
 var rumored sync.Map
 
 const MAX_RETRY = 3
+const N_PARALLEL_REQ = 32
+
+var semaphore chan struct{}
+
+func init() {
+  semaphore = make(chan struct{}, N_PARALLEL_REQ)
+}
 
 func dial() (net.Conn, error) {
 	return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second)
@@ -85,6 +92,8 @@ func doRumor(key string, out chan rumorResult) {
 	waitgroup.Add(1)
 	go func() {
 		defer waitgroup.Done()
+		semaphore<-struct{}{}
+		defer func() { <-semaphore }()
 		if _, known := rumored.LoadOrStore(key, true); known {
 			return
 		}

scripts/ratio.py

@@ -0,0 +1,41 @@
+#!/usr/bin/env python2.7
+
+import json
+
+with open("current", "r") as f:
+  data = json.loads(f.read())
+
+kinds = ["peers", "dht", "ratio"]
+results = dict()
+for kind in kinds:
+  results[kind] = dict()
+
+for k,v in data["yggnodes"].iteritems():
+  for kind in kinds:
+    if kind not in v: continue
+    num = len(v[kind])
+    if num not in results[kind]: results[kind][num] = 0
+    results[kind][num] += 1
+  # Added ratio part
+  if "dht" in v and "peers" in v:
+    ratio = float(len(v["dht"]))/len(v["peers"])
+    if ratio not in results["ratio"]: results["ratio"][ratio] = 0
+    results["ratio"][ratio] += 1
+
+import matplotlib.pyplot as plt
+fig, axs = plt.subplots(1, len(kinds), sharey=True, tight_layout=True)
+for kdx in xrange(len(kinds)):
+  kind = kinds[kdx]
+  bins = []
+  for num,count in results[kind].iteritems():
+    bins += [num]*count
+  nbins = max(results[kind].keys())+1
+  axs[kdx].set_title(kind)
+  if kind == "ratio":
+    nbins="auto"
+    axs[kdx].set_title("ratio (dht/peers)")
+  axs[kdx].hist(bins, bins=nbins)
+plt.savefig("fig.svg")
+
+print results
+

web/updateGraph.py

@@ -42,7 +42,7 @@ def generate_graph(time_limit=60*60*3):
         if 'nodeinfo' in data[key]:
           if 'name' in data[key]['nodeinfo']:
             label = str(data[key]['nodeinfo']['name'])
-            if len(label) <= 32:
+            if len(label) <= 64:
               info.label = label
       except: pass
       info.label = cgi.escape(info.label)