From 19cdf8de3438b48b764d8f8c223804c2cc02c938 Mon Sep 17 00:00:00 2001
From: Arceliar <Arceliar@users.noreply.github.com>
Date: Sat, 31 Jul 2021 10:34:52 -0500
Subject: [PATCH] add histogramming code, update crawler, allow longer names

---
 scripts/crawler.go |  9 +++++++++
 scripts/ratio.py   | 41 +++++++++++++++++++++++++++++++++++++++++
 web/updateGraph.py |  2 +-
 3 files changed, 51 insertions(+), 1 deletion(-)
 create mode 100644 scripts/ratio.py

diff --git a/scripts/crawler.go b/scripts/crawler.go
index d66cc9e..c1b5a8c 100644
--- a/scripts/crawler.go
+++ b/scripts/crawler.go
@@ -13,6 +13,13 @@ var visited sync.Map
 var rumored sync.Map
 
 const MAX_RETRY = 3
+const N_PARALLEL_REQ = 32
+
+var semaphore chan struct{}
+
+func init() {
+  semaphore = make(chan struct{}, N_PARALLEL_REQ)
+}
 
 func dial() (net.Conn, error) {
 	return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second)
@@ -85,6 +92,8 @@ func doRumor(key string, out chan rumorResult) {
 	waitgroup.Add(1)
 	go func() {
 		defer waitgroup.Done()
+		semaphore<-struct{}{}
+		defer func() { <-semaphore }()
 		if _, known := rumored.LoadOrStore(key, true); known {
 			return
 		}
diff --git a/scripts/ratio.py b/scripts/ratio.py
new file mode 100644
index 0000000..9a8bd58
--- /dev/null
+++ b/scripts/ratio.py
@@ -0,0 +1,41 @@
+#!/usr/bin/env python2.7
+
+import json
+
+with open("current", "r") as f:
+  data = json.loads(f.read())
+
+kinds = ["peers", "dht", "ratio"]
+results = dict()
+for kind in kinds:
+  results[kind] = dict()
+
+for k,v in data["yggnodes"].iteritems():
+  for kind in kinds:
+    if kind not in v: continue
+    num = len(v[kind])
+    if num not in results[kind]: results[kind][num] = 0
+    results[kind][num] += 1
+  # Added ratio part
+  if "dht" in v and "peers" in v:
+    ratio = float(len(v["dht"]))/len(v["peers"])
+    if ratio not in results["ratio"]: results["ratio"][ratio] = 0
+    results["ratio"][ratio] += 1
+
+import matplotlib.pyplot as plt
+fig, axs = plt.subplots(1, len(kinds), sharey=True, tight_layout=True)
+for kdx in xrange(len(kinds)):
+  kind = kinds[kdx]
+  bins = []
+  for num,count in results[kind].iteritems():
+    bins += [num]*count
+  nbins = max(results[kind].keys())+1
+  axs[kdx].set_title(kind)
+  if kind == "ratio":
+    nbins="auto"
+    axs[kdx].set_title("ratio (dht/peers)")
+  axs[kdx].hist(bins, bins=nbins)
+plt.savefig("fig.svg")
+
+print results
+
diff --git a/web/updateGraph.py b/web/updateGraph.py
index a7210ad..1bb9eac 100755
--- a/web/updateGraph.py
+++ b/web/updateGraph.py
@@ -42,7 +42,7 @@ def generate_graph(time_limit=60*60*3):
         if 'nodeinfo' in data[key]:
           if 'name' in data[key]['nodeinfo']:
             label = str(data[key]['nodeinfo']['name'])
-            if len(label) <= 32:
+            if len(label) <= 64:
               info.label = label
       except: pass
       info.label = cgi.escape(info.label)