diff --git a/scripts/crawler.go b/scripts/crawler.go index 77c9fb8..d4d06e2 100644 --- a/scripts/crawler.go +++ b/scripts/crawler.go @@ -1,11 +1,11 @@ package main import ( - "encoding/json" - "fmt" - "net" - "sync" - "time" + "encoding/json" + "fmt" + "net" + "sync" + "time" ) var waitgroup sync.WaitGroup @@ -13,155 +13,168 @@ var visited sync.Map var rumored sync.Map func dial() (net.Conn, error) { - return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second) + return net.DialTimeout("unix", "/var/run/yggdrasil.sock", time.Second) } func getRequest(key, request string) map[string]interface{} { - return map[string]interface{}{ - "keepalive": true, - "request": request, - "key": key, - } + return map[string]interface{}{ + "keepalive": true, + "request": request, + "key": key, + } } func doRequest(request map[string]interface{}) map[string]interface{} { - req, err := json.Marshal(request) - if err != nil { - panic(err) - } - sock, err := dial() - if err != nil { - panic(err) - } - if _,err = sock.Write(req); err != nil { - panic(err) - } - bs := make([]byte, 65535) - n, err := sock.Read(bs) - if err != nil { - panic(bs) - } - bs = bs[:n] - var res map[string]interface{} - if err = json.Unmarshal(bs, &res); err != nil { - panic(err) - } - return res + req, err := json.Marshal(request) + if err != nil { + panic(err) + } + sock, err := dial() + if err != nil { + panic(err) + } + if _, err = sock.Write(req); err != nil { + panic(err) + } + bs := make([]byte, 65535) + n, err := sock.Read(bs) + if err != nil { + panic(bs) + } + bs = bs[:n] + var res map[string]interface{} + if err = json.Unmarshal(bs, &res); err != nil { + panic(err) + } + return res } func getNodeInfo(key string) map[string]interface{} { - return doRequest(getRequest(key, "getNodeInfo")) + return doRequest(getRequest(key, "getNodeInfo")) } func getSelf(key string) map[string]interface{} { - return doRequest(getRequest(key, "debug_remoteGetSelf")) + return doRequest(getRequest(key, "debug_remoteGetSelf")) } func getPeers(key string) map[string]interface{} { - return doRequest(getRequest(key, "debug_remoteGetPeers")) + return doRequest(getRequest(key, "debug_remoteGetPeers")) } func getDHT(key string) map[string]interface{} { - return doRequest(getRequest(key, "debug_remoteGetDHT")) + return doRequest(getRequest(key, "debug_remoteGetDHT")) } type rumorResult struct { - key string - res map[string]interface{} + key string + res map[string]interface{} } func doRumor(key string, out chan rumorResult) { - waitgroup.Add(1) - go func() { - defer waitgroup.Done() - if _, known := rumored.LoadOrStore(key, true); known { - return - } - defer rumored.Delete(key) - if _, known := visited.Load(key); known { - return - } - results := make(map[string]interface{}) - if res, ok := getNodeInfo(key)["response"]; ok { - for addr,v := range res.(map[string]interface{}) { - results["address"] = addr - results["nodeinfo"] = v - } - } - if res, ok := getSelf(key)["response"]; ok { - for _,v := range res.(map[string]interface{}) { - vm := v.(map[string]interface{}) - if coords, ok := vm["coords"]; ok { - results["coords"] = coords + waitgroup.Add(1) + go func() { + defer waitgroup.Done() + if _, known := rumored.LoadOrStore(key, true); known { + return + } + defer rumored.Delete(key) + if _, known := visited.Load(key); known { + return + } + results := make(map[string]interface{}) + if res, ok := getNodeInfo(key)["response"]; ok { + for addr, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return } - } - } - if res, ok := getPeers(key)["response"]; ok { - for _,v := range res.(map[string]interface{}) { - vm := v.(map[string]interface{}) - if keys, ok := vm["keys"]; ok { - results["peers"] = keys - for _,key := range keys.([]interface{}) { - doRumor(key.(string), out) - } - } - } - } - if res, ok := getDHT(key)["response"]; ok { - for _,v := range res.(map[string]interface{}) { - vm := v.(map[string]interface{}) - if keys, ok := vm["keys"]; ok { - results["dht"] = keys - for _,key := range keys.([]interface{}) { - doRumor(key.(string), out) - } - } - } - } - if len(results) > 0 { - if _, known := visited.LoadOrStore(key, true); known { - return - } - results["time"] = time.Now().Unix() - out<-rumorResult{key, results} - } - }() + results["address"] = addr + results["nodeinfo"] = vm + } + } + if res, ok := getSelf(key)["response"]; ok { + for _, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return + } + if coords, ok := vm["coords"]; ok { + results["coords"] = coords + } + } + } + if res, ok := getPeers(key)["response"]; ok { + for _, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return + } + if keys, ok := vm["keys"]; ok { + results["peers"] = keys + for _, key := range keys.([]interface{}) { + doRumor(key.(string), out) + } + } + } + } + if res, ok := getDHT(key)["response"]; ok { + for _, v := range res.(map[string]interface{}) { + vm, ok := v.(map[string]interface{}) + if !ok { + return + } + if keys, ok := vm["keys"]; ok { + results["dht"] = keys + for _, key := range keys.([]interface{}) { + doRumor(key.(string), out) + } + } + } + } + if len(results) > 0 { + if _, known := visited.LoadOrStore(key, true); known { + return + } + results["time"] = time.Now().Unix() + out <- rumorResult{key, results} + } + }() } func doPrinter() (chan rumorResult, chan struct{}) { - results := make(chan rumorResult) - done := make(chan struct{}) - go func() { - defer close(done) - fmt.Println("{\"yggnodes\": {") - var notFirst bool - for result := range results { - // TODO correct output - res, err := json.Marshal(result.res) - if err != nil { - panic(err) - } - if notFirst { - fmt.Println(",") - } - fmt.Printf("\"%s\": %s", result.key, res) - notFirst = true - } - fmt.Println("\n}}") - }() - return results, done + results := make(chan rumorResult) + done := make(chan struct{}) + go func() { + defer close(done) + fmt.Println("{\"yggnodes\": {") + var notFirst bool + for result := range results { + // TODO correct output + res, err := json.Marshal(result.res) + if err != nil { + panic(err) + } + if notFirst { + fmt.Println(",") + } + fmt.Printf("\"%s\": %s", result.key, res) + notFirst = true + } + fmt.Println("\n}}") + }() + return results, done } func main() { - self := doRequest(map[string]interface{}{"keepalive": true, "request": "getSelf"}) - res := self["response"].(map[string]interface{})["self"].(map[string]interface{}) - var key string - for _,v := range res { - key = v.(map[string]interface{})["key"].(string) - } - results, done := doPrinter() - doRumor(key, results) - waitgroup.Wait() - close(results) - <-done + self := doRequest(map[string]interface{}{"keepalive": true, "request": "getSelf"}) + res := self["response"].(map[string]interface{})["self"].(map[string]interface{}) + var key string + for _, v := range res { + key = v.(map[string]interface{})["key"].(string) + } + results, done := doPrinter() + doRumor(key, results) + waitgroup.Wait() + close(results) + <-done } diff --git a/web/updateGraph.py b/web/updateGraph.py index e52a448..54f978a 100755 --- a/web/updateGraph.py +++ b/web/updateGraph.py @@ -40,8 +40,8 @@ def generate_graph(time_limit=60*60*3): info = NodeInfo(ip, coords) if 'nodeinfo' in data[key]: if 'name' in data[key]['nodeinfo']: - label = data[key]['nodeinfo']['name'] - if type(label) == str and len(label) <= 32: + label = str(data[key]['nodeinfo']['name']) + if len(label) <= 32: info.label = label info.label = cgi.escape(info.label) toAdd.append(info)