5
0
mirror of https://github.com/cwinfo/yggdrasil-map synced 2024-11-09 16:00:27 +00:00

fix crawlers (hopefully)

This commit is contained in:
Arceliar 2022-12-03 16:13:55 -06:00
parent 056d976821
commit a627042420
2 changed files with 43 additions and 29 deletions

View File

@ -16,23 +16,23 @@ else:
sockaddr = "/var/run/yggdrasil.sock" sockaddr = "/var/run/yggdrasil.sock"
def getNodeInfoRequest(key): def getNodeInfoRequest(key):
return '{{"keepalive":true, "request":"getNodeInfo", "key":"{}"}}'.format(key) return '{{"keepalive":true, "request":"getNodeInfo", "arguments": {{"key":"{}"}}}}'.format(key)
def getSelfRequest(key): def getSelfRequest(key):
return '{{"keepalive":true, "request":"debug_remoteGetSelf", "key":"{}"}}'.format(key) return '{{"keepalive":true, "request":"debug_remoteGetSelf", "arguments": {{"key":"{}"}}}}'.format(key)
def getPeersRequest(key): def getPeersRequest(key):
return '{{"keepalive":true, "request":"debug_remoteGetPeers", "key":"{}"}}'.format(key) return '{{"keepalive":true, "request":"debug_remoteGetPeers", "arguments": {{"key":"{}"}}}}'.format(key)
def getDHTRequest(key): def getDHTRequest(key):
return '{{"keepalive":true, "request":"debug_remoteGetDHT", "key":"{}"}}'.format(key) return '{{"keepalive":true, "request":"debug_remoteGetDHT", "arguments": {{"key":"{}"}}}}'.format(key)
def doRequest(req): def doRequest(req):
try: try:
ygg = socket.socket(socktype, socket.SOCK_STREAM) ygg = socket.socket(socktype, socket.SOCK_STREAM)
ygg.connect(sockaddr) ygg.connect(sockaddr)
ygg.send(req) ygg.send(req)
data = json.loads(ygg.recv(1024*15)) data = json.loads(ygg.recv(1048576))
return data return data
except: except:
return None return None
@ -85,7 +85,7 @@ def handleNodeInfoResponse(publicKey, data):
# Get self info # Get self info
selfInfo = doRequest('{"keepalive":true, "request":"getSelf"}') selfInfo = doRequest('{"keepalive":true, "request":"getSelf"}')
for k,v in selfInfo['response']['self'].iteritems(): rumored.add(v['key']) rumored.add(selfInfo['response']['key'])
# Initialize dicts of visited/rumored nodes # Initialize dicts of visited/rumored nodes
#for k,v in selfInfo['response']['self'].iteritems(): rumored[k] = v #for k,v in selfInfo['response']['self'].iteritems(): rumored[k] = v

View File

@ -9,8 +9,6 @@ import (
) )
var waitgroup sync.WaitGroup var waitgroup sync.WaitGroup
var visited sync.Map
var rumored sync.Map
const MAX_RETRY = 3 const MAX_RETRY = 3
const N_PARALLEL_REQ = 32 const N_PARALLEL_REQ = 32
@ -29,7 +27,9 @@ func getRequest(key, request string) map[string]interface{} {
return map[string]interface{}{ return map[string]interface{}{
"keepalive": true, "keepalive": true,
"request": request, "request": request,
"key": key, "arguments": map[string]interface{}{
"key": key,
},
} }
} }
@ -99,13 +99,6 @@ func doRumor(key string, out chan rumorResult) {
defer waitgroup.Done() defer waitgroup.Done()
semaphore <- struct{}{} semaphore <- struct{}{}
defer func() { <-semaphore }() defer func() { <-semaphore }()
if _, known := rumored.LoadOrStore(key, true); known {
return
}
defer rumored.Delete(key)
if _, known := visited.Load(key); known {
return
}
results := make(map[string]interface{}) results := make(map[string]interface{})
if res, ok := getNodeInfo(key)["response"]; ok { if res, ok := getNodeInfo(key)["response"]; ok {
for addr, v := range res.(map[string]interface{}) { for addr, v := range res.(map[string]interface{}) {
@ -136,9 +129,6 @@ func doRumor(key string, out chan rumorResult) {
} }
if keys, ok := vm["keys"]; ok { if keys, ok := vm["keys"]; ok {
results["peers"] = keys results["peers"] = keys
for _, key := range keys.([]interface{}) {
doRumor(key.(string), out)
}
} }
} }
} }
@ -150,18 +140,13 @@ func doRumor(key string, out chan rumorResult) {
} }
if keys, ok := vm["keys"]; ok { if keys, ok := vm["keys"]; ok {
results["dht"] = keys results["dht"] = keys
for _, key := range keys.([]interface{}) {
doRumor(key.(string), out)
}
} }
} }
} }
if len(results) > 0 { if len(results) > 0 {
if _, known := visited.LoadOrStore(key, true); known {
return
}
results["time"] = time.Now().Unix() results["time"] = time.Now().Unix()
out <- rumorResult{key, results} out <- rumorResult{key, results}
waitgroup.Add(1)
} }
}() }()
} }
@ -171,10 +156,16 @@ func doPrinter() (chan rumorResult, chan struct{}) {
done := make(chan struct{}) done := make(chan struct{})
go func() { go func() {
defer close(done) defer close(done)
visited := make(map[string]struct{})
fmt.Println("{\"yggnodes\": {") fmt.Println("{\"yggnodes\": {")
var notFirst bool var notFirst bool
for result := range results { for result := range results {
// TODO correct output // TODO correct output
if _, isIn := visited[result.key]; isIn {
waitgroup.Done()
continue
}
visited[result.key] = struct{}{}
res, err := json.Marshal(result.res) res, err := json.Marshal(result.res)
if err != nil { if err != nil {
panic(err) panic(err)
@ -184,6 +175,31 @@ func doPrinter() (chan rumorResult, chan struct{}) {
} }
fmt.Printf("\"%s\": %s", result.key, res) fmt.Printf("\"%s\": %s", result.key, res)
notFirst = true notFirst = true
toVisit := make(map[string]struct{})
if peers, isIn := result.res["peers"]; isIn {
if ks, ok := peers.([]interface{}); ok {
for _, k := range ks {
if key, ok := k.(string); ok {
toVisit[key] = struct{}{}
}
}
}
}
if dht, isIn := result.res["dht"]; isIn {
if ks, ok := dht.([]interface{}); ok {
for _, k := range ks {
if key, ok := k.(string); ok {
toVisit[key] = struct{}{}
}
}
}
}
for k := range toVisit {
if _, isIn := visited[k]; !isIn {
doRumor(k, results)
}
}
waitgroup.Done()
} }
fmt.Println("\n}}") fmt.Println("\n}}")
}() }()
@ -192,11 +208,9 @@ func doPrinter() (chan rumorResult, chan struct{}) {
func main() { func main() {
self := doRequest(map[string]interface{}{"keepalive": true, "request": "getSelf"}) self := doRequest(map[string]interface{}{"keepalive": true, "request": "getSelf"})
res := self["response"].(map[string]interface{})["self"].(map[string]interface{}) res := self["response"].(map[string]interface{})
var key string var key string
for _, v := range res { key = res["key"].(string)
key = v.(map[string]interface{})["key"].(string)
}
results, done := doPrinter() results, done := doPrinter()
doRumor(key, results) doRumor(key, results)
waitgroup.Wait() waitgroup.Wait()