5
0
mirror of https://github.com/cwinfo/yggdrasil-map synced 2024-11-09 16:00:27 +00:00

try to fix some long delays with the crawler

This commit is contained in:
Arceliar 2022-04-29 05:14:12 -05:00
parent 19cdf8de34
commit 6a307ab37e
2 changed files with 9 additions and 2 deletions

View File

@ -18,7 +18,7 @@ const N_PARALLEL_REQ = 32
var semaphore chan struct{} var semaphore chan struct{}
func init() { func init() {
semaphore = make(chan struct{}, N_PARALLEL_REQ) semaphore = make(chan struct{}, N_PARALLEL_REQ)
} }
func dial() (net.Conn, error) { func dial() (net.Conn, error) {
@ -48,12 +48,17 @@ func doRequest(request map[string]interface{}) map[string]interface{} {
panic(err) panic(err)
} }
bs := make([]byte, 65535) bs := make([]byte, 65535)
deadline := time.Now().Add(6 * time.Second)
sock.SetReadDeadline(deadline)
n, err := sock.Read(bs) n, err := sock.Read(bs)
sock.Close()
if err != nil { if err != nil {
continue
panic(bs) panic(bs)
} }
bs = bs[:n] bs = bs[:n]
if err = json.Unmarshal(bs, &res); err != nil { if err = json.Unmarshal(bs, &res); err != nil {
return nil
panic(err) panic(err)
} }
// TODO parse res, check if there's an error // TODO parse res, check if there's an error
@ -92,7 +97,7 @@ func doRumor(key string, out chan rumorResult) {
waitgroup.Add(1) waitgroup.Add(1)
go func() { go func() {
defer waitgroup.Done() defer waitgroup.Done()
semaphore<-struct{}{} semaphore <- struct{}{}
defer func() { <-semaphore }() defer func() { <-semaphore }()
if _, known := rumored.LoadOrStore(key, true); known { if _, known := rumored.LoadOrStore(key, true); known {
return return

View File

@ -36,6 +36,8 @@ for kdx in xrange(len(kinds)):
axs[kdx].set_title("ratio (dht/peers)") axs[kdx].set_title("ratio (dht/peers)")
axs[kdx].hist(bins, bins=nbins) axs[kdx].hist(bins, bins=nbins)
plt.savefig("fig.svg") plt.savefig("fig.svg")
plt.yscale('log')
plt.savefig("fig-logy.svg")
print results print results