mirror of
https://github.com/cwinfo/yggdrasil-map
synced 2024-11-09 16:00:27 +00:00
try to fix some long delays with the crawler
This commit is contained in:
parent
19cdf8de34
commit
6a307ab37e
@ -18,7 +18,7 @@ const N_PARALLEL_REQ = 32
|
|||||||
var semaphore chan struct{}
|
var semaphore chan struct{}
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
semaphore = make(chan struct{}, N_PARALLEL_REQ)
|
semaphore = make(chan struct{}, N_PARALLEL_REQ)
|
||||||
}
|
}
|
||||||
|
|
||||||
func dial() (net.Conn, error) {
|
func dial() (net.Conn, error) {
|
||||||
@ -48,12 +48,17 @@ func doRequest(request map[string]interface{}) map[string]interface{} {
|
|||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
bs := make([]byte, 65535)
|
bs := make([]byte, 65535)
|
||||||
|
deadline := time.Now().Add(6 * time.Second)
|
||||||
|
sock.SetReadDeadline(deadline)
|
||||||
n, err := sock.Read(bs)
|
n, err := sock.Read(bs)
|
||||||
|
sock.Close()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
continue
|
||||||
panic(bs)
|
panic(bs)
|
||||||
}
|
}
|
||||||
bs = bs[:n]
|
bs = bs[:n]
|
||||||
if err = json.Unmarshal(bs, &res); err != nil {
|
if err = json.Unmarshal(bs, &res); err != nil {
|
||||||
|
return nil
|
||||||
panic(err)
|
panic(err)
|
||||||
}
|
}
|
||||||
// TODO parse res, check if there's an error
|
// TODO parse res, check if there's an error
|
||||||
@ -92,7 +97,7 @@ func doRumor(key string, out chan rumorResult) {
|
|||||||
waitgroup.Add(1)
|
waitgroup.Add(1)
|
||||||
go func() {
|
go func() {
|
||||||
defer waitgroup.Done()
|
defer waitgroup.Done()
|
||||||
semaphore<-struct{}{}
|
semaphore <- struct{}{}
|
||||||
defer func() { <-semaphore }()
|
defer func() { <-semaphore }()
|
||||||
if _, known := rumored.LoadOrStore(key, true); known {
|
if _, known := rumored.LoadOrStore(key, true); known {
|
||||||
return
|
return
|
||||||
|
@ -36,6 +36,8 @@ for kdx in xrange(len(kinds)):
|
|||||||
axs[kdx].set_title("ratio (dht/peers)")
|
axs[kdx].set_title("ratio (dht/peers)")
|
||||||
axs[kdx].hist(bins, bins=nbins)
|
axs[kdx].hist(bins, bins=nbins)
|
||||||
plt.savefig("fig.svg")
|
plt.savefig("fig.svg")
|
||||||
|
plt.yscale('log')
|
||||||
|
plt.savefig("fig-logy.svg")
|
||||||
|
|
||||||
print results
|
print results
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user