diff --git a/scripts/crawler.go b/scripts/crawler.go index c1b5a8c..6ca1f99 100644 --- a/scripts/crawler.go +++ b/scripts/crawler.go @@ -18,7 +18,7 @@ const N_PARALLEL_REQ = 32 var semaphore chan struct{} func init() { - semaphore = make(chan struct{}, N_PARALLEL_REQ) + semaphore = make(chan struct{}, N_PARALLEL_REQ) } func dial() (net.Conn, error) { @@ -48,12 +48,17 @@ func doRequest(request map[string]interface{}) map[string]interface{} { panic(err) } bs := make([]byte, 65535) + deadline := time.Now().Add(6 * time.Second) + sock.SetReadDeadline(deadline) n, err := sock.Read(bs) + sock.Close() if err != nil { + continue panic(bs) } bs = bs[:n] if err = json.Unmarshal(bs, &res); err != nil { + return nil panic(err) } // TODO parse res, check if there's an error @@ -92,7 +97,7 @@ func doRumor(key string, out chan rumorResult) { waitgroup.Add(1) go func() { defer waitgroup.Done() - semaphore<-struct{}{} + semaphore <- struct{}{} defer func() { <-semaphore }() if _, known := rumored.LoadOrStore(key, true); known { return diff --git a/scripts/ratio.py b/scripts/ratio.py index 9a8bd58..e7bed86 100644 --- a/scripts/ratio.py +++ b/scripts/ratio.py @@ -36,6 +36,8 @@ for kdx in xrange(len(kinds)): axs[kdx].set_title("ratio (dht/peers)") axs[kdx].hist(bins, bins=nbins) plt.savefig("fig.svg") +plt.yscale('log') +plt.savefig("fig-logy.svg") print results