5
0
mirror of https://github.com/cwinfo/yggdrasil-go.git synced 2024-11-26 07:11:40 +00:00

more backpressure work, still needs testing

This commit is contained in:
Arceliar 2018-06-23 21:51:32 -05:00
parent 52a0027aea
commit 4b83efa218
4 changed files with 52 additions and 66 deletions

View File

@ -481,13 +481,17 @@ func DEBUG_simLinkPeers(p, q *peer) {
} }
}() }()
p.out = func(bs []byte) { p.out = func(bs []byte) {
p.core.switchTable.idleIn <- p.port
go q.handlePacket(bs) go q.handlePacket(bs)
} }
q.out = func(bs []byte) { q.out = func(bs []byte) {
q.core.switchTable.idleIn <- q.port
go p.handlePacket(bs) go p.handlePacket(bs)
} }
go p.linkLoop() go p.linkLoop()
go q.linkLoop() go q.linkLoop()
p.core.switchTable.idleIn <- p.port
q.core.switchTable.idleIn <- q.port
} }
func (c *Core) DEBUG_simFixMTU() { func (c *Core) DEBUG_simFixMTU() {

View File

@ -74,9 +74,8 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) {
ps.ports.Store(ports) ps.ports.Store(ports)
} }
// Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral), a handler for their outgoing traffic, and queue sizes for local backpressure. // Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
type peer struct { type peer struct {
queueSize int64 // used to track local backpressure
bytesSent uint64 // To track bandwidth usage for getPeers bytesSent uint64 // To track bandwidth usage for getPeers
bytesRecvd uint64 // To track bandwidth usage for getPeers bytesRecvd uint64 // To track bandwidth usage for getPeers
// BUG: sync/atomic, 32 bit platforms need the above to be the first element // BUG: sync/atomic, 32 bit platforms need the above to be the first element
@ -94,16 +93,6 @@ type peer struct {
close func() // Called when a peer is removed, to close the underlying connection, or via admin api close func() // Called when a peer is removed, to close the underlying connection, or via admin api
} }
// Size of the queue of packets to be sent to the node.
func (p *peer) getQueueSize() int64 {
return atomic.LoadInt64(&p.queueSize)
}
// Used to increment or decrement the queue.
func (p *peer) updateQueueSize(delta int64) {
atomic.AddInt64(&p.queueSize, delta)
}
// Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unocupied port number. // Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unocupied port number.
func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey, linkShared *boxSharedKey) *peer { func (ps *peers) newPeer(box *boxPubKey, sig *sigPubKey, linkShared *boxSharedKey) *peer {
now := time.Now() now := time.Now()

View File

@ -503,11 +503,12 @@ func (t *switchTable) lookup(dest []byte) switchPort {
if !(dist < myDist) { if !(dist < myDist) {
continue continue
} }
p, isIn := ports[info.port] //p, isIn := ports[info.port]
_, isIn := ports[info.port]
if !isIn { if !isIn {
continue continue
} }
cost := int64(dist) + p.getQueueSize() cost := int64(dist) // + p.getQueueSize()
if cost < bestCost { if cost < bestCost {
best = info.port best = info.port
bestCost = cost bestCost = cost
@ -573,6 +574,7 @@ func (t *switchTable) portIsCloser(dest []byte, port switchPort) bool {
// Handle an incoming packet // Handle an incoming packet
// Either send it to ourself, or to the first idle peer that's free // Either send it to ourself, or to the first idle peer that's free
// Returns true if the packet has been handled somehow, false if it should be queued
func (t *switchTable) handleIn(packet []byte, idle map[switchPort]struct{}) bool { func (t *switchTable) handleIn(packet []byte, idle map[switchPort]struct{}) bool {
// Get the coords, skipping the first byte (the pType) // Get the coords, skipping the first byte (the pType)
_, pTypeLen := wire_decode_uint64(packet) _, pTypeLen := wire_decode_uint64(packet)
@ -599,6 +601,27 @@ func (t *switchTable) handleIn(packet []byte, idle map[switchPort]struct{}) bool
return false return false
} }
// Handles incoming idle notifications
// Loops over packets and sends the newest one that's OK for this peer to send
// Returns true if the peer is no longer idle, false if it should be added to the idle list
func (t *switchTable) handleIdle(port switchPort, packets *[][]byte) bool {
to := t.core.peers.getPorts()[port]
if to == nil {
return true
}
for idx := len(*packets) - 1; idx >= 0; idx-- {
packet := (*packets)[idx]
_, pTypeLen := wire_decode_uint64(packet)
coords, _ := wire_decode_coords(packet[pTypeLen:])
if t.portIsCloser(coords, port) {
to.sendPacket(packet)
*packets = append((*packets)[:idx], (*packets)[idx+1:]...)
return true
}
}
return false
}
// The switch worker does routing lookups and sends packets to where they need to be // The switch worker does routing lookups and sends packets to where they need to be
func (t *switchTable) doWorker() { func (t *switchTable) doWorker() {
var packets [][]byte // Should really be a linked list var packets [][]byte // Should really be a linked list
@ -606,13 +629,9 @@ func (t *switchTable) doWorker() {
for { for {
select { select {
case packet := <-t.packetIn: case packet := <-t.packetIn:
idle = make(map[switchPort]struct{}) // Try to send it somewhere (or drop it if it's corrupt or at a dead end)
for port := range t.getTable().elems {
idle[port] = struct{}{}
}
// TODO correcty fill idle, so the above can be removed
if !t.handleIn(packet, idle) { if !t.handleIn(packet, idle) {
// There's nobody free to take it now, so queue it // There's nobody free to take it right now, so queue it for later
packets = append(packets, packet) packets = append(packets, packet)
for len(packets) > 32 { for len(packets) > 32 {
util_putBytes(packets[0]) util_putBytes(packets[0])
@ -620,9 +639,11 @@ func (t *switchTable) doWorker() {
} }
} }
case port := <-t.idleIn: case port := <-t.idleIn:
// TODO the part that loops over packets and finds something to send // Try to find something to send to this peer
// Didn't find anything to send, so add this port to the idle list if !t.handleIdle(port, &packets) {
idle[port] = struct{}{} // Didn't find anything ready to send yet, so stay idle
idle[port] = struct{}{}
}
} }
} }
} }

View File

@ -19,7 +19,6 @@ import (
"fmt" "fmt"
"math/rand" "math/rand"
"net" "net"
"sort"
"sync" "sync"
"sync/atomic" "sync/atomic"
"time" "time"
@ -243,26 +242,15 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) {
in := func(bs []byte) { in := func(bs []byte) {
p.handlePacket(bs) p.handlePacket(bs)
} }
out := make(chan []byte, 1024) // Should be effectively infinite, but gets fed into finite LIFO stack out := make(chan []byte, 1)
defer close(out) defer close(out)
go func() { go func() {
var shadow int64 // This goroutine waits for outgoing packets, link protocol traffic, or sends idle keep-alive traffic
var stack [][]byte
put := func(msg []byte) {
stack = append(stack, msg)
sort.SliceStable(stack, func(i, j int) bool {
// Sort in reverse order, with smallest messages at the end
return len(stack[i]) >= len(stack[j])
})
for len(stack) > 32 {
util_putBytes(stack[0])
stack = stack[1:]
shadow++
}
}
send := make(chan []byte) send := make(chan []byte)
defer close(send) defer close(send)
go func() { go func() {
// This goroutine does the actual socket write operations
// The parent goroutine aggregates things for it and feeds them in
for msg := range send { for msg := range send {
msgLen := wire_encode_uint64(uint64(len(msg))) msgLen := wire_encode_uint64(uint64(len(msg)))
buf := net.Buffers{tcp_msg[:], msgLen, msg} buf := net.Buffers{tcp_msg[:], msgLen, msg}
@ -275,10 +263,14 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) {
timer := time.NewTimer(timerInterval) timer := time.NewTimer(timerInterval)
defer timer.Stop() defer timer.Stop()
for { for {
if shadow != 0 { select {
p.updateQueueSize(-shadow) case msg := <-p.linkOut:
shadow = 0 // Always send outgoing link traffic first, if needed
send <- msg
continue
default:
} }
// Otherwise wait reset the timer and wait for something to do
timer.Stop() timer.Stop()
select { select {
case <-timer.C: case <-timer.C:
@ -294,34 +286,14 @@ func (iface *tcpInterface) handler(sock net.Conn, incoming bool) {
if !ok { if !ok {
return return
} }
put(msg) send <- msg // Block until the socket writer has the packet
} // Now inform the switch that we're ready for more traffic
for len(stack) > 0 { p.core.switchTable.idleIn <- p.port
// First make sure linkOut gets sent first, if it's non-empty
select {
case msg := <-p.linkOut:
send <- msg
continue
default:
}
// Then block until we send or receive something
select {
case msg := <-p.linkOut:
send <- msg
case msg, ok := <-out:
if !ok {
return
}
put(msg)
case send <- stack[len(stack)-1]:
stack = stack[:len(stack)-1]
p.updateQueueSize(-1)
}
} }
} }
}() }()
p.core.switchTable.idleIn <- p.port // Start in the idle state
p.out = func(msg []byte) { p.out = func(msg []byte) {
p.updateQueueSize(1)
defer func() { recover() }() defer func() { recover() }()
out <- msg out <- msg
} }