Merge pull request #519 from Arceliar/actors

Actors

cmd/yggdrasil/main.go

@@ -279,6 +279,7 @@ func main() {
 		case _ = <-r:
 			if *useconffile != "" {
 				cfg = readConfig(useconf, useconffile, normaliseconf)
+				logger.Infoln("Reloading configuration from", *useconffile)
 				n.core.UpdateConfig(cfg)
 				n.tuntap.UpdateConfig(cfg)
 				n.multicast.UpdateConfig(cfg)

go.mod

@@ -1,6 +1,7 @@
 module github.com/yggdrasil-network/yggdrasil-go
 
 require (
+	github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4
 	github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8
 	github.com/hashicorp/go-syslog v1.0.0
 	github.com/hjson/hjson-go v0.0.0-20181010104306-a25ecf6bd222

go.sum

@@ -1,3 +1,5 @@
+github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4 h1:OePImnPRBqS6JiHuVVq4Rfvt2yyhqMpWCB63PrwGrJE=
+github.com/Arceliar/phony v0.0.0-20190831050304-94a6d3da9ba4/go.mod h1:6Lkn+/zJilRMsKmbmG1RPoamiArC6HS73xbwRyp3UyI=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8 h1:WD8iJ37bRNwvETMfVTusVSAi0WdXTpfNVGY2aHycNKY=
 github.com/gologme/log v0.0.0-20181207131047-4e5d8ccb38e8/go.mod h1:gq31gQ8wEHkR+WekdWsqDuf8pXTUZA9BnnzTuPz1Y9U=
 github.com/hashicorp/go-syslog v1.0.0 h1:KaodqZuhUoZereWVIYmpUgZysurB1kBLX2j0MwMrUAE=

src/multicast/multicast.go

@@ -83,7 +83,6 @@ func (m *Multicast) Stop() error {
 func (m *Multicast) UpdateConfig(config *config.NodeConfig) {
 	m.log.Debugln("Reloading multicast configuration...")
 	m.config.Replace(*config)
-	m.log.Infoln("Multicast configuration reloaded successfully")
 }
 
 // GetInterfaces returns the currently known/enabled multicast interfaces. It is

src/tuntap/ckr.go

@@ -20,7 +20,6 @@ import (
 type cryptokey struct {
 	tun          *TunAdapter
 	enabled      atomic.Value // bool
-	reconfigure  chan chan error
 	ipv4remotes  []cryptokey_route
 	ipv6remotes  []cryptokey_route
 	ipv4cache    map[address.Address]cryptokey_route
@@ -40,25 +39,11 @@ type cryptokey_route struct {
 // Initialise crypto-key routing. This must be done before any other CKR calls.
 func (c *cryptokey) init(tun *TunAdapter) {
 	c.tun = tun
-	c.reconfigure = make(chan chan error, 1)
+	c.configure()
-	go func() {
-		for {
-			e := <-c.reconfigure
-			e <- nil
-		}
-	}()
-
-	c.tun.log.Debugln("Configuring CKR...")
-	if err := c.configure(); err != nil {
-		c.tun.log.Errorln("CKR configuration failed:", err)
-	} else {
-		c.tun.log.Debugln("CKR configured")
-	}
 }
 
-// Configure the CKR routes - this must only ever be called from the router
+// Configure the CKR routes. This should only ever be ran by the TUN/TAP actor.
-// goroutine, e.g. through router.doAdmin
+func (c *cryptokey) configure() {
-func (c *cryptokey) configure() error {
 	current := c.tun.config.GetCurrent()
 
 	// Set enabled/disabled state
@@ -73,14 +58,14 @@ func (c *cryptokey) configure() error {
 	// Add IPv6 routes
 	for ipv6, pubkey := range current.TunnelRouting.IPv6RemoteSubnets {
 		if err := c.addRemoteSubnet(ipv6, pubkey); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv6 remote subnet:", err)
 		}
 	}
 
 	// Add IPv4 routes
 	for ipv4, pubkey := range current.TunnelRouting.IPv4RemoteSubnets {
 		if err := c.addRemoteSubnet(ipv4, pubkey); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv4 remote subnet:", err)
 		}
 	}
 
@@ -94,7 +79,7 @@ func (c *cryptokey) configure() error {
 	c.ipv6locals = make([]net.IPNet, 0)
 	for _, source := range current.TunnelRouting.IPv6LocalSubnets {
 		if err := c.addLocalSubnet(source); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv6 local subnet:", err)
 		}
 	}
 
@@ -102,7 +87,7 @@ func (c *cryptokey) configure() error {
 	c.ipv4locals = make([]net.IPNet, 0)
 	for _, source := range current.TunnelRouting.IPv4LocalSubnets {
 		if err := c.addLocalSubnet(source); err != nil {
-			return err
+			c.tun.log.Errorln("Error adding CKR IPv4 local subnet:", err)
 		}
 	}
 
@@ -111,8 +96,6 @@ func (c *cryptokey) configure() error {
 	c.ipv4cache = make(map[address.Address]cryptokey_route, 0)
 	c.ipv6cache = make(map[address.Address]cryptokey_route, 0)
 	c.mutexcaches.Unlock()
-
-	return nil
 }
 
 // Enable or disable crypto-key routing.
@@ -182,19 +165,19 @@ func (c *cryptokey) addLocalSubnet(cidr string) error {
 	} else if prefixsize == net.IPv4len*8 {
 		routingsources = &c.ipv4locals
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Check if we already have this CIDR
 	for _, subnet := range *routingsources {
 		if subnet.String() == ipnet.String() {
-			return errors.New("Source subnet already configured")
+			return errors.New("local subnet already configured")
 		}
 	}
 
 	// Add the source subnet
 	*routingsources = append(*routingsources, *ipnet)
-	c.tun.log.Infoln("Added CKR source subnet", cidr)
+	c.tun.log.Infoln("Added CKR local subnet", cidr)
 	return nil
 }
 
@@ -227,7 +210,7 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 		routingtable = &c.ipv4remotes
 		routingcache = &c.ipv4cache
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Is the route an Yggdrasil destination?
@@ -236,19 +219,19 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 	copy(addr[:], ipaddr)
 	copy(snet[:], ipnet.IP)
 	if addr.IsValid() || snet.IsValid() {
-		return errors.New("Can't specify Yggdrasil destination as crypto-key route")
+		return errors.New("can't specify Yggdrasil destination as crypto-key route")
 	}
 	// Do we already have a route for this subnet?
 	for _, route := range *routingtable {
 		if route.subnet.String() == ipnet.String() {
-			return errors.New(fmt.Sprintf("Route already exists for %s", cidr))
+			return fmt.Errorf("remote subnet already exists for %s", cidr)
 		}
 	}
 	// Decode the public key
 	if bpk, err := hex.DecodeString(dest); err != nil {
 		return err
 	} else if len(bpk) != crypto.BoxPubKeyLen {
-		return errors.New(fmt.Sprintf("Incorrect key length for %s", dest))
+		return fmt.Errorf("incorrect key length for %s", dest)
 	} else {
 		// Add the new crypto-key route
 		var key crypto.BoxPubKey
@@ -271,7 +254,7 @@ func (c *cryptokey) addRemoteSubnet(cidr string, dest string) error {
 			delete(*routingcache, k)
 		}
 
-		c.tun.log.Infoln("Added CKR destination subnet", cidr)
+		c.tun.log.Infoln("Added CKR remote subnet", cidr)
 		return nil
 	}
 }
@@ -285,7 +268,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	// Check if the address is a valid Yggdrasil address - if so it
 	// is exempt from all CKR checking
 	if addr.IsValid() {
-		return crypto.BoxPubKey{}, errors.New("Cannot look up CKR for Yggdrasil addresses")
+		return crypto.BoxPubKey{}, errors.New("cannot look up CKR for Yggdrasil addresses")
 	}
 
 	// Build our references to the routing table and cache
@@ -298,7 +281,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	} else if addrlen == net.IPv4len {
 		routingcache = &c.ipv4cache
 	} else {
-		return crypto.BoxPubKey{}, errors.New("Unexpected prefix size")
+		return crypto.BoxPubKey{}, errors.New("unexpected prefix size")
 	}
 
 	// Check if there's a cache entry for this addr
@@ -318,7 +301,7 @@ func (c *cryptokey) getPublicKeyForAddress(addr address.Address, addrlen int) (c
 	} else if addrlen == net.IPv4len {
 		routingtable = &c.ipv4remotes
 	} else {
-		return crypto.BoxPubKey{}, errors.New("Unexpected prefix size")
+		return crypto.BoxPubKey{}, errors.New("unexpected prefix size")
 	}
 
 	// No cache was found - start by converting the address into a net.IP
@@ -379,18 +362,18 @@ func (c *cryptokey) removeLocalSubnet(cidr string) error {
 	} else if prefixsize == net.IPv4len*8 {
 		routingsources = &c.ipv4locals
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Check if we already have this CIDR
 	for idx, subnet := range *routingsources {
 		if subnet.String() == ipnet.String() {
 			*routingsources = append((*routingsources)[:idx], (*routingsources)[idx+1:]...)
-			c.tun.log.Infoln("Removed CKR source subnet", cidr)
+			c.tun.log.Infoln("Removed CKR local subnet", cidr)
 			return nil
 		}
 	}
-	return errors.New("Source subnet not found")
+	return errors.New("local subnet not found")
 }
 
 // Removes a destination route for the given CIDR to be tunnelled to the node
@@ -422,7 +405,7 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 		routingtable = &c.ipv4remotes
 		routingcache = &c.ipv4cache
 	} else {
-		return errors.New("Unexpected prefix size")
+		return errors.New("unexpected prefix size")
 	}
 
 	// Decode the public key
@@ -430,7 +413,7 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 	if err != nil {
 		return err
 	} else if len(bpk) != crypto.BoxPubKeyLen {
-		return errors.New(fmt.Sprintf("Incorrect key length for %s", dest))
+		return fmt.Errorf("incorrect key length for %s", dest)
 	}
 	netStr := ipnet.String()
 
@@ -440,9 +423,9 @@ func (c *cryptokey) removeRemoteSubnet(cidr string, dest string) error {
 			for k := range *routingcache {
 				delete(*routingcache, k)
 			}
-			c.tun.log.Infof("Removed CKR destination subnet %s via %s\n", cidr, dest)
+			c.tun.log.Infof("Removed CKR remote subnet %s via %s\n", cidr, dest)
 			return nil
 		}
 	}
-	return errors.New(fmt.Sprintf("Route does not exists for %s", cidr))
+	return fmt.Errorf("route does not exists for %s", cidr)
 }

src/tuntap/conn.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
@@ -16,22 +17,20 @@ import (
 const tunConnTimeout = 2 * time.Minute
 
 type tunConn struct {
+	phony.Inbox
 	tun   *TunAdapter
 	conn  *yggdrasil.Conn
 	addr  address.Address
 	snet  address.Subnet
-	send  chan []byte
 	stop  chan struct{}
-	alive chan struct{}
+	alive *time.Timer // From calling time.AfterFunc
 }
 
 func (s *tunConn) close() {
-	s.tun.mutex.Lock()
+	s.tun.Act(s, s._close_from_tun)
-	defer s.tun.mutex.Unlock()
-	s._close_nomutex()
 }
 
-func (s *tunConn) _close_nomutex() {
+func (s *tunConn) _close_from_tun() {
 	s.conn.Close()
 	delete(s.tun.addrToConn, s.addr)
 	delete(s.tun.subnetToConn, s.snet)
@@ -39,235 +38,197 @@ func (s *tunConn) _close_nomutex() {
 		defer func() { recover() }()
 		close(s.stop) // Closes reader/writer goroutines
 	}()
-	func() {
-		defer func() { recover() }()
-		close(s.alive) // Closes timeout goroutine
-	}()
 }
 
-func (s *tunConn) reader() (err error) {
+func (s *tunConn) _read(bs []byte) (err error) {
 	select {
-	case _, ok := <-s.stop:
+	case <-s.stop:
-		if !ok {
+		err = errors.New("session was already closed")
-			return errors.New("session was already closed")
+		util.PutBytes(bs)
-		}
+		return
 	default:
 	}
-	s.tun.log.Debugln("Starting conn reader for", s.conn.String())
+	if len(bs) == 0 {
-	defer s.tun.log.Debugln("Stopping conn reader for", s.conn.String())
+		err = errors.New("read packet with 0 size")
-	for {
+		util.PutBytes(bs)
-		select {
+		return
-		case <-s.stop:
+	}
-			return nil
+	ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
-		default:
+	ipv6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
+	isCGA := true
+	// Check source addresses
+	switch {
+	case ipv6 && bs[8] == 0x02 && bytes.Equal(s.addr[:16], bs[8:24]): // source
+	case ipv6 && bs[8] == 0x03 && bytes.Equal(s.snet[:8], bs[8:16]): // source
+	default:
+		isCGA = false
+	}
+	// Check destiantion addresses
+	switch {
+	case ipv6 && bs[24] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[24:40]): // destination
+	case ipv6 && bs[24] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[24:32]): // destination
+	default:
+		isCGA = false
+	}
+	// Decide how to handle the packet
+	var skip bool
+	switch {
+	case isCGA: // Allowed
+	case s.tun.ckr.isEnabled() && (ipv4 || ipv6):
+		var srcAddr address.Address
+		var dstAddr address.Address
+		var addrlen int
+		if ipv4 {
+			copy(srcAddr[:], bs[12:16])
+			copy(dstAddr[:], bs[16:20])
+			addrlen = 4
 		}
-		var bs []byte
+		if ipv6 {
-		if bs, err = s.conn.ReadNoCopy(); err != nil {
+			copy(srcAddr[:], bs[8:24])
-			if e, eok := err.(yggdrasil.ConnError); eok && !e.Temporary() {
+			copy(dstAddr[:], bs[24:40])
-				if e.Closed() {
+			addrlen = 16
-					s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn read debug:", err)
+		}
-				} else {
+		if !s.tun.ckr.isValidLocalAddress(dstAddr, addrlen) {
-					s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn read error:", err)
+			// The destination address isn't in our CKR allowed range
-				}
+			skip = true
-				return e
+		} else if key, err := s.tun.ckr.getPublicKeyForAddress(srcAddr, addrlen); err == nil {
-			}
+			srcNodeID := crypto.GetNodeID(&key)
-		} else if len(bs) > 0 {
+			if s.conn.RemoteAddr() == *srcNodeID {
-			ipv4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
+				// This is the one allowed CKR case, where source and destination addresses are both good
-			ipv6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
+			} else {
-			isCGA := true
+				// The CKR key associated with this address doesn't match the sender's NodeID
-			// Check source addresses
-			switch {
-			case ipv6 && bs[8] == 0x02 && bytes.Equal(s.addr[:16], bs[8:24]): // source
-			case ipv6 && bs[8] == 0x03 && bytes.Equal(s.snet[:8], bs[8:16]): // source
-			default:
-				isCGA = false
-			}
-			// Check destiantion addresses
-			switch {
-			case ipv6 && bs[24] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[24:40]): // destination
-			case ipv6 && bs[24] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[24:32]): // destination
-			default:
-				isCGA = false
-			}
-			// Decide how to handle the packet
-			var skip bool
-			switch {
-			case isCGA: // Allowed
-			case s.tun.ckr.isEnabled() && (ipv4 || ipv6):
-				var srcAddr address.Address
-				var dstAddr address.Address
-				var addrlen int
-				if ipv4 {
-					copy(srcAddr[:], bs[12:16])
-					copy(dstAddr[:], bs[16:20])
-					addrlen = 4
-				}
-				if ipv6 {
-					copy(srcAddr[:], bs[8:24])
-					copy(dstAddr[:], bs[24:40])
-					addrlen = 16
-				}
-				if !s.tun.ckr.isValidLocalAddress(dstAddr, addrlen) {
-					// The destination address isn't in our CKR allowed range
-					skip = true
-				} else if key, err := s.tun.ckr.getPublicKeyForAddress(srcAddr, addrlen); err == nil {
-					srcNodeID := crypto.GetNodeID(&key)
-					if s.conn.RemoteAddr() == *srcNodeID {
-						// This is the one allowed CKR case, where source and destination addresses are both good
-					} else {
-						// The CKR key associated with this address doesn't match the sender's NodeID
-						skip = true
-					}
-				} else {
-					// We have no CKR route for this source address
-					skip = true
-				}
-			default:
 				skip = true
 			}
-			if skip {
-				util.PutBytes(bs)
-				continue
-			}
-			s.tun.send <- bs
-			s.stillAlive()
 		} else {
-			util.PutBytes(bs)
+			// We have no CKR route for this source address
-		}
+			skip = true
-	}
-}
-
-func (s *tunConn) writer() error {
-	select {
-	case _, ok := <-s.stop:
-		if !ok {
-			return errors.New("session was already closed")
 		}
 	default:
+		skip = true
 	}
-	s.tun.log.Debugln("Starting conn writer for", s.conn.String())
+	if skip {
-	defer s.tun.log.Debugln("Stopping conn writer for", s.conn.String())
+		err = errors.New("address not allowed")
-	for {
+		util.PutBytes(bs)
-		select {
+		return
-		case <-s.stop:
+	}
-			return nil
+	s.tun.writer.writeFrom(s, bs)
-		case bs, ok := <-s.send:
+	s.stillAlive()
-			if !ok {
+	return
-				return errors.New("send closed")
+}
-			}
+
-			v4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
+func (s *tunConn) writeFrom(from phony.Actor, bs []byte) {
-			v6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
+	s.Act(from, func() {
-			isCGA := true
+		s._write(bs)
-			// Check source addresses
+	})
-			switch {
+}
-			case v6 && bs[8] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[8:24]): // source
+
-			case v6 && bs[8] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[8:16]): // source
+func (s *tunConn) _write(bs []byte) (err error) {
-			default:
+	select {
-				isCGA = false
+	case <-s.stop:
-			}
+		err = errors.New("session was already closed")
-			// Check destiantion addresses
+		util.PutBytes(bs)
-			switch {
+		return
-			case v6 && bs[24] == 0x02 && bytes.Equal(s.addr[:16], bs[24:40]): // destination
+	default:
-			case v6 && bs[24] == 0x03 && bytes.Equal(s.snet[:8], bs[24:32]): // destination
+	}
-			default:
+	v4 := len(bs) > 20 && bs[0]&0xf0 == 0x40
-				isCGA = false
+	v6 := len(bs) > 40 && bs[0]&0xf0 == 0x60
-			}
+	isCGA := true
-			// Decide how to handle the packet
+	// Check source addresses
-			var skip bool
+	switch {
-			switch {
+	case v6 && bs[8] == 0x02 && bytes.Equal(s.tun.addr[:16], bs[8:24]): // source
-			case isCGA: // Allowed
+	case v6 && bs[8] == 0x03 && bytes.Equal(s.tun.subnet[:8], bs[8:16]): // source
-			case s.tun.ckr.isEnabled() && (v4 || v6):
+	default:
-				var srcAddr address.Address
+		isCGA = false
-				var dstAddr address.Address
+	}
-				var addrlen int
+	// Check destiantion addresses
-				if v4 {
+	switch {
-					copy(srcAddr[:], bs[12:16])
+	case v6 && bs[24] == 0x02 && bytes.Equal(s.addr[:16], bs[24:40]): // destination
-					copy(dstAddr[:], bs[16:20])
+	case v6 && bs[24] == 0x03 && bytes.Equal(s.snet[:8], bs[24:32]): // destination
-					addrlen = 4
+	default:
-				}
+		isCGA = false
-				if v6 {
+	}
-					copy(srcAddr[:], bs[8:24])
+	// Decide how to handle the packet
-					copy(dstAddr[:], bs[24:40])
+	var skip bool
-					addrlen = 16
+	switch {
-				}
+	case isCGA: // Allowed
-				if !s.tun.ckr.isValidLocalAddress(srcAddr, addrlen) {
+	case s.tun.ckr.isEnabled() && (v4 || v6):
-					// The source address isn't in our CKR allowed range
+		var srcAddr address.Address
-					skip = true
+		var dstAddr address.Address
-				} else if key, err := s.tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
+		var addrlen int
-					dstNodeID := crypto.GetNodeID(&key)
+		if v4 {
-					if s.conn.RemoteAddr() == *dstNodeID {
+			copy(srcAddr[:], bs[12:16])
-						// This is the one allowed CKR case, where source and destination addresses are both good
+			copy(dstAddr[:], bs[16:20])
-					} else {
+			addrlen = 4
-						// The CKR key associated with this address doesn't match the sender's NodeID
+		}
-						skip = true
+		if v6 {
-					}
+			copy(srcAddr[:], bs[8:24])
-				} else {
+			copy(dstAddr[:], bs[24:40])
-					// We have no CKR route for this destination address... why do we have the packet in the first place?
+			addrlen = 16
-					skip = true
+		}
-				}
+		if !s.tun.ckr.isValidLocalAddress(srcAddr, addrlen) {
-			default:
+			// The source address isn't in our CKR allowed range
+			skip = true
+		} else if key, err := s.tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
+			dstNodeID := crypto.GetNodeID(&key)
+			if s.conn.RemoteAddr() == *dstNodeID {
+				// This is the one allowed CKR case, where source and destination addresses are both good
+			} else {
+				// The CKR key associated with this address doesn't match the sender's NodeID
 				skip = true
 			}
-			if skip {
+		} else {
-				util.PutBytes(bs)
+			// We have no CKR route for this destination address... why do we have the packet in the first place?
-				continue
+			skip = true
-			}
+		}
-			msg := yggdrasil.FlowKeyMessage{
+	default:
-				FlowKey: util.GetFlowKey(bs),
+		skip = true
-				Message: bs,
+	}
-			}
+	if skip {
-			if err := s.conn.WriteNoCopy(msg); err != nil {
+		err = errors.New("address not allowed")
-				if e, eok := err.(yggdrasil.ConnError); !eok {
+		util.PutBytes(bs)
-					if e.Closed() {
+		return
-						s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
+	}
-					} else {
+	msg := yggdrasil.FlowKeyMessage{
-						s.tun.log.Errorln(s.conn.String(), "TUN/TAP generic write error:", err)
+		FlowKey: util.GetFlowKey(bs),
-					}
+		Message: bs,
-				} else if e.PacketTooBig() {
+	}
-					// TODO: This currently isn't aware of IPv4 for CKR
+	s.conn.WriteFrom(s, msg, func(err error) {
-					ptb := &icmp.PacketTooBig{
+		if err == nil {
-						MTU:  int(e.PacketMaximumSize()),
+			// No point in wasting resources to send back an error if there was none
-						Data: bs[:900],
+			return
-					}
+		}
-					if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
+		s.Act(s.conn, func() {
-						s.tun.send <- packet
+			if e, eok := err.(yggdrasil.ConnError); !eok {
-					}
+				if e.Closed() {
+					s.tun.log.Debugln(s.conn.String(), "TUN/TAP generic write debug:", err)
 				} else {
-					if e.Closed() {
+					s.tun.log.Errorln(s.conn.String(), "TUN/TAP generic write error:", err)
-						s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn write debug:", err)
+				}
-					} else {
+			} else if e.PacketTooBig() {
-						s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn write error:", err)
+				// TODO: This currently isn't aware of IPv4 for CKR
-					}
+				ptb := &icmp.PacketTooBig{
+					MTU:  int(e.PacketMaximumSize()),
+					Data: bs[:900],
+				}
+				if packet, err := CreateICMPv6(bs[8:24], bs[24:40], ipv6.ICMPTypePacketTooBig, 0, ptb); err == nil {
+					s.tun.writer.writeFrom(s, packet)
 				}
 			} else {
-				s.stillAlive()
+				if e.Closed() {
+					s.tun.log.Debugln(s.conn.String(), "TUN/TAP conn write debug:", err)
+				} else {
+					s.tun.log.Errorln(s.conn.String(), "TUN/TAP conn write error:", err)
+				}
 			}
-		}
+		})
-	}
+	})
+	s.stillAlive()
+	return
 }
 
 func (s *tunConn) stillAlive() {
-	defer func() { recover() }()
+	if s.alive != nil {
-	select {
+		s.alive.Stop()
-	case s.alive <- struct{}{}:
-	default:
-	}
-}
-
-func (s *tunConn) checkForTimeouts() error {
-	timer := time.NewTimer(tunConnTimeout)
-	defer util.TimerStop(timer)
-	defer s.close()
-	for {
-		select {
-		case _, ok := <-s.alive:
-			if !ok {
-				return errors.New("connection closed")
-			}
-			util.TimerStop(timer)
-			timer.Reset(tunConnTimeout)
-		case <-timer.C:
-			return errors.New("timed out")
-		}
 	}
+	s.alive = time.AfterFunc(tunConnTimeout, s.close)
 }

src/tuntap/iface.go

@@ -9,264 +9,269 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
-func (tun *TunAdapter) writer() error {
+type tunWriter struct {
-	var w int
+	phony.Inbox
+	tun *TunAdapter
+}
+
+func (w *tunWriter) writeFrom(from phony.Actor, b []byte) {
+	w.Act(from, func() {
+		w._write(b)
+	})
+}
+
+// write is pretty loose with the memory safety rules, e.g. it assumes it can read w.tun.iface.IsTap() safely
+func (w *tunWriter) _write(b []byte) {
+	var written int
 	var err error
-	for {
+	n := len(b)
-		b := <-tun.send
+	if n == 0 {
-		n := len(b)
+		return
-		if n == 0 {
+	}
-			continue
+	if w.tun.iface.IsTAP() {
+		sendndp := func(dstAddr address.Address) {
+			neigh, known := w.tun.icmpv6.getNeighbor(dstAddr)
+			known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
+			if !known {
+				w.tun.icmpv6.Solicit(dstAddr)
+			}
 		}
-		if tun.iface.IsTAP() {
+		peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
-			sendndp := func(dstAddr address.Address) {
+		var dstAddr address.Address
-				neigh, known := tun.icmpv6.getNeighbor(dstAddr)
+		var peerknown bool
-				known = known && (time.Since(neigh.lastsolicitation).Seconds() < 30)
+		if b[0]&0xf0 == 0x40 {
-				if !known {
+			dstAddr = w.tun.addr
-					tun.icmpv6.Solicit(dstAddr)
+		} else if b[0]&0xf0 == 0x60 {
-				}
+			if !bytes.Equal(w.tun.addr[:16], dstAddr[:16]) && !bytes.Equal(w.tun.subnet[:8], dstAddr[:8]) {
-			}
+				dstAddr = w.tun.addr
-			peermac := net.HardwareAddr{0x00, 0x00, 0x00, 0x00, 0x00, 0x00}
-			var dstAddr address.Address
-			var peerknown bool
-			if b[0]&0xf0 == 0x40 {
-				dstAddr = tun.addr
-			} else if b[0]&0xf0 == 0x60 {
-				if !bytes.Equal(tun.addr[:16], dstAddr[:16]) && !bytes.Equal(tun.subnet[:8], dstAddr[:8]) {
-					dstAddr = tun.addr
-				}
-			}
-			if neighbor, ok := tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
-				// If we've learned the MAC of a 300::/7 address, for example, or a CKR
-				// address, use the MAC address of that
-				peermac = neighbor.mac
-				peerknown = true
-			} else if neighbor, ok := tun.icmpv6.getNeighbor(tun.addr); ok && neighbor.learned {
-				// Otherwise send directly to the MAC address of the host if that's
-				// known instead
-				peermac = neighbor.mac
-				peerknown = true
-			} else {
-				// Nothing has been discovered, try to discover the destination
-				sendndp(tun.addr)
-			}
-			if peerknown {
-				var proto ethernet.Ethertype
-				switch {
-				case b[0]&0xf0 == 0x60:
-					proto = ethernet.IPv6
-				case b[0]&0xf0 == 0x40:
-					proto = ethernet.IPv4
-				}
-				var frame ethernet.Frame
-				frame.Prepare(
-					peermac[:6],          // Destination MAC address
-					tun.icmpv6.mymac[:6], // Source MAC address
-					ethernet.NotTagged,   // VLAN tagging
-					proto,                // Ethertype
-					len(b))               // Payload length
-				copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
-				n += tun_ETHER_HEADER_LENGTH
-				w, err = tun.iface.Write(frame[:n])
-			} else {
-				tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
 			}
+		}
+		if neighbor, ok := w.tun.icmpv6.getNeighbor(dstAddr); ok && neighbor.learned {
+			// If we've learned the MAC of a 300::/7 address, for example, or a CKR
+			// address, use the MAC address of that
+			peermac = neighbor.mac
+			peerknown = true
+		} else if neighbor, ok := w.tun.icmpv6.getNeighbor(w.tun.addr); ok && neighbor.learned {
+			// Otherwise send directly to the MAC address of the host if that's
+			// known instead
+			peermac = neighbor.mac
+			peerknown = true
 		} else {
-			w, err = tun.iface.Write(b[:n])
+			// Nothing has been discovered, try to discover the destination
-			util.PutBytes(b)
+			sendndp(w.tun.addr)
 		}
-		if err != nil {
+		if peerknown {
-			if !tun.isOpen {
+			var proto ethernet.Ethertype
-				return err
+			switch {
+			case b[0]&0xf0 == 0x60:
+				proto = ethernet.IPv6
+			case b[0]&0xf0 == 0x40:
+				proto = ethernet.IPv4
 			}
-			tun.log.Errorln("TUN/TAP iface write error:", err)
+			var frame ethernet.Frame
-			continue
+			frame.Prepare(
-		}
+				peermac[:6],            // Destination MAC address
-		if w != n {
+				w.tun.icmpv6.mymac[:6], // Source MAC address
-			tun.log.Errorln("TUN/TAP iface write mismatch:", w, "bytes written vs", n, "bytes given")
+				ethernet.NotTagged,     // VLAN tagging
-			continue
+				proto,                  // Ethertype
+				len(b))                 // Payload length
+			copy(frame[tun_ETHER_HEADER_LENGTH:], b[:n])
+			n += tun_ETHER_HEADER_LENGTH
+			written, err = w.tun.iface.Write(frame[:n])
+		} else {
+			w.tun.log.Errorln("TUN/TAP iface write error: no peer MAC known for", net.IP(dstAddr[:]).String(), "- dropping packet")
 		}
+	} else {
+		written, err = w.tun.iface.Write(b[:n])
+		util.PutBytes(b)
+	}
+	if err != nil {
+		w.tun.Act(w, func() {
+			if !w.tun.isOpen {
+				w.tun.log.Errorln("TUN/TAP iface write error:", err)
+			}
+		})
+	}
+	if written != n {
+		w.tun.log.Errorln("TUN/TAP iface write mismatch:", written, "bytes written vs", n, "bytes given")
 	}
 }
 
-// Run in a separate goroutine by the reader
+type tunReader struct {
-// Does all of the per-packet ICMP checks, passes packets to the right Conn worker
+	phony.Inbox
-func (tun *TunAdapter) readerPacketHandler(ch chan []byte) {
+	tun *TunAdapter
-	for recvd := range ch {
+}
-		// If it's a TAP adapter, update the buffer slice so that we no longer
+
-		// include the ethernet headers
+func (r *tunReader) _read() {
-		offset := 0
+	// Get a slice to store the packet in
-		if tun.iface.IsTAP() {
+	recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
-			// Set our offset to beyond the ethernet headers
+	// Wait for a packet to be delivered to us through the TUN/TAP adapter
-			offset = tun_ETHER_HEADER_LENGTH
+	n, err := r.tun.iface.Read(recvd)
-			// Check first of all that we can go beyond the ethernet headers
+	if n == 0 {
-			if len(recvd) <= offset {
+		util.PutBytes(recvd)
-				continue
+	} else {
-			}
+		r.tun.handlePacketFrom(r, recvd[:n], err)
+	}
+	if err == nil {
+		// Now read again
+		r.Act(nil, r._read)
+	}
+}
+
+func (tun *TunAdapter) handlePacketFrom(from phony.Actor, packet []byte, err error) {
+	tun.Act(from, func() {
+		tun._handlePacket(packet, err)
+	})
+}
+
+// does the work of reading a packet and sending it to the correct tunConn
+func (tun *TunAdapter) _handlePacket(recvd []byte, err error) {
+	if err != nil {
+		tun.log.Errorln("TUN/TAP iface read error:", err)
+		return
+	}
+	// If it's a TAP adapter, update the buffer slice so that we no longer
+	// include the ethernet headers
+	offset := 0
+	if tun.iface.IsTAP() {
+		// Set our offset to beyond the ethernet headers
+		offset = tun_ETHER_HEADER_LENGTH
+		// Check first of all that we can go beyond the ethernet headers
+		if len(recvd) <= offset {
+			return
 		}
-		// Offset the buffer from now on so that we can ignore ethernet frames if
+	}
-		// they are present
+	// Offset the buffer from now on so that we can ignore ethernet frames if
-		bs := recvd[offset:]
+	// they are present
-		// If we detect an ICMP packet then hand it to the ICMPv6 module
+	bs := recvd[offset:]
-		if bs[6] == 58 {
+	// If we detect an ICMP packet then hand it to the ICMPv6 module
-			// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
+	if bs[6] == 58 {
-			// Ethernet frame rather than just the IPv6 packet as this is needed for
+		// Found an ICMPv6 packet - we need to make sure to give ICMPv6 the full
-			// NDP to work correctly
+		// Ethernet frame rather than just the IPv6 packet as this is needed for
-			if err := tun.icmpv6.ParsePacket(recvd); err == nil {
+		// NDP to work correctly
-				// We acted on the packet in the ICMPv6 module so don't forward or do
+		if err := tun.icmpv6.ParsePacket(recvd); err == nil {
-				// anything else with it
+			// We acted on the packet in the ICMPv6 module so don't forward or do
-				continue
+			// anything else with it
-			}
+			return
 		}
-		if offset != 0 {
+	}
-			// Shift forward to avoid leaking bytes off the front of the slice when we eventually store it
+	if offset != 0 {
-			bs = append(recvd[:0], bs...)
+		// Shift forward to avoid leaking bytes off the front of the slice when we eventually store it
+		bs = append(recvd[:0], bs...)
+	}
+	// From the IP header, work out what our source and destination addresses
+	// and node IDs are. We will need these in order to work out where to send
+	// the packet
+	var dstAddr address.Address
+	var dstSnet address.Subnet
+	var addrlen int
+	n := len(bs)
+	// Check the IP protocol - if it doesn't match then we drop the packet and
+	// do nothing with it
+	if bs[0]&0xf0 == 0x60 {
+		// Check if we have a fully-sized IPv6 header
+		if len(bs) < 40 {
+			return
 		}
-		// From the IP header, work out what our source and destination addresses
+		// Check the packet size
-		// and node IDs are. We will need these in order to work out where to send
+		if n-tun_IPv6_HEADER_LENGTH != 256*int(bs[4])+int(bs[5]) {
-		// the packet
+			return
-		var dstAddr address.Address
-		var dstSnet address.Subnet
-		var addrlen int
-		n := len(bs)
-		// Check the IP protocol - if it doesn't match then we drop the packet and
-		// do nothing with it
-		if bs[0]&0xf0 == 0x60 {
-			// Check if we have a fully-sized IPv6 header
-			if len(bs) < 40 {
-				continue
-			}
-			// Check the packet size
-			if n-tun_IPv6_HEADER_LENGTH != 256*int(bs[4])+int(bs[5]) {
-				continue
-			}
-			// IPv6 address
-			addrlen = 16
-			copy(dstAddr[:addrlen], bs[24:])
-			copy(dstSnet[:addrlen/2], bs[24:])
-		} else if bs[0]&0xf0 == 0x40 {
-			// Check if we have a fully-sized IPv4 header
-			if len(bs) < 20 {
-				continue
-			}
-			// Check the packet size
-			if n != 256*int(bs[2])+int(bs[3]) {
-				continue
-			}
-			// IPv4 address
-			addrlen = 4
-			copy(dstAddr[:addrlen], bs[16:])
-		} else {
-			// Unknown address length or protocol, so drop the packet and ignore it
-			tun.log.Traceln("Unknown packet type, dropping")
-			continue
 		}
-		if tun.ckr.isEnabled() {
+		// IPv6 address
-			if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
+		addrlen = 16
-				if key, err := tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
+		copy(dstAddr[:addrlen], bs[24:])
-					// A public key was found, get the node ID for the search
+		copy(dstSnet[:addrlen/2], bs[24:])
-					dstNodeID := crypto.GetNodeID(&key)
+	} else if bs[0]&0xf0 == 0x40 {
-					dstAddr = *address.AddrForNodeID(dstNodeID)
+		// Check if we have a fully-sized IPv4 header
-					dstSnet = *address.SubnetForNodeID(dstNodeID)
+		if len(bs) < 20 {
-					addrlen = 16
+			return
-				}
-			}
 		}
+		// Check the packet size
+		if n != 256*int(bs[2])+int(bs[3]) {
+			return
+		}
+		// IPv4 address
+		addrlen = 4
+		copy(dstAddr[:addrlen], bs[16:])
+	} else {
+		// Unknown address length or protocol, so drop the packet and ignore it
+		tun.log.Traceln("Unknown packet type, dropping")
+		return
+	}
+	if tun.ckr.isEnabled() {
 		if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
-			// Couldn't find this node's ygg IP
+			if key, err := tun.ckr.getPublicKeyForAddress(dstAddr, addrlen); err == nil {
-			continue
+				// A public key was found, get the node ID for the search
-		}
+				dstNodeID := crypto.GetNodeID(&key)
-		// Do we have an active connection for this node address?
+				dstAddr = *address.AddrForNodeID(dstNodeID)
-		var dstNodeID, dstNodeIDMask *crypto.NodeID
+				dstSnet = *address.SubnetForNodeID(dstNodeID)
-		tun.mutex.RLock()
+				addrlen = 16
-		session, isIn := tun.addrToConn[dstAddr]
-		if !isIn || session == nil {
-			session, isIn = tun.subnetToConn[dstSnet]
-			if !isIn || session == nil {
-				// Neither an address nor a subnet mapping matched, therefore populate
-				// the node ID and mask to commence a search
-				if dstAddr.IsValid() {
-					dstNodeID, dstNodeIDMask = dstAddr.GetNodeIDandMask()
-				} else {
-					dstNodeID, dstNodeIDMask = dstSnet.GetNodeIDandMask()
-				}
 			}
 		}
-		tun.mutex.RUnlock()
+	}
-		// If we don't have a connection then we should open one
+	if addrlen != 16 || (!dstAddr.IsValid() && !dstSnet.IsValid()) {
+		// Couldn't find this node's ygg IP
+		return
+	}
+	// Do we have an active connection for this node address?
+	var dstNodeID, dstNodeIDMask *crypto.NodeID
+	session, isIn := tun.addrToConn[dstAddr]
+	if !isIn || session == nil {
+		session, isIn = tun.subnetToConn[dstSnet]
 		if !isIn || session == nil {
-			// Check we haven't been given empty node ID, really this shouldn't ever
+			// Neither an address nor a subnet mapping matched, therefore populate
-			// happen but just to be sure...
+			// the node ID and mask to commence a search
-			if dstNodeID == nil || dstNodeIDMask == nil {
+			if dstAddr.IsValid() {
-				panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
+				dstNodeID, dstNodeIDMask = dstAddr.GetNodeIDandMask()
+			} else {
+				dstNodeID, dstNodeIDMask = dstSnet.GetNodeIDandMask()
 			}
-			// Dial to the remote node
+		}
+	}
+	// If we don't have a connection then we should open one
+	if !isIn || session == nil {
+		// Check we haven't been given empty node ID, really this shouldn't ever
+		// happen but just to be sure...
+		if dstNodeID == nil || dstNodeIDMask == nil {
+			panic("Given empty dstNodeID and dstNodeIDMask - this shouldn't happen")
+		}
+		_, known := tun.dials[*dstNodeID]
+		tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
+		for len(tun.dials[*dstNodeID]) > 32 {
+			util.PutBytes(tun.dials[*dstNodeID][0])
+			tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
+		}
+		if !known {
 			go func() {
-				// FIXME just spitting out a goroutine to do this is kind of ugly and means we drop packets until the dial finishes
+				conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask)
-				tun.mutex.Lock()
+				tun.Act(nil, func() {
-				_, known := tun.dials[*dstNodeID]
+					packets := tun.dials[*dstNodeID]
-				tun.dials[*dstNodeID] = append(tun.dials[*dstNodeID], bs)
+					delete(tun.dials, *dstNodeID)
-				for len(tun.dials[*dstNodeID]) > 32 {
+					if err != nil {
-					util.PutBytes(tun.dials[*dstNodeID][0])
+						return
-					tun.dials[*dstNodeID] = tun.dials[*dstNodeID][1:]
+					}
-				}
-				tun.mutex.Unlock()
-				if known {
-					return
-				}
-				var tc *tunConn
-				if conn, err := tun.dialer.DialByNodeIDandMask(dstNodeID, dstNodeIDMask); err == nil {
 					// We've been given a connection so prepare the session wrapper
-					if tc, err = tun.wrap(conn); err != nil {
+					var tc *tunConn
+					if tc, err = tun._wrap(conn); err != nil {
 						// Something went wrong when storing the connection, typically that
 						// something already exists for this address or subnet
 						tun.log.Debugln("TUN/TAP iface wrap:", err)
+						return
 					}
-				}
-				tun.mutex.Lock()
-				packets := tun.dials[*dstNodeID]
-				delete(tun.dials, *dstNodeID)
-				tun.mutex.Unlock()
-				if tc != nil {
 					for _, packet := range packets {
-						p := packet // Possibly required because of how range
+						tc.writeFrom(nil, packet)
-						tc.send <- p
 					}
-				}
+				})
+				return
 			}()
-			// While the dial is going on we can't do much else
-			// continuing this iteration - skip to the next one
-			continue
-		}
-		// If we have a connection now, try writing to it
-		if isIn && session != nil {
-			session.send <- bs
 		}
 	}
-}
+	// If we have a connection now, try writing to it
-
+	if isIn && session != nil {
-func (tun *TunAdapter) reader() error {
+		session.writeFrom(tun, bs)
-	toWorker := make(chan []byte, 32)
-	defer close(toWorker)
-	go tun.readerPacketHandler(toWorker)
-	for {
-		// Get a slice to store the packet in
-		recvd := util.ResizeBytes(util.GetBytes(), 65535+tun_ETHER_HEADER_LENGTH)
-		// Wait for a packet to be delivered to us through the TUN/TAP adapter
-		n, err := tun.iface.Read(recvd)
-		if err != nil {
-			if !tun.isOpen {
-				return err
-			}
-			panic(err)
-		}
-		if n == 0 {
-			util.PutBytes(recvd)
-			continue
-		}
-		// Send the packet to the worker
-		toWorker <- recvd[:n]
 	}
 }

src/tuntap/tun.go

@@ -13,8 +13,10 @@ import (
 	"errors"
 	"fmt"
 	"net"
-	"sync"
 
+	//"sync"
+
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 	"github.com/yggdrasil-network/water"
 
@@ -33,19 +35,21 @@ const tun_ETHER_HEADER_LENGTH = 14
 // you should pass this object to the yggdrasil.SetRouterAdapter() function
 // before calling yggdrasil.Start().
 type TunAdapter struct {
-	config       *config.NodeState
+	writer      tunWriter
-	log          *log.Logger
+	reader      tunReader
-	reconfigure  chan chan error
+	config      *config.NodeState
-	listener     *yggdrasil.Listener
+	log         *log.Logger
-	dialer       *yggdrasil.Dialer
+	reconfigure chan chan error
-	addr         address.Address
+	listener    *yggdrasil.Listener
-	subnet       address.Subnet
+	dialer      *yggdrasil.Dialer
-	ckr          cryptokey
+	addr        address.Address
-	icmpv6       ICMPv6
+	subnet      address.Subnet
-	mtu          int
+	ckr         cryptokey
-	iface        *water.Interface
+	icmpv6      ICMPv6
-	send         chan []byte
+	mtu         int
-	mutex        sync.RWMutex // Protects the below
+	iface       *water.Interface
+	phony.Inbox // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below
+	//mutex        sync.RWMutex // Protects the below
 	addrToConn   map[address.Address]*tunConn
 	subnetToConn map[address.Subnet]*tunConn
 	dials        map[crypto.NodeID][][]byte // Buffer of packets to send after dialing finishes
@@ -114,11 +118,21 @@ func (tun *TunAdapter) Init(config *config.NodeState, log *log.Logger, listener
 	tun.addrToConn = make(map[address.Address]*tunConn)
 	tun.subnetToConn = make(map[address.Subnet]*tunConn)
 	tun.dials = make(map[crypto.NodeID][][]byte)
+	tun.writer.tun = tun
+	tun.reader.tun = tun
 }
 
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
-// read/write goroutines to handle packets on that interface.
+// reader actor to handle packets on that interface.
 func (tun *TunAdapter) Start() error {
+	var err error
+	phony.Block(tun, func() {
+		err = tun._start()
+	})
+	return err
+}
+
+func (tun *TunAdapter) _start() error {
 	current := tun.config.GetCurrent()
 	if tun.config == nil || tun.listener == nil || tun.dialer == nil {
 		return errors.New("No configuration available to TUN/TAP")
@@ -145,11 +159,8 @@ func (tun *TunAdapter) Start() error {
 		tun.log.Debugln("Not starting TUN/TAP as ifname is none or dummy")
 		return nil
 	}
-	tun.mutex.Lock()
 	tun.isOpen = true
-	tun.send = make(chan []byte, 32) // TODO: is this a sensible value?
 	tun.reconfigure = make(chan chan error)
-	tun.mutex.Unlock()
 	go func() {
 		for {
 			e := <-tun.reconfigure
@@ -157,8 +168,7 @@ func (tun *TunAdapter) Start() error {
 		}
 	}()
 	go tun.handler()
-	go tun.reader()
+	tun.reader.Act(nil, tun.reader._read) // Start the reader
-	go tun.writer()
 	tun.icmpv6.Init(tun)
 	if iftapmode {
 		go tun.icmpv6.Solicit(tun.addr)
@@ -170,6 +180,14 @@ func (tun *TunAdapter) Start() error {
 // Start the setup process for the TUN/TAP adapter. If successful, starts the
 // read/write goroutines to handle packets on that interface.
 func (tun *TunAdapter) Stop() error {
+	var err error
+	phony.Block(tun, func() {
+		err = tun._stop()
+	})
+	return err
+}
+
+func (tun *TunAdapter) _stop() error {
 	tun.isOpen = false
 	// TODO: we have nothing that cleanly stops all the various goroutines opened
 	// by TUN/TAP, e.g. readers/writers, sessions
@@ -183,29 +201,11 @@ func (tun *TunAdapter) Stop() error {
 func (tun *TunAdapter) UpdateConfig(config *config.NodeConfig) {
 	tun.log.Debugln("Reloading TUN/TAP configuration...")
 
+	// Replace the active configuration with the supplied one
 	tun.config.Replace(*config)
 
-	errors := 0
+	// Notify children about the configuration change
-
+	tun.Act(nil, tun.ckr.configure)
-	components := []chan chan error{
-		tun.reconfigure,
-		tun.ckr.reconfigure,
-	}
-
-	for _, component := range components {
-		response := make(chan error)
-		component <- response
-		if err := <-response; err != nil {
-			tun.log.Errorln(err)
-			errors++
-		}
-	}
-
-	if errors > 0 {
-		tun.log.Warnln(errors, "TUN/TAP module(s) reported errors during configuration reload")
-	} else {
-		tun.log.Infoln("TUN/TAP configuration reloaded successfully")
-	}
 }
 
 func (tun *TunAdapter) handler() error {
@@ -216,22 +216,22 @@ func (tun *TunAdapter) handler() error {
 			tun.log.Errorln("TUN/TAP connection accept error:", err)
 			return err
 		}
-		if _, err := tun.wrap(conn); err != nil {
+		phony.Block(tun, func() {
-			// Something went wrong when storing the connection, typically that
+			if _, err := tun._wrap(conn); err != nil {
-			// something already exists for this address or subnet
+				// Something went wrong when storing the connection, typically that
-			tun.log.Debugln("TUN/TAP handler wrap:", err)
+				// something already exists for this address or subnet
-		}
+				tun.log.Debugln("TUN/TAP handler wrap:", err)
+			}
+		})
 	}
 }
 
-func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
+func (tun *TunAdapter) _wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	// Prepare a session wrapper for the given connection
 	s := tunConn{
-		tun:   tun,
+		tun:  tun,
-		conn:  conn,
+		conn: conn,
-		send:  make(chan []byte, 32), // TODO: is this a sensible value?
+		stop: make(chan struct{}),
-		stop:  make(chan struct{}),
-		alive: make(chan struct{}, 1),
 	}
 	c = &s
 	// Get the remote address and subnet of the other side
@@ -239,27 +239,28 @@ func (tun *TunAdapter) wrap(conn *yggdrasil.Conn) (c *tunConn, err error) {
 	s.addr = *address.AddrForNodeID(&remoteNodeID)
 	s.snet = *address.SubnetForNodeID(&remoteNodeID)
 	// Work out if this is already a destination we already know about
-	tun.mutex.Lock()
-	defer tun.mutex.Unlock()
 	atc, aok := tun.addrToConn[s.addr]
 	stc, sok := tun.subnetToConn[s.snet]
 	// If we know about a connection for this destination already then assume it
 	// is no longer valid and close it
 	if aok {
-		atc._close_nomutex()
+		atc._close_from_tun()
 		err = errors.New("replaced connection for address")
 	} else if sok {
-		stc._close_nomutex()
+		stc._close_from_tun()
 		err = errors.New("replaced connection for subnet")
 	}
 	// Save the session wrapper so that we can look it up quickly next time
 	// we receive a packet through the interface for this address
 	tun.addrToConn[s.addr] = &s
 	tun.subnetToConn[s.snet] = &s
-	// Start the connection goroutines
+	// Set the read callback and start the timeout
-	go s.reader()
+	conn.SetReadCallback(func(bs []byte) {
-	go s.writer()
+		s.Act(conn, func() {
-	go s.checkForTimeouts()
+			s._read(bs)
+		})
+	})
+	s.Act(nil, s.stillAlive)
 	// Return
 	return c, err
 }

src/yggdrasil/api.go

@@ -6,12 +6,13 @@ import (
 	"fmt"
 	"net"
 	"sort"
-	"sync/atomic"
 	"time"
 
 	"github.com/gologme/log"
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
+
+	"github.com/Arceliar/phony"
 )
 
 // Peer represents a single peer object. This contains information from the
@@ -106,15 +107,18 @@ func (c *Core) GetPeers() []Peer {
 	sort.Slice(ps, func(i, j int) bool { return ps[i] < ps[j] })
 	for _, port := range ps {
 		p := ports[port]
-		info := Peer{
+		var info Peer
-			Endpoint:   p.intf.name,
+		phony.Block(p, func() {
-			BytesSent:  atomic.LoadUint64(&p.bytesSent),
+			info = Peer{
-			BytesRecvd: atomic.LoadUint64(&p.bytesRecvd),
+				Endpoint:   p.intf.name,
-			Protocol:   p.intf.info.linkType,
+				BytesSent:  p.bytesSent,
-			Port:       uint64(port),
+				BytesRecvd: p.bytesRecvd,
-			Uptime:     time.Since(p.firstSeen),
+				Protocol:   p.intf.info.linkType,
-		}
+				Port:       uint64(port),
-		copy(info.PublicKey[:], p.box[:])
+				Uptime:     time.Since(p.firstSeen),
+			}
+			copy(info.PublicKey[:], p.box[:])
+		})
 		peers = append(peers, info)
 	}
 	return peers
@@ -135,15 +139,18 @@ func (c *Core) GetSwitchPeers() []SwitchPeer {
 			continue
 		}
 		coords := elem.locator.getCoords()
-		info := SwitchPeer{
+		var info SwitchPeer
-			Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
+		phony.Block(peer, func() {
-			BytesSent:  atomic.LoadUint64(&peer.bytesSent),
+			info = SwitchPeer{
-			BytesRecvd: atomic.LoadUint64(&peer.bytesRecvd),
+				Coords:     append([]uint64{}, wire_coordsBytestoUint64s(coords)...),
-			Port:       uint64(elem.port),
+				BytesSent:  peer.bytesSent,
-			Protocol:   peer.intf.info.linkType,
+				BytesRecvd: peer.bytesRecvd,
-			Endpoint:   peer.intf.info.remote,
+				Port:       uint64(elem.port),
-		}
+				Protocol:   peer.intf.info.linkType,
-		copy(info.PublicKey[:], peer.box[:])
+				Endpoint:   peer.intf.info.remote,
+			}
+			copy(info.PublicKey[:], peer.box[:])
+		})
 		switchpeers = append(switchpeers, info)
 	}
 	return switchpeers
@@ -156,11 +163,11 @@ func (c *Core) GetDHT() []DHTEntry {
 	getDHT := func() {
 		now := time.Now()
 		var dhtentry []*dhtInfo
-		for _, v := range c.dht.table {
+		for _, v := range c.router.dht.table {
 			dhtentry = append(dhtentry, v)
 		}
 		sort.SliceStable(dhtentry, func(i, j int) bool {
-			return dht_ordered(&c.dht.nodeID, dhtentry[i].getNodeID(), dhtentry[j].getNodeID())
+			return dht_ordered(&c.router.dht.nodeID, dhtentry[i].getNodeID(), dhtentry[j].getNodeID())
 		})
 		for _, v := range dhtentry {
 			info := DHTEntry{
@@ -171,7 +178,7 @@ func (c *Core) GetDHT() []DHTEntry {
 			dhtentries = append(dhtentries, info)
 		}
 	}
-	c.router.doAdmin(getDHT)
+	phony.Block(&c.router, getDHT)
 	return dhtentries
 }
 
@@ -186,7 +193,7 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			Size:         switchTable.queues.size,
 			HighestCount: uint64(switchTable.queues.maxbufs),
 			HighestSize:  switchTable.queues.maxsize,
-			MaximumSize:  switchTable.queueTotalMaxSize,
+			MaximumSize:  switchTable.queues.totalMaxSize,
 		}
 		for k, v := range switchTable.queues.bufs {
 			nexthop := switchTable.bestPortForCoords([]byte(k))
@@ -198,9 +205,8 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 			}
 			switchqueues.Queues = append(switchqueues.Queues, queue)
 		}
-
 	}
-	c.switchTable.doAdmin(getSwitchQueues)
+	phony.Block(&c.switchTable, getSwitchQueues)
 	return switchqueues
 }
 
@@ -208,12 +214,12 @@ func (c *Core) GetSwitchQueues() SwitchQueues {
 func (c *Core) GetSessions() []Session {
 	var sessions []Session
 	getSessions := func() {
-		for _, sinfo := range c.sessions.sinfos {
+		for _, sinfo := range c.router.sessions.sinfos {
 			var session Session
 			workerFunc := func() {
 				session = Session{
 					Coords:      append([]uint64{}, wire_coordsBytestoUint64s(sinfo.coords)...),
-					MTU:         sinfo.getMTU(),
+					MTU:         sinfo._getMTU(),
 					BytesSent:   sinfo.bytesSent,
 					BytesRecvd:  sinfo.bytesRecvd,
 					Uptime:      time.Now().Sub(sinfo.timeOpened),
@@ -221,39 +227,28 @@ func (c *Core) GetSessions() []Session {
 				}
 				copy(session.PublicKey[:], sinfo.theirPermPub[:])
 			}
-			var skip bool
+			phony.Block(sinfo, workerFunc)
-			func() {
-				defer func() {
-					if recover() != nil {
-						skip = true
-					}
-				}()
-				sinfo.doFunc(workerFunc)
-			}()
-			if skip {
-				continue
-			}
 			// TODO? skipped known but timed out sessions?
 			sessions = append(sessions, session)
 		}
 	}
-	c.router.doAdmin(getSessions)
+	phony.Block(&c.router, getSessions)
 	return sessions
 }
 
 // ConnListen returns a listener for Yggdrasil session connections.
 func (c *Core) ConnListen() (*Listener, error) {
-	c.sessions.listenerMutex.Lock()
+	c.router.sessions.listenerMutex.Lock()
-	defer c.sessions.listenerMutex.Unlock()
+	defer c.router.sessions.listenerMutex.Unlock()
-	if c.sessions.listener != nil {
+	if c.router.sessions.listener != nil {
 		return nil, errors.New("a listener already exists")
 	}
-	c.sessions.listener = &Listener{
+	c.router.sessions.listener = &Listener{
 		core:  c,
 		conn:  make(chan *Conn),
 		close: make(chan interface{}),
 	}
-	return c.sessions.listener, nil
+	return c.router.sessions.listener, nil
 }
 
 // ConnDialer returns a dialer for Yggdrasil session connections.
@@ -338,11 +333,9 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 		})
 		c.router.nodeinfo.sendNodeInfo(key, wire_coordsUint64stoBytes(coords), false)
 	}
-	c.router.doAdmin(sendNodeInfoRequest)
+	phony.Block(&c.router, sendNodeInfoRequest)
-	go func() {
+	timer := time.AfterFunc(6*time.Second, func() { close(response) })
-		time.Sleep(6 * time.Second)
+	defer timer.Stop()
-		close(response)
-	}()
 	for res := range response {
 		return *res, nil
 	}
@@ -356,10 +349,10 @@ func (c *Core) GetNodeInfo(key crypto.BoxPubKey, coords []uint64, nocache bool)
 // received an incoming session request. The function should return true to
 // allow the session or false to reject it.
 func (c *Core) SetSessionGatekeeper(f func(pubkey *crypto.BoxPubKey, initiator bool) bool) {
-	c.sessions.isAllowedMutex.Lock()
+	c.router.sessions.isAllowedMutex.Lock()
-	defer c.sessions.isAllowedMutex.Unlock()
+	defer c.router.sessions.isAllowedMutex.Unlock()
 
-	c.sessions.isAllowedHandler = f
+	c.router.sessions.isAllowedHandler = f
 }
 
 // SetLogger sets the output logger of the Yggdrasil node after startup. This
@@ -445,12 +438,12 @@ func (c *Core) DHTPing(key crypto.BoxPubKey, coords []uint64, target *crypto.Nod
 	}
 	rq := dhtReqKey{info.key, *target}
 	sendPing := func() {
-		c.dht.addCallback(&rq, func(res *dhtRes) {
+		c.router.dht.addCallback(&rq, func(res *dhtRes) {
 			resCh <- res
 		})
-		c.dht.ping(&info, &rq.dest)
+		c.router.dht.ping(&info, &rq.dest)
 	}
-	c.router.doAdmin(sendPing)
+	phony.Block(&c.router, sendPing)
 	// TODO: do something better than the below...
 	res := <-resCh
 	if res != nil {

src/yggdrasil/conn.go

@@ -3,12 +3,12 @@ package yggdrasil
 import (
 	"errors"
 	"fmt"
-	"sync"
-	"sync/atomic"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // ConnError implements the net.Error interface
@@ -54,37 +54,47 @@ func (e *ConnError) Closed() bool {
 }
 
 type Conn struct {
+	phony.Inbox
 	core          *Core
-	readDeadline  atomic.Value // time.Time // TODO timer
+	readDeadline  *time.Time
-	writeDeadline atomic.Value // time.Time // TODO timer
+	writeDeadline *time.Time
-	mutex         sync.RWMutex // protects the below
 	nodeID        *crypto.NodeID
 	nodeMask      *crypto.NodeID
 	session       *sessionInfo
+	mtu           uint16
+	readCallback  func([]byte)
+	readBuffer    chan []byte
 }
 
 // TODO func NewConn() that initializes additional fields as needed
 func newConn(core *Core, nodeID *crypto.NodeID, nodeMask *crypto.NodeID, session *sessionInfo) *Conn {
 	conn := Conn{
-		core:     core,
+		core:       core,
-		nodeID:   nodeID,
+		nodeID:     nodeID,
-		nodeMask: nodeMask,
+		nodeMask:   nodeMask,
-		session:  session,
+		session:    session,
+		readBuffer: make(chan []byte, 1024),
 	}
 	return &conn
 }
 
 func (c *Conn) String() string {
-	c.mutex.RLock()
+	var s string
-	defer c.mutex.RUnlock()
+	phony.Block(c, func() { s = fmt.Sprintf("conn=%p", c) })
-	return fmt.Sprintf("conn=%p", c)
+	return s
+}
+
+func (c *Conn) setMTU(from phony.Actor, mtu uint16) {
+	c.Act(from, func() { c.mtu = mtu })
 }
 
 // This should never be called from the router goroutine, used in the dial functions
 func (c *Conn) search() error {
 	var sinfo *searchInfo
 	var isIn bool
-	c.core.router.doAdmin(func() { sinfo, isIn = c.core.searches.searches[*c.nodeID] })
+	phony.Block(&c.core.router, func() {
+		sinfo, isIn = c.core.router.searches.searches[*c.nodeID]
+	})
 	if !isIn {
 		done := make(chan struct{}, 1)
 		var sess *sessionInfo
@@ -98,8 +108,8 @@ func (c *Conn) search() error {
 			default:
 			}
 		}
-		c.core.router.doAdmin(func() {
+		phony.Block(&c.core.router, func() {
-			sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
+			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			sinfo.continueSearch()
 		})
 		<-done
@@ -112,6 +122,7 @@ func (c *Conn) search() error {
 			for i := range c.nodeMask {
 				c.nodeMask[i] = 0xFF
 			}
+			c.session.conn = c
 		}
 		return err
 	} else {
@@ -120,27 +131,27 @@ func (c *Conn) search() error {
 	return nil
 }
 
-// Used in session keep-alive traffic in Conn.Write
+// Used in session keep-alive traffic
 func (c *Conn) doSearch() {
 	routerWork := func() {
 		// Check to see if there is a search already matching the destination
-		sinfo, isIn := c.core.searches.searches[*c.nodeID]
+		sinfo, isIn := c.core.router.searches.searches[*c.nodeID]
 		if !isIn {
 			// Nothing was found, so create a new search
 			searchCompleted := func(sinfo *sessionInfo, e error) {}
-			sinfo = c.core.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
+			sinfo = c.core.router.searches.newIterSearch(c.nodeID, c.nodeMask, searchCompleted)
 			c.core.log.Debugf("%s DHT search started: %p", c.String(), sinfo)
 			// Start the search
 			sinfo.continueSearch()
 		}
 	}
-	go func() { c.core.router.admin <- routerWork }()
+	c.core.router.Act(c.session, routerWork)
 }
 
-func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation, bool) {
+func (c *Conn) _getDeadlineCancellation(t *time.Time) (util.Cancellation, bool) {
-	if deadline, ok := value.Load().(time.Time); ok {
+	if t != nil {
 		// A deadline is set, so return a Cancellation that uses it
-		c := util.CancellationWithDeadline(c.session.cancel, deadline)
+		c := util.CancellationWithDeadline(c.session.cancel, *t)
 		return c, true
 	} else {
 		// No deadline was set, so just return the existinc cancellation and a dummy value
@@ -148,9 +159,45 @@ func (c *Conn) getDeadlineCancellation(value *atomic.Value) (util.Cancellation,
 	}
 }
 
+// SetReadCallback sets a callback which will be called whenever a packet is received.
+func (c *Conn) SetReadCallback(callback func([]byte)) {
+	c.Act(nil, func() {
+		c.readCallback = callback
+		c._drainReadBuffer()
+	})
+}
+
+func (c *Conn) _drainReadBuffer() {
+	if c.readCallback == nil {
+		return
+	}
+	select {
+	case bs := <-c.readBuffer:
+		c.readCallback(bs)
+		c.Act(nil, c._drainReadBuffer) // In case there's more
+	default:
+	}
+}
+
+// Called by the session to pass a new message to the Conn
+func (c *Conn) recvMsg(from phony.Actor, msg []byte) {
+	c.Act(from, func() {
+		if c.readCallback != nil {
+			c.readCallback(msg)
+		} else {
+			select {
+			case c.readBuffer <- msg:
+			default:
+			}
+		}
+	})
+}
+
 // Used internally by Read, the caller is responsible for util.PutBytes when they're done.
 func (c *Conn) ReadNoCopy() ([]byte, error) {
-	cancel, doCancel := c.getDeadlineCancellation(&c.readDeadline)
+	var cancel util.Cancellation
+	var doCancel bool
+	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.readDeadline) })
 	if doCancel {
 		defer cancel.Cancel(nil)
 	}
@@ -162,7 +209,7 @@ func (c *Conn) ReadNoCopy() ([]byte, error) {
 		} else {
 			return nil, ConnError{errors.New("session closed"), false, false, true, 0}
 		}
-	case bs := <-c.session.recv:
+	case bs := <-c.readBuffer:
 		return bs, nil
 	}
 }
@@ -185,49 +232,63 @@ func (c *Conn) Read(b []byte) (int, error) {
 	return n, err
 }
 
-// Used internally by Write, the caller must not reuse the argument bytes when no error occurs
+func (c *Conn) _write(msg FlowKeyMessage) error {
-func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
+	if len(msg.Message) > int(c.mtu) {
-	var err error
+		return ConnError{errors.New("packet too big"), true, false, false, int(c.mtu)}
-	sessionFunc := func() {
+	}
-		// Does the packet exceed the permitted size for the session?
+	c.session.Act(c, func() {
-		if uint16(len(msg.Message)) > c.session.getMTU() {
+		// Send the packet
-			err = ConnError{errors.New("packet too big"), true, false, false, int(c.session.getMTU())}
+		c.session._send(msg)
-			return
+		// Session keep-alive, while we wait for the crypto workers from send
-		}
-		// The rest of this work is session keep-alive traffic
 		switch {
 		case time.Since(c.session.time) > 6*time.Second:
 			if c.session.time.Before(c.session.pingTime) && time.Since(c.session.pingTime) > 6*time.Second {
 				// TODO double check that the above condition is correct
 				c.doSearch()
 			} else {
-				c.core.sessions.ping(c.session)
+				c.session.ping(c.session) // TODO send from self if this becomes an actor
 			}
 		case c.session.reset && c.session.pingTime.Before(c.session.time):
-			c.core.sessions.ping(c.session)
+			c.session.ping(c.session) // TODO send from self if this becomes an actor
 		default: // Don't do anything, to keep traffic throttled
 		}
-	}
+	})
-	c.session.doFunc(sessionFunc)
+	return nil
-	if err == nil {
+}
-		cancel, doCancel := c.getDeadlineCancellation(&c.writeDeadline)
+
-		if doCancel {
+// WriteFrom should be called by a phony.Actor, and tells the Conn to send a message.
-			defer cancel.Cancel(nil)
+// This is used internaly by WriteNoCopy and Write.
-		}
+// If the callback is called with a non-nil value, then it is safe to reuse the argument FlowKeyMessage.
-		select {
+func (c *Conn) WriteFrom(from phony.Actor, msg FlowKeyMessage, callback func(error)) {
-		case <-cancel.Finished():
+	c.Act(from, func() {
-			if cancel.Error() == util.CancellationTimeoutError {
+		callback(c._write(msg))
-				err = ConnError{errors.New("write timeout"), true, false, false, 0}
+	})
-			} else {
+}
-				err = ConnError{errors.New("session closed"), false, false, true, 0}
+
-			}
+// WriteNoCopy is used internally by Write and makes use of WriteFrom under the hood.
-		case c.session.send <- msg:
+// The caller must not reuse the argument FlowKeyMessage when a nil error is returned.
+func (c *Conn) WriteNoCopy(msg FlowKeyMessage) error {
+	var cancel util.Cancellation
+	var doCancel bool
+	phony.Block(c, func() { cancel, doCancel = c._getDeadlineCancellation(c.writeDeadline) })
+	var err error
+	select {
+	case <-cancel.Finished():
+		if cancel.Error() == util.CancellationTimeoutError {
+			err = ConnError{errors.New("write timeout"), true, false, false, 0}
+		} else {
+			err = ConnError{errors.New("session closed"), false, false, true, 0}
 		}
+	default:
+		done := make(chan struct{})
+		callback := func(e error) { err = e; close(done) }
+		c.WriteFrom(nil, msg, callback)
+		<-done
 	}
 	return err
 }
 
-// Implements net.Conn.Write
+// Write implement the Write function of a net.Conn, and makes use of WriteNoCopy under the hood.
 func (c *Conn) Write(b []byte) (int, error) {
 	written := len(b)
 	msg := FlowKeyMessage{Message: append(util.GetBytes(), b...)}
@@ -240,25 +301,28 @@ func (c *Conn) Write(b []byte) (int, error) {
 }
 
 func (c *Conn) Close() (err error) {
-	c.mutex.Lock()
+	phony.Block(c, func() {
-	defer c.mutex.Unlock()
+		if c.session != nil {
-	if c.session != nil {
+			// Close the session, if it hasn't been closed already
-		// Close the session, if it hasn't been closed already
+			if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
-		if e := c.session.cancel.Cancel(errors.New("connection closed")); e != nil {
+				err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
-			err = ConnError{errors.New("close failed, session already closed"), false, false, true, 0}
+			} else {
+				c.session.doRemove()
+			}
 		}
-	}
+	})
 	return
 }
 
 func (c *Conn) LocalAddr() crypto.NodeID {
-	return *crypto.GetNodeID(&c.session.core.boxPub)
+	return *crypto.GetNodeID(&c.core.boxPub)
 }
 
 func (c *Conn) RemoteAddr() crypto.NodeID {
-	c.mutex.RLock()
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
-	defer c.mutex.RUnlock()
+	var n crypto.NodeID
-	return *c.nodeID
+	phony.Block(c, func() { n = *c.nodeID })
+	return n
 }
 
 func (c *Conn) SetDeadline(t time.Time) error {
@@ -268,11 +332,13 @@ func (c *Conn) SetDeadline(t time.Time) error {
 }
 
 func (c *Conn) SetReadDeadline(t time.Time) error {
-	c.readDeadline.Store(t)
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
+	phony.Block(c, func() { c.readDeadline = &t })
 	return nil
 }
 
 func (c *Conn) SetWriteDeadline(t time.Time) error {
-	c.writeDeadline.Store(t)
+	// TODO warn that this can block while waiting for the Conn actor to run, so don't call it from other actors...
+	phony.Block(c, func() { c.writeDeadline = &t })
 	return nil
 }

src/yggdrasil/core.go

@@ -6,6 +6,7 @@ import (
 	"io/ioutil"
 	"time"
 
+	"github.com/Arceliar/phony"
 	"github.com/gologme/log"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/config"
@@ -19,6 +20,7 @@ type Core struct {
 	// This is the main data structure that holds everything else for a node
 	// We're going to keep our own copy of the provided config - that way we can
 	// guarantee that it will be covered by the mutex
+	phony.Inbox
 	config      config.NodeState // Config
 	boxPub      crypto.BoxPubKey
 	boxPriv     crypto.BoxPrivKey
@@ -26,15 +28,12 @@ type Core struct {
 	sigPriv     crypto.SigPrivKey
 	switchTable switchTable
 	peers       peers
-	sessions    sessions
 	router      router
-	dht         dht
-	searches    searches
 	link        link
 	log         *log.Logger
 }
 
-func (c *Core) init() error {
+func (c *Core) _init() error {
 	// TODO separate init and start functions
 	//  Init sets up structs
 	//  Start launches goroutines that depend on structs being set up
@@ -76,9 +75,6 @@ func (c *Core) init() error {
 		c.log.Warnln("SigningPublicKey in config is incorrect, should be", sp)
 	}
 
-	c.searches.init(c)
-	c.dht.init(c)
-	c.sessions.init(c)
 	c.peers.init(c)
 	c.router.init(c)
 	c.switchTable.init(c) // TODO move before peers? before router?
@@ -89,64 +85,44 @@ func (c *Core) init() error {
 // If any static peers were provided in the configuration above then we should
 // configure them. The loop ensures that disconnected peers will eventually
 // be reconnected with.
-func (c *Core) addPeerLoop() {
+func (c *Core) _addPeerLoop() {
-	for {
+	// Get the peers from the config - these could change!
-		//  the peers from the config - these could change!
+	current := c.config.GetCurrent()
-		current := c.config.GetCurrent()
 
-		// Add peers from the Peers section
+	// Add peers from the Peers section
-		for _, peer := range current.Peers {
+	for _, peer := range current.Peers {
-			go c.AddPeer(peer, "")
+		go c.AddPeer(peer, "") // TODO: this should be acted and not in a goroutine?
+		time.Sleep(time.Second)
+	}
+
+	// Add peers from the InterfacePeers section
+	for intf, intfpeers := range current.InterfacePeers {
+		for _, peer := range intfpeers {
+			go c.AddPeer(peer, intf) // TODO: this should be acted and not in a goroutine?
 			time.Sleep(time.Second)
 		}
-
-		// Add peers from the InterfacePeers section
-		for intf, intfpeers := range current.InterfacePeers {
-			for _, peer := range intfpeers {
-				go c.AddPeer(peer, intf)
-				time.Sleep(time.Second)
-			}
-		}
-
-		// Sit for a while
-		time.Sleep(time.Minute)
 	}
+
+	// Sit for a while
+	time.AfterFunc(time.Minute, func() {
+		c.Act(c, c._addPeerLoop)
+	})
 }
 
 // UpdateConfig updates the configuration in Core with the provided
 // config.NodeConfig and then signals the various module goroutines to
 // reconfigure themselves if needed.
 func (c *Core) UpdateConfig(config *config.NodeConfig) {
-	c.log.Debugln("Reloading node configuration...")
+	c.Act(nil, func() {
+		c.log.Debugln("Reloading node configuration...")
 
-	c.config.Replace(*config)
+		// Replace the active configuration with the supplied one
+		c.config.Replace(*config)
 
-	errors := 0
+		// Notify the router and switch about the new configuration
-
+		c.router.Act(c, c.router.reconfigure)
-	components := []chan chan error{
+		c.switchTable.Act(c, c.switchTable.reconfigure)
-		c.searches.reconfigure,
+	})
-		c.dht.reconfigure,
-		c.sessions.reconfigure,
-		c.peers.reconfigure,
-		c.router.reconfigure,
-		c.switchTable.reconfigure,
-		c.link.reconfigure,
-	}
-
-	for _, component := range components {
-		response := make(chan error)
-		component <- response
-		if err := <-response; err != nil {
-			c.log.Errorln(err)
-			errors++
-		}
-	}
-
-	if errors > 0 {
-		c.log.Warnln(errors, "node module(s) reported errors during configuration reload")
-	} else {
-		c.log.Infoln("Node configuration reloaded successfully")
-	}
 }
 
 // Start starts up Yggdrasil using the provided config.NodeConfig, and outputs
@@ -154,7 +130,15 @@ func (c *Core) UpdateConfig(config *config.NodeConfig) {
 // TCP and UDP sockets, a multicast discovery socket, an admin socket, router,
 // switch and DHT node. A config.NodeState is returned which contains both the
 // current and previous configurations (from reconfigures).
-func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState, error) {
+func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (conf *config.NodeState, err error) {
+	phony.Block(c, func() {
+		conf, err = c._start(nc, log)
+	})
+	return
+}
+
+// This function is unsafe and should only be ran by the core actor.
+func (c *Core) _start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState, error) {
 	c.log = log
 
 	c.config = config.NodeState{
@@ -170,20 +154,13 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 	}
 
 	c.log.Infoln("Starting up...")
-
+	c._init()
-	c.init()
 
 	if err := c.link.init(c); err != nil {
 		c.log.Errorln("Failed to start link interfaces")
 		return nil, err
 	}
 
-	c.config.Mutex.RLock()
-	if c.config.Current.SwitchOptions.MaxTotalQueueSize >= SwitchQueueTotalMinSize {
-		c.switchTable.queueTotalMaxSize = c.config.Current.SwitchOptions.MaxTotalQueueSize
-	}
-	c.config.Mutex.RUnlock()
-
 	if err := c.switchTable.start(); err != nil {
 		c.log.Errorln("Failed to start switch")
 		return nil, err
@@ -194,7 +171,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 		return nil, err
 	}
 
-	go c.addPeerLoop()
+	c.Act(c, c._addPeerLoop)
 
 	c.log.Infoln("Startup complete")
 	return &c.config, nil
@@ -202,5 +179,10 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) (*config.NodeState,
 
 // Stop shuts down the Yggdrasil node.
 func (c *Core) Stop() {
+	phony.Block(c, c._stop)
+}
+
+// This function is unsafe and should only be ran by the core actor.
+func (c *Core) _stop() {
 	c.log.Infoln("Stopping...")
 }

src/yggdrasil/debug.go

@@ -2,20 +2,7 @@
 
 package yggdrasil
 
-// These are functions that should not exist
-// They are (or were) used during development, to work around missing features
-// They're also used to configure things from the outside
-//  It would be better to define and export a few config functions elsewhere
-//  Or define some remote API and call it to send/request configuration info
-
-import _ "golang.org/x/net/ipv6" // TODO put this somewhere better
-
-//import "golang.org/x/net/proxy"
-
 import "fmt"
-import "net"
-import "regexp"
-import "encoding/hex"
 
 import _ "net/http/pprof"
 import "net/http"
@@ -24,11 +11,6 @@ import "os"
 
 import "github.com/gologme/log"
 
-import "github.com/yggdrasil-network/yggdrasil-go/src/address"
-import "github.com/yggdrasil-network/yggdrasil-go/src/config"
-import "github.com/yggdrasil-network/yggdrasil-go/src/crypto"
-import "github.com/yggdrasil-network/yggdrasil-go/src/defaults"
-
 // Start the profiler in debug builds, if the required environment variable is set.
 func init() {
 	envVarName := "PPROFLISTEN"
@@ -49,580 +31,3 @@ func StartProfiler(log *log.Logger) error {
 	go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }()
 	return nil
 }
-
-// This function is only called by the simulator to set up a node with random
-// keys. It should not be used and may be removed in the future.
-func (c *Core) Init() {
-	bpub, bpriv := crypto.NewBoxKeys()
-	spub, spriv := crypto.NewSigKeys()
-	hbpub := hex.EncodeToString(bpub[:])
-	hbpriv := hex.EncodeToString(bpriv[:])
-	hspub := hex.EncodeToString(spub[:])
-	hspriv := hex.EncodeToString(spriv[:])
-	cfg := config.NodeConfig{
-		EncryptionPublicKey:  hbpub,
-		EncryptionPrivateKey: hbpriv,
-		SigningPublicKey:     hspub,
-		SigningPrivateKey:    hspriv,
-	}
-	c.config = config.NodeState{
-		Current:  cfg,
-		Previous: cfg,
-	}
-	c.init()
-	c.switchTable.start()
-	c.router.start()
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-// Core
-
-func (c *Core) DEBUG_getSigningPublicKey() crypto.SigPubKey {
-	return (crypto.SigPubKey)(c.sigPub)
-}
-
-func (c *Core) DEBUG_getEncryptionPublicKey() crypto.BoxPubKey {
-	return (crypto.BoxPubKey)(c.boxPub)
-}
-
-/*
-func (c *Core) DEBUG_getSend() chan<- []byte {
-	return c.router.tun.send
-}
-
-func (c *Core) DEBUG_getRecv() <-chan []byte {
-	return c.router.tun.recv
-}
-*/
-
-// Peer
-
-func (c *Core) DEBUG_getPeers() *peers {
-	return &c.peers
-}
-
-func (ps *peers) DEBUG_newPeer(box crypto.BoxPubKey, sig crypto.SigPubKey, link crypto.BoxSharedKey) *peer {
-	sim := linkInterface{
-		name: "(simulator)",
-		info: linkInfo{
-			local:    "(simulator)",
-			remote:   "(simulator)",
-			linkType: "sim",
-		},
-	}
-	return ps.newPeer(&box, &sig, &link, &sim, nil)
-}
-
-/*
-func (ps *peers) DEBUG_startPeers() {
-  ps.mutex.RLock()
-  defer ps.mutex.RUnlock()
-  for _, p := range ps.ports {
-    if p == nil { continue }
-    go p.MainLoop()
-  }
-}
-*/
-
-func (ps *peers) DEBUG_hasPeer(key crypto.SigPubKey) bool {
-	ports := ps.ports.Load().(map[switchPort]*peer)
-	for _, p := range ports {
-		if p == nil {
-			continue
-		}
-		if p.sig == key {
-			return true
-		}
-	}
-	return false
-}
-
-func (ps *peers) DEBUG_getPorts() map[switchPort]*peer {
-	ports := ps.ports.Load().(map[switchPort]*peer)
-	newPeers := make(map[switchPort]*peer)
-	for port, p := range ports {
-		newPeers[port] = p
-	}
-	return newPeers
-}
-
-func (p *peer) DEBUG_getSigKey() crypto.SigPubKey {
-	return p.sig
-}
-
-func (p *peer) DEEBUG_getPort() switchPort {
-	return p.port
-}
-
-// Router
-
-func (c *Core) DEBUG_getSwitchTable() *switchTable {
-	return &c.switchTable
-}
-
-func (c *Core) DEBUG_getLocator() switchLocator {
-	return c.switchTable.getLocator()
-}
-
-func (l *switchLocator) DEBUG_getCoords() []byte {
-	return l.getCoords()
-}
-
-func (c *Core) DEBUG_switchLookup(dest []byte) switchPort {
-	return c.switchTable.DEBUG_lookup(dest)
-}
-
-// This does the switch layer lookups that decide how to route traffic.
-// Traffic uses greedy routing in a metric space, where the metric distance between nodes is equal to the distance between them on the tree.
-// Traffic must be routed to a node that is closer to the destination via the metric space distance.
-// In the event that two nodes are equally close, it gets routed to the one with the longest uptime (due to the order that things are iterated over).
-// The size of the outgoing packet queue is added to a node's tree distance when the cost of forwarding to a node, subject to the constraint that the real tree distance puts them closer to the destination than ourself.
-// Doing so adds a limited form of backpressure routing, based on local information, which allows us to forward traffic around *local* bottlenecks, provided that another greedy path exists.
-func (t *switchTable) DEBUG_lookup(dest []byte) switchPort {
-	table := t.getTable()
-	myDist := table.self.dist(dest)
-	if myDist == 0 {
-		return 0
-	}
-	// cost is in units of (expected distance) + (expected queue size), where expected distance is used as an approximation of the minimum backpressure gradient needed for packets to flow
-	ports := t.core.peers.getPorts()
-	var best switchPort
-	bestCost := int64(^uint64(0) >> 1)
-	for _, info := range table.elems {
-		dist := info.locator.dist(dest)
-		if !(dist < myDist) {
-			continue
-		}
-		//p, isIn := ports[info.port]
-		_, isIn := ports[info.port]
-		if !isIn {
-			continue
-		}
-		cost := int64(dist) // + p.getQueueSize()
-		if cost < bestCost {
-			best = info.port
-			bestCost = cost
-		}
-	}
-	return best
-}
-
-/*
-func (t *switchTable) DEBUG_isDirty() bool {
-  //data := t.data.Load().(*tabledata)
-  t.mutex.RLock()
-  defer t.mutex.RUnlock()
-  data := t.data
-  return data.dirty
-}
-*/
-
-func (t *switchTable) DEBUG_dumpTable() {
-	//data := t.data.Load().(*tabledata)
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	data := t.data
-	for _, peer := range data.peers {
-		//fmt.Println("DUMPTABLE:", t.treeID, peer.treeID, peer.port,
-		//            peer.locator.Root, peer.coords,
-		//            peer.reverse.Root, peer.reverse.Coords, peer.forward)
-		fmt.Println("DUMPTABLE:", t.key, peer.key, peer.locator.coords, peer.port /*, peer.forward*/)
-	}
-}
-
-func (t *switchTable) DEBUG_getReversePort(port switchPort) switchPort {
-	// Returns Port(0) if it cannot get the reverse peer for any reason
-	//data := t.data.Load().(*tabledata)
-	t.mutex.RLock()
-	defer t.mutex.RUnlock()
-	data := t.data
-	if port >= switchPort(len(data.peers)) {
-		return switchPort(0)
-	}
-	pinfo := data.peers[port]
-	if len(pinfo.locator.coords) < 1 {
-		return switchPort(0)
-	}
-	return pinfo.locator.coords[len(pinfo.locator.coords)-1]
-}
-
-// Wire
-
-func DEBUG_wire_encode_coords(coords []byte) []byte {
-	return wire_encode_coords(coords)
-}
-
-// DHT, via core
-
-func (c *Core) DEBUG_getDHTSize() int {
-	var total int
-	c.router.doAdmin(func() {
-		total = len(c.dht.table)
-	})
-	return total
-}
-
-// TUN defaults
-
-func (c *Core) DEBUG_GetTUNDefaultIfName() string {
-	return defaults.GetDefaults().DefaultIfName
-}
-
-func (c *Core) DEBUG_GetTUNDefaultIfMTU() int {
-	return defaults.GetDefaults().DefaultIfMTU
-}
-
-func (c *Core) DEBUG_GetTUNDefaultIfTAPMode() bool {
-	return defaults.GetDefaults().DefaultIfTAPMode
-}
-
-// udpInterface
-//  FIXME udpInterface isn't exported
-//  So debug functions need to work differently...
-
-/*
-func (c *Core) DEBUG_setupLoopbackUDPInterface() {
-  iface := udpInterface{}
-  iface.init(c, "[::1]:0")
-  c.ifaces = append(c.ifaces[:0], &iface)
-}
-*/
-
-/*
-func (c *Core) DEBUG_getLoopbackAddr() net.Addr {
-  iface := c.ifaces[0]
-  return iface.sock.LocalAddr()
-}
-*/
-
-/*
-func (c *Core) DEBUG_addLoopbackPeer(addr *net.UDPAddr,
-                                     in (chan<- []byte),
-                                     out (<-chan []byte)) {
-  iface := c.ifaces[0]
-  iface.addPeer(addr, in, out)
-}
-*/
-
-/*
-func (c *Core) DEBUG_startLoopbackUDPInterface() {
-  iface := c.ifaces[0]
-  go iface.reader()
-  for addr, chs := range iface.peers {
-    udpAddr, err := net.ResolveUDPAddr("udp6", addr)
-    if err != nil { panic(err) }
-    go iface.writer(udpAddr, chs.out)
-  }
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_getAddr() *address.Address {
-	return address.AddrForNodeID(&c.dht.nodeID)
-}
-
-/*
-func (c *Core) DEBUG_startTun(ifname string, iftapmode bool) {
-	c.DEBUG_startTunWithMTU(ifname, iftapmode, 1280)
-}
-
-func (c *Core) DEBUG_startTunWithMTU(ifname string, iftapmode bool, mtu int) {
-	addr := c.DEBUG_getAddr()
-	straddr := fmt.Sprintf("%s/%v", net.IP(addr[:]).String(), 8*len(address.GetPrefix()))
-	if ifname != "none" {
-		err := c.router.tun.setup(ifname, iftapmode, straddr, mtu)
-		if err != nil {
-			panic(err)
-		}
-		c.log.Println("Setup TUN/TAP:", c.router.tun.iface.Name(), straddr)
-		go func() { panic(c.router.tun.read()) }()
-	}
-	go func() { panic(c.router.tun.write()) }()
-}
-
-func (c *Core) DEBUG_stopTun() {
-	c.router.tun.close()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_newBoxKeys() (*crypto.BoxPubKey, *crypto.BoxPrivKey) {
-	return crypto.NewBoxKeys()
-}
-
-func (c *Core) DEBUG_getSharedKey(myPrivKey *crypto.BoxPrivKey, othersPubKey *crypto.BoxPubKey) *crypto.BoxSharedKey {
-	return crypto.GetSharedKey(myPrivKey, othersPubKey)
-}
-
-func (c *Core) DEBUG_newSigKeys() (*crypto.SigPubKey, *crypto.SigPrivKey) {
-	return crypto.NewSigKeys()
-}
-
-func (c *Core) DEBUG_getNodeID(pub *crypto.BoxPubKey) *crypto.NodeID {
-	return crypto.GetNodeID(pub)
-}
-
-func (c *Core) DEBUG_getTreeID(pub *crypto.SigPubKey) *crypto.TreeID {
-	return crypto.GetTreeID(pub)
-}
-
-func (c *Core) DEBUG_addrForNodeID(nodeID *crypto.NodeID) string {
-	return net.IP(address.AddrForNodeID(nodeID)[:]).String()
-}
-
-func (c *Core) DEBUG_init(bpub []byte,
-	bpriv []byte,
-	spub []byte,
-	spriv []byte) {
-	/*var boxPub crypto.BoxPubKey
-	var boxPriv crypto.BoxPrivKey
-	var sigPub crypto.SigPubKey
-	var sigPriv crypto.SigPrivKey
-	copy(boxPub[:], bpub)
-	copy(boxPriv[:], bpriv)
-	copy(sigPub[:], spub)
-	copy(sigPriv[:], spriv)
-	c.init(&boxPub, &boxPriv, &sigPub, &sigPriv)*/
-	hbpub := hex.EncodeToString(bpub[:])
-	hbpriv := hex.EncodeToString(bpriv[:])
-	hspub := hex.EncodeToString(spub[:])
-	hspriv := hex.EncodeToString(spriv[:])
-	cfg := config.NodeConfig{
-		EncryptionPublicKey:  hbpub,
-		EncryptionPrivateKey: hbpriv,
-		SigningPublicKey:     hspub,
-		SigningPrivateKey:    hspriv,
-	}
-	c.config = config.NodeState{
-		Current:  cfg,
-		Previous: cfg,
-	}
-	c.init()
-
-	if err := c.router.start(); err != nil {
-		panic(err)
-	}
-
-}
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalUDPInterface(addrport string) {
-	if err := c.udp.init(c, addrport); err != nil {
-		c.log.Println("Failed to start UDP interface:", err)
-		panic(err)
-	}
-}
-
-func (c *Core) DEBUG_getGlobalUDPAddr() *net.UDPAddr {
-	return c.udp.sock.LocalAddr().(*net.UDPAddr)
-}
-
-func (c *Core) DEBUG_maybeSendUDPKeys(saddr string) {
-	udpAddr, err := net.ResolveUDPAddr("udp", saddr)
-	if err != nil {
-		panic(err)
-	}
-	var addr connAddr
-	addr.fromUDPAddr(udpAddr)
-	c.udp.mutex.RLock()
-	_, isIn := c.udp.conns[addr]
-	c.udp.mutex.RUnlock()
-	if !isIn {
-		c.udp.sendKeys(addr)
-	}
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-/*
-func (c *Core) DEBUG_addPeer(addr string) {
-	err := c.admin.addPeer(addr, "")
-	if err != nil {
-		panic(err)
-	}
-}
-*/
-/*
-func (c *Core) DEBUG_addSOCKSConn(socksaddr, peeraddr string) {
-	go func() {
-		dialer, err := proxy.SOCKS5("tcp", socksaddr, nil, proxy.Direct)
-		if err == nil {
-			conn, err := dialer.Dial("tcp", peeraddr)
-			if err == nil {
-				c.tcp.callWithConn(&wrappedConn{
-					c: conn,
-					raddr: &wrappedAddr{
-						network: "tcp",
-						addr:    peeraddr,
-					},
-				})
-			}
-		}
-	}()
-}
-*/
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalTCPInterface(addrport string) {
-	c.config.Listen = []string{addrport}
-	if err := c.link.init(c); err != nil {
-		c.log.Println("Failed to start interfaces:", err)
-		panic(err)
-	}
-}
-
-func (c *Core) DEBUG_getGlobalTCPAddr() *net.TCPAddr {
-	return c.link.tcp.getAddr()
-}
-
-func (c *Core) DEBUG_addTCPConn(saddr string) {
-	c.link.tcp.call(saddr, nil, "")
-}
-
-//*/
-
-/*
-func (c *Core) DEBUG_startSelfPeer() {
-  c.Peers.mutex.RLock()
-  defer c.Peers.mutex.RUnlock()
-  p := c.Peers.ports[0]
-  go p.MainLoop()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartGlobalKCPInterface(addrport string) {
-  iface := kcpInterface{}
-  iface.init(c, addrport)
-  c.kcp = &iface
-}
-
-func (c *Core) DEBUG_getGlobalKCPAddr() net.Addr {
-  return c.kcp.serv.Addr()
-}
-
-func (c *Core) DEBUG_addKCPConn(saddr string) {
-  c.kcp.call(saddr)
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-/*
-func (c *Core) DEBUG_setupAndStartAdminInterface(addrport string) {
-	a := admin{}
-	c.config.AdminListen = addrport
-	a.init()
-	c.admin = a
-}
-
-func (c *Core) DEBUG_setupAndStartMulticastInterface() {
-	m := multicast{}
-	m.init(c)
-	c.multicast = m
-	m.start()
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func (c *Core) DEBUG_setLogger(log *log.Logger) {
-	c.log = log
-}
-
-func (c *Core) DEBUG_setIfceExpr(expr *regexp.Regexp) {
-	c.log.Println("DEBUG_setIfceExpr no longer implemented")
-}
-
-/*
-func (c *Core) DEBUG_addAllowedEncryptionPublicKey(boxStr string) {
-	err := c.admin.addAllowedEncryptionPublicKey(boxStr)
-	if err != nil {
-		panic(err)
-	}
-}
-*/
-////////////////////////////////////////////////////////////////////////////////
-
-func DEBUG_simLinkPeers(p, q *peer) {
-	// Sets q.out() to point to p and starts p.linkLoop()
-	goWorkers := func(source, dest *peer) {
-		source.linkOut = make(chan []byte, 1)
-		send := make(chan []byte, 1)
-		source.out = func(bss [][]byte) {
-			for _, bs := range bss {
-				send <- bs
-			}
-		}
-		go source.linkLoop()
-		go func() {
-			var packets [][]byte
-			for {
-				select {
-				case packet := <-source.linkOut:
-					packets = append(packets, packet)
-					continue
-				case packet := <-send:
-					packets = append(packets, packet)
-					source.core.switchTable.idleIn <- source.port
-					continue
-				default:
-				}
-				if len(packets) > 0 {
-					dest.handlePacket(packets[0])
-					packets = packets[1:]
-					continue
-				}
-				select {
-				case packet := <-source.linkOut:
-					packets = append(packets, packet)
-				case packet := <-send:
-					packets = append(packets, packet)
-					source.core.switchTable.idleIn <- source.port
-				}
-			}
-		}()
-	}
-	goWorkers(p, q)
-	goWorkers(q, p)
-	p.core.switchTable.idleIn <- p.port
-	q.core.switchTable.idleIn <- q.port
-}
-
-/*
-func (c *Core) DEBUG_simFixMTU() {
-	c.router.tun.mtu = 65535
-}
-*/
-
-////////////////////////////////////////////////////////////////////////////////
-
-func Util_testAddrIDMask() {
-	for idx := 0; idx < 16; idx++ {
-		var orig crypto.NodeID
-		orig[8] = 42
-		for bidx := 0; bidx < idx; bidx++ {
-			orig[bidx/8] |= (0x80 >> uint8(bidx%8))
-		}
-		addr := address.AddrForNodeID(&orig)
-		nid, mask := addr.GetNodeIDandMask()
-		for b := 0; b < len(mask); b++ {
-			nid[b] &= mask[b]
-			orig[b] &= mask[b]
-		}
-		if *nid != orig {
-			fmt.Println(orig)
-			fmt.Println(*addr)
-			fmt.Println(*nid)
-			fmt.Println(*mask)
-			panic(idx)
-		}
-	}
-}

src/yggdrasil/dht.go

@@ -65,33 +65,27 @@ type dhtReqKey struct {
 
 // The main DHT struct.
 type dht struct {
-	core        *Core
+	router    *router
-	reconfigure chan chan error
+	nodeID    crypto.NodeID
-	nodeID      crypto.NodeID
+	reqs      map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
-	peers       chan *dhtInfo                    // other goroutines put incoming dht updates here
+	callbacks map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
-	reqs        map[dhtReqKey]time.Time          // Keeps track of recent outstanding requests
-	callbacks   map[dhtReqKey][]dht_callbackInfo // Search and admin lookup callbacks
 	// These next two could be replaced by a single linked list or similar...
 	table map[crypto.NodeID]*dhtInfo
 	imp   []*dhtInfo
 }
 
 // Initializes the DHT.
-func (t *dht) init(c *Core) {
+func (t *dht) init(r *router) {
-	t.core = c
+	t.router = r
-	t.reconfigure = make(chan chan error, 1)
+	t.nodeID = *t.router.core.NodeID()
-	go func() {
-		for {
-			e := <-t.reconfigure
-			e <- nil
-		}
-	}()
-	t.nodeID = *t.core.NodeID()
-	t.peers = make(chan *dhtInfo, 1024)
 	t.callbacks = make(map[dhtReqKey][]dht_callbackInfo)
 	t.reset()
 }
 
+func (t *dht) reconfigure() {
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // Resets the DHT in response to coord changes.
 // This empties all info from the DHT and drops outstanding requests.
 func (t *dht) reset() {
@@ -192,10 +186,10 @@ func dht_ordered(first, second, third *crypto.NodeID) bool {
 // Update info about the node that sent the request.
 func (t *dht) handleReq(req *dhtReq) {
 	// Send them what they asked for
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	res := dhtRes{
-		Key:    t.core.boxPub,
+		Key:    t.router.core.boxPub,
 		Coords: coords,
 		Dest:   req.Dest,
 		Infos:  t.lookup(&req.Dest, false),
@@ -223,17 +217,17 @@ func (t *dht) handleReq(req *dhtReq) {
 func (t *dht) sendRes(res *dhtRes, req *dhtReq) {
 	// Send a reply for a dhtReq
 	bs := res.encode()
-	shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &req.Key)
+	shared := t.router.sessions.getSharedKey(&t.router.core.boxPriv, &req.Key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  req.Coords,
 		ToKey:   req.Key,
-		FromKey: t.core.boxPub,
+		FromKey: t.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	t.core.router.out(packet)
+	t.router.out(packet)
 }
 
 type dht_callbackInfo struct {
@@ -287,17 +281,17 @@ func (t *dht) handleRes(res *dhtRes) {
 func (t *dht) sendReq(req *dhtReq, dest *dhtInfo) {
 	// Send a dhtReq to the node in dhtInfo
 	bs := req.encode()
-	shared := t.core.sessions.getSharedKey(&t.core.boxPriv, &dest.key)
+	shared := t.router.sessions.getSharedKey(&t.router.core.boxPriv, &dest.key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  dest.coords,
 		ToKey:   dest.key,
-		FromKey: t.core.boxPub,
+		FromKey: t.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	t.core.router.out(packet)
+	t.router.out(packet)
 	rq := dhtReqKey{dest.key, req.Dest}
 	t.reqs[rq] = time.Now()
 }
@@ -308,10 +302,10 @@ func (t *dht) ping(info *dhtInfo, target *crypto.NodeID) {
 	if target == nil {
 		target = &t.nodeID
 	}
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
 	req := dhtReq{
-		Key:    t.core.boxPub,
+		Key:    t.router.core.boxPub,
 		Coords: coords,
 		Dest:   *target,
 	}
@@ -386,7 +380,7 @@ func (t *dht) getImportant() []*dhtInfo {
 		})
 		// Keep the ones that are no further than the closest seen so far
 		minDist := ^uint64(0)
-		loc := t.core.switchTable.getLocator()
+		loc := t.router.core.switchTable.getLocator()
 		important := infos[:0]
 		for _, info := range infos {
 			dist := uint64(loc.dist(info.coords))
@@ -415,12 +409,12 @@ func (t *dht) getImportant() []*dhtInfo {
 
 // Returns true if this is a node we need to keep track of for the DHT to work.
 func (t *dht) isImportant(ninfo *dhtInfo) bool {
-	if ninfo.key == t.core.boxPub {
+	if ninfo.key == t.router.core.boxPub {
 		return false
 	}
 	important := t.getImportant()
 	// Check if ninfo is of equal or greater importance to what we already know
-	loc := t.core.switchTable.getLocator()
+	loc := t.router.core.switchTable.getLocator()
 	ndist := uint64(loc.dist(ninfo.coords))
 	minDist := ^uint64(0)
 	for _, info := range important {

src/yggdrasil/dialer.go

@@ -65,6 +65,7 @@ func (d *Dialer) DialByNodeIDandMask(nodeID, nodeMask *crypto.NodeID) (*Conn, er
 		conn.Close()
 		return nil, err
 	}
+	conn.session.setConn(nil, conn)
 	t := time.NewTimer(6 * time.Second) // TODO use a context instead
 	defer t.Stop()
 	select {

src/yggdrasil/link.go

@@ -16,14 +16,15 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 type link struct {
-	core        *Core
+	core       *Core
-	reconfigure chan chan error
+	mutex      sync.RWMutex // protects interfaces below
-	mutex       sync.RWMutex // protects interfaces below
+	interfaces map[linkInfo]*linkInterface
-	interfaces  map[linkInfo]*linkInterface
+	tcp        tcp // TCP interface support
-	tcp         tcp // TCP interface support
 	// TODO timeout (to remove from switch), read from config.ReadTimeout
 }
 
@@ -45,21 +46,29 @@ type linkInterfaceMsgIO interface {
 }
 
 type linkInterface struct {
-	name     string
+	name           string
-	link     *link
+	link           *link
-	peer     *peer
+	peer           *peer
-	msgIO    linkInterfaceMsgIO
+	msgIO          linkInterfaceMsgIO
-	info     linkInfo
+	info           linkInfo
-	incoming bool
+	incoming       bool
-	force    bool
+	force          bool
-	closed   chan struct{}
+	closed         chan struct{}
+	reader         linkReader  // Reads packets, notifies this linkInterface, passes packets to switch
+	writer         linkWriter  // Writes packets, notifies this linkInterface
+	phony.Inbox                // Protects the below
+	sendTimer      *time.Timer // Fires to signal that sending is blocked
+	keepAliveTimer *time.Timer // Fires to send keep-alive traffic
+	stallTimer     *time.Timer // Fires to signal that no incoming traffic (including keep-alive) has been seen
+	closeTimer     *time.Timer // Fires when the link has been idle so long we need to close it
+	inSwitch       bool        // True if the switch is tracking this link
+	stalled        bool        // True if we haven't been receiving any response traffic
 }
 
 func (l *link) init(c *Core) error {
 	l.core = c
 	l.mutex.Lock()
 	l.interfaces = make(map[linkInfo]*linkInterface)
-	l.reconfigure = make(chan chan error)
 	l.mutex.Unlock()
 
 	if err := l.tcp.init(l); err != nil {
@@ -67,22 +76,13 @@ func (l *link) init(c *Core) error {
 		return err
 	}
 
-	go func() {
-		for {
-			e := <-l.reconfigure
-			tcpresponse := make(chan error)
-			l.tcp.reconfigure <- tcpresponse
-			if err := <-tcpresponse; err != nil {
-				e <- err
-				continue
-			}
-			e <- nil
-		}
-	}()
-
 	return nil
 }
 
+func (l *link) reconfigure() {
+	l.tcp.reconfigure()
+}
+
 func (l *link) call(uri string, sintf string) error {
 	u, err := url.Parse(uri)
 	if err != nil {
@@ -128,6 +128,9 @@ func (l *link) create(msgIO linkInterfaceMsgIO, name, linkType, local, remote st
 		incoming: incoming,
 		force:    force,
 	}
+	intf.writer.intf = &intf
+	intf.reader.intf = &intf
+	intf.reader.err = make(chan error)
 	return &intf, nil
 }
 
@@ -206,213 +209,187 @@ func (intf *linkInterface) handler() error {
 		// More cleanup can go here
 		intf.link.core.peers.removePeer(intf.peer.port)
 	}()
-	// Finish setting up the peer struct
-	out := make(chan [][]byte, 1)
-	defer close(out)
 	intf.peer.out = func(msgs [][]byte) {
-		defer func() { recover() }()
+		intf.writer.sendFrom(intf.peer, msgs, false)
-		out <- msgs
+	}
+	intf.peer.linkOut = func(bs []byte) {
+		intf.writer.sendFrom(intf.peer, [][]byte{bs}, true)
 	}
-	intf.peer.linkOut = make(chan []byte, 1)
 	themAddr := address.AddrForNodeID(crypto.GetNodeID(&intf.info.box))
 	themAddrString := net.IP(themAddr[:]).String()
 	themString := fmt.Sprintf("%s@%s", themAddrString, intf.info.remote)
 	intf.link.core.log.Infof("Connected %s: %s, source %s",
 		strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-	// Start the link loop
+	// Start things
-	go intf.peer.linkLoop()
+	go intf.peer.start()
-	// Start the writer
+	intf.reader.Act(nil, intf.reader._read)
-	signalReady := make(chan struct{}, 1)
+	// Wait for the reader to finish
-	signalSent := make(chan bool, 1)
+	err = <-intf.reader.err
-	sendAck := make(chan struct{}, 1)
+	if err != nil {
-	sendBlocked := time.NewTimer(time.Second)
-	defer util.TimerStop(sendBlocked)
-	util.TimerStop(sendBlocked)
-	go func() {
-		defer close(signalReady)
-		defer close(signalSent)
-		interval := 4 * time.Second
-		tcpTimer := time.NewTimer(interval) // used for backwards compat with old tcp
-		defer util.TimerStop(tcpTimer)
-		send := func(bss [][]byte) {
-			sendBlocked.Reset(time.Second)
-			size, _ := intf.msgIO.writeMsgs(bss)
-			util.TimerStop(sendBlocked)
-			select {
-			case signalSent <- size > 0:
-			default:
-			}
-		}
-		for {
-			// First try to send any link protocol traffic
-			select {
-			case msg := <-intf.peer.linkOut:
-				send([][]byte{msg})
-				continue
-			default:
-			}
-			// No protocol traffic to send, so reset the timer
-			util.TimerStop(tcpTimer)
-			tcpTimer.Reset(interval)
-			// Now block until something is ready or the timer triggers keepalive traffic
-			select {
-			case <-tcpTimer.C:
-				intf.link.core.log.Tracef("Sending (legacy) keep-alive to %s: %s, source %s",
-					strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				send([][]byte{nil})
-			case <-sendAck:
-				intf.link.core.log.Tracef("Sending ack to %s: %s, source %s",
-					strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				send([][]byte{nil})
-			case msg := <-intf.peer.linkOut:
-				send([][]byte{msg})
-			case msgs, ok := <-out:
-				if !ok {
-					return
-				}
-				send(msgs)
-				for _, msg := range msgs {
-					util.PutBytes(msg)
-				}
-				select {
-				case signalReady <- struct{}{}:
-				default:
-				}
-				//intf.link.core.log.Tracef("Sending packet to %s: %s, source %s",
-				//	strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-			}
-		}
-	}()
-	//intf.link.core.switchTable.idleIn <- intf.peer.port // notify switch that we're idle
-	// Used to enable/disable activity in the switch
-	signalAlive := make(chan bool, 1) // True = real packet, false = keep-alive
-	defer close(signalAlive)
-	ret := make(chan error, 1) // How we signal the return value when multiple goroutines are involved
-	go func() {
-		var isAlive bool
-		var isReady bool
-		var sendTimerRunning bool
-		var recvTimerRunning bool
-		recvTime := 6 * time.Second     // TODO set to ReadTimeout from the config, reset if it gets changed
-		closeTime := 2 * switch_timeout // TODO or maybe this makes more sense for ReadTimeout?...
-		sendTime := time.Second
-		sendTimer := time.NewTimer(sendTime)
-		defer util.TimerStop(sendTimer)
-		recvTimer := time.NewTimer(recvTime)
-		defer util.TimerStop(recvTimer)
-		closeTimer := time.NewTimer(closeTime)
-		defer util.TimerStop(closeTimer)
-		for {
-			//intf.link.core.log.Debugf("State of %s: %s, source %s :: isAlive %t isReady %t sendTimerRunning %t recvTimerRunning %t",
-			//	strings.ToUpper(intf.info.linkType), themString, intf.info.local,
-			//	isAlive, isReady, sendTimerRunning, recvTimerRunning)
-			select {
-			case gotMsg, ok := <-signalAlive:
-				if !ok {
-					return
-				}
-				util.TimerStop(closeTimer)
-				closeTimer.Reset(closeTime)
-				util.TimerStop(recvTimer)
-				recvTimerRunning = false
-				isAlive = true
-				if !isReady {
-					// (Re-)enable in the switch
-					intf.link.core.switchTable.idleIn <- intf.peer.port
-					isReady = true
-				}
-				if gotMsg && !sendTimerRunning {
-					// We got a message
-					// Start a timer, if it expires then send a 0-sized ack to let them know we're alive
-					util.TimerStop(sendTimer)
-					sendTimer.Reset(sendTime)
-					sendTimerRunning = true
-				}
-				if !gotMsg {
-					intf.link.core.log.Tracef("Received ack from %s: %s, source %s",
-						strings.ToUpper(intf.info.linkType), themString, intf.info.local)
-				}
-			case sentMsg, ok := <-signalSent:
-				// Stop any running ack timer
-				if !ok {
-					return
-				}
-				util.TimerStop(sendTimer)
-				sendTimerRunning = false
-				if sentMsg && !recvTimerRunning {
-					// We sent a message
-					// Start a timer, if it expires and we haven't gotten any return traffic (including a 0-sized ack), then assume there's a problem
-					util.TimerStop(recvTimer)
-					recvTimer.Reset(recvTime)
-					recvTimerRunning = true
-				}
-			case _, ok := <-signalReady:
-				if !ok {
-					return
-				}
-				if !isAlive {
-					// Disable in the switch
-					isReady = false
-				} else {
-					// Keep enabled in the switch
-					intf.link.core.switchTable.idleIn <- intf.peer.port
-					isReady = true
-				}
-			case <-sendBlocked.C:
-				// We blocked while trying to send something
-				isReady = false
-				intf.link.core.switchTable.blockPeer(intf.peer.port)
-			case <-sendTimer.C:
-				// We haven't sent anything, so signal a send of a 0 packet to let them know we're alive
-				select {
-				case sendAck <- struct{}{}:
-				default:
-				}
-			case <-recvTimer.C:
-				// We haven't received anything, so assume there's a problem and don't return this node to the switch until they start responding
-				isAlive = false
-				intf.link.core.switchTable.blockPeer(intf.peer.port)
-			case <-closeTimer.C:
-				// We haven't received anything in a really long time, so things have died at the switch level and then some...
-				// Just close the connection at this point...
-				select {
-				case ret <- errors.New("timeout"):
-				default:
-				}
-				intf.msgIO.close()
-			}
-		}
-	}()
-	// Run reader loop
-	for {
-		msg, err := intf.msgIO.readMsg()
-		if len(msg) > 0 {
-			intf.peer.handlePacket(msg)
-		}
-		if err != nil {
-			if err != io.EOF {
-				select {
-				case ret <- err:
-				default:
-				}
-			}
-			break
-		}
-		select {
-		case signalAlive <- len(msg) > 0:
-		default:
-		}
-	}
-	////////////////////////////////////////////////////////////////////////////////
-	// Remember to set `err` to something useful before returning
-	select {
-	case err = <-ret:
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s; error: %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local, err)
-	default:
+	} else {
-		err = nil
 		intf.link.core.log.Infof("Disconnected %s: %s, source %s",
 			strings.ToUpper(intf.info.linkType), themString, intf.info.local)
 	}
 	return err
 }
+
+////////////////////////////////////////////////////////////////////////////////
+
+const (
+	sendTime      = 1 * time.Second    // How long to wait before deciding a send is blocked
+	keepAliveTime = 2 * time.Second    // How long to wait before sending a keep-alive response if we have no real traffic to send
+	stallTime     = 6 * time.Second    // How long to wait for response traffic before deciding the connection has stalled
+	closeTime     = 2 * switch_timeout // How long to wait before closing the link
+)
+
+// notify the intf that we're currently sending
+func (intf *linkInterface) notifySending(size int, isLinkTraffic bool) {
+	intf.Act(&intf.writer, func() {
+		if !isLinkTraffic {
+			intf.inSwitch = false
+		}
+		intf.sendTimer = time.AfterFunc(sendTime, intf.notifyBlockedSend)
+		intf._cancelStallTimer()
+	})
+}
+
+// we just sent something, so cancel any pending timer to send keep-alive traffic
+func (intf *linkInterface) _cancelStallTimer() {
+	if intf.stallTimer != nil {
+		intf.stallTimer.Stop()
+		intf.stallTimer = nil
+	}
+}
+
+// called by an AfterFunc if we appear to have timed out
+func (intf *linkInterface) notifyBlockedSend() {
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
+		if intf.sendTimer != nil {
+			//As far as we know, we're still trying to send, and the timer fired.
+			intf.link.core.switchTable.blockPeer(intf.peer.port)
+		}
+	})
+}
+
+// notify the intf that we've finished sending, returning the peer to the switch
+func (intf *linkInterface) notifySent(size int, isLinkTraffic bool) {
+	intf.Act(&intf.writer, func() {
+		intf.sendTimer.Stop()
+		intf.sendTimer = nil
+		if !isLinkTraffic {
+			intf._notifySwitch()
+		}
+		if size > 0 && intf.stallTimer == nil {
+			intf.stallTimer = time.AfterFunc(stallTime, intf.notifyStalled)
+		}
+	})
+}
+
+// Notify the switch that we're ready for more traffic, assuming we're not in a stalled state
+func (intf *linkInterface) _notifySwitch() {
+	if !intf.inSwitch && !intf.stalled {
+		intf.inSwitch = true
+		intf.link.core.switchTable.Act(intf, func() {
+			intf.link.core.switchTable._idleIn(intf.peer.port)
+		})
+	}
+}
+
+// Set the peer as stalled, to prevent them from returning to the switch until a read succeeds
+func (intf *linkInterface) notifyStalled() {
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
+		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
+			intf.stallTimer = nil
+			intf.stalled = true
+			intf.link.core.switchTable.blockPeer(intf.peer.port)
+		}
+	})
+}
+
+// reset the close timer
+func (intf *linkInterface) notifyReading() {
+	intf.Act(&intf.reader, func() {
+		if intf.closeTimer != nil {
+			intf.closeTimer.Stop()
+		}
+		intf.closeTimer = time.AfterFunc(closeTime, func() { intf.msgIO.close() })
+	})
+}
+
+// wake up the link if it was stalled, and (if size > 0) prepare to send keep-alive traffic
+func (intf *linkInterface) notifyRead(size int) {
+	intf.Act(&intf.reader, func() {
+		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
+			intf.stallTimer = nil
+		}
+		intf.stalled = false
+		intf._notifySwitch()
+		if size > 0 && intf.stallTimer == nil {
+			intf.stallTimer = time.AfterFunc(keepAliveTime, intf.notifyDoKeepAlive)
+		}
+	})
+}
+
+// We need to send keep-alive traffic now
+func (intf *linkInterface) notifyDoKeepAlive() {
+	intf.Act(nil, func() { // Sent from a time.AfterFunc
+		if intf.stallTimer != nil {
+			intf.stallTimer.Stop()
+			intf.stallTimer = nil
+			intf.writer.sendFrom(nil, [][]byte{nil}, true) // Empty keep-alive traffic
+		}
+	})
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+type linkWriter struct {
+	phony.Inbox
+	intf *linkInterface
+}
+
+func (w *linkWriter) sendFrom(from phony.Actor, bss [][]byte, isLinkTraffic bool) {
+	w.Act(from, func() {
+		var size int
+		for _, bs := range bss {
+			size += len(bs)
+		}
+		w.intf.notifySending(size, isLinkTraffic)
+		w.intf.msgIO.writeMsgs(bss)
+		w.intf.notifySent(size, isLinkTraffic)
+		// Cleanup
+		for _, bs := range bss {
+			util.PutBytes(bs)
+		}
+	})
+}
+
+////////////////////////////////////////////////////////////////////////////////
+
+type linkReader struct {
+	phony.Inbox
+	intf *linkInterface
+	err  chan error
+}
+
+func (r *linkReader) _read() {
+	r.intf.notifyReading()
+	msg, err := r.intf.msgIO.readMsg()
+	r.intf.notifyRead(len(msg))
+	if len(msg) > 0 {
+		r.intf.peer.handlePacketFrom(r, msg)
+	}
+	if err != nil {
+		if err != io.EOF {
+			r.err <- err
+		}
+		close(r.err)
+		return
+	}
+	// Now try to read again
+	r.Act(nil, r._read)
+}

src/yggdrasil/listener.go

@@ -31,8 +31,8 @@ func (l *Listener) Close() (err error) {
 		recover()
 		err = errors.New("already closed")
 	}()
-	if l.core.sessions.listener == l {
+	if l.core.router.sessions.listener == l {
-		l.core.sessions.listener = nil
+		l.core.router.sessions.listener = nil
 	}
 	close(l.close)
 	close(l.conn)

src/yggdrasil/nodeinfo.go

@@ -47,25 +47,25 @@ func (m *nodeinfo) init(core *Core) {
 	m.callbacks = make(map[crypto.BoxPubKey]nodeinfoCallback)
 	m.cache = make(map[crypto.BoxPubKey]nodeinfoCached)
 
-	go func() {
+	var f func()
-		for {
+	f = func() {
-			m.callbacksMutex.Lock()
+		m.callbacksMutex.Lock()
-			for boxPubKey, callback := range m.callbacks {
+		for boxPubKey, callback := range m.callbacks {
-				if time.Since(callback.created) > time.Minute {
+			if time.Since(callback.created) > time.Minute {
-					delete(m.callbacks, boxPubKey)
+				delete(m.callbacks, boxPubKey)
-				}
 			}
-			m.callbacksMutex.Unlock()
-			m.cacheMutex.Lock()
-			for boxPubKey, cache := range m.cache {
-				if time.Since(cache.created) > time.Hour {
-					delete(m.cache, boxPubKey)
-				}
-			}
-			m.cacheMutex.Unlock()
-			time.Sleep(time.Second * 30)
 		}
-	}()
+		m.callbacksMutex.Unlock()
+		m.cacheMutex.Lock()
+		for boxPubKey, cache := range m.cache {
+			if time.Since(cache.created) > time.Hour {
+				delete(m.cache, boxPubKey)
+			}
+		}
+		m.cacheMutex.Unlock()
+		time.AfterFunc(time.Second*30, f)
+	}
+	go f()
 }
 
 // Add a callback for a nodeinfo lookup
@@ -172,7 +172,7 @@ func (m *nodeinfo) sendNodeInfo(key crypto.BoxPubKey, coords []byte, isResponse
 		NodeInfo:   m.getNodeInfo(),
 	}
 	bs := nodeinfo.encode()
-	shared := m.core.sessions.getSharedKey(&m.core.boxPriv, &key)
+	shared := m.core.router.sessions.getSharedKey(&m.core.boxPriv, &key)
 	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  coords,

src/yggdrasil/peer.go

@@ -12,6 +12,8 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // The peers struct represents peers with an active connection.
@@ -19,10 +21,9 @@ import (
 // In most cases, this involves passing the packet to the handler for outgoing traffic to another peer.
 // In other cases, it's link protocol traffic used to build the spanning tree, in which case this checks signatures and passes the message along to the switch.
 type peers struct {
-	core        *Core
+	core  *Core
-	reconfigure chan chan error
+	mutex sync.Mutex   // Synchronize writes to atomic
-	mutex       sync.Mutex   // Synchronize writes to atomic
+	ports atomic.Value //map[switchPort]*peer, use CoW semantics
-	ports       atomic.Value //map[switchPort]*peer, use CoW semantics
 }
 
 // Initializes the peers struct.
@@ -31,13 +32,10 @@ func (ps *peers) init(c *Core) {
 	defer ps.mutex.Unlock()
 	ps.putPorts(make(map[switchPort]*peer))
 	ps.core = c
-	ps.reconfigure = make(chan chan error, 1)
+}
-	go func() {
+
-		for {
+func (ps *peers) reconfigure() {
-			e := <-ps.reconfigure
+	// This is where reconfiguration would go, if we had anything to do
-			e <- nil
-		}
-	}()
 }
 
 // Returns true if an incoming peer connection to a key is allowed, either
@@ -94,9 +92,7 @@ func (ps *peers) putPorts(ports map[switchPort]*peer) {
 
 // Information known about a peer, including thier box/sig keys, precomputed shared keys (static and ephemeral) and a handler for their outgoing traffic
 type peer struct {
-	bytesSent  uint64 // To track bandwidth usage for getPeers
+	phony.Inbox
-	bytesRecvd uint64 // To track bandwidth usage for getPeers
-	// BUG: sync/atomic, 32 bit platforms need the above to be the first element
 	core       *Core
 	intf       *linkInterface
 	port       switchPort
@@ -106,11 +102,14 @@ type peer struct {
 	linkShared crypto.BoxSharedKey
 	endpoint   string
 	firstSeen  time.Time       // To track uptime for getPeers
-	linkOut    (chan []byte)   // used for protocol traffic (to bypass queues)
+	linkOut    func([]byte)    // used for protocol traffic (bypasses the switch)
-	doSend     (chan struct{}) // tell the linkLoop to send a switchMsg
+	dinfo      *dhtInfo        // used to keep the DHT working
-	dinfo      (chan *dhtInfo) // used to keep the DHT working
 	out        func([][]byte)  // Set up by whatever created the peers struct, used to send packets to other nodes
+	done       (chan struct{}) // closed to exit the linkLoop
 	close      func()          // Called when a peer is removed, to close the underlying connection, or via admin api
+	// The below aren't actually useful internally, they're just gathered for getPeers statistics
+	bytesSent  uint64
+	bytesRecvd uint64
 }
 
 // Creates a new peer with the specified box, sig, and linkShared keys, using the lowest unoccupied port number.
@@ -121,8 +120,7 @@ func (ps *peers) newPeer(box *crypto.BoxPubKey, sig *crypto.SigPubKey, linkShare
 		shared:     *crypto.GetSharedKey(&ps.core.boxPriv, box),
 		linkShared: *linkShared,
 		firstSeen:  now,
-		doSend:     make(chan struct{}, 1),
+		done:       make(chan struct{}),
-		dinfo:      make(chan *dhtInfo, 1),
 		close:      closer,
 		core:       ps.core,
 		intf:       intf,
@@ -150,7 +148,7 @@ func (ps *peers) removePeer(port switchPort) {
 	if port == 0 {
 		return
 	} // Can't remove self peer
-	ps.core.router.doAdmin(func() {
+	phony.Block(&ps.core.router, func() {
 		ps.core.switchTable.forgetPeer(port)
 	})
 	ps.mutex.Lock()
@@ -167,103 +165,106 @@ func (ps *peers) removePeer(port switchPort) {
 		if p.close != nil {
 			p.close()
 		}
-		close(p.doSend)
+		close(p.done)
 	}
 }
 
 // If called, sends a notification to each peer that they should send a new switch message.
 // Mainly called by the switch after an update.
-func (ps *peers) sendSwitchMsgs() {
+func (ps *peers) sendSwitchMsgs(from phony.Actor) {
 	ports := ps.getPorts()
 	for _, p := range ports {
 		if p.port == 0 {
 			continue
 		}
-		p.doSendSwitchMsgs()
+		p.Act(from, p._sendSwitchMsg)
-	}
-}
-
-// If called, sends a notification to the peer's linkLoop to trigger a switchMsg send.
-// Mainly called by sendSwitchMsgs or during linkLoop startup.
-func (p *peer) doSendSwitchMsgs() {
-	defer func() { recover() }() // In case there's a race with close(p.doSend)
-	select {
-	case p.doSend <- struct{}{}:
-	default:
 	}
 }
 
 // This must be launched in a separate goroutine by whatever sets up the peer struct.
 // It handles link protocol traffic.
-func (p *peer) linkLoop() {
+func (p *peer) start() {
-	tick := time.NewTicker(time.Second)
+	var updateDHT func()
-	defer tick.Stop()
+	updateDHT = func() {
-	p.doSendSwitchMsgs()
+		phony.Block(p, func() {
-	var dinfo *dhtInfo
+			select {
-	for {
+			case <-p.done:
-		select {
+			default:
-		case _, ok := <-p.doSend:
+				p._updateDHT()
-			if !ok {
+				time.AfterFunc(time.Second, updateDHT)
-				return
 			}
-			p.sendSwitchMsg()
+		})
-		case dinfo = <-p.dinfo:
-		case _ = <-tick.C:
-			if dinfo != nil {
-				p.core.dht.peers <- dinfo
-			}
-		}
 	}
+	updateDHT()
+	// Just for good measure, immediately send a switch message to this peer when we start
+	p.Act(nil, p._sendSwitchMsg)
+}
+
+func (p *peer) _updateDHT() {
+	if p.dinfo != nil {
+		p.core.router.insertPeer(p, p.dinfo)
+	}
+}
+
+func (p *peer) handlePacketFrom(from phony.Actor, packet []byte) {
+	p.Act(from, func() {
+		p._handlePacket(packet)
+	})
 }
 
 // Called to handle incoming packets.
 // Passes the packet to a handler for that packet type.
-func (p *peer) handlePacket(packet []byte) {
+func (p *peer) _handlePacket(packet []byte) {
 	// FIXME this is off by stream padding and msg length overhead, should be done in tcp.go
-	atomic.AddUint64(&p.bytesRecvd, uint64(len(packet)))
+	p.bytesRecvd += uint64(len(packet))
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
 	}
 	switch pType {
 	case wire_Traffic:
-		p.handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet)
 	case wire_ProtocolTraffic:
-		p.handleTraffic(packet, pTypeLen)
+		p._handleTraffic(packet)
 	case wire_LinkProtocolTraffic:
-		p.handleLinkTraffic(packet)
+		p._handleLinkTraffic(packet)
 	default:
 		util.PutBytes(packet)
 	}
-	return
 }
 
 // Called to handle traffic or protocolTraffic packets.
 // In either case, this reads from the coords of the packet header, does a switch lookup, and forwards to the next node.
-func (p *peer) handleTraffic(packet []byte, pTypeLen int) {
+func (p *peer) _handleTraffic(packet []byte) {
 	table := p.core.switchTable.getTable()
 	if _, isIn := table.elems[p.port]; !isIn && p.port != 0 {
 		// Drop traffic if the peer isn't in the switch
 		return
 	}
-	p.core.switchTable.packetIn <- packet
+	p.core.switchTable.packetInFrom(p, packet)
+}
+
+func (p *peer) sendPacketsFrom(from phony.Actor, packets [][]byte) {
+	p.Act(from, func() {
+		p._sendPackets(packets)
+	})
 }
 
 // This just calls p.out(packet) for now.
-func (p *peer) sendPackets(packets [][]byte) {
+func (p *peer) _sendPackets(packets [][]byte) {
 	// Is there ever a case where something more complicated is needed?
 	// What if p.out blocks?
 	var size int
 	for _, packet := range packets {
 		size += len(packet)
 	}
-	atomic.AddUint64(&p.bytesSent, uint64(size))
+	p.bytesSent += uint64(size)
 	p.out(packets)
 }
 
 // This wraps the packet in the inner (ephemeral) and outer (permanent) crypto layers.
 // It sends it to p.linkOut, which bypasses the usual packet queues.
-func (p *peer) sendLinkPacket(packet []byte) {
+func (p *peer) _sendLinkPacket(packet []byte) {
 	innerPayload, innerNonce := crypto.BoxSeal(&p.linkShared, packet, nil)
 	innerLinkPacket := wire_linkProtoTrafficPacket{
 		Nonce:   *innerNonce,
@@ -276,12 +277,12 @@ func (p *peer) sendLinkPacket(packet []byte) {
 		Payload: bs,
 	}
 	packet = linkPacket.encode()
-	p.linkOut <- packet
+	p.linkOut(packet)
 }
 
 // Decrypts the outer (permanent) and inner (ephemeral) crypto layers on link traffic.
 // Identifies the link traffic type and calls the appropriate handler.
-func (p *peer) handleLinkTraffic(bs []byte) {
+func (p *peer) _handleLinkTraffic(bs []byte) {
 	packet := wire_linkProtoTrafficPacket{}
 	if !packet.decode(bs) {
 		return
@@ -304,14 +305,14 @@ func (p *peer) handleLinkTraffic(bs []byte) {
 	}
 	switch pType {
 	case wire_SwitchMsg:
-		p.handleSwitchMsg(payload)
+		p._handleSwitchMsg(payload)
 	default:
 		util.PutBytes(bs)
 	}
 }
 
 // Gets a switchMsg from the switch, adds signed next-hop info for this peer, and sends it to them.
-func (p *peer) sendSwitchMsg() {
+func (p *peer) _sendSwitchMsg() {
 	msg := p.core.switchTable.getMsg()
 	if msg == nil {
 		return
@@ -323,12 +324,12 @@ func (p *peer) sendSwitchMsg() {
 		Sig:  *crypto.Sign(&p.core.sigPriv, bs),
 	})
 	packet := msg.encode()
-	p.sendLinkPacket(packet)
+	p._sendLinkPacket(packet)
 }
 
 // Handles a switchMsg from the peer, checking signatures and passing good messages to the switch.
 // Also creates a dhtInfo struct and arranges for it to be added to the dht (this is how dht bootstrapping begins).
-func (p *peer) handleSwitchMsg(packet []byte) {
+func (p *peer) _handleSwitchMsg(packet []byte) {
 	var msg switchMsg
 	if !msg.decode(packet) {
 		return
@@ -352,16 +353,16 @@ func (p *peer) handleSwitchMsg(packet []byte) {
 	p.core.switchTable.handleMsg(&msg, p.port)
 	if !p.core.switchTable.checkRoot(&msg) {
 		// Bad switch message
-		p.dinfo <- nil
+		p.dinfo = nil
 		return
 	}
 	// Pass a mesage to the dht informing it that this peer (still) exists
 	loc.coords = loc.coords[:len(loc.coords)-1]
-	dinfo := dhtInfo{
+	p.dinfo = &dhtInfo{
 		key:    p.box,
 		coords: loc.getCoords(),
 	}
-	p.dinfo <- &dinfo
+	p._updateDHT()
 }
 
 // This generates the bytes that we sign or check the signature of for a switchMsg.

src/yggdrasil/router.go

@@ -30,29 +30,29 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // The router struct has channels to/from the adapter device and a self peer (0), which is how messages are passed between this node and the peers/switch layer.
-// The router's mainLoop goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
+// The router's phony.Inbox goroutine is responsible for managing all information related to the dht, searches, and crypto sessions.
 type router struct {
-	core        *Core
+	phony.Inbox
-	reconfigure chan chan error
+	core     *Core
-	addr        address.Address
+	addr     address.Address
-	subnet      address.Subnet
+	subnet   address.Subnet
-	in          <-chan [][]byte // packets we received from the network, link to peer's "out"
+	out      func([]byte) // packets we're sending to the network, link to peer's "in"
-	out         func([]byte)    // packets we're sending to the network, link to peer's "in"
+	dht      dht
-	reset       chan struct{}   // signal that coords changed (re-init sessions/dht)
+	nodeinfo nodeinfo
-	admin       chan func()     // pass a lambda for the admin socket to query stuff
+	searches searches
-	nodeinfo    nodeinfo
+	sessions sessions
 }
 
 // Initializes the router struct, which includes setting up channels to/from the adapter.
 func (r *router) init(core *Core) {
 	r.core = core
-	r.reconfigure = make(chan chan error, 1)
+	r.addr = *address.AddrForNodeID(&r.dht.nodeID)
-	r.addr = *address.AddrForNodeID(&r.core.dht.nodeID)
+	r.subnet = *address.SubnetForNodeID(&r.dht.nodeID)
-	r.subnet = *address.SubnetForNodeID(&r.core.dht.nodeID)
-	in := make(chan [][]byte, 1) // TODO something better than this...
 	self := linkInterface{
 		name: "(self)",
 		info: linkInfo{
@@ -62,120 +62,109 @@ func (r *router) init(core *Core) {
 		},
 	}
 	p := r.core.peers.newPeer(&r.core.boxPub, &r.core.sigPub, &crypto.BoxSharedKey{}, &self, nil)
-	p.out = func(packets [][]byte) { in <- packets }
+	p.out = func(packets [][]byte) { r.handlePackets(p, packets) }
-	r.in = in
+	r.out = func(bs []byte) { p.handlePacketFrom(r, bs) }
-	out := make(chan []byte, 32)
-	go func() {
-		for packet := range out {
-			p.handlePacket(packet)
-		}
-	}()
-	out2 := make(chan []byte, 32)
-	go func() {
-		// This worker makes sure r.out never blocks
-		// It will buffer traffic long enough for the switch worker to take it
-		// If (somehow) you can send faster than the switch can receive, then this would use unbounded memory
-		// But crypto slows sends enough that the switch should always be able to take the packets...
-		var buf [][]byte
-		for {
-			buf = append(buf, <-out2)
-			for len(buf) > 0 {
-				select {
-				case bs := <-out2:
-					buf = append(buf, bs)
-				case out <- buf[0]:
-					buf = buf[1:]
-				}
-			}
-		}
-	}()
-	r.out = func(packet []byte) { out2 <- packet }
-	r.reset = make(chan struct{}, 1)
-	r.admin = make(chan func(), 32)
 	r.nodeinfo.init(r.core)
 	r.core.config.Mutex.RLock()
 	r.nodeinfo.setNodeInfo(r.core.config.Current.NodeInfo, r.core.config.Current.NodeInfoPrivacy)
 	r.core.config.Mutex.RUnlock()
+	r.dht.init(r)
+	r.searches.init(r)
+	r.sessions.init(r)
 }
 
-// Starts the mainLoop goroutine.
+// Reconfigures the router and any child modules. This should only ever be run
+// by the router actor.
+func (r *router) reconfigure() {
+	// Reconfigure the router
+	current := r.core.config.GetCurrent()
+	if err := r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy); err != nil {
+		r.core.log.Errorln("Error reloading NodeInfo:", err)
+	} else {
+		r.core.log.Infoln("NodeInfo updated")
+	}
+	// Reconfigure children
+	r.dht.reconfigure()
+	r.searches.reconfigure()
+	r.sessions.reconfigure()
+}
+
+// Starts the tickerLoop goroutine.
 func (r *router) start() error {
 	r.core.log.Infoln("Starting router")
-	go r.mainLoop()
+	go r.doMaintenance()
 	return nil
 }
 
-// Takes traffic from the adapter and passes it to router.send, or from r.in and handles incoming traffic.
+// In practice, the switch will call this with 1 packet
-// Also adds new peer info to the DHT.
+func (r *router) handlePackets(from phony.Actor, packets [][]byte) {
-// Also resets the DHT and sesssions in the event of a coord change.
+	r.Act(from, func() {
-// Also does periodic maintenance stuff.
+		for _, packet := range packets {
-func (r *router) mainLoop() {
+			r._handlePacket(packet)
-	ticker := time.NewTicker(time.Second)
-	defer ticker.Stop()
-	for {
-		select {
-		case ps := <-r.in:
-			for _, p := range ps {
-				r.handleIn(p)
-			}
-		case info := <-r.core.dht.peers:
-			r.core.dht.insertPeer(info)
-		case <-r.reset:
-			r.core.sessions.reset()
-			r.core.dht.reset()
-		case <-ticker.C:
-			{
-				// Any periodic maintenance stuff goes here
-				r.core.switchTable.doMaintenance()
-				r.core.dht.doMaintenance()
-				r.core.sessions.cleanup()
-			}
-		case f := <-r.admin:
-			f()
-		case e := <-r.reconfigure:
-			current := r.core.config.GetCurrent()
-			e <- r.nodeinfo.setNodeInfo(current.NodeInfo, current.NodeInfoPrivacy)
 		}
-	}
+	})
+}
+
+// Insert a peer info into the dht, TODO? make the dht a separate actor
+func (r *router) insertPeer(from phony.Actor, info *dhtInfo) {
+	r.Act(from, func() {
+		r.dht.insertPeer(info)
+	})
+}
+
+// Reset sessions and DHT after the switch sees our coords change
+func (r *router) reset(from phony.Actor) {
+	r.Act(from, func() {
+		r.sessions.reset()
+		r.dht.reset()
+	})
+}
+
+// TODO remove reconfigure so this is just a ticker loop
+// and then find something better than a ticker loop to schedule things...
+func (r *router) doMaintenance() {
+	phony.Block(r, func() {
+		// Any periodic maintenance stuff goes here
+		r.core.switchTable.doMaintenance()
+		r.dht.doMaintenance()
+		r.sessions.cleanup()
+	})
+	time.AfterFunc(time.Second, r.doMaintenance)
 }
 
 // Checks incoming traffic type and passes it to the appropriate handler.
-func (r *router) handleIn(packet []byte) {
+func (r *router) _handlePacket(packet []byte) {
 	pType, pTypeLen := wire_decode_uint64(packet)
 	if pTypeLen == 0 {
 		return
 	}
 	switch pType {
 	case wire_Traffic:
-		r.handleTraffic(packet)
+		r._handleTraffic(packet)
 	case wire_ProtocolTraffic:
-		r.handleProto(packet)
+		r._handleProto(packet)
 	default:
 	}
 }
 
 // Handles incoming traffic, i.e. encapuslated ordinary IPv6 packets.
 // Passes them to the crypto session worker to be decrypted and sent to the adapter.
-func (r *router) handleTraffic(packet []byte) {
+func (r *router) _handleTraffic(packet []byte) {
 	defer util.PutBytes(packet)
 	p := wire_trafficPacket{}
 	if !p.decode(packet) {
 		return
 	}
-	sinfo, isIn := r.core.sessions.getSessionForHandle(&p.Handle)
+	sinfo, isIn := r.sessions.getSessionForHandle(&p.Handle)
 	if !isIn {
 		util.PutBytes(p.Payload)
 		return
 	}
-	select {
+	sinfo.recv(r, &p)
-	case sinfo.fromRouter <- p:
-	case <-sinfo.cancel.Finished():
-		util.PutBytes(p.Payload)
-	}
 }
 
 // Handles protocol traffic by decrypting it, checking its type, and passing it to the appropriate handler for that traffic type.
-func (r *router) handleProto(packet []byte) {
+func (r *router) _handleProto(packet []byte) {
 	// First parse the packet
 	p := wire_protoTrafficPacket{}
 	if !p.decode(packet) {
@@ -185,7 +174,7 @@ func (r *router) handleProto(packet []byte) {
 	var sharedKey *crypto.BoxSharedKey
 	if p.ToKey == r.core.boxPub {
 		// Try to open using our permanent key
-		sharedKey = r.core.sessions.getSharedKey(&r.core.boxPriv, &p.FromKey)
+		sharedKey = r.sessions.getSharedKey(&r.core.boxPriv, &p.FromKey)
 	} else {
 		return
 	}
@@ -202,59 +191,59 @@ func (r *router) handleProto(packet []byte) {
 	}
 	switch bsType {
 	case wire_SessionPing:
-		r.handlePing(bs, &p.FromKey)
+		r._handlePing(bs, &p.FromKey)
 	case wire_SessionPong:
-		r.handlePong(bs, &p.FromKey)
+		r._handlePong(bs, &p.FromKey)
 	case wire_NodeInfoRequest:
 		fallthrough
 	case wire_NodeInfoResponse:
-		r.handleNodeInfo(bs, &p.FromKey)
+		r._handleNodeInfo(bs, &p.FromKey)
 	case wire_DHTLookupRequest:
-		r.handleDHTReq(bs, &p.FromKey)
+		r._handleDHTReq(bs, &p.FromKey)
 	case wire_DHTLookupResponse:
-		r.handleDHTRes(bs, &p.FromKey)
+		r._handleDHTRes(bs, &p.FromKey)
 	default:
 		util.PutBytes(packet)
 	}
 }
 
 // Decodes session pings from wire format and passes them to sessions.handlePing where they either create or update a session.
-func (r *router) handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handlePing(bs []byte, fromKey *crypto.BoxPubKey) {
 	ping := sessionPing{}
 	if !ping.decode(bs) {
 		return
 	}
 	ping.SendPermPub = *fromKey
-	r.core.sessions.handlePing(&ping)
+	r.sessions.handlePing(&ping)
 }
 
 // Handles session pongs (which are really pings with an extra flag to prevent acknowledgement).
-func (r *router) handlePong(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handlePong(bs []byte, fromKey *crypto.BoxPubKey) {
-	r.handlePing(bs, fromKey)
+	r._handlePing(bs, fromKey)
 }
 
 // Decodes dht requests and passes them to dht.handleReq to trigger a lookup/response.
-func (r *router) handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleDHTReq(bs []byte, fromKey *crypto.BoxPubKey) {
 	req := dhtReq{}
 	if !req.decode(bs) {
 		return
 	}
 	req.Key = *fromKey
-	r.core.dht.handleReq(&req)
+	r.dht.handleReq(&req)
 }
 
 // Decodes dht responses and passes them to dht.handleRes to update the DHT table and further pass them to the search code (if applicable).
-func (r *router) handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleDHTRes(bs []byte, fromKey *crypto.BoxPubKey) {
 	res := dhtRes{}
 	if !res.decode(bs) {
 		return
 	}
 	res.Key = *fromKey
-	r.core.dht.handleRes(&res)
+	r.dht.handleRes(&res)
 }
 
 // Decodes nodeinfo request
-func (r *router) handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
+func (r *router) _handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	req := nodeinfoReqRes{}
 	if !req.decode(bs) {
 		return
@@ -262,18 +251,3 @@ func (r *router) handleNodeInfo(bs []byte, fromKey *crypto.BoxPubKey) {
 	req.SendPermPub = *fromKey
 	r.nodeinfo.handleNodeInfo(&req)
 }
-
-// Passed a function to call.
-// This will send the function to r.admin and block until it finishes.
-// It's used by the admin socket to ask the router mainLoop goroutine about information in the session or dht structs, which cannot be read safely from outside that goroutine.
-func (r *router) doAdmin(f func()) {
-	// Pass this a function that needs to be run by the router's main goroutine
-	// It will pass the function to the router and wait for the router to finish
-	done := make(chan struct{})
-	newF := func() {
-		f()
-		close(done)
-	}
-	r.admin <- newF
-	<-done
-}

src/yggdrasil/search.go

@@ -33,7 +33,7 @@ const search_RETRY_TIME = time.Second
 // Information about an ongoing search.
 // Includes the target NodeID, the bitmask to match it to an IP, and the list of nodes to visit / already visited.
 type searchInfo struct {
-	core     *Core
+	searches *searches
 	dest     crypto.NodeID
 	mask     crypto.NodeID
 	time     time.Time
@@ -45,28 +45,24 @@ type searchInfo struct {
 
 // This stores a map of active searches.
 type searches struct {
-	core        *Core
+	router   *router
-	reconfigure chan chan error
+	searches map[crypto.NodeID]*searchInfo
-	searches    map[crypto.NodeID]*searchInfo
 }
 
 // Initializes the searches struct.
-func (s *searches) init(core *Core) {
+func (s *searches) init(r *router) {
-	s.core = core
+	s.router = r
-	s.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-s.reconfigure
-			e <- nil
-		}
-	}()
 	s.searches = make(map[crypto.NodeID]*searchInfo)
 }
 
+func (s *searches) reconfigure() {
+	// This is where reconfiguration would go, if we had anything to do
+}
+
 // Creates a new search info, adds it to the searches struct, and returns a pointer to the info.
 func (s *searches) createSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	info := searchInfo{
-		core:     s.core,
+		searches: s,
 		dest:     *dest,
 		mask:     *mask,
 		time:     time.Now(),
@@ -100,7 +96,7 @@ func (sinfo *searchInfo) addToSearch(res *dhtRes) {
 	from := dhtInfo{key: res.Key, coords: res.Coords}
 	sinfo.visited[*from.getNodeID()] = true
 	for _, info := range res.Infos {
-		if *info.getNodeID() == sinfo.core.dht.nodeID || sinfo.visited[*info.getNodeID()] {
+		if *info.getNodeID() == sinfo.searches.router.dht.nodeID || sinfo.visited[*info.getNodeID()] {
 			continue
 		}
 		if dht_ordered(&sinfo.dest, info.getNodeID(), from.getNodeID()) {
@@ -134,7 +130,7 @@ func (sinfo *searchInfo) doSearchStep() {
 	if len(sinfo.toVisit) == 0 {
 		if time.Since(sinfo.time) > search_RETRY_TIME {
 			// Dead end and no response in too long, do cleanup
-			delete(sinfo.core.searches.searches, sinfo.dest)
+			delete(sinfo.searches.searches, sinfo.dest)
 			sinfo.callback(nil, errors.New("search reached dead end"))
 		}
 		return
@@ -143,8 +139,8 @@ func (sinfo *searchInfo) doSearchStep() {
 	var next *dhtInfo
 	next, sinfo.toVisit = sinfo.toVisit[0], sinfo.toVisit[1:]
 	rq := dhtReqKey{next.key, sinfo.dest}
-	sinfo.core.dht.addCallback(&rq, sinfo.handleDHTRes)
+	sinfo.searches.router.dht.addCallback(&rq, sinfo.handleDHTRes)
-	sinfo.core.dht.ping(next, &sinfo.dest)
+	sinfo.searches.router.dht.ping(next, &sinfo.dest)
 	sinfo.time = time.Now()
 }
 
@@ -155,27 +151,25 @@ func (sinfo *searchInfo) continueSearch() {
 	// In case the search dies, try to spawn another thread later
 	// Note that this will spawn multiple parallel searches as time passes
 	// Any that die aren't restarted, but a new one will start later
-	retryLater := func() {
+	time.AfterFunc(search_RETRY_TIME, func() {
-		// FIXME this keeps the search alive forever if not for the searches map, fix that
+		sinfo.searches.router.Act(nil, func() {
-		newSearchInfo := sinfo.core.searches.searches[sinfo.dest]
+			// FIXME this keeps the search alive forever if not for the searches map, fix that
-		if newSearchInfo != sinfo {
+			newSearchInfo := sinfo.searches.searches[sinfo.dest]
-			return
+			if newSearchInfo != sinfo {
-		}
+				return
-		sinfo.continueSearch()
+			}
-	}
+			sinfo.continueSearch()
-	go func() {
+		})
-		time.Sleep(search_RETRY_TIME)
+	})
-		sinfo.core.router.admin <- retryLater
-	}()
 }
 
 // Calls create search, and initializes the iterative search parts of the struct before returning it.
 func (s *searches) newIterSearch(dest *crypto.NodeID, mask *crypto.NodeID, callback func(*sessionInfo, error)) *searchInfo {
 	sinfo := s.createSearch(dest, mask, callback)
 	sinfo.visited = make(map[crypto.NodeID]bool)
-	loc := s.core.switchTable.getLocator()
+	loc := s.router.core.switchTable.getLocator()
 	sinfo.toVisit = append(sinfo.toVisit, &dhtInfo{
-		key:    s.core.boxPub,
+		key:    s.router.core.boxPub,
 		coords: loc.getCoords(),
 	}) // Start the search by asking ourself, useful if we're the destination
 	return sinfo
@@ -196,26 +190,26 @@ func (sinfo *searchInfo) checkDHTRes(res *dhtRes) bool {
 		return false
 	}
 	// They match, so create a session and send a sessionRequest
-	sess, isIn := sinfo.core.sessions.getByTheirPerm(&res.Key)
+	sess, isIn := sinfo.searches.router.sessions.getByTheirPerm(&res.Key)
 	if !isIn {
-		sess = sinfo.core.sessions.createSession(&res.Key)
+		sess = sinfo.searches.router.sessions.createSession(&res.Key)
 		if sess == nil {
 			// nil if the DHT search finished but the session wasn't allowed
 			sinfo.callback(nil, errors.New("session not allowed"))
 			// Cleanup
-			delete(sinfo.core.searches.searches, res.Dest)
+			delete(sinfo.searches.searches, res.Dest)
 			return true
 		}
-		_, isIn := sinfo.core.sessions.getByTheirPerm(&res.Key)
+		_, isIn := sinfo.searches.router.sessions.getByTheirPerm(&res.Key)
 		if !isIn {
 			panic("This should never happen")
 		}
 	}
 	// FIXME (!) replay attacks could mess with coords? Give it a handle (tstamp)?
 	sess.coords = res.Coords
-	sinfo.core.sessions.ping(sess)
+	sess.ping(sinfo.searches.router)
 	sinfo.callback(sess, nil)
 	// Cleanup
-	delete(sinfo.core.searches.searches, res.Dest)
+	delete(sinfo.searches.searches, res.Dest)
 	return true
 }

src/yggdrasil/session.go

@@ -7,13 +7,14 @@ package yggdrasil
 import (
 	"bytes"
 	"container/heap"
-	"errors"
 	"sync"
 	"time"
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/address"
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 // Duration that we keep track of old nonces per session, to allow some out-of-order packet delivery
@@ -37,15 +38,15 @@ func (h nonceHeap) peek() *crypto.BoxNonce { return &h[0] }
 // All the information we know about an active session.
 // This includes coords, permanent and ephemeral keys, handles and nonces, various sorts of timing information for timeout and maintenance, and some metadata for the admin API.
 type sessionInfo struct {
-	mutex          sync.Mutex                    // Protects all of the below, use it any time you read/chance the contents of a session
+	phony.Inbox                                  // Protects all of the below, use it any time you read/change the contents of a session
-	core           *Core                         //
+	sessions       *sessions                     //
-	reconfigure    chan chan error               //
 	theirAddr      address.Address               //
 	theirSubnet    address.Subnet                //
 	theirPermPub   crypto.BoxPubKey              //
 	theirSesPub    crypto.BoxPubKey              //
 	mySesPub       crypto.BoxPubKey              //
 	mySesPriv      crypto.BoxPrivKey             //
+	sharedPermKey  crypto.BoxSharedKey           // used for session pings
 	sharedSesKey   crypto.BoxSharedKey           // derived from session keys
 	theirHandle    crypto.Handle                 //
 	myHandle       crypto.Handle                 //
@@ -68,15 +69,12 @@ type sessionInfo struct {
 	bytesRecvd     uint64                        // Bytes of real traffic received in this session
 	init           chan struct{}                 // Closed when the first session pong arrives, used to signal that the session is ready for initial use
 	cancel         util.Cancellation             // Used to terminate workers
-	fromRouter     chan wire_trafficPacket       // Received packets go here, to be decrypted by the session
+	conn           *Conn                         // The associated Conn object
-	recv           chan []byte                   // Decrypted packets go here, picked up by the associated Conn
+	callbacks      []chan func()                 // Finished work from crypto workers
-	send           chan FlowKeyMessage           // Packets with optional flow key go here, to be encrypted and sent
 }
 
-func (sinfo *sessionInfo) doFunc(f func()) {
+func (sinfo *sessionInfo) reconfigure() {
-	sinfo.mutex.Lock()
+	// This is where reconfiguration would go, if we had anything to do
-	defer sinfo.mutex.Unlock()
-	f()
 }
 
 // Represents a session ping/pong packet, andincludes information like public keys, a session handle, coords, a timestamp to prevent replays, and the tun/tap MTU.
@@ -92,7 +90,7 @@ type sessionPing struct {
 
 // Updates session info in response to a ping, after checking that the ping is OK.
 // Returns true if the session was updated, or false otherwise.
-func (s *sessionInfo) update(p *sessionPing) bool {
+func (s *sessionInfo) _update(p *sessionPing) bool {
 	if !(p.Tstamp > s.tstamp) {
 		// To protect against replay attacks
 		return false
@@ -112,6 +110,9 @@ func (s *sessionInfo) update(p *sessionPing) bool {
 	}
 	if p.MTU >= 1280 || p.MTU == 0 {
 		s.theirMTU = p.MTU
+		if s.conn != nil {
+			s.conn.setMTU(s, s._getMTU())
+		}
 	}
 	if !bytes.Equal(s.coords, p.Coords) {
 		// allocate enough space for additional coords
@@ -134,10 +135,9 @@ func (s *sessionInfo) update(p *sessionPing) bool {
 // Sessions are indexed by handle.
 // Additionally, stores maps of address/subnet onto keys, and keys onto handles.
 type sessions struct {
-	core             *Core
+	router           *router
 	listener         *Listener
 	listenerMutex    sync.Mutex
-	reconfigure      chan chan error
 	lastCleanup      time.Time
 	isAllowedHandler func(pubkey *crypto.BoxPubKey, initiator bool) bool // Returns true or false if session setup is allowed
 	isAllowedMutex   sync.RWMutex                                        // Protects the above
@@ -147,32 +147,20 @@ type sessions struct {
 }
 
 // Initializes the session struct.
-func (ss *sessions) init(core *Core) {
+func (ss *sessions) init(r *router) {
-	ss.core = core
+	ss.router = r
-	ss.reconfigure = make(chan chan error, 1)
-	go func() {
-		for {
-			e := <-ss.reconfigure
-			responses := make(map[crypto.Handle]chan error)
-			for index, session := range ss.sinfos {
-				responses[index] = make(chan error)
-				session.reconfigure <- responses[index]
-			}
-			for _, response := range responses {
-				if err := <-response; err != nil {
-					e <- err
-					continue
-				}
-			}
-			e <- nil
-		}
-	}()
 	ss.permShared = make(map[crypto.BoxPubKey]*crypto.BoxSharedKey)
 	ss.sinfos = make(map[crypto.Handle]*sessionInfo)
 	ss.byTheirPerm = make(map[crypto.BoxPubKey]*crypto.Handle)
 	ss.lastCleanup = time.Now()
 }
 
+func (ss *sessions) reconfigure() {
+	for _, session := range ss.sinfos {
+		session.reconfigure()
+	}
+}
+
 // Determines whether the session with a given publickey is allowed based on
 // session firewall rules.
 func (ss *sessions) isSessionAllowed(pubkey *crypto.BoxPubKey, initiator bool) bool {
@@ -211,17 +199,17 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 		return nil
 	}
 	sinfo := sessionInfo{}
-	sinfo.core = ss.core
+	sinfo.sessions = ss
-	sinfo.reconfigure = make(chan chan error, 1)
 	sinfo.theirPermPub = *theirPermKey
+	sinfo.sharedPermKey = *ss.getSharedKey(&ss.router.core.boxPriv, &sinfo.theirPermPub)
 	pub, priv := crypto.NewBoxKeys()
 	sinfo.mySesPub = *pub
 	sinfo.mySesPriv = *priv
 	sinfo.myNonce = *crypto.NewBoxNonce()
 	sinfo.theirMTU = 1280
-	ss.core.config.Mutex.RLock()
+	ss.router.core.config.Mutex.RLock()
-	sinfo.myMTU = uint16(ss.core.config.Current.IfMTU)
+	sinfo.myMTU = uint16(ss.router.core.config.Current.IfMTU)
-	ss.core.config.Mutex.RUnlock()
+	ss.router.core.config.Mutex.RUnlock()
 	now := time.Now()
 	sinfo.timeOpened = now
 	sinfo.time = now
@@ -231,11 +219,11 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.init = make(chan struct{})
 	sinfo.cancel = util.NewCancellation()
 	higher := false
-	for idx := range ss.core.boxPub {
+	for idx := range ss.router.core.boxPub {
-		if ss.core.boxPub[idx] > sinfo.theirPermPub[idx] {
+		if ss.router.core.boxPub[idx] > sinfo.theirPermPub[idx] {
 			higher = true
 			break
-		} else if ss.core.boxPub[idx] < sinfo.theirPermPub[idx] {
+		} else if ss.router.core.boxPub[idx] < sinfo.theirPermPub[idx] {
 			break
 		}
 	}
@@ -249,17 +237,8 @@ func (ss *sessions) createSession(theirPermKey *crypto.BoxPubKey) *sessionInfo {
 	sinfo.myHandle = *crypto.NewHandle()
 	sinfo.theirAddr = *address.AddrForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
 	sinfo.theirSubnet = *address.SubnetForNodeID(crypto.GetNodeID(&sinfo.theirPermPub))
-	sinfo.fromRouter = make(chan wire_trafficPacket, 1)
-	sinfo.recv = make(chan []byte, 32)
-	sinfo.send = make(chan FlowKeyMessage, 32)
 	ss.sinfos[sinfo.myHandle] = &sinfo
 	ss.byTheirPerm[sinfo.theirPermPub] = &sinfo.myHandle
-	go func() {
-		// Run cleanup when the session is canceled
-		<-sinfo.cancel.Finished()
-		sinfo.core.router.doAdmin(sinfo.close)
-	}()
-	go sinfo.startWorkers()
 	return &sinfo
 }
 
@@ -291,20 +270,26 @@ func (ss *sessions) cleanup() {
 	ss.lastCleanup = time.Now()
 }
 
+func (sinfo *sessionInfo) doRemove() {
+	sinfo.sessions.router.Act(nil, func() {
+		sinfo.sessions.removeSession(sinfo)
+	})
+}
+
 // Closes a session, removing it from sessions maps.
-func (sinfo *sessionInfo) close() {
+func (ss *sessions) removeSession(sinfo *sessionInfo) {
-	if s := sinfo.core.sessions.sinfos[sinfo.myHandle]; s == sinfo {
+	if s := sinfo.sessions.sinfos[sinfo.myHandle]; s == sinfo {
-		delete(sinfo.core.sessions.sinfos, sinfo.myHandle)
+		delete(sinfo.sessions.sinfos, sinfo.myHandle)
-		delete(sinfo.core.sessions.byTheirPerm, sinfo.theirPermPub)
+		delete(sinfo.sessions.byTheirPerm, sinfo.theirPermPub)
 	}
 }
 
 // Returns a session ping appropriate for the given session info.
-func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing {
+func (sinfo *sessionInfo) _getPing() sessionPing {
-	loc := ss.core.switchTable.getLocator()
+	loc := sinfo.sessions.router.core.switchTable.getLocator()
 	coords := loc.getCoords()
-	ref := sessionPing{
+	ping := sessionPing{
-		SendPermPub: ss.core.boxPub,
+		SendPermPub: sinfo.sessions.router.core.boxPub,
 		Handle:      sinfo.myHandle,
 		SendSesPub:  sinfo.mySesPub,
 		Tstamp:      time.Now().Unix(),
@@ -312,7 +297,7 @@ func (ss *sessions) getPing(sinfo *sessionInfo) sessionPing {
 		MTU:         sinfo.myMTU,
 	}
 	sinfo.myNonce.Increment()
-	return ref
+	return ping
 }
 
 // Gets the shared key for a pair of box keys.
@@ -339,41 +324,50 @@ func (ss *sessions) getSharedKey(myPriv *crypto.BoxPrivKey,
 }
 
 // Sends a session ping by calling sendPingPong in ping mode.
-func (ss *sessions) ping(sinfo *sessionInfo) {
+func (sinfo *sessionInfo) ping(from phony.Actor) {
-	ss.sendPingPong(sinfo, false)
+	sinfo.Act(from, func() {
+		sinfo._sendPingPong(false)
+	})
 }
 
 // Calls getPing, sets the appropriate ping/pong flag, encodes to wire format, and send it.
 // Updates the time the last ping was sent in the session info.
-func (ss *sessions) sendPingPong(sinfo *sessionInfo, isPong bool) {
+func (sinfo *sessionInfo) _sendPingPong(isPong bool) {
-	ping := ss.getPing(sinfo)
+	ping := sinfo._getPing()
 	ping.IsPong = isPong
 	bs := ping.encode()
-	shared := ss.getSharedKey(&ss.core.boxPriv, &sinfo.theirPermPub)
+	payload, nonce := crypto.BoxSeal(&sinfo.sharedPermKey, bs, nil)
-	payload, nonce := crypto.BoxSeal(shared, bs, nil)
 	p := wire_protoTrafficPacket{
 		Coords:  sinfo.coords,
 		ToKey:   sinfo.theirPermPub,
-		FromKey: ss.core.boxPub,
+		FromKey: sinfo.sessions.router.core.boxPub,
 		Nonce:   *nonce,
 		Payload: payload,
 	}
 	packet := p.encode()
-	ss.core.router.out(packet)
+	// TODO rewrite the below if/when the peer struct becomes an actor, to not go through the router first
+	sinfo.sessions.router.Act(sinfo, func() { sinfo.sessions.router.out(packet) })
 	if sinfo.pingTime.Before(sinfo.time) {
 		sinfo.pingTime = time.Now()
 	}
 }
 
+func (sinfo *sessionInfo) setConn(from phony.Actor, conn *Conn) {
+	sinfo.Act(from, func() {
+		sinfo.conn = conn
+		sinfo.conn.setMTU(sinfo, sinfo._getMTU())
+	})
+}
+
 // Handles a session ping, creating a session if needed and calling update, then possibly responding with a pong if the ping was in ping mode and the update was successful.
 // If the session has a packet cached (common when first setting up a session), it will be sent.
 func (ss *sessions) handlePing(ping *sessionPing) {
 	// Get the corresponding session (or create a new session)
 	sinfo, isIn := ss.getByTheirPerm(&ping.SendPermPub)
 	switch {
+	case ping.IsPong: // This is a response, not an initial ping, so ignore it.
 	case isIn: // Session already exists
 	case !ss.isSessionAllowed(&ping.SendPermPub, false): // Session is not allowed
-	case ping.IsPong: // This is a response, not an initial ping, so ignore it.
 	default:
 		ss.listenerMutex.Lock()
 		if ss.listener != nil {
@@ -383,23 +377,24 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 			if s, _ := ss.getByTheirPerm(&ping.SendPermPub); s != sinfo {
 				panic("This should not happen")
 			}
-			conn := newConn(ss.core, crypto.GetNodeID(&sinfo.theirPermPub), &crypto.NodeID{}, sinfo)
+			conn := newConn(ss.router.core, crypto.GetNodeID(&sinfo.theirPermPub), &crypto.NodeID{}, sinfo)
 			for i := range conn.nodeMask {
 				conn.nodeMask[i] = 0xFF
 			}
+			sinfo.setConn(ss.router, conn)
 			c := ss.listener.conn
 			go func() { c <- conn }()
 		}
 		ss.listenerMutex.Unlock()
 	}
 	if sinfo != nil {
-		sinfo.doFunc(func() {
+		sinfo.Act(ss.router, func() {
 			// Update the session
-			if !sinfo.update(ping) { /*panic("Should not happen in testing")*/
+			if !sinfo._update(ping) { /*panic("Should not happen in testing")*/
 				return
 			}
 			if !ping.IsPong {
-				ss.sendPingPong(sinfo, true)
+				sinfo._sendPingPong(true)
 			}
 		})
 	}
@@ -408,7 +403,7 @@ func (ss *sessions) handlePing(ping *sessionPing) {
 // Get the MTU of the session.
 // Will be equal to the smaller of this node's MTU or the remote node's MTU.
 // If sending over links with a maximum message size (this was a thing with the old UDP code), it could be further lowered, to a minimum of 1280.
-func (sinfo *sessionInfo) getMTU() uint16 {
+func (sinfo *sessionInfo) _getMTU() uint16 {
 	if sinfo.theirMTU == 0 || sinfo.myMTU == 0 {
 		return 0
 	}
@@ -419,7 +414,7 @@ func (sinfo *sessionInfo) getMTU() uint16 {
 }
 
 // Checks if a packet's nonce is recent enough to fall within the window of allowed packets, and not already received.
-func (sinfo *sessionInfo) nonceIsOK(theirNonce *crypto.BoxNonce) bool {
+func (sinfo *sessionInfo) _nonceIsOK(theirNonce *crypto.BoxNonce) bool {
 	// The bitmask is to allow for some non-duplicate out-of-order packets
 	if theirNonce.Minus(&sinfo.theirNonce) > 0 {
 		// This is newer than the newest nonce we've seen
@@ -437,7 +432,7 @@ func (sinfo *sessionInfo) nonceIsOK(theirNonce *crypto.BoxNonce) bool {
 }
 
 // Updates the nonce mask by (possibly) shifting the bitmask and setting the bit corresponding to this nonce to 1, and then updating the most recent nonce
-func (sinfo *sessionInfo) updateNonce(theirNonce *crypto.BoxNonce) {
+func (sinfo *sessionInfo) _updateNonce(theirNonce *crypto.BoxNonce) {
 	// Start with some cleanup
 	for len(sinfo.theirNonceHeap) > 64 {
 		if time.Since(sinfo.theirNonceMap[*sinfo.theirNonceHeap.peek()]) < nonceWindow {
@@ -461,7 +456,7 @@ func (sinfo *sessionInfo) updateNonce(theirNonce *crypto.BoxNonce) {
 // Called after coord changes, so attemtps to use a session will trigger a new ping and notify the remote end of the coord change.
 func (ss *sessions) reset() {
 	for _, sinfo := range ss.sinfos {
-		sinfo.doFunc(func() {
+		sinfo.Act(ss.router, func() {
 			sinfo.reset = true
 		})
 	}
@@ -471,198 +466,119 @@ func (ss *sessions) reset() {
 //////////////////////////// Worker Functions Below ////////////////////////////
 ////////////////////////////////////////////////////////////////////////////////
 
-func (sinfo *sessionInfo) startWorkers() {
+type sessionCryptoManager struct {
-	go sinfo.recvWorker()
+	phony.Inbox
-	go sinfo.sendWorker()
 }
 
+func (m *sessionCryptoManager) workerGo(from phony.Actor, f func()) {
+	m.Act(from, func() {
+		util.WorkerGo(f)
+	})
+}
+
+var manager = sessionCryptoManager{}
+
 type FlowKeyMessage struct {
 	FlowKey uint64
 	Message []byte
 }
 
-func (sinfo *sessionInfo) recvWorker() {
+func (sinfo *sessionInfo) recv(from phony.Actor, packet *wire_trafficPacket) {
-	// TODO move theirNonce etc into a struct that gets stored here, passed in over a channel
+	sinfo.Act(from, func() {
-	//  Since there's no reason for anywhere else in the session code to need to *read* it...
+		sinfo._recvPacket(packet)
-	//  Only needs to be updated from the outside if a ping resets it...
+	})
-	//  That would get rid of the need to take a mutex for the sessionFunc
-	var callbacks []chan func()
-	doRecv := func(p wire_trafficPacket) {
-		var bs []byte
-		var err error
-		var k crypto.BoxSharedKey
-		sessionFunc := func() {
-			if !sinfo.nonceIsOK(&p.Nonce) {
-				err = ConnError{errors.New("packet dropped due to invalid nonce"), false, true, false, 0}
-				return
-			}
-			k = sinfo.sharedSesKey
-		}
-		sinfo.doFunc(sessionFunc)
-		if err != nil {
-			util.PutBytes(p.Payload)
-			return
-		}
-		var isOK bool
-		ch := make(chan func(), 1)
-		poolFunc := func() {
-			bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
-			callback := func() {
-				util.PutBytes(p.Payload)
-				if !isOK {
-					util.PutBytes(bs)
-					return
-				}
-				sessionFunc = func() {
-					if k != sinfo.sharedSesKey || !sinfo.nonceIsOK(&p.Nonce) {
-						// The session updated in the mean time, so return an error
-						err = ConnError{errors.New("session updated during crypto operation"), false, true, false, 0}
-						return
-					}
-					sinfo.updateNonce(&p.Nonce)
-					sinfo.time = time.Now()
-					sinfo.bytesRecvd += uint64(len(bs))
-				}
-				sinfo.doFunc(sessionFunc)
-				if err != nil {
-					// Not sure what else to do with this packet, I guess just drop it
-					util.PutBytes(bs)
-				} else {
-					// Pass the packet to the buffer for Conn.Read
-					select {
-					case <-sinfo.cancel.Finished():
-						util.PutBytes(bs)
-					case sinfo.recv <- bs:
-					}
-				}
-			}
-			ch <- callback
-		}
-		// Send to the worker and wait for it to finish
-		util.WorkerGo(poolFunc)
-		callbacks = append(callbacks, ch)
-	}
-	fromHelper := make(chan wire_trafficPacket, 1)
-	go func() {
-		var buf []wire_trafficPacket
-		for {
-			for len(buf) > 0 {
-				select {
-				case <-sinfo.cancel.Finished():
-					return
-				case p := <-sinfo.fromRouter:
-					buf = append(buf, p)
-					for len(buf) > 64 { // Based on nonce window size
-						util.PutBytes(buf[0].Payload)
-						buf = buf[1:]
-					}
-				case fromHelper <- buf[0]:
-					buf = buf[1:]
-				}
-			}
-			select {
-			case <-sinfo.cancel.Finished():
-				return
-			case p := <-sinfo.fromRouter:
-				buf = append(buf, p)
-			}
-		}
-	}()
-	select {
-	case <-sinfo.cancel.Finished():
-		return
-	case <-sinfo.init:
-		// Wait until the session has finished initializing before processing any packets
-	}
-	for {
-		for len(callbacks) > 0 {
-			select {
-			case f := <-callbacks[0]:
-				callbacks = callbacks[1:]
-				f()
-			case <-sinfo.cancel.Finished():
-				return
-			case p := <-fromHelper:
-				doRecv(p)
-			}
-		}
-		select {
-		case <-sinfo.cancel.Finished():
-			return
-		case p := <-fromHelper:
-			doRecv(p)
-		}
-	}
 }
 
-func (sinfo *sessionInfo) sendWorker() {
+func (sinfo *sessionInfo) _recvPacket(p *wire_trafficPacket) {
-	// TODO move info that this worker needs here, send updates via a channel
-	//  Otherwise we need to take a mutex to avoid races with update()
-	var callbacks []chan func()
-	doSend := func(msg FlowKeyMessage) {
-		var p wire_trafficPacket
-		var k crypto.BoxSharedKey
-		sessionFunc := func() {
-			sinfo.bytesSent += uint64(len(msg.Message))
-			p = wire_trafficPacket{
-				Coords: append([]byte(nil), sinfo.coords...),
-				Handle: sinfo.theirHandle,
-				Nonce:  sinfo.myNonce,
-			}
-			if msg.FlowKey != 0 {
-				// Helps ensure that traffic from this flow ends up in a separate queue from other flows
-				// The zero padding relies on the fact that the self-peer is always on port 0
-				p.Coords = append(p.Coords, 0)
-				p.Coords = wire_put_uint64(msg.FlowKey, p.Coords)
-			}
-			sinfo.myNonce.Increment()
-			k = sinfo.sharedSesKey
-		}
-		// Get the mutex-protected info needed to encrypt the packet
-		sinfo.doFunc(sessionFunc)
-		ch := make(chan func(), 1)
-		poolFunc := func() {
-			// Encrypt the packet
-			p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
-			// The callback will send the packet
-			callback := func() {
-				// Encoding may block on a util.GetBytes(), so kept out of the worker pool
-				packet := p.encode()
-				// Cleanup
-				util.PutBytes(msg.Message)
-				util.PutBytes(p.Payload)
-				// Send the packet
-				sinfo.core.router.out(packet)
-			}
-			ch <- callback
-		}
-		// Send to the worker and wait for it to finish
-		util.WorkerGo(poolFunc)
-		callbacks = append(callbacks, ch)
-	}
 	select {
-	case <-sinfo.cancel.Finished():
-		return
 	case <-sinfo.init:
-		// Wait until the session has finished initializing before processing any packets
+	default:
+		// TODO find a better way to drop things until initialized
+		util.PutBytes(p.Payload)
+		return
 	}
-	for {
+	if !sinfo._nonceIsOK(&p.Nonce) {
-		for len(callbacks) > 0 {
+		util.PutBytes(p.Payload)
-			select {
+		return
-			case f := <-callbacks[0]:
+	}
-				callbacks = callbacks[1:]
+	k := sinfo.sharedSesKey
-				f()
+	var isOK bool
-			case <-sinfo.cancel.Finished():
+	var bs []byte
+	ch := make(chan func(), 1)
+	poolFunc := func() {
+		bs, isOK = crypto.BoxOpen(&k, p.Payload, &p.Nonce)
+		callback := func() {
+			util.PutBytes(p.Payload)
+			if !isOK || k != sinfo.sharedSesKey || !sinfo._nonceIsOK(&p.Nonce) {
+				// Either we failed to decrypt, or the session was updated, or we received this packet in the mean time
+				util.PutBytes(bs)
 				return
-			case msg := <-sinfo.send:
+			}
-				doSend(msg)
+			sinfo._updateNonce(&p.Nonce)
+			sinfo.time = time.Now()
+			sinfo.bytesRecvd += uint64(len(bs))
+			sinfo.conn.recvMsg(sinfo, bs)
+		}
+		ch <- callback
+		sinfo.checkCallbacks()
+	}
+	sinfo.callbacks = append(sinfo.callbacks, ch)
+	manager.workerGo(sinfo, poolFunc)
+}
+
+func (sinfo *sessionInfo) _send(msg FlowKeyMessage) {
+	select {
+	case <-sinfo.init:
+	default:
+		// TODO find a better way to drop things until initialized
+		util.PutBytes(msg.Message)
+		return
+	}
+	sinfo.bytesSent += uint64(len(msg.Message))
+	coords := append([]byte(nil), sinfo.coords...)
+	if msg.FlowKey != 0 {
+		coords = append(coords, 0)
+		coords = append(coords, wire_encode_uint64(msg.FlowKey)...)
+	}
+	p := wire_trafficPacket{
+		Coords: coords,
+		Handle: sinfo.theirHandle,
+		Nonce:  sinfo.myNonce,
+	}
+	sinfo.myNonce.Increment()
+	k := sinfo.sharedSesKey
+	ch := make(chan func(), 1)
+	poolFunc := func() {
+		p.Payload, _ = crypto.BoxSeal(&k, msg.Message, &p.Nonce)
+		callback := func() {
+			// Encoding may block on a util.GetBytes(), so kept out of the worker pool
+			packet := p.encode()
+			// Cleanup
+			util.PutBytes(msg.Message)
+			util.PutBytes(p.Payload)
+			// Send the packet
+			// TODO replace this with a send to the peer struct if that becomes an actor
+			sinfo.sessions.router.Act(sinfo, func() {
+				sinfo.sessions.router.out(packet)
+			})
+		}
+		ch <- callback
+		sinfo.checkCallbacks()
+	}
+	sinfo.callbacks = append(sinfo.callbacks, ch)
+	manager.workerGo(sinfo, poolFunc)
+}
+
+func (sinfo *sessionInfo) checkCallbacks() {
+	sinfo.Act(nil, func() {
+		if len(sinfo.callbacks) > 0 {
+			select {
+			case callback := <-sinfo.callbacks[0]:
+				sinfo.callbacks = sinfo.callbacks[1:]
+				callback()
+				sinfo.checkCallbacks()
+			default:
 			}
 		}
-		select {
+	})
-		case <-sinfo.cancel.Finished():
-			return
-		case bs := <-sinfo.send:
-			doSend(bs)
-		}
-	}
 }

src/yggdrasil/switch.go

@@ -19,6 +19,8 @@ import (
 
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 	"github.com/yggdrasil-network/yggdrasil-go/src/util"
+
+	"github.com/Arceliar/phony"
 )
 
 const (
@@ -162,22 +164,18 @@ type switchData struct {
 
 // All the information stored by the switch.
 type switchTable struct {
-	core              *Core
+	core        *Core
-	reconfigure       chan chan error
+	key         crypto.SigPubKey           // Our own key
-	key               crypto.SigPubKey           // Our own key
+	time        time.Time                  // Time when locator.tstamp was last updated
-	time              time.Time                  // Time when locator.tstamp was last updated
+	drop        map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
-	drop              map[crypto.SigPubKey]int64 // Tstamp associated with a dropped root
+	mutex       sync.RWMutex               // Lock for reads/writes of switchData
-	mutex             sync.RWMutex               // Lock for reads/writes of switchData
+	parent      switchPort                 // Port of whatever peer is our parent, or self if we're root
-	parent            switchPort                 // Port of whatever peer is our parent, or self if we're root
+	data        switchData                 //
-	data              switchData                 //
+	updater     atomic.Value               // *sync.Once
-	updater           atomic.Value               // *sync.Once
+	table       atomic.Value               // lookupTable
-	table             atomic.Value               // lookupTable
+	phony.Inbox                            // Owns the below
-	packetIn          chan []byte                // Incoming packets for the worker to handle
+	queues      switch_buffers             // Queues - not atomic so ONLY use through the actor
-	idleIn            chan switchPort            // Incoming idle notifications from peer links
+	idle        map[switchPort]time.Time   // idle peers - not atomic so ONLY use through the actor
-	admin             chan func()                // Pass a lambda for the admin socket to query stuff
-	queues            switch_buffers             // Queues - not atomic so ONLY use through admin chan
-	queueTotalMaxSize uint64                     // Maximum combined size of queues
-	toRouter          chan []byte                // Packets to be sent to the router
 }
 
 // Minimum allowed total size of switch queues.
@@ -187,7 +185,6 @@ const SwitchQueueTotalMinSize = 4 * 1024 * 1024
 func (t *switchTable) init(core *Core) {
 	now := time.Now()
 	t.core = core
-	t.reconfigure = make(chan chan error, 1)
 	t.key = t.core.sigPub
 	locator := switchLocator{root: t.key, tstamp: now.Unix()}
 	peers := make(map[switchPort]peerInfo)
@@ -195,11 +192,23 @@ func (t *switchTable) init(core *Core) {
 	t.updater.Store(&sync.Once{})
 	t.table.Store(lookupTable{})
 	t.drop = make(map[crypto.SigPubKey]int64)
-	t.packetIn = make(chan []byte, 1024)
+	phony.Block(t, func() {
-	t.idleIn = make(chan switchPort, 1024)
+		core.config.Mutex.RLock()
-	t.admin = make(chan func())
+		if core.config.Current.SwitchOptions.MaxTotalQueueSize > SwitchQueueTotalMinSize {
-	t.queueTotalMaxSize = SwitchQueueTotalMinSize
+			t.queues.totalMaxSize = core.config.Current.SwitchOptions.MaxTotalQueueSize
-	t.toRouter = make(chan []byte, 1)
+		} else {
+			t.queues.totalMaxSize = SwitchQueueTotalMinSize
+		}
+		core.config.Mutex.RUnlock()
+		t.queues.bufs = make(map[string]switch_buffer)
+		t.idle = make(map[switchPort]time.Time)
+	})
+}
+
+func (t *switchTable) reconfigure() {
+	// This is where reconfiguration would go, if we had anything useful to do.
+	t.core.link.reconfigure()
+	t.core.peers.reconfigure()
 }
 
 // Safely gets a copy of this node's locator.
@@ -245,13 +254,10 @@ func (t *switchTable) cleanRoot() {
 		if t.data.locator.root != t.key {
 			t.data.seq++
 			t.updater.Store(&sync.Once{})
-			select {
+			t.core.router.reset(nil)
-			case t.core.router.reset <- struct{}{}:
-			default:
-			}
 		}
 		t.data.locator = switchLocator{root: t.key, tstamp: now.Unix()}
-		t.core.peers.sendSwitchMsgs()
+		t.core.peers.sendSwitchMsgs(t)
 	}
 }
 
@@ -279,7 +285,7 @@ func (t *switchTable) blockPeer(port switchPort) {
 }
 
 // Removes a peer.
-// Must be called by the router mainLoop goroutine, e.g. call router.doAdmin with a lambda that calls this.
+// Must be called by the router actor with a lambda that calls this.
 // If the removed peer was this node's parent, it immediately tries to find a new parent.
 func (t *switchTable) forgetPeer(port switchPort) {
 	t.mutex.Lock()
@@ -511,17 +517,14 @@ func (t *switchTable) unlockedHandleMsg(msg *switchMsg, fromPort switchPort, rep
 		if !equiv(&sender.locator, &t.data.locator) {
 			doUpdate = true
 			t.data.seq++
-			select {
+			t.core.router.reset(nil)
-			case t.core.router.reset <- struct{}{}:
-			default:
-			}
 		}
 		if t.data.locator.tstamp != sender.locator.tstamp {
 			t.time = now
 		}
 		t.data.locator = sender.locator
 		t.parent = sender.port
-		t.core.peers.sendSwitchMsgs()
+		t.core.peers.sendSwitchMsgs(t)
 	}
 	if doUpdate {
 		t.updater.Store(&sync.Once{})
@@ -573,7 +576,7 @@ func (t *switchTable) getTable() lookupTable {
 // Starts the switch worker
 func (t *switchTable) start() error {
 	t.core.log.Infoln("Starting switch")
-	go t.doWorker()
+	// There's actually nothing to do to start it...
 	return nil
 }
 
@@ -659,12 +662,13 @@ func (t *switchTable) bestPortForCoords(coords []byte) switchPort {
 // Handle an incoming packet
 // Either send it to ourself, or to the first idle peer that's free
 // Returns true if the packet has been handled somehow, false if it should be queued
-func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) bool {
+func (t *switchTable) _handleIn(packet []byte, idle map[switchPort]time.Time) bool {
 	coords := switch_getPacketCoords(packet)
 	closer := t.getCloser(coords)
 	if len(closer) == 0 {
 		// TODO? call the router directly, and remove the whole concept of a self peer?
-		t.toRouter <- packet
+		self := t.core.peers.getPorts()[0]
+		self.sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
 	var best *peer
@@ -709,7 +713,7 @@ func (t *switchTable) handleIn(packet []byte, idle map[switchPort]time.Time) boo
 	if best != nil {
 		// Send to the best idle next hop
 		delete(idle, best.port)
-		best.sendPackets([][]byte{packet})
+		best.sendPacketsFrom(t, [][]byte{packet})
 		return true
 	}
 	// Didn't find anyone idle to send it to
@@ -729,15 +733,15 @@ type switch_buffer struct {
 }
 
 type switch_buffers struct {
-	switchTable *switchTable
+	totalMaxSize uint64
-	bufs        map[string]switch_buffer // Buffers indexed by StreamID
+	bufs         map[string]switch_buffer // Buffers indexed by StreamID
-	size        uint64                   // Total size of all buffers, in bytes
+	size         uint64                   // Total size of all buffers, in bytes
-	maxbufs     int
+	maxbufs      int
-	maxsize     uint64
+	maxsize      uint64
-	closer      []closerInfo // Scratch space
+	closer       []closerInfo // Scratch space
 }
 
-func (b *switch_buffers) cleanup(t *switchTable) {
+func (b *switch_buffers) _cleanup(t *switchTable) {
 	for streamID, buf := range b.bufs {
 		// Remove queues for which we have no next hop
 		packet := buf.packets[0]
@@ -751,7 +755,7 @@ func (b *switch_buffers) cleanup(t *switchTable) {
 		}
 	}
 
-	for b.size > b.switchTable.queueTotalMaxSize {
+	for b.size > b.totalMaxSize {
 		// Drop a random queue
 		target := rand.Uint64() % b.size
 		var size uint64 // running total
@@ -779,14 +783,14 @@ func (b *switch_buffers) cleanup(t *switchTable) {
 // Handles incoming idle notifications
 // Loops over packets and sends the newest one that's OK for this peer to send
 // Returns true if the peer is no longer idle, false if it should be added to the idle list
-func (t *switchTable) handleIdle(port switchPort) bool {
+func (t *switchTable) _handleIdle(port switchPort) bool {
 	to := t.core.peers.getPorts()[port]
 	if to == nil {
 		return true
 	}
 	var packets [][]byte
 	var psize int
-	t.queues.cleanup(t)
+	t.queues._cleanup(t)
 	now := time.Now()
 	for psize < 65535 {
 		var best string
@@ -823,102 +827,49 @@ func (t *switchTable) handleIdle(port switchPort) bool {
 		}
 	}
 	if len(packets) > 0 {
-		to.sendPackets(packets)
+		to.sendPacketsFrom(t, packets)
 		return true
 	}
 	return false
 }
 
-// The switch worker does routing lookups and sends packets to where they need to be
+func (t *switchTable) packetInFrom(from phony.Actor, bytes []byte) {
-func (t *switchTable) doWorker() {
+	t.Act(from, func() {
-	sendingToRouter := make(chan []byte, 1)
+		t._packetIn(bytes)
-	go func() {
+	})
-		// Keep sending packets to the router
+}
-		self := t.core.peers.getPorts()[0]
+
-		for bs := range sendingToRouter {
+func (t *switchTable) _packetIn(bytes []byte) {
-			self.sendPackets([][]byte{bs})
+	// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
+	if !t._handleIn(bytes, t.idle) {
+		// There's nobody free to take it right now, so queue it for later
+		packet := switch_packetInfo{bytes, time.Now()}
+		streamID := switch_getPacketStreamID(packet.bytes)
+		buf, bufExists := t.queues.bufs[streamID]
+		buf.packets = append(buf.packets, packet)
+		buf.size += uint64(len(packet.bytes))
+		t.queues.size += uint64(len(packet.bytes))
+		// Keep a track of the max total queue size
+		if t.queues.size > t.queues.maxsize {
+			t.queues.maxsize = t.queues.size
 		}
-	}()
+		t.queues.bufs[streamID] = buf
-	go func() {
+		if !bufExists {
-		// Keep taking packets from the idle worker and sending them to the above whenever it's idle, keeping anything extra in a (fifo, head-drop) buffer
+			// Keep a track of the max total queue count. Only recalculate this
-		var buf [][]byte
+			// when the queue is new because otherwise repeating len(dict) might
-		var size int
+			// cause unnecessary processing overhead
-		for {
+			if len(t.queues.bufs) > t.queues.maxbufs {
-			bs := <-t.toRouter
+				t.queues.maxbufs = len(t.queues.bufs)
-			size += len(bs)
-			buf = append(buf, bs)
-			for len(buf) > 0 {
-				select {
-				case bs := <-t.toRouter:
-					size += len(bs)
-					buf = append(buf, bs)
-					for size > int(t.queueTotalMaxSize) {
-						size -= len(buf[0])
-						util.PutBytes(buf[0])
-						buf = buf[1:]
-					}
-				case sendingToRouter <- buf[0]:
-					size -= len(buf[0])
-					buf = buf[1:]
-				}
 			}
 		}
-	}()
+		t.queues._cleanup(t)
-	t.queues.switchTable = t
-	t.queues.bufs = make(map[string]switch_buffer) // Packets per PacketStreamID (string)
-	idle := make(map[switchPort]time.Time)         // this is to deduplicate things
-	for {
-		//t.core.log.Debugf("Switch state: idle = %d, buffers = %d", len(idle), len(t.queues.bufs))
-		select {
-		case bytes := <-t.packetIn:
-			// Try to send it somewhere (or drop it if it's corrupt or at a dead end)
-			if !t.handleIn(bytes, idle) {
-				// There's nobody free to take it right now, so queue it for later
-				packet := switch_packetInfo{bytes, time.Now()}
-				streamID := switch_getPacketStreamID(packet.bytes)
-				buf, bufExists := t.queues.bufs[streamID]
-				buf.packets = append(buf.packets, packet)
-				buf.size += uint64(len(packet.bytes))
-				t.queues.size += uint64(len(packet.bytes))
-				// Keep a track of the max total queue size
-				if t.queues.size > t.queues.maxsize {
-					t.queues.maxsize = t.queues.size
-				}
-				t.queues.bufs[streamID] = buf
-				if !bufExists {
-					// Keep a track of the max total queue count. Only recalculate this
-					// when the queue is new because otherwise repeating len(dict) might
-					// cause unnecessary processing overhead
-					if len(t.queues.bufs) > t.queues.maxbufs {
-						t.queues.maxbufs = len(t.queues.bufs)
-					}
-				}
-				t.queues.cleanup(t)
-			}
-		case port := <-t.idleIn:
-			// Try to find something to send to this peer
-			if !t.handleIdle(port) {
-				// Didn't find anything ready to send yet, so stay idle
-				idle[port] = time.Now()
-			}
-		case f := <-t.admin:
-			f()
-		case e := <-t.reconfigure:
-			e <- nil
-		}
 	}
 }
 
-// Passed a function to call.
+func (t *switchTable) _idleIn(port switchPort) {
-// This will send the function to t.admin and block until it finishes.
+	// Try to find something to send to this peer
-func (t *switchTable) doAdmin(f func()) {
+	if !t._handleIdle(port) {
-	// Pass this a function that needs to be run by the router's main goroutine
+		// Didn't find anything ready to send yet, so stay idle
-	// It will pass the function to the router and wait for the router to finish
+		t.idle[port] = time.Now()
-	done := make(chan struct{})
-	newF := func() {
-		f()
-		close(done)
 	}
-	t.admin <- newF
-	<-done
 }

src/yggdrasil/tcp.go

@@ -33,12 +33,11 @@ const tcp_ping_interval = (default_timeout * 2 / 3)
 
 // The TCP listener and information about active TCP connections, to avoid duplication.
 type tcp struct {
-	link        *link
+	link      *link
-	reconfigure chan chan error
+	mutex     sync.Mutex // Protecting the below
-	mutex       sync.Mutex // Protecting the below
+	listeners map[string]*TcpListener
-	listeners   map[string]*TcpListener
+	calls     map[string]struct{}
-	calls       map[string]struct{}
+	conns     map[linkInfo](chan struct{})
-	conns       map[linkInfo](chan struct{})
 }
 
 // TcpListener is a stoppable TCP listener interface. These are typically
@@ -76,49 +75,12 @@ func (t *tcp) getAddr() *net.TCPAddr {
 // Initializes the struct.
 func (t *tcp) init(l *link) error {
 	t.link = l
-	t.reconfigure = make(chan chan error, 1)
 	t.mutex.Lock()
 	t.calls = make(map[string]struct{})
 	t.conns = make(map[linkInfo](chan struct{}))
 	t.listeners = make(map[string]*TcpListener)
 	t.mutex.Unlock()
 
-	go func() {
-		for {
-			e := <-t.reconfigure
-			t.link.core.config.Mutex.RLock()
-			added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
-			deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
-			t.link.core.config.Mutex.RUnlock()
-			if len(added) > 0 || len(deleted) > 0 {
-				for _, a := range added {
-					if a[:6] != "tcp://" {
-						continue
-					}
-					if _, err := t.listen(a[6:]); err != nil {
-						e <- err
-						continue
-					}
-				}
-				for _, d := range deleted {
-					if d[:6] != "tcp://" {
-						continue
-					}
-					t.mutex.Lock()
-					if listener, ok := t.listeners[d[6:]]; ok {
-						t.mutex.Unlock()
-						listener.Stop <- true
-					} else {
-						t.mutex.Unlock()
-					}
-				}
-				e <- nil
-			} else {
-				e <- nil
-			}
-		}
-	}()
-
 	t.link.core.config.Mutex.RLock()
 	defer t.link.core.config.Mutex.RUnlock()
 	for _, listenaddr := range t.link.core.config.Current.Listen {
@@ -133,6 +95,38 @@ func (t *tcp) init(l *link) error {
 	return nil
 }
 
+func (t *tcp) reconfigure() {
+	t.link.core.config.Mutex.RLock()
+	added := util.Difference(t.link.core.config.Current.Listen, t.link.core.config.Previous.Listen)
+	deleted := util.Difference(t.link.core.config.Previous.Listen, t.link.core.config.Current.Listen)
+	t.link.core.config.Mutex.RUnlock()
+	if len(added) > 0 || len(deleted) > 0 {
+		for _, a := range added {
+			if a[:6] != "tcp://" {
+				continue
+			}
+			if _, err := t.listen(a[6:]); err != nil {
+				t.link.core.log.Errorln("Error adding TCP", a[6:], "listener:", err)
+			} else {
+				t.link.core.log.Infoln("Started TCP listener:", a[6:])
+			}
+		}
+		for _, d := range deleted {
+			if d[:6] != "tcp://" {
+				continue
+			}
+			t.mutex.Lock()
+			if listener, ok := t.listeners[d[6:]]; ok {
+				t.mutex.Unlock()
+				listener.Stop <- true
+				t.link.core.log.Infoln("Stopped TCP listener:", d[6:])
+			} else {
+				t.mutex.Unlock()
+			}
+		}
+	}
+}
+
 func (t *tcp) listen(listenaddr string) (*TcpListener, error) {
 	var err error