From 02d92ff81c2dd986323492ef9f6af26a2bab5ec5 Mon Sep 17 00:00:00 2001 From: Neil Date: Sat, 20 Jul 2024 15:24:30 +0100 Subject: [PATCH] TUN vectorised reads/writes (#1145) This PR updates the Wireguard dependency and updates to use new vectorised reads/writes, which should reduce the number of syscalls and improve performance. This will only make a difference on Linux as this is the only platform for which the Wireguard TUN library supports vectorised reads/writes. For other platforms, single reads and writes will be performed as usual. --------- Co-authored-by: Neil Alexander --- go.mod | 2 +- go.sum | 8 ++++-- src/tun/iface.go | 56 ++++++++++++++++++++++++++++-------------- src/tun/tun.go | 32 ++++++++++++++++++++++-- src/tun/tun_bsd.go | 3 +++ src/tun/tun_darwin.go | 6 +++++ src/tun/tun_linux.go | 3 +++ src/tun/tun_other.go | 3 +++ src/tun/tun_windows.go | 3 +++ 9 files changed, 92 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 3dd8c84..eec9626 100644 --- a/go.mod +++ b/go.mod @@ -17,7 +17,7 @@ require ( golang.org/x/net v0.25.0 golang.org/x/sys v0.20.0 golang.org/x/text v0.15.0 - golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675 + golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 golang.zx2c4.com/wireguard/windows v0.5.3 ) diff --git a/go.sum b/go.sum index 7ee7686..b7d9798 100644 --- a/go.sum +++ b/go.sum @@ -27,6 +27,8 @@ github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/gologme/log v1.3.0 h1:l781G4dE+pbigClDSDzSaaYKtiueHCILUa/qSDsmHAo= github.com/gologme/log v1.3.0/go.mod h1:yKT+DvIPdDdDoPtqFrFxheooyVmoqi0BAsw+erN3wA4= +github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= +github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38 h1:yAJXTCF9TqKcTiHJAE8dj7HMvPfh66eeA2JYW7eFpSE= @@ -137,8 +139,8 @@ golang.org/x/tools v0.21.0/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 h1:B82qJJgjvYKsXS9jeunTOisW56dUokqW/FOteYJJ/yg= golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2/go.mod h1:deeaetjYA+DHMHg+sMSMI58GrEteJUUzzw7en6TJQcI= -golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675 h1:/J/RVnr7ng4fWPRH3xa4WtBJ1Jp+Auu4YNLmGiPv5QU= -golang.zx2c4.com/wireguard v0.0.0-20230223181233-21636207a675/go.mod h1:whfbyDBt09xhCYQWtO2+3UVjlaq6/9hDZrjg2ZE6SyA= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173 h1:/jFs0duh4rdb8uIfPMv78iAJGcPKDeqAFnaLBropIC4= +golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173/go.mod h1:tkCQ4FQXmpAgYVh++1cq16/dH4QJtmvpRv19DWGAHSA= golang.zx2c4.com/wireguard/windows v0.5.3 h1:On6j2Rpn3OEMXqBq00QEDC7bWSZrPIHKIus8eIuExIE= golang.zx2c4.com/wireguard/windows v0.5.3/go.mod h1:9TEe8TJmtwyQebdFwAkEWOPr3prrtqm+REGFifP60hI= google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= @@ -147,3 +149,5 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259 h1:TbRPT0HtzFP3Cno1zZo7yPzEEnfu8EjLfl6IU9VfqkQ= +gvisor.dev/gvisor v0.0.0-20230927004350-cbd86285d259/go.mod h1:AVgIgHMwK63XvmAzWG9vLQ41YnVHN0du0tEC46fI7yY= diff --git a/src/tun/iface.go b/src/tun/iface.go index e06f3a5..3a4c55f 100644 --- a/src/tun/iface.go +++ b/src/tun/iface.go @@ -1,42 +1,60 @@ package tun -const TUN_OFFSET_BYTES = 4 +const TUN_OFFSET_BYTES = 80 // sizeof(virtio_net_hdr) func (tun *TunAdapter) read() { - var buf [TUN_OFFSET_BYTES + 65535]byte + vs := tun.iface.BatchSize() + bufs := make([][]byte, vs) + sizes := make([]int, vs) + for i := range bufs { + bufs[i] = make([]byte, TUN_OFFSET_BYTES+65535) + } for { - n, err := tun.iface.Read(buf[:], TUN_OFFSET_BYTES) - if n <= TUN_OFFSET_BYTES || err != nil { + n, err := tun.iface.Read(bufs, sizes, TUN_OFFSET_BYTES) + if err != nil { tun.log.Errorln("Error reading TUN:", err) - ferr := tun.iface.Flush() - if ferr != nil { - tun.log.Errorln("Unable to flush packets:", ferr) - } return } - begin := TUN_OFFSET_BYTES - end := begin + n - bs := buf[begin:end] - if _, err := tun.rwc.Write(bs); err != nil { - tun.log.Debugln("Unable to send packet:", err) + for i, b := range bufs[:n] { + if _, err := tun.rwc.Write(b[TUN_OFFSET_BYTES : TUN_OFFSET_BYTES+sizes[i]]); err != nil { + tun.log.Debugln("Unable to send packet:", err) + } } } } -func (tun *TunAdapter) write() { - var buf [TUN_OFFSET_BYTES + 65535]byte +func (tun *TunAdapter) queue() { for { - bs := buf[TUN_OFFSET_BYTES:] - n, err := tun.rwc.Read(bs) + p := bufPool.Get().([]byte)[:bufPoolSize] + n, err := tun.rwc.Read(p) if err != nil { tun.log.Errorln("Exiting TUN writer due to core read error:", err) return } + tun.ch <- p[:n] + } +} + +func (tun *TunAdapter) write() { + vs := cap(tun.ch) + bufs := make([][]byte, vs) + for i := range bufs { + bufs[i] = make([]byte, TUN_OFFSET_BYTES+65535) + } + for { + n := len(tun.ch) + if n == 0 { + n = 1 // Nothing queued up yet, wait for it instead + } + for i := 0; i < n; i++ { + msg := <-tun.ch + bufs[i] = append(bufs[i][:TUN_OFFSET_BYTES], msg...) + bufPool.Put(msg) // nolint:staticcheck + } if !tun.isEnabled { continue // Nothing to do, the tun isn't enabled } - bs = buf[:TUN_OFFSET_BYTES+n] - if _, err = tun.iface.Write(bs, TUN_OFFSET_BYTES); err != nil { + if _, err := tun.iface.Write(bufs[:n], TUN_OFFSET_BYTES); err != nil { tun.Act(nil, func() { if !tun.isOpen { tun.log.Errorln("TUN iface write error:", err) diff --git a/src/tun/tun.go b/src/tun/tun.go index 83c8267..e6795c6 100644 --- a/src/tun/tun.go +++ b/src/tun/tun.go @@ -10,9 +10,11 @@ import ( "fmt" "io" "net" + "sync" + "time" "github.com/Arceliar/phony" - "golang.zx2c4.com/wireguard/tun" + wgtun "golang.zx2c4.com/wireguard/tun" "github.com/yggdrasil-network/yggdrasil-go/src/address" "github.com/yggdrasil-network/yggdrasil-go/src/config" @@ -39,7 +41,7 @@ type TunAdapter struct { addr address.Address subnet address.Subnet mtu uint64 - iface tun.Device + iface wgtun.Device phony.Inbox // Currently only used for _handlePacket from the reader, TODO: all the stuff that currently needs a mutex below isOpen bool isEnabled bool // Used by the writer to drop sessionTraffic if not enabled @@ -48,6 +50,7 @@ type TunAdapter struct { name InterfaceName mtu InterfaceMTU } + ch chan []byte } // Gets the maximum supported MTU for the platform based on the defaults in @@ -62,6 +65,20 @@ func getSupportedMTU(mtu uint64) uint64 { return mtu } +func waitForTUNUp(ch <-chan wgtun.Event) bool { + t := time.After(time.Second * 5) + for { + select { + case ev := <-ch: + if ev == wgtun.EventUp { + return true + } + case <-t: + return false + } + } +} + // Name returns the name of the adapter, e.g. "tun0". On Windows, this may // return a canonical adapter name instead. func (tun *TunAdapter) Name() string { @@ -145,6 +162,8 @@ func (tun *TunAdapter) _start() error { tun.rwc.SetMTU(tun.MTU()) tun.isOpen = true tun.isEnabled = true + tun.ch = make(chan []byte, tun.iface.BatchSize()) + go tun.queue() go tun.read() go tun.write() return nil @@ -178,3 +197,12 @@ func (tun *TunAdapter) _stop() error { } return nil } + +const bufPoolSize = TUN_OFFSET_BYTES + 65535 + +var bufPool = sync.Pool{ + New: func() any { + b := [bufPoolSize]byte{} + return b[:] + }, +} diff --git a/src/tun/tun_bsd.go b/src/tun/tun_bsd.go index da5b329..7f26260 100644 --- a/src/tun/tun_bsd.go +++ b/src/tun/tun_bsd.go @@ -80,6 +80,9 @@ func (tun *TunAdapter) setup(ifname string, addr string, mtu uint64) error { if err != nil { return fmt.Errorf("failed to create TUN: %w", err) } + if !waitForTUNUp(iface.Events()) { + return fmt.Errorf("TUN did not come up in time") + } tun.iface = iface if mtu, err := iface.MTU(); err == nil { tun.mtu = getSupportedMTU(uint64(mtu)) diff --git a/src/tun/tun_darwin.go b/src/tun/tun_darwin.go index deeb115..a9d734b 100644 --- a/src/tun/tun_darwin.go +++ b/src/tun/tun_darwin.go @@ -27,6 +27,9 @@ func (tun *TunAdapter) setup(ifname string, addr string, mtu uint64) error { if err != nil { return fmt.Errorf("failed to create TUN: %w", err) } + if !waitForTUNUp(iface.Events()) { + return fmt.Errorf("TUN did not come up in time") + } tun.iface = iface if m, err := iface.MTU(); err == nil { tun.mtu = getSupportedMTU(uint64(m)) @@ -55,6 +58,9 @@ func (tun *TunAdapter) setupFD(fd int32, addr string, mtu uint64) error { unix.Close(dfd) return fmt.Errorf("failed to create TUN from FD: %w", err) } + if !waitForTUNUp(iface.Events()) { + return fmt.Errorf("TUN did not come up in time") + } tun.iface = iface if m, err := iface.MTU(); err == nil { tun.mtu = getSupportedMTU(uint64(m)) diff --git a/src/tun/tun_linux.go b/src/tun/tun_linux.go index 98d63db..c98cdd7 100644 --- a/src/tun/tun_linux.go +++ b/src/tun/tun_linux.go @@ -21,6 +21,9 @@ func (tun *TunAdapter) setup(ifname string, addr string, mtu uint64) error { if err != nil { return fmt.Errorf("failed to create TUN: %w", err) } + if !waitForTUNUp(iface.Events()) { + return fmt.Errorf("TUN did not come up in time") + } tun.iface = iface if mtu, err := iface.MTU(); err == nil { tun.mtu = getSupportedMTU(uint64(mtu)) diff --git a/src/tun/tun_other.go b/src/tun/tun_other.go index 0ddd0c1..fc94081 100644 --- a/src/tun/tun_other.go +++ b/src/tun/tun_other.go @@ -18,6 +18,9 @@ func (tun *TunAdapter) setup(ifname string, addr string, mtu uint64) error { if err != nil { return fmt.Errorf("failed to create TUN: %w", err) } + if !waitForTUNUp(iface.Events()) { + return fmt.Errorf("TUN did not come up in time") + } tun.iface = iface if mtu, err := iface.MTU(); err == nil { tun.mtu = getSupportedMTU(uint64(mtu)) diff --git a/src/tun/tun_windows.go b/src/tun/tun_windows.go index 19d9248..7bcdb7a 100644 --- a/src/tun/tun_windows.go +++ b/src/tun/tun_windows.go @@ -34,6 +34,9 @@ func (tun *TunAdapter) setup(ifname string, addr string, mtu uint64) error { if iface, err = wgtun.CreateTUNWithRequestedGUID(ifname, &guid, int(mtu)); err != nil { return err } + if !waitForTUNUp(iface.Events()) { + return fmt.Errorf("TUN did not come up in time") + } tun.iface = iface if addr != "" { if err = tun.setupAddress(addr); err != nil {