Merge pull request #291 from yggdrasil-network/develop

Version 0.3.2

.circleci/config.yml

@@ -103,7 +103,7 @@ jobs:
           name: Create tags (master branch only)
           command: >
               if [ "${CIRCLE_BRANCH}" == "master" ]; then
-                git tag -f -a $(sh contrib/semver/version.sh) -m "Created by CircleCI" && git push -f --tags;
+                (git tag -a $(sh contrib/semver/version.sh) -m "Created by CircleCI" && git push --tags) || true;
               else
                 echo "Only runs for master branch (this is ${CIRCLE_BRANCH})";
               fi;

CHANGELOG.md

@@ -25,6 +25,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.
 - in case of vulnerabilities.
 -->
 
+## [0.3.2] - 2018-12-26
+### Added
+- The admin socket is now multithreaded, greatly improving performance of the crawler and allowing concurrent lookups to take place
+- The ability to hide NodeInfo defaults through either setting the `NodeInfoPrivacy` option or through setting individual `NodeInfo` attributes to `null`
+
+### Changed
+- The `armhf` build now targets ARMv6 instead of ARMv7, adding support for Raspberry Pi Zero and other older models, amongst others
+
+### Fixed
+- DHT entries are now populated using a copy in memory to fix various potential DHT bugs
+- DHT traffic should now throttle back exponentially to reduce idle traffic
+- Adjust how nodes are inserted into the DHT which should help to reduce some incorrect DHT traffic
+- In TAP mode, the NDP target address is now correctly used when populating the peer MAC table. This fixes serious connectivity problems when in TAP mode, particularly on BSD
+- In TUN mode, ICMPv6 packets are now ignored whereas they were incorrectly processed before
+
 ## [0.3.1] - 2018-12-17
 ### Added
 - Build name and version is now imprinted onto the binaries if available/specified during build

clean

@@ -1,2 +1,2 @@
-#!/bin/bash
+#!/bin/sh
 git clean -dxf

cmd/yggdrasil/main.go

@@ -71,6 +71,7 @@ func generateConfig(isAutoconf bool) *nodeConfig {
 	cfg.SessionFirewall.AllowFromDirect = true
 	cfg.SessionFirewall.AllowFromRemote = true
 	cfg.SwitchOptions.MaxTotalQueueSize = yggdrasil.SwitchQueueTotalMinSize
+	cfg.NodeInfoPrivacy = false
 
 	return &cfg
 }

contrib/deb/generate.sh

@@ -25,7 +25,7 @@ if [ $PKGARCH = "amd64" ]; then GOARCH=amd64 GOOS=linux ./build
 elif [ $PKGARCH = "i386" ]; then GOARCH=386 GOOS=linux ./build
 elif [ $PKGARCH = "mipsel" ]; then GOARCH=mipsle GOOS=linux ./build
 elif [ $PKGARCH = "mips" ]; then GOARCH=mips64 GOOS=linux ./build
-elif [ $PKGARCH = "armhf" ]; then GOARCH=arm GOOS=linux GOARM=7 ./build
+elif [ $PKGARCH = "armhf" ]; then GOARCH=arm GOOS=linux GOARM=6 ./build
 elif [ $PKGARCH = "arm64" ]; then GOARCH=arm64 GOOS=linux ./build
 else
   echo "Specify PKGARCH=amd64,i386,mips,mipsel,armhf,arm64"

contrib/semver/version.sh

@@ -33,9 +33,6 @@ if [ $? != 0 ]; then
   exit 1
 fi
 
-# Get the number of merges on the current branch since the last tag
-BUILD=$(git rev-list $TAG..HEAD --count --merges)
-
 # Split out into major, minor and patch numbers
 MAJOR=$(echo $TAG | cut -c 2- | cut -d "." -f 1)
 MINOR=$(echo $TAG | cut -c 2- | cut -d "." -f 2)
@@ -54,9 +51,17 @@ else
   printf '%s%d.%d.%d' "$PREPEND" "$MAJOR" "$MINOR" "$PATCH"
 fi
 
+# Get the number of merges on the current branch since the last tag
+TAG=$(git describe --abbrev=0 --tags --match="v[0-9]*\.[0-9]*\.[0-9]*" --first-parent master 2>/dev/null)
+BUILD=$(git rev-list $TAG.. --count)
+
 # Add the build tag on non-master branches
 if [ $BRANCH != "master" ]; then
   if [ $BUILD != 0 ]; then
     printf -- "-%04d" "$BUILD"
   fi
+else
+  if [ $BUILD != 0 ]; then
+    printf -- "-%d" "$(($BUILD+1))"
+  fi
 fi

doc/Whitepaper.md

@@ -63,7 +63,7 @@ These coordinates are used as a distance label.
 Given the coordinates of any two nodes, it is possible to calculate the length of some real path through the network between the two nodes.
 
 Traffic is forwarded using a [greedy routing](https://en.wikipedia.org/wiki/Small-world_routing#Greedy_routing) scheme, where each node forwards the packet to a one-hop neighbor that is closer to the destination (according to this distance metric) than the current node.
-In particular, when a packet needs to be forward, a node will forward it to whatever peer is closest to the destination in the greedy [metric space](https://en.wikipedia.org/wiki/Metric_space) used by the network, provided that the peer is closer to the destination than the current node.
+In particular, when a packet needs to be forwarded, a node will forward it to whatever peer is closest to the destination in the greedy [metric space](https://en.wikipedia.org/wiki/Metric_space) used by the network, provided that the peer is closer to the destination than the current node.
 
 If no closer peers are idle, then the packet is queued in FIFO order, with separate queues per destination coords (currently, as a bit of a hack, IPv6 flow labels are embedeed after the end of the significant part of the coords, so queues distinguish between different traffic streams with the same destination).
 Whenever the node finishes forwarding a packet to a peer, it checks the queues, and will forward the first packet from the queue with the maximum `<age of first packet>/<queue size in bytes>`, i.e. the bandwidth the queue is attempting to use, subject to the constraint that the peer is a valid next hop (i.e. closer to the destination than the current node).

src/config/config.go

@@ -19,6 +19,7 @@ type NodeConfig struct {
 	SessionFirewall             SessionFirewall        `comment:"The session firewall controls who can send/receive network traffic\nto/from. This is useful if you want to protect this node without\nresorting to using a real firewall. This does not affect traffic\nbeing routed via this node to somewhere else. Rules are prioritised as\nfollows: blacklist, whitelist, always allow outgoing, direct, remote."`
 	TunnelRouting               TunnelRouting          `comment:"Allow tunneling non-Yggdrasil traffic over Yggdrasil. This effectively\nallows you to use Yggdrasil to route to, or to bridge other networks,\nsimilar to a VPN tunnel. Tunnelling works between any two nodes and\ndoes not require them to be directly peered."`
 	SwitchOptions               SwitchOptions          `comment:"Advanced options for tuning the switch. Normally you will not need\nto edit these options."`
+	NodeInfoPrivacy             bool                   `comment:"By default, nodeinfo contains some defaults including the platform,\narchitecture and Yggdrasil version. These can help when surveying\nthe network and diagnosing network routing problems. Enabling\nnodeinfo privacy prevents this, so that only items specified in\n\"NodeInfo\" are sent back if specified."`
 	NodeInfo                    map[string]interface{} `comment:"Optional node info. This must be a { \"key\": \"value\", ... } map\nor set as null. This is entirely optional but, if set, is visible\nto the whole network on request."`
 	//Net                         NetConfig `comment:"Extended options for connecting to peers over other networks."`
 }

src/yggdrasil/core.go

@@ -125,7 +125,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) error {
 	c.admin.init(c, nc.AdminListen)
 
 	c.nodeinfo.init(c)
-	c.nodeinfo.setNodeInfo(nc.NodeInfo)
+	c.nodeinfo.setNodeInfo(nc.NodeInfo, nc.NodeInfoPrivacy)
 
 	if err := c.tcp.init(c, nc.Listen, nc.ReadTimeout); err != nil {
 		c.log.Println("Failed to start TCP interface")
@@ -248,8 +248,8 @@ func (c *Core) GetNodeInfo() nodeinfoPayload {
 }
 
 // Sets the nodeinfo.
-func (c *Core) SetNodeInfo(nodeinfo interface{}) {
+func (c *Core) SetNodeInfo(nodeinfo interface{}, nodeinfoprivacy bool) {
-	c.nodeinfo.setNodeInfo(nodeinfo)
+	c.nodeinfo.setNodeInfo(nodeinfo, nodeinfoprivacy)
 }
 
 // Sets the output logger of the Yggdrasil node after startup. This may be

src/yggdrasil/dht.go

@@ -12,7 +12,12 @@ import (
 	"github.com/yggdrasil-network/yggdrasil-go/src/crypto"
 )
 
-const dht_lookup_size = 16
+const (
+	dht_lookup_size     = 16
+	dht_timeout         = 6 * time.Minute
+	dht_max_delay       = 5 * time.Minute
+	dht_max_delay_dirty = 30 * time.Second
+)
 
 // dhtInfo represents everything we know about a node in the DHT.
 // This includes its key, a cache of it's NodeID, coords, and timing/ping related info for deciding who/when to ping nodes for maintenance.
@@ -23,6 +28,7 @@ type dhtInfo struct {
 	recv          time.Time // When we last received a message
 	pings         int       // Time out if at least 3 consecutive maintenance pings drop
 	throttle      time.Duration
+	dirty         bool // Set to true if we've used this node in ping responses (for queries about someone other than the person doing the asking, i.e. real searches) since the last time we heard from the node
 }
 
 // Returns the *NodeID associated with dhtInfo.key, calculating it on the fly the first time or from a cache all subsequent times.
@@ -134,6 +140,16 @@ func (t *dht) insert(info *dhtInfo) {
 	t.table[*info.getNodeID()] = info
 }
 
+// Insert a peer into the table if it hasn't been pinged lately, to keep peers from dropping
+func (t *dht) insertPeer(info *dhtInfo) {
+	oldInfo, isIn := t.table[*info.getNodeID()]
+	if !isIn || time.Since(oldInfo.recv) > dht_max_delay+30*time.Second {
+		// TODO? also check coords?
+		newInfo := *info // Insert a copy
+		t.insert(&newInfo)
+	}
+}
+
 // Return true if first/second/third are (partially) ordered correctly.
 func dht_ordered(first, second, third *crypto.NodeID) bool {
 	lessOrEqual := func(first, second *crypto.NodeID) bool {
@@ -185,6 +201,14 @@ func (t *dht) handleReq(req *dhtReq) {
 	if _, isIn := t.table[*info.getNodeID()]; !isIn && t.isImportant(&info) {
 		t.ping(&info, nil)
 	}
+	// Maybe mark nodes from lookup as dirty
+	if req.Dest != *info.getNodeID() {
+		// This node asked about someone other than themself, so this wasn't just idle traffic.
+		for _, info := range res.Infos {
+			// Mark nodes dirty so we're sure to check up on them again later
+			info.dirty = true
+		}
+	}
 }
 
 // Sends a lookup response to the specified node.
@@ -302,19 +326,32 @@ func (t *dht) doMaintenance() {
 	}
 	t.callbacks = newCallbacks
 	for infoID, info := range t.table {
-		if now.Sub(info.recv) > time.Minute || info.pings > 3 {
+		switch {
+		case info.pings > 6:
+			// It failed to respond to too many pings
+			fallthrough
+		case now.Sub(info.recv) > dht_timeout:
+			// It's too old
+			fallthrough
+		case info.dirty && now.Sub(info.recv) > dht_max_delay_dirty && !t.isImportant(info):
+			// We won't ping it to refresh it, so just drop it
 			delete(t.table, infoID)
 			t.imp = nil
 		}
 	}
 	for _, info := range t.getImportant() {
-		if now.Sub(info.recv) > info.throttle {
+		switch {
+		case now.Sub(info.recv) > info.throttle:
+			info.throttle *= 2
+			if info.throttle < time.Second {
+				info.throttle = time.Second
+			} else if info.throttle > dht_max_delay {
+				info.throttle = dht_max_delay
+			}
+			fallthrough
+		case info.dirty && now.Sub(info.recv) > dht_max_delay_dirty:
 			t.ping(info, nil)
 			info.pings++
-			info.throttle += time.Second
-			if info.throttle > 30*time.Second {
-				info.throttle = 30 * time.Second
-			}
 		}
 	}
 }

src/yggdrasil/icmpv6.go

@@ -156,6 +156,9 @@ func (i *icmpv6) parse_packet_tun(datain []byte, datamac *[]byte) ([]byte, error
 	// Check for a supported message type
 	switch icmpv6Header.Type {
 	case ipv6.ICMPTypeNeighborSolicitation:
+		if !i.tun.iface.IsTAP() {
+			return nil, errors.New("Ignoring Neighbor Solicitation in TUN mode")
+		}
 		response, err := i.handle_ndp(datain[ipv6.HeaderLen:])
 		if err == nil {
 			// Create our ICMPv6 response
@@ -173,16 +176,22 @@ func (i *icmpv6) parse_packet_tun(datain []byte, datamac *[]byte) ([]byte, error
 			return nil, err
 		}
 	case ipv6.ICMPTypeNeighborAdvertisement:
+		if !i.tun.iface.IsTAP() {
+			return nil, errors.New("Ignoring Neighbor Advertisement in TUN mode")
+		}
 		if datamac != nil {
 			var addr address.Address
+			var target address.Address
 			var mac macAddress
 			copy(addr[:], ipv6Header.Src[:])
+			copy(target[:], datain[48:64])
 			copy(mac[:], (*datamac)[:])
-			neighbor := i.peermacs[addr]
+			// i.tun.core.log.Printf("Learning peer MAC %x for %x\n", mac, target)
+			neighbor := i.peermacs[target]
 			neighbor.mac = mac
 			neighbor.learned = true
 			neighbor.lastadvertisement = time.Now()
-			i.peermacs[addr] = neighbor
+			i.peermacs[target] = neighbor
 		}
 		return nil, errors.New("No response needed")
 	}

src/yggdrasil/nodeinfo.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"runtime"
+	"strings"
 	"sync"
 	"time"
 
@@ -96,18 +97,27 @@ func (m *nodeinfo) getNodeInfo() nodeinfoPayload {
 }
 
 // Set the current node's nodeinfo
-func (m *nodeinfo) setNodeInfo(given interface{}) error {
+func (m *nodeinfo) setNodeInfo(given interface{}, privacy bool) error {
 	m.myNodeInfoMutex.Lock()
 	defer m.myNodeInfoMutex.Unlock()
-	newnodeinfo := map[string]interface{}{
+	defaults := map[string]interface{}{
 		"buildname":     GetBuildName(),
 		"buildversion":  GetBuildVersion(),
 		"buildplatform": runtime.GOOS,
 		"buildarch":     runtime.GOARCH,
 	}
+	newnodeinfo := make(map[string]interface{})
+	if !privacy {
+		for k, v := range defaults {
+			newnodeinfo[k] = v
+		}
+	}
 	if nodeinfomap, ok := given.(map[string]interface{}); ok {
 		for key, value := range nodeinfomap {
-			if _, ok := newnodeinfo[key]; ok {
+			if _, ok := defaults[key]; ok {
+				if strvalue, strok := value.(string); strok && strings.EqualFold(strvalue, "null") || value == nil {
+					delete(newnodeinfo, key)
+				}
 				continue
 			}
 			newnodeinfo[key] = value

src/yggdrasil/router.go

@@ -110,12 +110,7 @@ func (r *router) mainLoop() {
 		case p := <-r.send:
 			r.sendPacket(p)
 		case info := <-r.core.dht.peers:
-			now := time.Now()
+			r.core.dht.insertPeer(info)
-			oldInfo, isIn := r.core.dht.table[*info.getNodeID()]
-			r.core.dht.insert(info)
-			if isIn && now.Sub(oldInfo.recv) < 45*time.Second {
-				info.recv = oldInfo.recv
-			}
 		case <-r.reset:
 			r.core.sessions.resetInits()
 			r.core.dht.reset()

src/yggdrasil/tun.go

@@ -214,11 +214,12 @@ func (tun *tunAdapter) read() error {
 			continue
 		}
 		if buf[o+6] == 58 {
-			// Found an ICMPv6 packet
+			if tun.iface.IsTAP() {
-			b := make([]byte, n)
+				// Found an ICMPv6 packet
-			copy(b, buf)
+				b := make([]byte, n)
-			// tun.icmpv6.recv <- b
+				copy(b, buf)
-			go tun.icmpv6.parse_packet(b)
+				go tun.icmpv6.parse_packet(b)
+			}
 		}
 		packet := append(util.GetBytes(), buf[o:n]...)
 		tun.send <- packet

src/yggdrasil/wire.go

@@ -70,7 +70,7 @@ func wire_decode_uint64(bs []byte) (uint64, int) {
 
 // Converts an int64 into uint64 so it can be written to the wire.
 // Non-negative integers are mapped to even integers: 0 -> 0, 1 -> 2, etc.
-// Negative integres are mapped to odd integes: -1 -> 1, -2 -> 3, etc.
+// Negative integers are mapped to odd integers: -1 -> 1, -2 -> 3, etc.
 // This means the least significant bit is a sign bit.
 func wire_intToUint(i int64) uint64 {
 	return ((uint64(-(i+1))<<1)|0x01)*(uint64(i)>>63) + (uint64(i)<<1)*(^uint64(i)>>63)