diff --git a/.circleci/config.yml b/.circleci/config.yml index f3092ba..e548aa1 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -103,7 +103,7 @@ jobs: name: Create tags (master branch only) command: > if [ "${CIRCLE_BRANCH}" == "master" ]; then - git tag -f -a $(sh contrib/semver/version.sh) -m "Created by CircleCI" && git push -f --tags; + (git tag -a $(sh contrib/semver/version.sh) -m "Created by CircleCI" && git push --tags) || true; else echo "Only runs for master branch (this is ${CIRCLE_BRANCH})"; fi; diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c8994f..84b5c38 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,21 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - in case of vulnerabilities. --> +## [0.3.2] - 2018-12-26 +### Added +- The admin socket is now multithreaded, greatly improving performance of the crawler and allowing concurrent lookups to take place +- The ability to hide NodeInfo defaults through either setting the `NodeInfoPrivacy` option or through setting individual `NodeInfo` attributes to `null` + +### Changed +- The `armhf` build now targets ARMv6 instead of ARMv7, adding support for Raspberry Pi Zero and other older models, amongst others + +### Fixed +- DHT entries are now populated using a copy in memory to fix various potential DHT bugs +- DHT traffic should now throttle back exponentially to reduce idle traffic +- Adjust how nodes are inserted into the DHT which should help to reduce some incorrect DHT traffic +- In TAP mode, the NDP target address is now correctly used when populating the peer MAC table. This fixes serious connectivity problems when in TAP mode, particularly on BSD +- In TUN mode, ICMPv6 packets are now ignored whereas they were incorrectly processed before + ## [0.3.1] - 2018-12-17 ### Added - Build name and version is now imprinted onto the binaries if available/specified during build diff --git a/clean b/clean index 4361a7b..a103676 100755 --- a/clean +++ b/clean @@ -1,2 +1,2 @@ -#!/bin/bash +#!/bin/sh git clean -dxf diff --git a/cmd/yggdrasil/main.go b/cmd/yggdrasil/main.go index 5a9db26..2b6d2f0 100644 --- a/cmd/yggdrasil/main.go +++ b/cmd/yggdrasil/main.go @@ -71,6 +71,7 @@ func generateConfig(isAutoconf bool) *nodeConfig { cfg.SessionFirewall.AllowFromDirect = true cfg.SessionFirewall.AllowFromRemote = true cfg.SwitchOptions.MaxTotalQueueSize = yggdrasil.SwitchQueueTotalMinSize + cfg.NodeInfoPrivacy = false return &cfg } diff --git a/contrib/deb/generate.sh b/contrib/deb/generate.sh index ed2c1c9..6c8f955 100644 --- a/contrib/deb/generate.sh +++ b/contrib/deb/generate.sh @@ -25,7 +25,7 @@ if [ $PKGARCH = "amd64" ]; then GOARCH=amd64 GOOS=linux ./build elif [ $PKGARCH = "i386" ]; then GOARCH=386 GOOS=linux ./build elif [ $PKGARCH = "mipsel" ]; then GOARCH=mipsle GOOS=linux ./build elif [ $PKGARCH = "mips" ]; then GOARCH=mips64 GOOS=linux ./build -elif [ $PKGARCH = "armhf" ]; then GOARCH=arm GOOS=linux GOARM=7 ./build +elif [ $PKGARCH = "armhf" ]; then GOARCH=arm GOOS=linux GOARM=6 ./build elif [ $PKGARCH = "arm64" ]; then GOARCH=arm64 GOOS=linux ./build else echo "Specify PKGARCH=amd64,i386,mips,mipsel,armhf,arm64" diff --git a/contrib/semver/version.sh b/contrib/semver/version.sh index a53ed32..03b5da2 100644 --- a/contrib/semver/version.sh +++ b/contrib/semver/version.sh @@ -33,9 +33,6 @@ if [ $? != 0 ]; then exit 1 fi -# Get the number of merges on the current branch since the last tag -BUILD=$(git rev-list $TAG..HEAD --count --merges) - # Split out into major, minor and patch numbers MAJOR=$(echo $TAG | cut -c 2- | cut -d "." -f 1) MINOR=$(echo $TAG | cut -c 2- | cut -d "." -f 2) @@ -54,9 +51,17 @@ else printf '%s%d.%d.%d' "$PREPEND" "$MAJOR" "$MINOR" "$PATCH" fi +# Get the number of merges on the current branch since the last tag +TAG=$(git describe --abbrev=0 --tags --match="v[0-9]*\.[0-9]*\.[0-9]*" --first-parent master 2>/dev/null) +BUILD=$(git rev-list $TAG.. --count) + # Add the build tag on non-master branches if [ $BRANCH != "master" ]; then if [ $BUILD != 0 ]; then printf -- "-%04d" "$BUILD" fi +else + if [ $BUILD != 0 ]; then + printf -- "-%d" "$(($BUILD+1))" + fi fi diff --git a/doc/Whitepaper.md b/doc/Whitepaper.md index 674f6dc..26d49a5 100644 --- a/doc/Whitepaper.md +++ b/doc/Whitepaper.md @@ -63,7 +63,7 @@ These coordinates are used as a distance label. Given the coordinates of any two nodes, it is possible to calculate the length of some real path through the network between the two nodes. Traffic is forwarded using a [greedy routing](https://en.wikipedia.org/wiki/Small-world_routing#Greedy_routing) scheme, where each node forwards the packet to a one-hop neighbor that is closer to the destination (according to this distance metric) than the current node. -In particular, when a packet needs to be forward, a node will forward it to whatever peer is closest to the destination in the greedy [metric space](https://en.wikipedia.org/wiki/Metric_space) used by the network, provided that the peer is closer to the destination than the current node. +In particular, when a packet needs to be forwarded, a node will forward it to whatever peer is closest to the destination in the greedy [metric space](https://en.wikipedia.org/wiki/Metric_space) used by the network, provided that the peer is closer to the destination than the current node. If no closer peers are idle, then the packet is queued in FIFO order, with separate queues per destination coords (currently, as a bit of a hack, IPv6 flow labels are embedeed after the end of the significant part of the coords, so queues distinguish between different traffic streams with the same destination). Whenever the node finishes forwarding a packet to a peer, it checks the queues, and will forward the first packet from the queue with the maximum `/`, i.e. the bandwidth the queue is attempting to use, subject to the constraint that the peer is a valid next hop (i.e. closer to the destination than the current node). diff --git a/src/config/config.go b/src/config/config.go index b5a1f89..192f435 100644 --- a/src/config/config.go +++ b/src/config/config.go @@ -19,6 +19,7 @@ type NodeConfig struct { SessionFirewall SessionFirewall `comment:"The session firewall controls who can send/receive network traffic\nto/from. This is useful if you want to protect this node without\nresorting to using a real firewall. This does not affect traffic\nbeing routed via this node to somewhere else. Rules are prioritised as\nfollows: blacklist, whitelist, always allow outgoing, direct, remote."` TunnelRouting TunnelRouting `comment:"Allow tunneling non-Yggdrasil traffic over Yggdrasil. This effectively\nallows you to use Yggdrasil to route to, or to bridge other networks,\nsimilar to a VPN tunnel. Tunnelling works between any two nodes and\ndoes not require them to be directly peered."` SwitchOptions SwitchOptions `comment:"Advanced options for tuning the switch. Normally you will not need\nto edit these options."` + NodeInfoPrivacy bool `comment:"By default, nodeinfo contains some defaults including the platform,\narchitecture and Yggdrasil version. These can help when surveying\nthe network and diagnosing network routing problems. Enabling\nnodeinfo privacy prevents this, so that only items specified in\n\"NodeInfo\" are sent back if specified."` NodeInfo map[string]interface{} `comment:"Optional node info. This must be a { \"key\": \"value\", ... } map\nor set as null. This is entirely optional but, if set, is visible\nto the whole network on request."` //Net NetConfig `comment:"Extended options for connecting to peers over other networks."` } diff --git a/src/yggdrasil/core.go b/src/yggdrasil/core.go index 99330e1..e38274f 100644 --- a/src/yggdrasil/core.go +++ b/src/yggdrasil/core.go @@ -125,7 +125,7 @@ func (c *Core) Start(nc *config.NodeConfig, log *log.Logger) error { c.admin.init(c, nc.AdminListen) c.nodeinfo.init(c) - c.nodeinfo.setNodeInfo(nc.NodeInfo) + c.nodeinfo.setNodeInfo(nc.NodeInfo, nc.NodeInfoPrivacy) if err := c.tcp.init(c, nc.Listen, nc.ReadTimeout); err != nil { c.log.Println("Failed to start TCP interface") @@ -248,8 +248,8 @@ func (c *Core) GetNodeInfo() nodeinfoPayload { } // Sets the nodeinfo. -func (c *Core) SetNodeInfo(nodeinfo interface{}) { - c.nodeinfo.setNodeInfo(nodeinfo) +func (c *Core) SetNodeInfo(nodeinfo interface{}, nodeinfoprivacy bool) { + c.nodeinfo.setNodeInfo(nodeinfo, nodeinfoprivacy) } // Sets the output logger of the Yggdrasil node after startup. This may be diff --git a/src/yggdrasil/dht.go b/src/yggdrasil/dht.go index af104f5..b52a820 100644 --- a/src/yggdrasil/dht.go +++ b/src/yggdrasil/dht.go @@ -12,7 +12,12 @@ import ( "github.com/yggdrasil-network/yggdrasil-go/src/crypto" ) -const dht_lookup_size = 16 +const ( + dht_lookup_size = 16 + dht_timeout = 6 * time.Minute + dht_max_delay = 5 * time.Minute + dht_max_delay_dirty = 30 * time.Second +) // dhtInfo represents everything we know about a node in the DHT. // This includes its key, a cache of it's NodeID, coords, and timing/ping related info for deciding who/when to ping nodes for maintenance. @@ -23,6 +28,7 @@ type dhtInfo struct { recv time.Time // When we last received a message pings int // Time out if at least 3 consecutive maintenance pings drop throttle time.Duration + dirty bool // Set to true if we've used this node in ping responses (for queries about someone other than the person doing the asking, i.e. real searches) since the last time we heard from the node } // Returns the *NodeID associated with dhtInfo.key, calculating it on the fly the first time or from a cache all subsequent times. @@ -134,6 +140,16 @@ func (t *dht) insert(info *dhtInfo) { t.table[*info.getNodeID()] = info } +// Insert a peer into the table if it hasn't been pinged lately, to keep peers from dropping +func (t *dht) insertPeer(info *dhtInfo) { + oldInfo, isIn := t.table[*info.getNodeID()] + if !isIn || time.Since(oldInfo.recv) > dht_max_delay+30*time.Second { + // TODO? also check coords? + newInfo := *info // Insert a copy + t.insert(&newInfo) + } +} + // Return true if first/second/third are (partially) ordered correctly. func dht_ordered(first, second, third *crypto.NodeID) bool { lessOrEqual := func(first, second *crypto.NodeID) bool { @@ -185,6 +201,14 @@ func (t *dht) handleReq(req *dhtReq) { if _, isIn := t.table[*info.getNodeID()]; !isIn && t.isImportant(&info) { t.ping(&info, nil) } + // Maybe mark nodes from lookup as dirty + if req.Dest != *info.getNodeID() { + // This node asked about someone other than themself, so this wasn't just idle traffic. + for _, info := range res.Infos { + // Mark nodes dirty so we're sure to check up on them again later + info.dirty = true + } + } } // Sends a lookup response to the specified node. @@ -302,19 +326,32 @@ func (t *dht) doMaintenance() { } t.callbacks = newCallbacks for infoID, info := range t.table { - if now.Sub(info.recv) > time.Minute || info.pings > 3 { + switch { + case info.pings > 6: + // It failed to respond to too many pings + fallthrough + case now.Sub(info.recv) > dht_timeout: + // It's too old + fallthrough + case info.dirty && now.Sub(info.recv) > dht_max_delay_dirty && !t.isImportant(info): + // We won't ping it to refresh it, so just drop it delete(t.table, infoID) t.imp = nil } } for _, info := range t.getImportant() { - if now.Sub(info.recv) > info.throttle { + switch { + case now.Sub(info.recv) > info.throttle: + info.throttle *= 2 + if info.throttle < time.Second { + info.throttle = time.Second + } else if info.throttle > dht_max_delay { + info.throttle = dht_max_delay + } + fallthrough + case info.dirty && now.Sub(info.recv) > dht_max_delay_dirty: t.ping(info, nil) info.pings++ - info.throttle += time.Second - if info.throttle > 30*time.Second { - info.throttle = 30 * time.Second - } } } } diff --git a/src/yggdrasil/icmpv6.go b/src/yggdrasil/icmpv6.go index baae3ab..52ca50c 100644 --- a/src/yggdrasil/icmpv6.go +++ b/src/yggdrasil/icmpv6.go @@ -156,6 +156,9 @@ func (i *icmpv6) parse_packet_tun(datain []byte, datamac *[]byte) ([]byte, error // Check for a supported message type switch icmpv6Header.Type { case ipv6.ICMPTypeNeighborSolicitation: + if !i.tun.iface.IsTAP() { + return nil, errors.New("Ignoring Neighbor Solicitation in TUN mode") + } response, err := i.handle_ndp(datain[ipv6.HeaderLen:]) if err == nil { // Create our ICMPv6 response @@ -173,16 +176,22 @@ func (i *icmpv6) parse_packet_tun(datain []byte, datamac *[]byte) ([]byte, error return nil, err } case ipv6.ICMPTypeNeighborAdvertisement: + if !i.tun.iface.IsTAP() { + return nil, errors.New("Ignoring Neighbor Advertisement in TUN mode") + } if datamac != nil { var addr address.Address + var target address.Address var mac macAddress copy(addr[:], ipv6Header.Src[:]) + copy(target[:], datain[48:64]) copy(mac[:], (*datamac)[:]) - neighbor := i.peermacs[addr] + // i.tun.core.log.Printf("Learning peer MAC %x for %x\n", mac, target) + neighbor := i.peermacs[target] neighbor.mac = mac neighbor.learned = true neighbor.lastadvertisement = time.Now() - i.peermacs[addr] = neighbor + i.peermacs[target] = neighbor } return nil, errors.New("No response needed") } diff --git a/src/yggdrasil/nodeinfo.go b/src/yggdrasil/nodeinfo.go index f525bec..b907632 100644 --- a/src/yggdrasil/nodeinfo.go +++ b/src/yggdrasil/nodeinfo.go @@ -4,6 +4,7 @@ import ( "encoding/json" "errors" "runtime" + "strings" "sync" "time" @@ -96,18 +97,27 @@ func (m *nodeinfo) getNodeInfo() nodeinfoPayload { } // Set the current node's nodeinfo -func (m *nodeinfo) setNodeInfo(given interface{}) error { +func (m *nodeinfo) setNodeInfo(given interface{}, privacy bool) error { m.myNodeInfoMutex.Lock() defer m.myNodeInfoMutex.Unlock() - newnodeinfo := map[string]interface{}{ + defaults := map[string]interface{}{ "buildname": GetBuildName(), "buildversion": GetBuildVersion(), "buildplatform": runtime.GOOS, "buildarch": runtime.GOARCH, } + newnodeinfo := make(map[string]interface{}) + if !privacy { + for k, v := range defaults { + newnodeinfo[k] = v + } + } if nodeinfomap, ok := given.(map[string]interface{}); ok { for key, value := range nodeinfomap { - if _, ok := newnodeinfo[key]; ok { + if _, ok := defaults[key]; ok { + if strvalue, strok := value.(string); strok && strings.EqualFold(strvalue, "null") || value == nil { + delete(newnodeinfo, key) + } continue } newnodeinfo[key] = value diff --git a/src/yggdrasil/router.go b/src/yggdrasil/router.go index 6c92869..87da882 100644 --- a/src/yggdrasil/router.go +++ b/src/yggdrasil/router.go @@ -110,12 +110,7 @@ func (r *router) mainLoop() { case p := <-r.send: r.sendPacket(p) case info := <-r.core.dht.peers: - now := time.Now() - oldInfo, isIn := r.core.dht.table[*info.getNodeID()] - r.core.dht.insert(info) - if isIn && now.Sub(oldInfo.recv) < 45*time.Second { - info.recv = oldInfo.recv - } + r.core.dht.insertPeer(info) case <-r.reset: r.core.sessions.resetInits() r.core.dht.reset() diff --git a/src/yggdrasil/tun.go b/src/yggdrasil/tun.go index b6bc913..8ed5333 100644 --- a/src/yggdrasil/tun.go +++ b/src/yggdrasil/tun.go @@ -214,11 +214,12 @@ func (tun *tunAdapter) read() error { continue } if buf[o+6] == 58 { - // Found an ICMPv6 packet - b := make([]byte, n) - copy(b, buf) - // tun.icmpv6.recv <- b - go tun.icmpv6.parse_packet(b) + if tun.iface.IsTAP() { + // Found an ICMPv6 packet + b := make([]byte, n) + copy(b, buf) + go tun.icmpv6.parse_packet(b) + } } packet := append(util.GetBytes(), buf[o:n]...) tun.send <- packet diff --git a/src/yggdrasil/wire.go b/src/yggdrasil/wire.go index 782af43..8891f1a 100644 --- a/src/yggdrasil/wire.go +++ b/src/yggdrasil/wire.go @@ -70,7 +70,7 @@ func wire_decode_uint64(bs []byte) (uint64, int) { // Converts an int64 into uint64 so it can be written to the wire. // Non-negative integers are mapped to even integers: 0 -> 0, 1 -> 2, etc. -// Negative integres are mapped to odd integes: -1 -> 1, -2 -> 3, etc. +// Negative integers are mapped to odd integers: -1 -> 1, -2 -> 3, etc. // This means the least significant bit is a sign bit. func wire_intToUint(i int64) uint64 { return ((uint64(-(i+1))<<1)|0x01)*(uint64(i)>>63) + (uint64(i)<<1)*(^uint64(i)>>63)