5
0
mirror of https://github.com/cwinfo/yggdrasil-map synced 2024-11-10 20:40:27 +00:00

Pushing nodes to a database and generating graph from it

This commit is contained in:
Vanhala Antti 2014-05-30 17:34:00 +03:00
parent d6b0b97528
commit f51cf2025e
13 changed files with 293 additions and 318 deletions

3
.gitignore vendored
View File

@ -1,6 +1,5 @@
.cache .cache
*.pyc *.pyc
graph.json graph.json
mapper/mapper-confs/* web_config.cfg
conf_sh.py
lighttp.conf lighttp.conf

View File

@ -1,19 +0,0 @@
#
# This file should be runnable by bash and python!
#
cjdns_path="/home/user/cjdns"
graph_output="../web/static/graph.json"
num_of_nodes=30
# Where mapper nodes connect to
peer_ip="127.0.0.1"
peer_port="33333"
peer_pw="mapper-peers-hunter2qwertyuiopoiuytrewq"
peer_pk="osufn28fjjduan29dajsdnasiqlqn8ahasoasa.k"
# Admin RPC of mapper nodes
rpc_bind="127.0.0.1"
rpc_connect="127.0.0.1"
rpc_pw="Kjs8HuaKu2afdw"
rpc_firstport=11244

View File

@ -1,231 +0,0 @@
#!/usr/bin/env python
import conf_sh as conf
import sys
sys.path.append(conf.cjdns_path + '/contrib/python/cjdnsadmin/')
import adminTools as admin
from collections import deque
import pygraphviz as pgv
import json
import time
import httplib2
import traceback
class Node:
def __init__(self, ip):
self.ip = ip
self.version = -1
self.label = ip[-4:]
def __lt__(self, b):
return self.ip < b.ip
class Edge:
def __init__(self, a, b):
self.a, self.b = sorted([a, b])
def is_in(self, edges):
for e in edges:
if e.a.ip == self.a.ip and e.b.ip == self.b.ip:
return True
return False
def get_network_from_cjdns(ip, port, password):
nodes = dict()
edges = []
cjdns = admin.connect(ip, port, password)
me = admin.whoami(cjdns)
my_ip = me['IP']
nodes[my_ip] = Node(my_ip)
nodes_to_check = deque()
nodes_to_check.append(my_ip)
while len(nodes_to_check) != 0:
current_ip = nodes_to_check.popleft()
resp = cjdns.NodeStore_nodeForAddr(current_ip)
if not 'result' in resp or not 'linkCount' in resp['result']:
continue
result = resp['result']
link_count = result['linkCount']
if 'protocolVersion' in result:
nodes[current_ip].version = result['protocolVersion']
for i in range(0, link_count):
result = cjdns.NodeStore_getLink(current_ip, i)['result']
if not 'child' in result:
continue
child_ip = result['child']
# Add links with one hop only
if result['isOneHop'] != 1:
continue
# Add node
if not child_ip in nodes:
nodes[child_ip] = Node(child_ip)
nodes_to_check.append(child_ip)
# Add edge
e = Edge(nodes[current_ip], nodes[child_ip])
if not e.is_in(edges):
edges.append(e)
return (nodes, edges)
def get_full_network():
all_nodes = dict()
all_edges = []
for i in range(0, conf.num_of_nodes):
port = conf.rpc_firstport + i
print '[%d/%d] Connecting to %s:%d...' % (i + 1, conf.num_of_nodes, conf.rpc_connect, port),
sys.stdout.flush()
try:
nodes, edges = get_network_from_cjdns(conf.rpc_connect, port, conf.rpc_pw)
except Exception as ex:
print 'Fail! Node unresponsive!'
continue
print '%d nodes, %d edges' % (len(nodes), len(edges))
for ip, n in nodes.iteritems():
all_nodes[ip] = n
for e in edges:
if not e.is_in(all_edges):
all_edges.append(e)
return (all_nodes, all_edges)
def download_names_from_nameinfo():
page = 'http://[fc5d:baa5:61fc:6ffd:9554:67f0:e290:7535]/nodes/list.json'
print 'Downloading names from Mikey\'s nodelist...',
sys.stdout.flush()
ip_dict = dict()
http = httplib2.Http('.cache', timeout=15.0)
r, content = http.request(page, 'GET')
name_and_ip = json.loads(content)['nodes']
for node in name_and_ip:
ip_dict[node['ip']] = node['name']
print 'Done!'
return ip_dict
def set_node_names(nodes):
try:
ip_dict = download_names_from_nameinfo()
except Exception as ex:
print 'Fail!'
# TODO use cache
print traceback.format_exc()
return
for ip, node in nodes.iteritems():
if ip in ip_dict:
node.label = ip_dict[ip]
def build_graph(nodes, edges):
G = pgv.AGraph(strict=True, directed=False, size='10!')
for n in nodes.values():
G.add_node(n.ip, label=n.label, version=n.version)
for e in edges:
G.add_edge(e.a.ip, e.b.ip, len=1.0)
G.layout(prog='neato', args='-Gepsilon=0.0001 -Gmaxiter=100000')
return G
def gradient_color(ratio, colors):
jump = 1.0 / (len(colors) - 1)
gap_num = int(ratio / (jump + 0.0000001))
a = colors[gap_num]
b = colors[gap_num + 1]
ratio = (ratio - gap_num * jump) * (len(colors) - 1)
r = a[0] + (b[0] - a[0]) * ratio
g = a[1] + (b[1] - a[1]) * ratio
b = a[2] + (b[2] - a[2]) * ratio
return '#%02x%02x%02x' % (r, g, b)
def get_graph_json(G):
max_neighbors = 1
for n in G.iternodes():
neighbors = len(G.neighbors(n))
if neighbors > max_neighbors:
max_neighbors = neighbors
print 'Max neighbors: %d' % max_neighbors
out_data = {
'created': int(time.time()),
'nodes': [],
'edges': []
}
for n in G.iternodes():
neighbor_ratio = len(G.neighbors(n)) / float(max_neighbors)
pos = n.attr['pos'].split(',', 1)
out_data['nodes'].append({
'id': n.name,
'label': n.attr['label'],
'version': n.attr['version'],
'x': float(pos[0]),
'y': float(pos[1]),
'color': gradient_color(neighbor_ratio, [(100, 100, 100), (0, 0, 0)]),
'size': neighbor_ratio
})
for e in G.iteredges():
out_data['edges'].append({
'sourceID': e[0],
'targetID': e[1]
})
return json.dumps(out_data)
if __name__ == '__main__':
nodes, edges = get_full_network()
print 'Total:'
print '%d nodes, %d edges' % (len(nodes), len(edges))
set_node_names(nodes)
G = build_graph(nodes, edges)
output = get_graph_json(G)
with open(conf.graph_output, 'w') as f:
f.write(output)

View File

@ -1,56 +0,0 @@
#!/bin/bash
source conf_sh.py
mkdir -p mapper-confs
# Generate configurations and collect their publick keys and ports
for i in $(seq 1 $num_of_nodes)
do
file=mapper-confs/node$i.conf
$cjdns_path/cjdroute --genconf > $file
# Get connecting info
publicKey=$(grep -oP -m1 '(?<="publicKey": ").*(?=",)' $file)
connectPort=$(grep -oP -m1 '(?<="0.0.0.0:).*(?=",)' $file)
connectToInfo[i]='"127.0.0.1:'"$connectPort"'":{"password":"'"$rpc_pw"'","publicKey":"'"$publicKey"'"},'
done
# Modify configurations
for i in $(seq 1 $num_of_nodes)
do
echo "Starting mapper node $i/$num_of_nodes"
file=mapper-confs/node$i.conf
rpcport=$(($rpc_firstport + $i - 1))
# Connect to all mapper nodes except itself
connectInfo=""
for j in $(seq 1 $num_of_nodes)
do
if [[ $i != $j ]]; then
connectInfo+="${connectToInfo[j]}"
fi
done
# Set peer credentials
sed -i 's/\/\/ Add connection credentials here to join the network/'"$connectInfo"'/g' $file
sed -i 's/\/\/ Ask somebody who is already connected./"'"${peer_ip}"':'"${peer_port}"'":{"password":"'"${peer_pw}"'","publicKey":"'"${peer_pk}"'"}/g' $file
# Set admin rpc credentials
sed -i 's/127.0.0.1:11234/'"${rpc_bind}"':'"${rpcport}"'/g' $file
sed -i 's/"password": ".*"/"password": "'"${rpc_pw}"'"/g' $file
# Disable tun interface
sed -i 's/"type": "TUNInterface"/\/\/"type": "TUNInterface"/g' $file
# Start mappers
if [[ $* == *-d* ]]; then
# Log to stdout
sed -i 's/\/\/ "logTo":"stdout"/"logTo":"stdout"/g' $file
gdb $cjdns_path/cjdroute -ex 'set follow-fork-mode child' -ex 'run < '"${file}" -ex 'thread apply all bt' -ex 'quit' > gdb-$i.log 2>&1 &
else
$cjdns_path/cjdroute < $file
fi
done

View File

@ -40,13 +40,19 @@ import cjdnsadmin
def main(): def main():
print "Connecting to cjdns...",; sys.stdout.flush()
cjdns = cjdns_connect() cjdns = cjdns_connect()
print "Done!"
success = generate_and_send_graph(cjdns) success = generate_and_send_graph(cjdns)
sys.exit(0 if success else 1) sys.exit(0 if success else 1)
def generate_and_send_graph(cjdns): def generate_and_send_graph(cjdns):
source_nodes = cjdns_get_node_store(cjdns) source_nodes = cjdns_get_node_store(cjdns)
print "Found %d source nodes." % len(source_nodes)
nodes, edges = cjdns_graph_from_nodes(cjdns, source_nodes) nodes, edges = cjdns_graph_from_nodes(cjdns, source_nodes)
print "Found %d nodes and %d links." % (len(nodes), len(edges))
graph_data = { graph_data = {
'nodes': [], 'nodes': [],
@ -56,7 +62,6 @@ def generate_and_send_graph(cjdns):
for n in nodes.values(): for n in nodes.values():
graph_data['nodes'].append({ graph_data['nodes'].append({
'ip': n.ip, 'ip': n.ip,
'key': n.key,
'version': n.version 'version': n.version
}) })
@ -67,15 +72,19 @@ def generate_and_send_graph(cjdns):
}) })
json_str = json.dumps(graph_data) json_str = json.dumps(graph_data)
return send_data(json_str)
print "Sending data...",; sys.stdout.flush()
success = send_data(json_str)
print ("Done!" if success else "Failed!")
return success
class Node: class Node:
def __init__(self, ip, version=None, key=None): def __init__(self, ip, version=None):
self.ip = ip self.ip = ip
self.version = version self.version = version
self.key = key
class Edge: class Edge:
def __init__(self, a, b): def __init__(self, a, b):
@ -140,9 +149,6 @@ def cjdns_graph_from_nodes(cjdns, source_nodes):
continue continue
res = resp['result'] res = resp['result']
if 'key' in res:
node.key = res['key']
if 'protocolVersion' in res: if 'protocolVersion' in res:
node.version = res['protocolVersion'] node.version = res['protocolVersion']

80
web/database.py Normal file
View File

@ -0,0 +1,80 @@
import MySQLdb as mdb
from graph import Node, Edge
import time
class NodeDB:
def __init__(self, config):
self.con = mdb.connect(
config['MYSQL_DATABASE_HOST'],
config['MYSQL_DATABASE_USER'],
config['MYSQL_DATABASE_PASSWORD'],
config['MYSQL_DATABASE_DB'])
self.cur = self.con.cursor()
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.con.commit()
self.con.close()
def insert_node(self, node):
now = int(time.time())
self.cur.execute('''
INSERT INTO nodes (ip, name, version, first_seen, last_seen)
VALUES (%s, %s, %s, %s, %s)
ON DUPLICATE KEY
UPDATE name = %s, version = %s, last_seen = %s''', (
node.ip, node.label, node.version, now, now,
node.label, node.version, now))
def insert_edge(self, edge):
now = int(time.time())
self.cur.execute('''
INSERT INTO edges (a, b, first_seen, last_seen)
VALUES (%s, %s, %s, %s)
ON DUPLICATE KEY
UPDATE last_seen = %s''', (
edge.a.ip, edge.b.ip, now, now,
now))
def insert_graph(self, nodes, edges):
for n in nodes.itervalues():
self.insert_node(n)
for e in edges:
self.insert_edge(e)
def get_nodes(self, time_limit):
since = int(time.time() - time_limit)
cur = self.con.cursor(mdb.cursors.DictCursor)
cur.execute("SELECT ip, version, name FROM nodes WHERE last_seen > %s", (since,))
db_nodes = cur.fetchall()
nodes = dict()
for n in db_nodes:
nodes[n['ip']] = Node(n['ip'], n['version'], n['name'])
return nodes
def get_edges(self, nodes, time_limit):
since = int(time.time() - time_limit)
cur = self.con.cursor(mdb.cursors.DictCursor)
cur.execute("SELECT a, b FROM edges WHERE last_seen > %s", (since,))
db_edges = cur.fetchall()
edges = []
for e in db_edges:
edges.append(Edge(nodes[e['a']], nodes[e['b']]))
return edges
def get_graph(self, time_limit):
nodes = self.get_nodes(time_limit)
edges = self.get_edges(nodes, time_limit)
return (nodes, edges)

31
web/database.sql Normal file
View File

@ -0,0 +1,31 @@
CREATE DATABASE IF NOT EXISTS `fc00`;
USE `fc00`;
--
-- Table structure for table `edges`
--
DROP TABLE IF EXISTS `edges`;
CREATE TABLE `edges` (
`a` varchar(39) NOT NULL,
`b` varchar(39) NOT NULL,
`first_seen` int(11) NOT NULL,
`last_seen` int(11) NOT NULL,
PRIMARY KEY (`a`,`b`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
--
-- Table structure for table `nodes`
--
DROP TABLE IF EXISTS `nodes`;
CREATE TABLE `nodes` (
`ip` varchar(39) NOT NULL,
`name` varchar(64) DEFAULT NULL,
`version` int(11) DEFAULT NULL,
`first_seen` int(11) NOT NULL,
`last_seen` int(11) NOT NULL,
PRIMARY KEY (`ip`)
) ENGINE=InnoDB DEFAULT CHARSET=utf8;

29
web/graph.py Normal file
View File

@ -0,0 +1,29 @@
class Node:
def __init__(self, ip, version=None, label=None):
self.ip = ip
self.version = version
self.label = ip[-4:] if label == None else label
def __lt__(self, b):
return self.ip < b.ip
def __repr__(self):
return 'Node(ip="%s", version=%s, label="%s")' % (
self.ip,
self.version,
self.label)
class Edge:
def __init__(self, a, b):
self.a, self.b = sorted([a, b])
def is_in(self, edges):
for e in edges:
if e.a.ip == self.a.ip and e.b.ip == self.b.ip:
return True
return False
def __repr__(self):
return 'Edge(a.ip="%s", b.ip="%s")' % (
self.a.ip,
self.b.ip)

View File

@ -1,9 +1,35 @@
import json import json
from database import NodeDB
from graph import Node, Edge
def insert_graph_data(json_str): def insert_graph_data(config, json_str):
try: try:
graph_data = json.loads(json_str) graph_data = json.loads(json_str)
except ValueError: except ValueError:
return False return False
nodes = dict()
edges = []
if not 'nodes' in graph_data or not 'edges' in graph_data:
return False
try:
for n in graph_data['nodes']:
node = Node(n['ip'], version=n['version'])
nodes[n['ip']] = node
for e in graph_data['edges']:
edge = Edge(nodes[e['a']], nodes[e['b']])
edges.append(edge)
except TypeError:
return False
print "Received %d nodes and %d links." % (len(nodes), len(edges))
with NodeDB(config) as db:
db.insert_graph(nodes, edges)
return True return True

74
web/graphPlotter.py Normal file
View File

@ -0,0 +1,74 @@
import pygraphviz as pgv
import time
import json
def position_nodes(nodes, edges):
G = pgv.AGraph(strict=True, directed=False, size='10!')
for n in nodes.values():
G.add_node(n.ip, label=n.label, version=n.version)
for e in edges:
G.add_edge(e.a.ip, e.b.ip, len=1.0)
G.layout(prog='neato', args='-Gepsilon=0.0001 -Gmaxiter=100000')
return G
def get_graph_json(G):
max_neighbors = 1
for n in G.iternodes():
neighbors = len(G.neighbors(n))
if neighbors > max_neighbors:
max_neighbors = neighbors
print 'Max neighbors: %d' % max_neighbors
out_data = {
'created': int(time.time()),
'nodes': [],
'edges': []
}
for n in G.iternodes():
neighbor_ratio = len(G.neighbors(n)) / float(max_neighbors)
pos = n.attr['pos'].split(',', 1)
out_data['nodes'].append({
'id': n.name,
'label': n.attr['label'],
'version': n.attr['version'],
'x': float(pos[0]),
'y': float(pos[1]),
'color': _gradient_color(neighbor_ratio, [(100, 100, 100), (0, 0, 0)]),
'size': neighbor_ratio
})
for e in G.iteredges():
out_data['edges'].append({
'sourceID': e[0],
'targetID': e[1]
})
return json.dumps(out_data)
def _gradient_color(ratio, colors):
jump = 1.0 / (len(colors) - 1)
gap_num = int(ratio / (jump + 0.0000001))
a = colors[gap_num]
b = colors[gap_num + 1]
ratio = (ratio - gap_num * jump) * (len(colors) - 1)
r = a[0] + (b[0] - a[0]) * ratio
g = a[1] + (b[1] - a[1]) * ratio
b = a[2] + (b[2] - a[2]) * ratio
return '#%02x%02x%02x' % (r, g, b)

26
web/updateGraph.py Executable file
View File

@ -0,0 +1,26 @@
#!/usr/bin/env python
from flask import Config
from database import NodeDB
import graphPlotter
def generate_graph(time_limit=60*60*3):
nodes, edges = load_graph_from_db(time_limit)
graph = graphPlotter.position_nodes(nodes, edges)
json = graphPlotter.get_graph_json(graph)
with open('static/graph.json', 'w') as f:
f.write(json)
def load_graph_from_db(time_limit):
config = Config('./')
config.from_pyfile('web_config.cfg')
with NodeDB(config) as db:
return db.get_graph(time_limit)
if __name__ == '__main__':
generate_graph()

View File

@ -2,7 +2,7 @@ from flask import Flask, render_template, request
from graphData import insert_graph_data from graphData import insert_graph_data
app = Flask(__name__) app = Flask(__name__)
app.debug = False app.config.from_pyfile('web_config.cfg')
@app.context_processor @app.context_processor
@ -21,8 +21,10 @@ def page_about():
@app.route('/sendGraph', methods=['POST']) @app.route('/sendGraph', methods=['POST'])
def page_sendGraph(): def page_sendGraph():
print "Receiving graph from %s" % (request.remote_addr)
data = request.form['data'] data = request.form['data']
ret = insert_graph_data(data) ret = insert_graph_data(app.config, data)
if ret: if ret:
return 'OK' return 'OK'
else: else:

View File

@ -0,0 +1,8 @@
DEBUG = False
MYSQL_DATABASE_HOST = 'localhost'
MYSQL_DATABASE_PORT = 3306
MYSQL_DATABASE_USER = 'fc00'
MYSQL_DATABASE_PASSWORD = 'hunter2'
MYSQL_DATABASE_DB = 'fc00'
MYSQL_DATABASE_CHARSET = 'utf8'