#!/usr/bin/env python3
"""
Copyright 2019 Pascal Engélibert <tuxmain@zettascript.org>, Pierre-Jean Chancellier <paidge@normandie-libre.fr>
License GNU AGPL v3 (http://www.gnu.org/licenses/#AGPL)

Dependances: python3-plyvel, python3-igraph
Command for installing:
$ sudo pip3 install plyvel python-igraph
"""

import sys, os, time, math, json, urllib.request, colorsys, plyvel, igraph, subprocess

VERSION = "0.1.0"
STEPMAX = 5

def getargv(arg, default=""):
	if arg in sys.argv and len(sys.argv) > sys.argv.index(arg)+1:
		return sys.argv[sys.argv.index(arg)+1]
	else:
		return default

LOG_TRACE = 1
LOG_INFO = 2
LOG_WARN = 4
LOG_ERROR = 8
LOGMSG_TYPES = {LOG_INFO:"\033[96minfo\033[0m", LOG_TRACE:"\033[39mtrace\033[0m", LOG_WARN:"\033[93mwarn\033[0m", LOG_ERROR:"\033[91merror\033[0m"}
VERBOSITY = LOG_INFO | LOG_WARN | LOG_ERROR

def log(msg, msgtype=LOG_TRACE, end="\n", header=True):
	if msgtype & VERBOSITY:
		if header:
			print(time.strftime("%Y-%m-%d %H:%M:%S")+" ["+LOGMSG_TYPES[msgtype]+"] "+msg, end=end)
		else:
			print(msg, end=end)

class Node:
	def __init__(self, pubkey, member=False, uid="", written_on=None):
		self.pubkey = pubkey
		self.member = member
		self.uid = uid
		self.written_on = written_on
		self.nb = None
		self.certs_issued = []
		self.neighbors = []
		self.degree = 0
		self.in_degree = 0
		self.out_degree = 0
		self.referent = 0

if __name__ == "__main__":
	if "--help" in sys.argv or "-h" in sys.argv:
		print("""This script extracts data from Duniter DB and generates output JSON files, with stats, positionned members (colored with communities) and certs on a graph,.

Options:
 -i <dir>    Input duniter db (LevelDB)
 --ow <file> Output JSON (wot)
 --os <file> Output JSON (stats)
 -e <url>    URL of the ElasticSearch server
 -d          Download avatars
 -a <dir>    Avatars local path
 -A <dir>    Avatars public local path
 -m <int>    Avatars cache max age (s)
 -M <file>   Default avatar public local path
 -s          Do not stop & restart Duniter-ts
 --no-pos    Do not compute node coords
 -v          Verbose

Default options:
 -i   ~/.config/duniter/duniter_default/data/leveldb
 --ow ~/wotmap/data/wot.json
 --os ~/wotmap/data/stats.json
 -e   https://g1.data.le-sou.org
 -a   ../img/photos
 -A   img/photos
 -m   604800  (7 days)
 -M   img/logo-g1.png""")
		exit()
	
	if "-v" in sys.argv:
		VERBOSITY |= LOG_TRACE
	
	# Initialize
	if not "-s" in sys.argv:
		log(subprocess.run(["duniter", "stop"], stdout=subprocess.PIPE).stdout.decode("utf-8"))
	
	outdata_wot = {"nodes":[], "edges":[]}
	outdata_stats = {"stats":{}, "instance":{"software": "wotmap", "version": VERSION, "working": True, "maintainers": [], "custom": {}}}
	
	# Read args
	dbpath = os.path.expanduser(getargv("-i", "~/.config/duniter/duniter_default/data/leveldb"))
	outpath_wot = os.path.expanduser(getargv("--ow", "~/wotmap/data/wot.json"))
	outpath_stats = os.path.expanduser(getargv("--os", "~/wotmap/data/stats.json"))
	url_es = getargv("-e", "https://g1.data.duniter.fr")
	download_avatars = "-d" in sys.argv
	avatars_path = os.path.expanduser(getargv("-a", os.path.dirname(os.path.abspath(__file__))+"/../img/photos"))
	avatars_url_path = getargv("-A", "img/photos")
	avatars_max_age = time.time() - int(getargv("-m", 604800))# 7 days
	avatars_default_url_path = getargv("-M", "img/logo-g1.png")
	gencoords = not "--no-pos"  in sys.argv
	verbose = "-v" in sys.argv
	
	log("Opening DBs...")
	iindex, cindex, mindex, bindex = 0, 0, 0, 0
	while not (iindex and cindex):
		try:
			iindex = plyvel.DB(dbpath+"/level_iindex/")
			cindex = plyvel.DB(dbpath+"/level_cindex/")
			mindex = plyvel.DB(dbpath+"/level_mindex/")
			bindex = plyvel.DB(dbpath+"/level_bindex/")
		except plyvel._plyvel.IOError:
			time.sleep(0.5)
	
	# Get latest block
	latest_block = json.loads(list(bindex.iterator(include_key=False, reverse=True))[0].decode())
	outdata_stats["instance"]["custom"]["currency"] = latest_block["currency"]
	outdata_stats["instance"]["custom"]["block"] = latest_block["number"]
	median_time = latest_block["medianTime"]
	outdata_stats["stats"]["median_time"] = median_time
	
	log("Generating nodes...")
	graph = igraph.Graph()
	
	# List all nodes
	tmp_nodes = []
	tmp_index = {}
	for pub, row in iindex:
		idty = json.loads(row.decode())[0]
		
		node = Node(idty["pub"], idty["member"], idty["uid"], idty["writtenOn"])
		
		# Index issued certs
		certs = json.loads(cindex.get(pub).decode())
		for cert in certs["issued"]:
			if not "expires_on" in cert:
				continue
			if cert["expires_on"] > median_time:
				node.certs_issued.append(cert)
		
		tmp_index[node.pubkey] = node
		tmp_nodes.append(node)
	
	# Link nodes
	for node in tmp_nodes:
		for cert in node.certs_issued:
			receiver = tmp_index[cert["receiver"]]
			
			if not node in receiver.neighbors:
				receiver.neighbors.append(node)
			if not receiver in node.neighbors:
				node.neighbors.append(receiver)
			
			receiver.in_degree += 1
			node.out_degree += 1
	
	# Index nodes
	index = {}
	nodes = []
	nb_nodes = 0
	nb_members = 0
	for node in tmp_nodes:
		node.degree = len(node.neighbors)
		if node.degree == 0:
			continue
		
		node.nb = nb_nodes
		nb_nodes += 1
		if node.member:
			nb_members += 1
		index[node.pubkey] = node
		nodes.append(node)
		
		written_on = bindex.get(str(node.written_on).zfill(10).encode())
		if written_on != None:
			block = json.loads(written_on.decode())
			written_time = block["time"]
		else:
			written_time = None
		
		avatar_path = avatars_path+"/"+node.pubkey+".png"
		avatar_exists = os.path.isfile(avatar_path)
		if download_avatars:
			if not avatar_exists or os.path.getmtime(avatar_path) < avatars_max_age:
				try:
					log("Download avatar: " + node.pubkey, end=" ")
					response = urllib.request.urlopen(url_es+"/user/profile/"+node.pubkey+"/_image/avatar.png")
					f = open(avatar_path, "wb")
					f.write(response.read())
					f.close()
					log("OK", header=False)
					avatar_url = avatars_url_path+"/"+node.pubkey+".png"
				except urllib.error.HTTPError:
					avatar_url = avatars_default_url_path
					log("ERR", header=False)
			else:
				avatar_url = avatars_url_path+"/"+node.pubkey+".png"
		elif avatar_exists:
			avatar_url = avatars_url_path+"/"+node.pubkey+".png"
		else:
			avatar_url = avatars_default_url_path
		
		outdata_wot["nodes"].append({
				"id": str(node.nb),
				"label": node.uid,
				"url": avatar_url,
				"x": 0,
				"y": 0,
				"my_community": 0,
				"size": node.degree,
				"color": "rgba(255,0,0,1)",
				"attributes": {
					"pubkey": node.pubkey,
					"member": node.member,
					"degree": node.degree,
					"referent": False,
					"outDegree": node.in_degree,
					"inDegree": node.out_degree,
					"written_time": written_time
				}
			})
	
	graph.add_vertices(nb_nodes)
	
	# Generate edges
	nb_edges = 0
	for node in nodes:
		for cert in node.certs_issued:
			receiver = index[cert["receiver"]]
			graph.add_edges([(node.nb, receiver.nb)])
			outdata_wot["edges"].append({
				"id": str(nb_edges),
				"size": 1,
				"type": "curvedArrow",
				"source": str(node.nb),
				"target": str(receiver.nb),
				"start_cert": cert["chainable_on"],
				"end_cert": cert["expires_on"]
			})
			nb_edges += 1
	
	iindex.close()
	outdata_stats["stats"]["total_edges"] = nb_edges
	outdata_stats["stats"]["total_nodes"] = nb_nodes
	outdata_stats["stats"]["total_members"] = nb_members
	
	log("Generating communities...")
	clusters = graph.community_label_propagation()
	membership = clusters.membership
	nb_communities = max(membership)+1
	outdata_stats["stats"]["total_communities"] = nb_communities
	log('> Total communities = ' + str(nb_communities), LOG_INFO)
	
	log("Updating referents and communities...")
	crit_referent = math.ceil((nb_members+1) ** (1/STEPMAX))
	outdata_stats["stats"]["crit_referent"] = crit_referent
	nb_referents = 0
	nb_noreferents = 0
	
	for i in range(nb_nodes):
		outdata_wot["nodes"][i]["my_community"] = membership[i]
		(r, g, b) = colorsys.hsv_to_rgb(float(membership[i]) / nb_communities, 1, 1)
		R, G, B = int(255 * r), int(255 * g), int(255 * b)
		outdata_wot["nodes"][i]["color"] = "rgba(" + str(R) + "," + str(G) + "," + str(B) + ",1)"
		
		if outdata_wot["nodes"][i]["attributes"]["member"] and \
		   outdata_wot["nodes"][i]["attributes"]["inDegree"] >= crit_referent and \
		   outdata_wot["nodes"][i]["attributes"]["outDegree"] >= crit_referent:
			outdata_wot["nodes"][i]["attributes"]["referent"] = True
			nb_referents += 1
			
	nb_noreferents = nb_members - nb_referents
	outdata_stats["stats"]["total_referents"] = nb_referents
	outdata_stats["stats"]["total_noreferents"] = nb_noreferents
	
	if gencoords:
		log("Updating coords...")
		layout = graph.layout_fruchterman_reingold()
		i = 0
		for dot in layout.coords:
			outdata_wot["nodes"][i]["x"] = dot[0]
			outdata_wot["nodes"][i]["y"] = dot[1]
			i += 1
	
	log("Saving output...")
	outfile_wot = open(outpath_wot, "w")
	json.dump(outdata_wot, outfile_wot)
	outfile_wot.close()
	outfile_stats = open(outpath_stats, "w")
	json.dump(outdata_stats, outfile_stats)
	outfile_stats.close()
	
	if not "-s" in sys.argv:
		log(subprocess.run(["duniter", "start"], stdout=subprocess.PIPE).stdout.decode("utf-8"))