20171203 src/graph2prom.sml (Konverter graph2prom (graph.json -> .prom) für TQ-Grafik und Verlinkung zu Nachbarknoten im status-Dashboard)

Makefile.in

@@ -9,7 +9,7 @@ conf/substitutions.sed: conf/Makefile conf/substitutions.conf
 conf/substitutions.conf:
 	cd conf && ${MAKE}
 
-build: src/Makefile src/nodes2prom src/json-pp
+build: src/Makefile src/nodes2prom src/graph2prom src/json-pp
 
 src/Makefile: src/Makefile.in
 	conf/substitute.sh $> $@
@@ -17,6 +17,9 @@ src/Makefile: src/Makefile.in
 src/nodes2prom: src/Makefile
 	cd src && ${MAKE} nodes2prom
 
+src/graph2prom: src/Makefile
+	cd src && ${MAKE} graph2prom
+
 src/json-pp: src/Makefile
 	cd src && ${MAKE} json-pp
 
@@ -57,7 +60,7 @@ dist/nodes2grafana.txz:
 
 install: install-bin install-export install-crontab install-dashboard
 
-install-bin: src/nodes2prom src/json-pp etc/nodes2prometheus.sh %%HEAP_INSTALL_FILES%%
+install-bin: src/nodes2prom src/graph2prom src/json-pp etc/nodes2prometheus.sh %%HEAP_INSTALL_FILES%%
 	%%INSTALL_BIN_CMD%% $> %%BIN_DIR%%
 
 install-export:

conf/substitutions.conf.Linux

@@ -5,7 +5,7 @@ EXPORT_DIR_INSTALL	install -d -o nobody -g nobody -m 1755
 #EXPORT_DIR_INSTALL	mkdir -p
 RUN_SML			/usr/lib/smlnj/bin/.run-sml
 HEAP_SUFFIX		x86-linux
-HEAP_INSTALL_FILES	src/nodes2prom.x86-linux src/json-pp.x86-linux
+HEAP_INSTALL_FILES	src/nodes2prom.x86-linux src/graph2prom.x86-linux src/json-pp.x86-linux
 BIN_DIR			/usr/local/bin
 # keep this:
 ETC_DIR			/etc

conf/substitutions.conf.franchise

@@ -1,5 +1,6 @@
 # siehe auch ../dashboard/groups.prom.in zwecks Anpassung fuer die jeweilige Community
 NODES_URL		https://map.ffdo.de/data/nodes.json
+GRAPH_URL		https://map.ffdo.de/data/graph.json
 PROM_SEPERATOR		_
 #PROM_SEPERATOR		:
 PROM_PREFIX		ffdo_nodes
@@ -7,9 +8,12 @@ PROM_SUMMARY_PREFIX	ffdo_nodes_summary
 PROM_INFO_PREFIX	ffdo_nodes_info
 PROM_STATS_PREFIX	ffdo_nodes_detail
 PROM_GROUPS_PREFIX	ffdo_groups
+PROM_GRAPH_PREFIX	ffdo_graph
+DASHBOARD_PATH		/dashboard/file/
 DASHBOARD_PREFIX	FF-DO-
 COMMUNITY_BRIEF		FF-DO
 COMMUNITY_FULL		Freifunk Dortmund
 COMMUNITY_URL		http://www.freifunk-dortmund.de/
 MAP_NODE_URL		https://map.ffdo.de/meshviewer/#!v:m;n:
+MAP_GRAPH_URL		https://map.ffdo.de/meshviewer/#!v:m;l:
 N2G_COMMUNITY_URL	http://url.free.de/ffdo/Technik/Netzinfrastruktur/Status

dist/Makefile.in

@@ -7,6 +7,7 @@ DISTDIR=	tmp/${DISTTARGET}
 DISTEXCLUDE=	--exclude=${DISTTARGET}/src/Makefile \
 		--exclude=${DISTTARGET}/src/promconfig.sml \
 		--exclude=${DISTTARGET}/src/nodes2prom \
+		--exclude=${DISTTARGET}/src/graph2prom \
 		--exclude=${DISTTARGET}/src/json-pp \
 		--exclude=${DISTTARGET}/src/.cm \
 		--exclude=${DISTTARGET}/src/JSON/.cm \

etc/nodes2prometheus.sh.in

@@ -1,27 +1,49 @@
 #!/bin/sh
 
-# Einzeldaten und Summen von nodes.json als labeled metrics for prometheus
+# Einzeldaten und Summen von nodes.json sowie Verbindungsdaten aus graph.json
+# als labeled metrics fuer prometheus
 
 ME="`basename $0`"
 
-PROM_PREFIX=%%PROM_PREFIX%%
 PROMDIR=%%EXPORT_DIR%%
 [ -d $PROMDIR -a -w $PROMDIR ] || PROMDIR=/tmp
-PROMFILE=$PROMDIR/$PROM_PREFIX.prom
 
+NODES_PROM=$PROMDIR/%%PROM_PREFIX%%.prom
 NODES_URL="%%NODES_URL%%"
-NODESFILE=/tmp/nodes.json.$$
+NODES_FILE=/tmp/nodes.json.$$
 NODES2PROM=%%BIN_DIR%%/nodes2prom
 [ -f $NODES2PROM -a -x $NODES2PROM ] || NODES2PROM=./nodes2prom
 
+GRAPH_PROM=$PROMDIR/%%PROM_GRAPH_PREFIX%%.prom
+GRAPH_URL="%%GRAPH_URL%%"
+GRAPH_FILE=/tmp/graph.json.$$
+GRAPH2PROM=%%BIN_DIR%%/graph2prom
+[ -f $GRAPH2PROM -a -x $GRAPH2PROM ] || GRAPH2PROM=./graph2prom
+
+result=0
+
 if timeout -s HUP -k 2 50 \
-   %%FETCH_CMD%% $NODESFILE "$NODES_URL"
+   %%FETCH_CMD%% $NODES_FILE "$NODES_URL"
 then
-	$NODES2PROM $NODESFILE > $PROMFILE.new && \
-	mv $PROMFILE.new $PROMFILE
-	rm -f $NODESFILE
+	$NODES2PROM $NODES_FILE > $NODES_PROM.new && \
+	mv $NODES_PROM.new $NODES_PROM
+	rm -f $NODES_FILE
 else
 	logger -t "$ME" "failed to fetch $NODES_URL"
-	rm -f $NODESFILE
-	exit 1
+	rm -f $NODES_FILE
+	result=1
+fi
+
+if timeout -s HUP -k 2 50 \
+   %%FETCH_CMD%% $GRAPH_FILE "$GRAPH_URL"
+then
+	$GRAPH2PROM $GRAPH_FILE > $GRAPH_PROM.new && \
+	mv $GRAPH_PROM.new $GRAPH_PROM
+	rm -f $GRAPH_FILE
+else
+	logger -t "$ME" "failed to fetch $GRAPH_URL"
+	rm -f $GRAPH_FILE
+	result=1
 fi
+
+return $result

src/Makefile.in

@@ -1,10 +1,11 @@
 BINFILES=
 BINFILES+=	nodes2prom nodes2prom.%%HEAP_SUFFIX%%
+BINFILES+=	graph2prom graph2prom.%%HEAP_SUFFIX%%
 BINFILES+=	json-pp json-pp.%%HEAP_SUFFIX%%
 
 CLEANFILES=	promconfig.sml .cm JSON/.cm
 
-all:	promconfig.sml nodes2prom json-pp
+all:	promconfig.sml nodes2prom graph2prom json-pp
 
 promconfig.sml: promconfig.sml.in
 	../conf/substitute.sh $> $@
@@ -15,6 +16,12 @@ nodes2prom: nodes2prom.%%HEAP_SUFFIX%%
 nodes2prom.%%HEAP_SUFFIX%%: promconfig.sml nodes2prom.cm nodes2prom.sml json_lib
 	ml-build nodes2prom.cm Main.main nodes2prom
 
+graph2prom: graph2prom.%%HEAP_SUFFIX%%
+	heap2exec graph2prom.%%HEAP_SUFFIX%% graph2prom || { ../conf/substitute.sh runheap.in $@ && chmod +x $@ ; }
+
+graph2prom.%%HEAP_SUFFIX%%: promconfig.sml graph2prom.cm graph2prom.sml json_lib
+	ml-build graph2prom.cm Main.main graph2prom
+
 json-pp: json-pp.%%HEAP_SUFFIX%%
 	heap2exec json-pp.%%HEAP_SUFFIX%% json-pp || { ../conf/substitute.sh runheap.in $@ && chmod +x $@ ; }
 

src/graph2prom.cm

@@ -0,0 +1,8 @@
+library
+	structure Main
+is
+	$/basis.cm
+	$/smlnj-lib.cm
+	JSON/json-lib.cm
+	promconfig.sml
+	graph2prom.sml

src/graph2prom.sml

@@ -0,0 +1,120 @@
+structure Main :
+sig
+	val main: string * string list -> OS.Process.status
+end =
+struct
+	structure J = JSON
+	structure JP = JSONParser
+	structure JU = JSONUtil
+
+	structure PC = PromConfig
+	val link_prefix = PC.link_prefix
+	val mesh_prefix = PC.mesh_prefix
+
+	(* val timestamp = LargeInt.toString (Time.toMilliseconds (Time.now ())) *)
+
+	val newline = String.str #"\n"
+	val link_header =
+		"# HELP " ^ link_prefix ^ " link quality between two nodes" ^ newline ^
+		"# TYPE " ^ link_prefix ^ " gauge" ^ newline
+	val mesh_header =
+		"# HELP " ^ mesh_prefix ^ " path length between two nodes in a mesh" ^ newline ^
+		"# TYPE " ^ mesh_prefix ^ " gauge" ^ newline
+
+	fun prom2string (metric, labels, scalar) =
+		let fun esc #"\"" = "\\\""
+		      | esc #"\\" = "\\\\"
+		      | esc #"\n" = "\\n"
+		      | esc c = str c
+		in metric ^
+		   (ListFormat.fmt {init = "{", sep= ",", final = "}",
+				    fmt = fn (label, value) => label ^ "=\"" ^ (String.translate esc value) ^ "\""}
+				   labels) ^ " " ^
+		   scalar ^ (* " " ^
+		   timestamp ^ *) newline
+		end
+
+	(* exnMessage from JSONUtil - slightly extended *)
+
+	fun v2bs' (J.ARRAY []) = "[]"
+	  | v2bs' (J.ARRAY _) = "[...]"
+	  | v2bs' (J.BOOL v) = Bool.toString v
+	  | v2bs' (J.FLOAT v) = Real.toString v
+	  | v2bs' (J.INT v) = IntInf.toString v
+	  | v2bs' J.NULL = "null"
+	  | v2bs' (J.OBJECT []) = "{}"
+	  | v2bs' (J.OBJECT _) = "{...}"
+	  | v2bs' (J.STRING v) = v
+
+	fun v2bs (J.ARRAY []) = "[]"
+	  | v2bs (J.ARRAY vl) =
+		ListFormat.fmt { init = "[", sep = ",", final = "]", fmt = v2bs' } vl
+	  | v2bs (J.OBJECT []) = "{}"
+	  | v2bs (J.OBJECT fl) =
+		ListFormat.fmt { init = "{", sep = ",", final = "}",
+				 fmt = fn (n, v) => n ^ ":" ^ (v2bs' v) } fl
+	  | v2bs v = v2bs' v
+
+	fun json_handler logstring exn =
+		(TextIO.output (TextIO.stdErr, 
+		 "json_handler(" ^ logstring ^ "): " ^ (JU.exnMessage exn) ^ ": " ^
+		 (v2bs (case exn
+			 of JU.NotBool v => v
+			  | JU.NotInt v => v
+			  | JU.NotNumber v => v
+		 	  | JU.NotString v => v
+		 	  | JU.NotObject v => v
+		 	  | JU.FieldNotFound(v, fld) => v
+		 	  | JU.NotArray v => v
+			  | _ => raise exn)) ^
+		 "\n") ; ())
+
+	fun get_version obj = JU.asInt (JU.lookupField obj "version")
+
+	fun get_fields (J.OBJECT flds) = flds
+	  | get_fields v = raise Fail ("value is not an object (" ^ (v2bs v) ^ ")")
+
+	fun get_list (J.OBJECT flds) = map #2 flds
+	  | get_list (J.ARRAY nds) = nds
+	  | get_list v = raise Fail ("value does not contain a list (" ^ (v2bs v) ^ ")")
+
+	fun get_nodes obj = get_list (JU.lookupField (JU.lookupField obj "batadv") "nodes")
+	fun get_links obj = get_list (JU.lookupField (JU.lookupField obj "batadv") "links")
+
+	fun extract_node obj = { id = JU.asString (JU.lookupField obj "id"),
+				 node_id = JU.asString (JU.lookupField obj "node_id") }
+
+	fun extract_link obj = { source = JU.asInt (JU.lookupField obj "source"),
+				 target = JU.asInt (JU.lookupField obj "target"),
+				 tq = JU.asNumber (JU.lookupField obj "tq"),
+				 vpn = JU.asBool (JU.lookupField obj "vpn") }
+
+	fun link2prom nodes_id_vector { source, target, tq, vpn } =
+		let val source_id = Vector.sub (nodes_id_vector, source)
+		    val target_id = Vector.sub (nodes_id_vector, target)
+		in prom2string (link_prefix,
+				[("source", source_id), ("target", target_id), ("vpn", Bool.toString vpn)],
+				Real.toString tq)
+		end
+
+	fun complain (p, s) =
+		(TextIO.output (TextIO.stdErr, concat [p, ": ", s, "\n"]);
+		 OS.Process.failure)
+
+	fun main (p, [inf]) =
+		(let val json = JP.parseFile inf
+		     val _ = get_version json = 1 orelse raise Fail "version must be 1"
+		     val nodes_json = get_nodes json
+				      handle exn => (json_handler "get_nodes" exn ; raise Fail "get_nodes")
+		     val nodes_vector = Vector.fromList (map extract_node nodes_json)
+		     val nodes_id_vector = Vector.map #node_id nodes_vector
+		     val links_json = get_links json
+				      handle exn => (json_handler "get_links" exn ; raise Fail "get_links")
+		     val links_extract = map extract_link links_json
+		     val links_prom = map (link2prom nodes_id_vector) links_extract
+		 in (print link_header ;
+		     app print links_prom ;
+		     OS.Process.success)
+		 end handle e => complain (p, exnMessage e))
+	  | main (p, _) = complain (p, "usage: " ^ p ^ " nodes.json")
+end

src/promconfig.sml.in

@@ -5,4 +5,7 @@ struct
 	val summary_prefix = "%%PROM_SUMMARY_PREFIX%%"
 	val info_prefix = "%%PROM_INFO_PREFIX%%"
 	val stats_prefix = "%%PROM_STATS_PREFIX%%" ^ seperator
+	val graph_prefix = "%%PROM_GRAPH_PREFIX%%" ^ seperator
+	val link_prefix = graph_prefix ^ "link"
+	val mesh_prefix = graph_prefix ^ "mesh"
 end

test/Makefile.in

@@ -1,6 +1,6 @@
 all:	test
 
-test:	tmp tmp/%%PROM_PREFIX%%.prom tmp/nodes.json.pp
+test:	tmp tmp/%%PROM_PREFIX%%.prom tmp/nodes.json.pp tmp/%%PROM_GRAPH_PREFIX%%.prom tmp/graph.json.pp
 
 tmp:
 	mkdir $@
@@ -9,12 +9,23 @@ tmp/%%PROM_PREFIX%%.prom: ../src/nodes2prom tmp/nodes.json
 	$> > $@.new && \
 	mv $@.new $@
 
+tmp/nodes.json:
+	%%FETCH_CMD%% $@.tmp "%%NODES_URL%%" && \
+	mv $@.tmp $@
+
 tmp/nodes.json.pp: ../src/json-pp tmp/nodes.json
 	$> $@
 
-tmp/nodes.json:
-	%%FETCH_CMD%% $@.tmp "%%NODES_URL%%" && \
+tmp/%%PROM_GRAPH_PREFIX%%.prom: ../src/graph2prom tmp/graph.json
+	$> > $@.new && \
+	mv $@.new $@
+
+tmp/graph.json:
+	%%FETCH_CMD%% $@.tmp "%%GRAPH_URL%%" && \
 	mv $@.tmp $@
 
+tmp/graph.json.pp: ../src/json-pp tmp/graph.json
+	$> $@
+
 clean:
 	rm -rf tmp