nodes2prom.sml 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299
  1. structure Main :
  2. sig
  3. val main: string * string list -> OS.Process.status
  4. end =
  5. struct
  6. structure J = JSON
  7. structure JP = JSONParser
  8. structure JU = JSONUtil
  9. structure PC = PromConfig
  10. val prom_prefix = PC.prom_prefix
  11. val summary_prefix = PC.summary_prefix
  12. val info_prefix = PC.info_prefix
  13. val stats_prefix = PC.stats_prefix
  14. val timestamp = ref (LargeInt.toString (Time.toMilliseconds (Time.now ())))
  15. val newline = String.str #"\n"
  16. val summary_header = [
  17. "# HELP " ^ summary_prefix ^ " sum of nodes and of some attributes" ^ newline ^
  18. "# TYPE " ^ summary_prefix ^ " gauge" ^ newline ]
  19. val info_header = [
  20. "# HELP " ^ info_prefix ^ " static attributes of a node" ^ newline ^
  21. "# TYPE " ^ info_prefix ^ " counter" ^ newline,
  22. "# HELP " ^ info_prefix ^ "_fw_version firmware version as an integer value" ^ newline ^
  23. "# TYPE " ^ info_prefix ^ "_fw_version gauge" ^ newline,
  24. "# HELP " ^ info_prefix ^ "_lastseen metric for the lastseen attribute of a node" ^ newline ^
  25. "# TYPE " ^ info_prefix ^ "_lastseen gauge" ^ newline,
  26. "# HELP " ^ info_prefix ^ "_firstseen metric for the firtseen attribute of a node" ^ newline ^
  27. "# TYPE " ^ info_prefix ^ "_firstseen gauge" ^ newline ]
  28. fun stats_header key =
  29. "# HELP " ^ stats_prefix ^ key ^ " metric for the " ^ key ^ " attribute of a node" ^ newline ^
  30. "# TYPE " ^ stats_prefix ^ key ^ " gauge" ^ newline
  31. fun prom2string (metric, labels, scalar) =
  32. let fun esc #"\"" = "\\\""
  33. | esc #"\\" = "\\\\"
  34. | esc #"\n" = "\\n"
  35. | esc c = str c
  36. in metric ^
  37. (ListFormat.fmt {init = "{", sep= ",", final = "}",
  38. fmt = fn (label, value) => label ^ "=\"" ^ (String.translate esc value) ^ "\""}
  39. labels) ^ " " ^
  40. scalar ^ " " ^
  41. (!timestamp) ^ newline
  42. end
  43. (* exnMessage from JSONUtil - slightly extended *)
  44. fun v2bs' (J.ARRAY []) = "[]"
  45. | v2bs' (J.ARRAY _) = "[...]"
  46. | v2bs' (J.BOOL v) = Bool.toString v
  47. | v2bs' (J.FLOAT v) = Real.toString v
  48. | v2bs' (J.INT v) = IntInf.toString v
  49. | v2bs' J.NULL = "null"
  50. | v2bs' (J.OBJECT []) = "{}"
  51. | v2bs' (J.OBJECT _) = "{...}"
  52. | v2bs' (J.STRING v) = v
  53. fun v2bs (J.ARRAY []) = "[]"
  54. | v2bs (J.ARRAY vl) =
  55. ListFormat.fmt { init = "[", sep = ",", final = "]", fmt = v2bs' } vl
  56. | v2bs (J.OBJECT []) = "{}"
  57. | v2bs (J.OBJECT fl) =
  58. ListFormat.fmt { init = "{", sep = ",", final = "}",
  59. fmt = fn (n, v) => n ^ ":" ^ (v2bs' v) } fl
  60. | v2bs v = v2bs' v
  61. fun json_handler logstring exn =
  62. (TextIO.output (TextIO.stdErr,
  63. "json_handler(" ^ logstring ^ "): " ^ (JU.exnMessage exn) ^ ": " ^
  64. (v2bs (case exn
  65. of JU.NotBool v => v
  66. | JU.NotInt v => v
  67. | JU.NotNumber v => v
  68. | JU.NotString v => v
  69. | JU.NotObject v => v
  70. | JU.FieldNotFound(v, fld) => v
  71. | JU.NotArray v => v
  72. | _ => raise exn)) ^
  73. "\n") ; ())
  74. fun firmware2int firmware =
  75. foldl (fn (s, i) => 100 * i + valOf (Int.fromString s))
  76. 0
  77. (String.tokens (fn c => c = #".") firmware)
  78. fun timestring2time ts = (* "timestamp" : "2017-10-01T18:49:11+02:00" *)
  79. let val tsfields = String.tokens (fn c => c = #"-" orelse c = #"T" orelse c = #":" orelse c = #"+") ts
  80. val tsints = map (valOf o Int.fromString) tsfields
  81. fun int2month i =
  82. let open Date
  83. val months = [ Jan, Feb, Mar, Apr, May, Jun, Jul, Aug, Sep, Oct, Nov, Dec ]
  84. in List.nth (months, i - 1)
  85. end
  86. val tsdate =
  87. Date.date {
  88. year = List.nth (tsints, 0),
  89. month = int2month (List.nth (tsints, 1)),
  90. day = List.nth (tsints, 2),
  91. hour = List.nth (tsints, 3),
  92. minute = List.nth (tsints, 4),
  93. second = List.nth (tsints, 5),
  94. (* Basis/date.html:
  95. ... offset reports time zone information as the amount of time west [sic!] of UTC. *)
  96. offset = SOME (Time.fromSeconds (~1 * 60 * Int.toLarge ((60 * List.nth (tsints, 6) + List.nth (tsints, 7))))) }
  97. in Date.toTime tsdate
  98. end
  99. fun get_timestamp obj =
  100. let val tsstring = JU.asString (JU.lookupField obj "timestamp")
  101. val tstime = timestring2time tsstring
  102. in timestamp := LargeInt.toString (Time.toMilliseconds tstime)
  103. end
  104. handle exn => json_handler ("get_timestamp") exn
  105. fun get_fields (J.OBJECT flds) = flds
  106. | get_fields v = raise Fail ("value is not an object (" ^ (v2bs v) ^ ")")
  107. fun get_list (J.OBJECT flds) = map #2 flds
  108. | get_list (J.ARRAY nds) = nds
  109. | get_list v = raise Fail ("value does not contain a list (" ^ (v2bs v) ^ ")")
  110. fun get_nodes obj = get_list (JU.lookupField obj "nodes")
  111. fun extract node =
  112. let val nodeinfo = JU.lookupField node "nodeinfo"
  113. val hostname = JU.asString (JU.lookupField nodeinfo "hostname")
  114. val node_id = JU.asString (JU.lookupField nodeinfo "node_id")
  115. local val interfaces' = get_fields (JU.lookupField (JU.lookupField (JU.lookupField (JU.lookupField nodeinfo "network") "mesh") "bat0") "interfaces")
  116. val interfaces'' = map (fn i => (#1 i) ^ ":" ^ (Int.toString (length (get_list (#2 i))))) interfaces'
  117. val interfaces''' = ListMergeSort.sort String.> interfaces''
  118. in val interfaces = ListFormat.fmt { init = "", sep = ",", final = "", fmt = (fn s => s) } interfaces'''
  119. end
  120. val location = JU.findField nodeinfo "location"
  121. val (longitude, latitude) =
  122. case location
  123. of SOME l => (SOME (JU.asNumber (JU.lookupField l "longitude")),
  124. SOME (JU.asNumber (JU.lookupField l "latitude")))
  125. | NONE => (NONE, NONE)
  126. val software = JU.lookupField nodeinfo "software"
  127. val autoupdater = JU.asBool (JU.lookupField (JU.lookupField software "autoupdater") "enabled")
  128. val fastd = JU.asBool (JU.lookupField (JU.lookupField software "fastd") "enabled")
  129. val firmware = JU.asString (JU.lookupField (JU.lookupField software "firmware") "release")
  130. val base = JU.asString (JU.lookupField (JU.lookupField software "firmware") "base")
  131. val model = JU.asString (JU.lookupField (JU.lookupField nodeinfo "hardware") "model")
  132. val statistics = JU.lookupField node "statistics"
  133. val clients = JU.asInt (JU.lookupField statistics "clients")
  134. val gateway = JU.asString (JU.lookupField statistics "gateway")
  135. val loadavg = JU.asNumber (JU.lookupField statistics "loadavg")
  136. val memory_usage = JU.asNumber (JU.lookupField statistics "memory_usage")
  137. val rootfs_usage = JU.asNumber (JU.lookupField statistics "rootfs_usage")
  138. val uptime = JU.asNumber (JU.lookupField statistics "uptime")
  139. val traffic = JU.lookupField statistics "traffic"
  140. val tx = SOME (JU.asNumber (JU.lookupField (JU.lookupField traffic "tx") "bytes"))
  141. handle exn => NONE
  142. val rx = SOME (JU.asNumber (JU.lookupField (JU.lookupField traffic "rx") "bytes"))
  143. handle exn => NONE
  144. val forward = SOME (JU.asNumber (JU.lookupField (JU.lookupField traffic "forward") "bytes"))
  145. handle exn => NONE
  146. val mgmt_tx = SOME (JU.asNumber (JU.lookupField (JU.lookupField traffic "mgmt_tx") "bytes"))
  147. handle exn => NONE
  148. val mgmt_rx = SOME (JU.asNumber (JU.lookupField (JU.lookupField traffic "mgmt_rx") "bytes"))
  149. handle exn => NONE
  150. val flags = JU.lookupField node "flags"
  151. val online = JU.asBool (JU.lookupField flags "online")
  152. val uplink = JU.asBool (JU.lookupField flags "uplink")
  153. val lastseen = timestring2time (JU.asString (JU.lookupField node "lastseen"))
  154. val firstseen = timestring2time (JU.asString (JU.lookupField node "firstseen"))
  155. in SOME
  156. ({ clients = clients,
  157. online = online,
  158. uplink = uplink },
  159. { hostname = hostname,
  160. online = online,
  161. node_id = node_id,
  162. interfaces = interfaces,
  163. longitude = longitude,
  164. latitude = latitude,
  165. autoupdater = autoupdater,
  166. gateway = gateway,
  167. fastd = fastd,
  168. firmware = firmware,
  169. base = base,
  170. model = model,
  171. lastseen = lastseen,
  172. firstseen = firstseen },
  173. { hostname = hostname,
  174. node_id = node_id,
  175. clients = clients,
  176. online = online,
  177. uplink = uplink,
  178. loadavg = loadavg,
  179. memory_usage = memory_usage,
  180. rootfs_usage = rootfs_usage,
  181. uptime = uptime,
  182. tx = tx,
  183. rx = rx,
  184. forward = forward,
  185. mgmt_tx = mgmt_tx,
  186. mgmt_rx = mgmt_rx }
  187. )
  188. end
  189. handle exn => (json_handler "extract" exn ; NONE)
  190. fun nodes_summary node_summaries =
  191. let fun summate (node_summary, (hcount, ccount, ocount, ucount)) =
  192. let val { clients, online, uplink } = node_summary
  193. in (hcount + 1,
  194. ccount + clients,
  195. if online then ocount + 1 else ocount,
  196. if uplink then ucount + 1 else ucount)
  197. end
  198. val (hc, cc, oc, uc) = foldl summate (0, 0, 0, 0) node_summaries
  199. val scalars = map Int.toString [hc, cc, oc, uc]
  200. val labelvalues = ["nodes", "clients", "online", "uplink"]
  201. val items = ListPair.zip (labelvalues, scalars)
  202. in foldr (fn ((count, scalar), result) =>
  203. (prom2string (summary_prefix, [("count", count)], scalar)) ^
  204. result)
  205. ""
  206. items
  207. end
  208. fun node_info { hostname, online, node_id, interfaces, longitude, latitude, autoupdater, fastd, gateway, firmware, base, model, lastseen, firstseen } =
  209. let val minitems = [
  210. ("hostname", hostname),
  211. ("node_id", node_id)]
  212. val allitems = minitems @ [
  213. ("interfaces", interfaces),
  214. ("longitude", case longitude
  215. of NONE => "NaN"
  216. | SOME l => Real.toString l),
  217. ("latitude", case latitude
  218. of NONE => "NaN"
  219. | SOME l => Real.toString l),
  220. ("autoupdater", Bool.toString autoupdater),
  221. ("fastd", Bool.toString fastd),
  222. ("gateway", gateway),
  223. ("firmware", firmware),
  224. ("base", base),
  225. ("model", model)]
  226. in [("!info", prom2string (info_prefix, allitems, "1")),
  227. ("fw_version", prom2string (info_prefix ^ "_fw_version", minitems, Int.toString (firmware2int firmware))),
  228. ("lastseen", prom2string (info_prefix ^ "_lastseen", minitems, LargeInt.toString (Time.toSeconds lastseen))),
  229. ("firstseen", prom2string (info_prefix ^ "_firstseen", minitems, LargeInt.toString (Time.toSeconds firstseen)))]
  230. end
  231. fun node_stats { hostname, node_id, clients, online, uplink, loadavg, memory_usage, uptime, rootfs_usage, tx, rx, forward, mgmt_tx, mgmt_rx } =
  232. let val items =
  233. [SOME ("clients", Int.toString clients),
  234. SOME ("online", if online then "1" else "0"),
  235. SOME ("uplink", if uplink then "1" else "0"),
  236. SOME ("loadavg", Real.toString loadavg),
  237. SOME ("memory_usage", Real.toString memory_usage),
  238. SOME ("rootfs_usage", Real.toString rootfs_usage),
  239. SOME ("uptime", Real.toString(uptime)),
  240. case tx of SOME tx' => SOME ("tx", Real.toString(tx')) | NONE => NONE,
  241. case rx of SOME rx' => SOME ("rx", Real.toString(rx')) | NONE => NONE,
  242. case forward of SOME forward' => SOME ("forward", Real.toString(forward')) | NONE => NONE,
  243. case mgmt_tx of SOME mgmt_tx' => SOME ("mgmt_tx", Real.toString(mgmt_tx')) | NONE => NONE,
  244. case mgmt_rx of SOME mgmt_rx' => SOME ("mgmt_rx", Real.toString(mgmt_rx')) | NONE => NONE]
  245. fun item2prom (key, value) = (key, stats_prefix ^ key ^ "{hostname=\"" ^ hostname ^ "\",node_id=\"" ^ node_id ^ "\"} " ^ value ^ " " ^ !timestamp ^ "\n")
  246. in List.mapPartial (fn (SOME i) => SOME (item2prom i) | NONE => NONE) items
  247. end
  248. fun complain (p, s) =
  249. (TextIO.output (TextIO.stdErr, concat [p, ": ", s, "\n"]);
  250. OS.Process.failure)
  251. fun main (p, [inf]) =
  252. (let val json = JP.parseFile inf
  253. val _ = get_timestamp json
  254. val nodes_json = get_nodes json
  255. handle exn => (json_handler "get_nodes" exn ; raise Fail "get_nodes")
  256. val nodes_extract = List.mapPartial extract nodes_json
  257. val summaries = map #1 nodes_extract
  258. val infos = map #2 nodes_extract
  259. val (infokeys', infos) =
  260. ListPair.unzip (ListMergeSort.sort
  261. (fn ((k1,_), (k2,_)) => String.> (k1, k2))
  262. (List.concat (List.map node_info infos)))
  263. val infokeys = ListMergeSort.uniqueSort String.compare infokeys'
  264. val stats' = List.filter (fn stts => #online stts) (map #3 nodes_extract)
  265. val (statskeys', stats) =
  266. ListPair.unzip (ListMergeSort.sort
  267. (fn ((k1,_), (k2,_)) => String.> (k1, k2))
  268. (List.concat (List.map node_stats stats')))
  269. val statskeys = ListMergeSort.uniqueSort String.compare statskeys'
  270. in (app print (List.concat [summary_header, info_header, map stats_header statskeys]) ;
  271. print (nodes_summary summaries) ;
  272. app print infos ;
  273. app print stats ;
  274. OS.Process.success)
  275. end handle e => complain (p, exnMessage e))
  276. | main (p, _) = complain (p, "usage: " ^ p ^ " nodes.json")
  277. end