A Lua script to check the health of Devuan Linux package mirrors. The master repo is at https://sledjhamr.org/cgit/apt-panopticon/ and the master issues tracker is at https://sledjhamr.org/mantisbt/project_page.php?project_id=13 https://sledjhamr.org/cgit/apt-panopticon/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

829 lines
26 KiB

  1. local APT = {}
  2. -- https://oss.oetiker.ch/rrdtool/prog/rrdlua.en.html
  3. APT.rrd = require 'rrd'
  4. APT.protocols = {"ftp", "http", "https", "rsync"}
  5. APT.tests = {'raw', 'Integrity', 'Protocol', 'Redirects', 'Updated', 'URLSanity', 'Speed'}
  6. APT.releases = {"ascii", "beowulf", "chimaera", "ceres"}
  7. APT.subRels = {'backports', 'proposed-updates', 'security', 'updates'}
  8. APT.notExist =
  9. {
  10. 'chimaera-backports',
  11. 'chimaera-security',
  12. 'ceres-backports', -- These will never exist, it's our code name for the testing suite.
  13. 'ceres-proposed-updates',
  14. 'ceres-updates',
  15. 'ceres-security',
  16. }
  17. APT.verbosity = -1
  18. APT.origin = false
  19. APT.redir = false
  20. APT.keep = false
  21. APT.IPv46 = ''
  22. APT.options =
  23. {
  24. bandwidth =
  25. {
  26. typ = "number",
  27. help = '\n 0 = low - HTTP tests for all IPs of all mirrors (HEAD tests), but not URL sanity tests.\n' ..
  28. ' 1 = medium - Also HTTP(S) redirect tests, other protocols, download Release files over HTTP, and check them.\n' ..
  29. ' 2 = high - Also download Packages.xz files that changed, unpack and check them.\n' ..
  30. ' Also download and check InRelease files.\n' ..
  31. ' Pick a few small packages, download them, check their SHA512.\n' ..
  32. ' 3 = more - Also pick more files and packages, some for each release / arch.\n' ..
  33. ' 4 = all - Do absolutely everything.\n' ..
  34. ' Actually download some Contents files, and some more Packages.xz, and package files.',
  35. value = 2,
  36. },
  37. referenceSite =
  38. {
  39. typ = "string",
  40. help = "The mirror that is used as a reference, coz the others sync to it.",
  41. value = "pkgmaster.devuan.org",
  42. },
  43. roundRobin =
  44. {
  45. typ = "string",
  46. help = "The round robin DNS name.",
  47. value = "deb.devuan.org",
  48. },
  49. tests =
  50. {
  51. typ = "table",
  52. help = "A list of the tests, see the documentation for details.",
  53. value =
  54. {
  55. "IPv4",
  56. "IPv6",
  57. -- "ftp",
  58. "http",
  59. "https",
  60. -- "rsync",
  61. "DNSRR",
  62. "Protocol",
  63. "Redirects",
  64. "URLSanity",
  65. "Integrity",
  66. "Updated",
  67. },
  68. },
  69. maxtime =
  70. {
  71. typ = "number",
  72. help = "",
  73. value = 240,
  74. },
  75. timeout =
  76. {
  77. typ = "number",
  78. help = "",
  79. value = 5,
  80. },
  81. timeouts =
  82. {
  83. typ = "number",
  84. help = "",
  85. value = 3,
  86. },
  87. refresh =
  88. {
  89. typ = "number",
  90. help = "",
  91. value = 300,
  92. },
  93. retries =
  94. {
  95. typ = "number",
  96. help = "",
  97. value = 2,
  98. },
  99. reports =
  100. {
  101. typ = "table",
  102. help = "A list of the reports generated.",
  103. value =
  104. {
  105. "RRD", -- RRD has to be before web, coz web creates a graph from the RRD data.
  106. "email-web",
  107. -- "Prometheus",
  108. },
  109. },
  110. cgi =
  111. {
  112. typ = "boolean",
  113. help = "",
  114. value = false,
  115. },
  116. }
  117. APT.args = {}
  118. APT.parseArgs = function(args)
  119. APT.args = args
  120. local arg = {}
  121. local sendArgs = ""
  122. -- A special test to disable IPv6 tests if IPv6 isn't available.
  123. if 1 == APT.exe('ip -6 addr | grep inet6 | grep " global"'):Do().status then
  124. table.insert(args, '--tests=-IPv6')
  125. end
  126. if 0 ~= #(args) then
  127. local option = ""
  128. for i, a in pairs(args) do
  129. if ("--help" == a) or ("-h" == a) then
  130. print("Check the health of Devuan Linux package mirrors.")
  131. for k, v in pairs(APT.options) do
  132. print("")
  133. print('--' .. k .. ' ' .. v.help)
  134. end
  135. os.exit()
  136. elseif "--version" == a then
  137. print("apt-panopticon version 0.2 alpha")
  138. os.exit()
  139. elseif "-v" == a then
  140. APT.verbosity = APT.verbosity + 1
  141. sendArgs = sendArgs .. a .. " "
  142. elseif "-q" == a then
  143. APT.verbosity = -1
  144. sendArgs = sendArgs .. a .. " "
  145. elseif ("-4" == a) or ("-6" == a) then
  146. APT.IPv46 = a
  147. sendArgs = sendArgs .. a .. " "
  148. elseif "-k" == a then
  149. APT.keep = true
  150. sendArgs = sendArgs .. a .. " "
  151. elseif "-o" == a then
  152. APT.origin = true
  153. -- Not adding to sendArgs.
  154. elseif "-r" == a then
  155. APT.redir = true
  156. sendArgs = sendArgs .. a .. " "
  157. elseif "--cgi" == a then
  158. APT.options.cgi.value = true
  159. sendArgs = sendArgs .. a .. " "
  160. elseif "--low" == a then
  161. APT.options.bandwidth.value = 0
  162. APT.options.timeout.value = 2
  163. APT.options.timeouts.value = 1
  164. APT.options.retries.value = 1
  165. sendArgs = sendArgs .. a .. " "
  166. elseif "--medium" == a then
  167. APT.options.bandwidth.value = 1
  168. sendArgs = sendArgs .. a .. " "
  169. elseif "--high" == a then
  170. APT.options.bandwidth.value = 2
  171. sendArgs = sendArgs .. a .. " "
  172. elseif "--more" == a then
  173. APT.options.bandwidth.value = 3
  174. sendArgs = sendArgs .. a .. " "
  175. elseif "--all" == a then
  176. APT.options.bandwidth.value = 4
  177. sendArgs = sendArgs .. a .. " "
  178. elseif "--" == a:sub(1, 2) then
  179. local s, e = a:find("=")
  180. if nil == s then e = -1 end
  181. option = a:sub(3, e - 1)
  182. local o = APT.options[option]
  183. if nil == o then
  184. print("Unknown option --" .. option)
  185. option = ""
  186. else
  187. option = a
  188. sendArgs = sendArgs .. a .. " "
  189. local s, e = a:find("=")
  190. if nil == s then e = 0 end
  191. option = a:sub(3, e - 1)
  192. if "table" == APT.options[option].typ then
  193. local result = {}
  194. for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
  195. local f = t:sub(1, 1)
  196. local n = t:sub(2, -1)
  197. if ("+" ~= f) and ("-" ~= f) then
  198. table.insert(result, t)
  199. end
  200. end
  201. if 0 ~= #result then
  202. APT.options[option].value = result
  203. else
  204. for t in (a:sub(e + 1) .. ","):gmatch("([+%-]?%w*),") do
  205. local f = t:sub(1, 1)
  206. local n = t:sub(2, -1)
  207. if "+" == f then
  208. table.insert(APT.options[option].value, n)
  209. elseif "-" == f then
  210. local r = {}
  211. for i, k in pairs(APT.options[option].value) do
  212. if k ~= n then table.insert(r, k) end
  213. end
  214. APT.options[option].value = r
  215. end
  216. end
  217. end
  218. else
  219. APT.options[option].value = a:sub(e + 1, -1)
  220. end
  221. option = ""
  222. end
  223. elseif "-" == a:sub(1, 1) then
  224. print("Unknown option " .. a)
  225. else
  226. table.insert(arg, a)
  227. end
  228. end
  229. end
  230. return arg, sendArgs
  231. end
  232. --[[ Ordered table iterator, allow to iterate on the natural order of the keys of a table.
  233. From http://lua-users.org/wiki/SortedIteration
  234. ]]
  235. function __genOrderedIndex( t )
  236. local orderedIndex = {}
  237. for key in pairs(t) do
  238. table.insert( orderedIndex, key )
  239. end
  240. table.sort( orderedIndex )
  241. return orderedIndex
  242. end
  243. function orderedNext(t, state)
  244. -- Equivalent of the next function, but returns the keys in the alphabetic
  245. -- order. We use a temporary ordered key table that is stored in the
  246. -- table being iterated.
  247. local key = nil
  248. --print("orderedNext: state = "..tostring(state) )
  249. if state == nil then
  250. -- the first time, generate the index
  251. t.__orderedIndex = __genOrderedIndex( t )
  252. key = t.__orderedIndex[1]
  253. else
  254. -- fetch the next value
  255. for i = 1,table.getn(t.__orderedIndex) do
  256. if t.__orderedIndex[i] == state then
  257. key = t.__orderedIndex[i+1]
  258. end
  259. end
  260. end
  261. if key then
  262. return key, t[key]
  263. end
  264. -- no more value to return, cleanup
  265. t.__orderedIndex = nil
  266. return
  267. end
  268. function APT.orderedPairs(t)
  269. -- Equivalent of the pairs() function on tables. Allows to iterate
  270. -- in order
  271. return orderedNext, t, nil
  272. end
  273. -- Use this to dump a table to a string, with HTML.
  274. APT.dumpTableHTML = function (table, name, space)
  275. if nil == space then space = '' end
  276. local r = name .. "\n"
  277. r = r .. dumpTableHTMLSub(table, space .. " ")
  278. r = r .. space .. ""
  279. return r
  280. end
  281. dumpTableHTMLSub = function (table, space)
  282. local r = ""
  283. for k, v in APT.orderedPairs(table) do
  284. if type(v) == "table" then
  285. if " " == space then
  286. r = r .. space .. APT.dumpTableHTML(v, k .. "<ul>", space) .. "</ul>\n"
  287. else
  288. r = r .. "<li>" .. space .. APT.dumpTableHTML(v, k .. "<ul>", space) .. "</ul></li>\n"
  289. end
  290. else
  291. r = r .. space .. "<li>" .. k .. "</li>\n"
  292. end
  293. end
  294. return r
  295. end
  296. -- Use this to dump a table to a string.
  297. APT.dumpTable = function (table, name, space)
  298. if nil == space then space = '' end
  299. local r = ""
  300. if "" == space then r = r .. space .. name .. " =\n" else r = r .. space .. "[" .. name .. "] =\n" end
  301. r = r .. space .. "{\n"
  302. r = r .. dumpTableSub(table, space .. " ")
  303. if "" == space then r = r .. space .. "}\n" else r = r .. space .. "},\n" end
  304. return r
  305. end
  306. dumpTableSub = function (table, space)
  307. local r = ""
  308. for k, v in pairs(table) do
  309. if type(k) == "string" then k = '"' .. k .. '"' end
  310. if type(v) == "table" then
  311. r = r .. APT.dumpTable(v, k, space)
  312. elseif type(v) == "string" then
  313. local bq = '"'
  314. local eq = '"'
  315. if nil ~= v:match(bq) then
  316. bq, eq = '"', '"'
  317. if nil ~= v:match(bq) then
  318. bq, eq = '[[', ']]'
  319. mbq, meq = '%[%[', '%]%]'
  320. while (nil ~= v:match(mbq)) or (nil ~= v:match(meq)) do
  321. bq = '[' .. '=' .. bq:sub(2, -1)
  322. eq = ']' .. '=' .. eq:sub(2, -1)
  323. mbq = '%[' .. '=' .. bq:sub(3, -1)
  324. meq = '%]' .. '=' .. eq:sub(3, -1)
  325. end
  326. end
  327. end
  328. r = r .. space .. "[" .. k .. "] = " .. bq .. v .. eq .. ";\n"
  329. elseif type(v) == "function" then
  330. r = r .. space .. "[" .. k .. "] = function ();\n"
  331. elseif type(v) == "userdata" then
  332. r = r .. space .. "userdata " .. "[" .. k .. "];\n"
  333. elseif type(v) == "boolean" then
  334. if (v) then
  335. r = r .. space .. "[" .. k .. "] = true;\n"
  336. else
  337. r = r .. space .. "[" .. k .. "] = false;\n"
  338. end
  339. else
  340. r = r .. space .. "[" .. k .. "] = " .. v .. ";\n"
  341. end
  342. end
  343. return r
  344. end
  345. APT.allpairs = function(tbl, func)
  346. for k, v in pairs(tbl) do
  347. if 'table' == type(v) then
  348. for i, w in pairs(v) do
  349. func(i, w, k, v)
  350. end
  351. else
  352. func(k, v)
  353. end
  354. end
  355. end
  356. APT.lnk = function(URL)
  357. return '<a href="' .. URL .. '">' .. URL .. '</a>'
  358. end
  359. APT.search = function(t, s)
  360. for i, v in pairs(t) do
  361. if v == s then return true end
  362. end
  363. return false
  364. end
  365. APT.results = {}
  366. APT.logFile = nil
  367. APT.html = false
  368. APT.logName = function(host, a2, a3)
  369. local name = host
  370. if (nil ~= a2) and ('' ~= a2) then name = name .. "_" .. a2 end
  371. if (nil ~= a3) and ('' ~= a3) then name = name .. "_" .. a3 end
  372. name = 'LOG_' .. name .. '.html'
  373. return {'results/' .. name, '<a href="' .. name:gsub("/", "_") .. '">' .. name:gsub("/", "_") .. '</a>'}
  374. end
  375. APT.logOpen = function(host, a2, a3)
  376. local name = APT.logName(host, a2, a3)[1]
  377. if APT.checkFile(name) then return false end
  378. APT.logFile, e = io.open(name, "a+")
  379. if nil == APT.logFile then C('opening log file (' .. name .. ') - ' .. e); return false end
  380. if nil ~= APT.logFile then
  381. APT.logFile:write("<html><head>\n")
  382. APT.logFile:write("</head><body bgcolor='black' text='white' alink='red' link='aqua' vlink='fuchsia'>\n")
  383. APT.logFile:write("<pre>\n")
  384. APT.logFile:write(APT.dumpTable(APT.args, 'Arguments'))
  385. APT.logFile:write("</pre>\n")
  386. else
  387. return false
  388. end
  389. return true
  390. end
  391. APT.logPost = function()
  392. if nil ~= APT.logFile then
  393. APT.logFile:write("</body></html> \n")
  394. APT.logFile:close()
  395. end
  396. end
  397. local log = function(v, t, s, prot, test, host)
  398. local x = ""
  399. if nil == prot then prot = "" end
  400. if nil == test then test = "" end
  401. x = x .. prot
  402. if "" ~= test then
  403. if #x > 0 then x = x .. " " end
  404. x = x .. test
  405. end
  406. if nil ~= host then
  407. if #x > 0 then x = x .. " " end
  408. x = x .. host
  409. end
  410. if #x > 0 then
  411. t = t .. "(" .. x .. ")"
  412. if "" ~= prot then
  413. if "" == test then
  414. if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
  415. if v == 0 then APT.results[prot].errors = APT.results[prot].errors + 1 end
  416. if v == 1 then APT.results[prot].warnings = APT.results[prot].warnings + 1 end
  417. if v == 2 then APT.results[prot].timeouts = APT.results[prot].timeouts + 1 end
  418. else
  419. if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
  420. if nil == APT.results[prot][test] then APT.results[prot][test] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
  421. if v == 0 then APT.results[prot][test].errors = APT.results[prot][test].errors + 1 end
  422. if v == 1 then APT.results[prot][test].warnings = APT.results[prot][test].warnings + 1 end
  423. if v == 2 then APT.results[prot][test].timeouts = APT.results[prot][test].timeouts + 1 end
  424. end
  425. end
  426. end
  427. if v <= APT.verbosity then
  428. if 3 <= APT.verbosity then t = os.date('!%F %T') .. " " .. t end
  429. print(t .. ": " .. s)
  430. end
  431. if nil ~= APT.logFile then
  432. if APT.html then
  433. local colour = "white"
  434. if -1 == v then colour = "fuchsia" end -- CRITICAL
  435. if 0 == v then colour = "red " end -- ERROR
  436. if 1 == v then colour = "yellow " end -- WARNING
  437. if 2 == v then colour = "blue " end -- TIMEOUT
  438. if 3 == v then colour = "white " end -- INFO
  439. if 4 == v then colour = "gray " end -- DEBUG
  440. APT.logFile:write(os.date('!%F %T') .. " <font color='" .. colour .. "'><b>" .. t .. "</b></font>: " .. s .. "</br>\n")
  441. else
  442. APT.logFile:write(os.date('!%F %T') .. " " .. t .. ": " .. s .. "\n")
  443. end
  444. APT.logFile:flush()
  445. end
  446. end
  447. APT.D = function(s, h) log(4, "DEBUG ", s, nil, nil, h) end
  448. APT.I = function(s, h) log(3, "INFO ", s, nil, nil, h) end
  449. APT.T = function(s, p, t, h) log(2, "TIMEOUT ", s, p, t, h) end
  450. APT.W = function(s, p, t, h) log(1, "WARNING ", s, p, t, h) end
  451. APT.E = function(s, p, t, h) log(0, "ERROR ", s, p, t, h) end
  452. APT.C = function(s) log(-1, "CRITICAL", s) end
  453. local D = APT.D
  454. local I = APT.I
  455. local T = APT.T
  456. local W = APT.W
  457. local E = APT.E
  458. local C = APT.C
  459. APT.debians = {}
  460. APT.mirrors = {}
  461. APT.testing = function(t, host)
  462. for i, v in pairs(APT.options.tests.value) do
  463. if t == v then
  464. local h = APT.mirrors[host]
  465. if nil == h then return true end
  466. if true == h["Protocols"][t] then return true else I("Skipping " .. t .. " checks for " .. host) end
  467. end
  468. end
  469. return false
  470. end
  471. APT.tested = function(prot, test, host)
  472. if "" == test then
  473. if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
  474. APT.results[prot].tested = APT.results[prot].tested + 1;
  475. else
  476. if nil == APT.results[prot] then APT.results[prot] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
  477. if nil == APT.results[prot][test] then APT.results[prot][test] = {tested = 0; errors = 0; warnings = 0; timeouts = 0} end
  478. APT.results[prot][test].tested = APT.results[prot][test].tested + 1;
  479. end
  480. end
  481. APT.exe = function(c)
  482. local exe = {status = 0, result = '', log = true, cmd = c .. ' '}
  483. function exe:log()
  484. self.log = true
  485. return self
  486. end
  487. function exe:Nice(c)
  488. if nil == c then
  489. self.cmd = 'ionice -c3 nice -n 19 ' .. self.cmd
  490. else
  491. self.cmd = self.cmd .. 'ionice -c3 nice -n 19 ' .. c .. ' '
  492. end
  493. return self
  494. end
  495. function exe:also(c)
  496. if nil == c then c = '' else c = ' ' .. c end
  497. self.cmd = self.cmd .. ';' .. c .. ' '
  498. return self
  499. end
  500. function exe:And(c)
  501. if nil == c then c = '' else c = ' ' .. c end
  502. self.cmd = self.cmd .. '&&' .. c .. ' '
  503. return self
  504. end
  505. function exe:Or(c)
  506. if nil == c then c = '' end
  507. self.cmd = self.cmd .. '|| ' .. c .. ' '
  508. return self
  509. end
  510. function exe:noErr()
  511. self.cmd = self.cmd .. '2>/dev/null '
  512. return self
  513. end
  514. function exe:wait(w)
  515. self.cmd = self.cmd .. '&& touch ' .. w .. ' '
  516. return self
  517. end
  518. function exe:Do()
  519. --[[ "The condition expression of a control structure can return any
  520. value. Both false and nil are considered false. All values different
  521. from nil and false are considered true (in particular, the number 0
  522. and the empty string are also true)."
  523. says the docs, I beg to differ.]]
  524. if true == self.log then D(" executing - &nbsp; <code>" .. self.cmd .. "</code>") end
  525. --[[ Damn os.execute()
  526. Lua 5.1 says it returns "a status code, which is system-dependent"
  527. Lua 5.2 says it returns true/nil, "exit"/"signal", the status code.
  528. I'm getting 7168 or 0. No idea what the fuck that is.
  529. local ok, rslt, status = os.execute(s)
  530. ]]
  531. local f = io.popen(self.cmd .. ' ; echo "$?"', 'r')
  532. -- The last line will be the command's returned status, collect everything else in result.
  533. self.status = '' -- Otherwise the result starts with 0.
  534. for l in f:lines() do
  535. self.result = self.result .. self.status .. "\n"
  536. self.status = l
  537. end
  538. self.status = tonumber(self.status)
  539. return self
  540. end
  541. function exe:fork()
  542. self.cmd = '{ ' .. self.cmd .. '; } &'
  543. if true == self.log then D(" forking - &nbsp; <code>" .. self.cmd .. "</code>") end
  544. os.execute(self.cmd)
  545. return self
  546. end
  547. return exe
  548. end
  549. APT.checkExes = function (exe)
  550. local count = io.popen('ps x | grep "' .. exe .. '" | grep -v " grep " | grep -v "flock -n apt-panopticon.lock " | wc -l'):read("*l")
  551. D(count .. " " .. exe .. " commands still running.")
  552. return tonumber(count)
  553. end
  554. APT.checkFile = function(f)
  555. local h, e = io.open(f, "r")
  556. if nil == h then return false else h:close(); return true end
  557. end
  558. APT.plurals = function(e, w, t)
  559. local result = ""
  560. if 1 == e then
  561. result = e .. " error"
  562. elseif e ~= 0 then
  563. result = e .. " errors"
  564. end
  565. if ("" ~= result) and APT.html then result = "<font color='red'><b>" .. result .. "</b></font>" end
  566. if 0 < w then
  567. if 0 < e then result = result .. ", " end
  568. if 1 == w then
  569. result = result .. w .. " warning"
  570. else
  571. result = result .. w .. " warnings"
  572. end
  573. if ("" ~= result) and APT.html then result = "<font color='yellow'><b>" .. result .. "</b></font>" end
  574. end
  575. if 0 < t then
  576. if (0 < e) or (0 < w) then result = result .. ", " end
  577. if 1 == t then
  578. result = result .. t .. " timeout"
  579. else
  580. result = result .. t .. " timeouts"
  581. end
  582. if ("" ~= result) and APT.html then result = "<font color='blue'><b>" .. result .. "</b></font>" end
  583. end
  584. if "" ~= result then result = " " .. result end
  585. return result
  586. end
  587. local typs = {'tested', 'errors', 'warnings', 'timeouts'}
  588. APT.padResults = function(results)
  589. local c = 0
  590. if nil == results then results = {}; c = 999 end
  591. for k, v in pairs(APT.protocols) do
  592. tests = results[v]
  593. if nil == tests then tests = {} end
  594. for m, x in pairs(typs) do
  595. if nil == tests[x] then tests[x] = c end
  596. end
  597. for l, w in pairs(APT.tests) do
  598. if ('raw' ~= w) and ('Speed' ~= w) then
  599. if nil == tests[w] then tests[w] = {} end
  600. for m, x in pairs(typs) do
  601. if nil == tests[w][x] then tests[w][x] = c end
  602. end
  603. end
  604. end
  605. if nil == tests.redirects then
  606. tests.redirects = {}
  607. end
  608. results[v] = tests
  609. end
  610. if nil == results.timeout then results.timeout = false end
  611. if nil == results.speed then results.speed = {min = 999999999999; max = 0} end
  612. return results
  613. end
  614. APT.collate = function(l, host, ip, results)
  615. results = APT.padResults(results)
  616. local f = l .. "/" .. host .. "_" .. ip .. ".lua"
  617. if APT.checkFile(f) then
  618. local rs = loadfile(f)()
  619. rs = APT.padResults(rs)
  620. for k, v in pairs(rs) do
  621. if "table" == type(v) then
  622. if ("speed" == k) and (nil ~= results.speed) then
  623. if v.min < results.speed.min then results.speed.min = v.min end
  624. if v.max > results.speed.max then results.speed.max = v.max end
  625. elseif 'IPs' ~= k then
  626. for i, u in pairs(v) do
  627. if 'redirects' ~= i then
  628. if "table" == type(u) then
  629. for h, t in pairs(u) do
  630. local a = results[k]
  631. if nil == a then results[k] = {i = {}}; a = results[k] end
  632. a = a[i]
  633. if nil == a then results[k][i] = {h = {}}; a = results[k][i] end
  634. a = a[h]
  635. if nil == a then a = 0 end
  636. results[k][i][h] = a + t
  637. end
  638. else
  639. local a = results[k]
  640. if nil == a then a = 0; results[k] = {} else a = a[i] end
  641. if nil == a then a = 0 end
  642. results[k][i] = a + u
  643. end
  644. end
  645. end
  646. end
  647. elseif "timeout" ~= k then
  648. local a = results[k]
  649. if nil == a then a = 0 end
  650. results[k] = a + v
  651. end
  652. end
  653. end
  654. return results
  655. end
  656. APT.collateAll = function(hosts, l, host, func)
  657. results = {}
  658. local f = l .. "/" .. host .. ".lua"
  659. if APT.checkFile(f) then
  660. results = loadfile(f)()
  661. results = APT.padResults(results)
  662. if nil ~= func then func(results) end
  663. results = APT.collate(l, host, 'R', results)
  664. local v = hosts[host]
  665. if nil ~= v then
  666. local IPs = results.IPs
  667. if nil == IPs then W('No IPs for ' .. host .. ' in ' .. l) else
  668. for i, u in pairs(IPs) do
  669. if "table" == type(u) then
  670. for h, t in pairs(u) do
  671. results = APT.collate(l, host, h, results)
  672. results = APT.collate(l, host, h .. '_R', results)
  673. if nil ~= func then func(results, h) end
  674. end
  675. else
  676. results = APT.collate(l, host, i .. '_R', results)
  677. if nil ~= func then func(results, i) end
  678. end
  679. end
  680. end
  681. end
  682. end
  683. results = APT.padResults(results)
  684. return results
  685. end
  686. APT.now = 0
  687. local status
  688. APT.now = tonumber(APT.exe('TZ="GMT" ls -l --time-style="+%s" results/stamp | cut -d " " -f 6-6'):Do().result)
  689. local start = 'now-1month'
  690. local step = '10min'
  691. local hb = '150min'
  692. local DSIe = 'DS:IntegrityErrors:GAUGE:' .. hb .. ':0:U'
  693. local DSIw = 'DS:IntegrityWarnings:GAUGE:' .. hb .. ':0:U'
  694. local DSIt = 'DS:IntegrityTimeouts:GAUGE:' .. hb .. ':0:U'
  695. local DSPe = 'DS:ProtocolErrors:GAUGE:' .. hb .. ':0:U'
  696. local DSPw = 'DS:ProtocolWarnings:GAUGE:' .. hb .. ':0:U'
  697. local DSPt = 'DS:ProtocolTimeouts:GAUGE:' .. hb .. ':0:U'
  698. local DSRe = 'DS:RedirectsErrors:GAUGE:' .. hb .. ':0:U'
  699. local DSRw = 'DS:RedirectsWarnings:GAUGE:' .. hb .. ':0:U'
  700. local DSRt = 'DS:RedirectsTimeouts:GAUGE:' .. hb .. ':0:U'
  701. local DSUe = 'DS:UpdatedErrors:GAUGE:' .. hb .. ':0:U'
  702. local DSUw = 'DS:UpdatedWarnings:GAUGE:' .. hb .. ':0:U'
  703. local DSUt = 'DS:UpdatedTimeouts:GAUGE:' .. hb .. ':0:U'
  704. local DSSe = 'DS:URLSanityErrors:GAUGE:' .. hb .. ':0:U'
  705. local DSSw = 'DS:URLSanityWarnings:GAUGE:' .. hb .. ':0:U'
  706. local DSSt = 'DS:URLSanityTimeouts:GAUGE:' .. hb .. ':0:U'
  707. local DSx = 'DS:max:GAUGE:' .. hb .. ':0:U'
  708. local DSn = 'DS:min:GAUGE:' .. hb .. ':0:U'
  709. -- What Collectd uses.
  710. local RRAc0 = 'RRA:AVERAGE:0.9:1:1200'
  711. local RRAc1 = 'RRA:MIN:0.9:1:1200'
  712. local RRAc2 = 'RRA:MAX:0.9:1:1200'
  713. local RRAc3 = 'RRA:AVERAGE:0.9:7:1235'
  714. local RRAc4 = 'RRA:MIN:0.9:7:1235'
  715. local RRAc5 = 'RRA:MAX:0.9:7:1235'
  716. local RRAc6 = 'RRA:AVERAGE:0.9:50:1210'
  717. local RRAc7 = 'RRA:MIN:0.9:50:1210'
  718. local RRAc8 = 'RRA:MAX:0.9:50:1210'
  719. local RRAc9 = 'RRA:AVERAGE:0.9:223:1202'
  720. local RRAc10 = 'RRA:MIN:0.9:223:1202'
  721. local RRAc11 = 'RRA:MAX:0.9:223:1202'
  722. local RRAc12 = 'RRA:AVERAGE:0.9:2635:1201'
  723. local RRAc13 = 'RRA:MIN:0.9:2635:1201'
  724. local RRAc14 = 'RRA:MAX:0.9:2635:1201'
  725. -- Try LAST.
  726. local RRAl0 = 'RRA:LAST:0.9:1:1200'
  727. local RRAl1 = 'RRA:LAST:0.9:7:1235'
  728. local RRAl2 = 'RRA:LAST:0.9:50:1210'
  729. local RRAl3 = 'RRA:LAST:0.9:223:1202'
  730. local RRAl4 = 'RRA:LAST:0.9:2635:1201'
  731. APT.createRRD = function(host, ip, o)
  732. if nil ~= o then start = o end
  733. if nil ~= ip then host = host .. '_' .. ip end
  734. for i, p in pairs(APT.protocols) do
  735. os.execute( 'mkdir -p rrd/' .. host .. '/' .. p:upper())
  736. if not APT.checkFile('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd') then
  737. D('Creating ' .. 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd')
  738. APT.rrd.create( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', '--start', start, '--step', step,
  739. DSIe, DSIw, DSIt, DSPe, DSPw, DSPt, DSRe, DSRw, DSRt, DSUe, DSUw, DSUt, DSSe, DSSw, DSSt,
  740. RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
  741. -- Start them at 0 so the average has something to work on.
  742. APT.rrd.update( 'rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', (APT.now - 600) .. ':0:0:0:0:0:0:0:0:0:0:0:0:0:0:0')
  743. end
  744. end
  745. os.execute( 'mkdir -p rrd/' .. host .. '/Speed')
  746. if not APT.checkFile('rrd/' .. host .. '/Speed/Speed.rrd') then
  747. D('Creating ' .. 'rrd/' .. host .. '/Speed/Speed.rrd')
  748. APT.rrd.create( 'rrd/' .. host .. '/Speed/Speed.rrd', '--start', start, '--step', step, DSx, DSn,
  749. RRAc0, RRAc1, RRAc2, RRAl0, RRAc3, RRAc4, RRAc5, RRAl1, RRAc6, RRAc7, RRAc8, RRAl2, RRAc9, RRAc10, RRAc11, RRAl3, RRAc12, RRAc13, RRAc14, RRAl4)
  750. APT.rrd.update( 'rrd/' .. host .. '/Speed/Speed.rrd', (APT.now - 600) .. ':0:0')
  751. end
  752. end
  753. APT.updateRRD = function(results, host, ip)
  754. if nil ~= ip then host = host .. '_' .. ip end
  755. for i, p in pairs(APT.protocols) do
  756. if nil ~= results[p] then
  757. APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':' ..
  758. results[p]['Integrity'].errors .. ':' .. results[p]['Integrity'].warnings .. ':' .. results[p]['Integrity'].timeouts .. ':' ..
  759. results[p]['Protocol'].errors .. ':' .. results[p]['Protocol'].warnings .. ':' .. results[p]['Protocol'].timeouts .. ':' ..
  760. results[p]['Redirects'].errors .. ':' .. results[p]['Redirects'].warnings .. ':' .. results[p]['Redirects'].timeouts .. ':' ..
  761. results[p]['Updated'].errors .. ':' .. results[p]['Updated'].warnings .. ':' .. results[p]['Updated'].timeouts .. ':' ..
  762. results[p]['URLSanity'].errors .. ':' .. results[p]['URLSanity'].warnings .. ':' .. results[p]['URLSanity'].timeouts)
  763. else
  764. APT.rrd.update('rrd/' .. host .. '/' .. p:upper() .. '/Tests.rrd', APT.now .. ':U:U:U:U:U:U:U:U:U:U:U:U:U:U:U')
  765. end
  766. end
  767. if nil ~= results.speed then
  768. if 0 ~= results.speed.max then
  769. APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':' .. results.speed.max .. ':' .. results.speed.min)
  770. else
  771. APT.rrd.update('rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':0:0')
  772. end
  773. else
  774. APT.rrd.update( 'rrd/' .. host .. '/Speed/Speed.rrd', APT.now .. ':U:U')
  775. end
  776. end
  777. APT.doRRD = function(l, k, v, o)
  778. -- TODO - Quick hack, fix it later.
  779. if k == APT.options.roundRobin.value then return end
  780. APT.collateAll(APT.mirrors, l, k,
  781. function(results, ip)
  782. APT.createRRD(k, ip, o)
  783. APT.updateRRD(results, k, ip)
  784. end)
  785. end
  786. return APT