A Lua script to check the health of Devuan Linux package mirrors. The master repo is at https://sledjhamr.org/cgit/apt-panopticon/ and the master issues tracker is at https://sledjhamr.org/mantisbt/project_page.php?project_id=13 https://sledjhamr.org/cgit/apt-panopticon/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

442 lines
16 KiB

  1. #!/usr/bin/env luajit
  2. local APT = require 'apt-panopticommon'
  3. local D = APT.D
  4. local I = APT.I
  5. local W = APT.W
  6. local E = APT.E
  7. local C = APT.C
  8. local arg, sendArgs = APT.parseArgs({...})
  9. local results = {}
  10. APT.mirrors = loadfile("results/mirrors.lua")()
  11. local revDNS = function(dom, IP)
  12. if "deb.devuan.org" ~= dom then
  13. if nil ~= APT.mirrors["deb.devuan.org"] then
  14. if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][IP] then
  15. if APT.html then
  16. return "<font color='purple'><b>DNS-RR</b></font>"
  17. else
  18. return "DNS-RR"
  19. end
  20. end
  21. end
  22. else
  23. for k, v in pairs(APT.mirrors) do
  24. if "deb.devuan.org" ~= k then
  25. local IPs = v.IPs
  26. for i, u in pairs(IPs) do
  27. if "table" == type(u) then
  28. for h, t in pairs(u) do
  29. if IP == h then return k end
  30. end
  31. else
  32. if IP == i then return k end
  33. end
  34. end
  35. end
  36. end
  37. end
  38. return ""
  39. end
  40. local faulty = ""
  41. local status = function(host, results, typ)
  42. local result = ""
  43. local e = 0
  44. local w = 0
  45. local s = nil ~= APT.mirrors[host].Protocols[typ]
  46. local to = results.timeout
  47. if ('http' ~= typ) and ('https' ~= typ) and ('ftp' ~= typ) and ('rsync' ~= typ) then s = true end
  48. if nil ~= results[typ] then
  49. e = results[typ].errors
  50. w = results[typ].warnings
  51. for k, v in pairs(results[typ]) do
  52. if "table" == type(v) then
  53. if 0 <= v.errors then e = e + v.errors else to = true end
  54. if 0 <= v.warnings then w = w + v.warnings else to = true end
  55. end
  56. end
  57. else
  58. for k, v in pairs(results) do
  59. if "table" == type(v) then
  60. for i, u in pairs(v) do
  61. if "table" == type(u) then
  62. if typ == i then
  63. if 0 <= u.errors then e = e + u.errors end
  64. if 0 <= u.warnings then w = w + u.warnings end
  65. end
  66. end
  67. end
  68. end
  69. end
  70. end
  71. if to then
  72. result = "[TIMEOUT"
  73. if not s then result = result .. "*" end
  74. if APT.html then
  75. if s then
  76. result = "[<font color='blue'><b>TIMEOUT</b></font>"
  77. else
  78. result = "[<font color='darkblue'><b>TIMEOUT*</b></font>"
  79. end
  80. end
  81. elseif 0 < e then
  82. result = "[FAILED"
  83. if not s then result = result .. "*" end
  84. if APT.html then
  85. if s then
  86. result = "[<font color='red'><b>FAILED</b></font>"
  87. else
  88. result = "[<font color='darkred'><b>FAILED*</b></font>"
  89. end
  90. end
  91. if APT.html then
  92. faulty = faulty .. host .. " (" .. typ .. ")<br>\n"
  93. else
  94. faulty = faulty .. host .. " (" .. typ .. ")\n"
  95. end
  96. else
  97. result = "[OK"
  98. if not s then result = result .. "*" end
  99. if APT.html then
  100. if s then
  101. result = "[<font color='lime'><b>OK</b></font>"
  102. else
  103. result = "[<font color='darkgreen'><b>OK*</b></font>"
  104. end
  105. end
  106. end
  107. return result .. APT.plurals(e, w) .. "]"
  108. end
  109. local m = {}
  110. local logCount = function(domain, ip)
  111. local nm = "LOG_" .. domain
  112. local log = ""
  113. local extra = ""
  114. if nil ~= ip then nm = nm .. "_" .. ip end
  115. nm = nm .. ".html"
  116. local rfile, e = io.open("results/" .. nm, "r")
  117. if nil ~= rfile then
  118. local errors = 0
  119. local warnings = 0
  120. for l in rfile:lines() do
  121. if nil ~= l:match("><b>ERROR ") then errors = errors + 1 end
  122. if nil ~= l:match("><b>WARNING ") then warnings = warnings + 1 end
  123. end
  124. rfile:close()
  125. if APT.html then
  126. if nil == ip then
  127. log = "<a href='" .. nm .. "'>" .. domain .. "</a>"
  128. else
  129. log = "<a href='" .. nm .. "'>" .. ip .. "</a>"
  130. end
  131. end
  132. log = log .. APT.plurals(errors, warnings)
  133. end
  134. return log
  135. end
  136. APT.html = false
  137. local email, e = io.open("results/Report-email.txt", "w+")
  138. if nil == email then C("opening mirrors file - " .. e) else
  139. email:write( "Dear Mirror Admins,\n\n" ..
  140. "This is the status of the mirror servers in the Devuan package mirror network.\n\n" ..
  141. "EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important." ..
  142. "The full list of Devuan package mirrors is available at the URL:\n\n" ..
  143. " https://pkgmaster.devuan.org/mirror_list.txt\n\n" ..
  144. 'Please contact "mirrors@devuan.org" if any of the information \nin the file above needs to be amended. \n\n' ..
  145. "The full results of the mirror checking is available at the URL:\n\n" ..
  146. " https://sledjhamr.org/apt-panopticon/results/Report-web.html\n\n" ..
  147. "Due to the nature of the tests, some errors or warnings will be \ncounted several times. " ..
  148. "Refer to the logs on the web page for details.\n\n" ..
  149. "Please see below the current status of the Devuan Package Mirror \nnetwork:\n\n" ..
  150. "==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====\n" ..
  151. "[skip] means that the test hasn't been written yet.\n\n")
  152. for k, v in APT.orderedPairs(APT.mirrors) do
  153. local results = loadfile("results/" .. k .. ".lua")()
  154. email:write(k .. "....\n")
  155. local IPs = v.IPs
  156. for i, u in pairs(IPs) do
  157. if "table" == type(u) then
  158. for h, t in pairs(u) do
  159. results = APT.collate(k, h, results)
  160. end
  161. else
  162. results = APT.collate(k, i, results)
  163. end
  164. end
  165. local ftp = "[skip]"
  166. local http = status(k, results, "http")
  167. local https = status(k, results, "https")
  168. local rsync = "[skip]"
  169. local dns = ""
  170. local protocol = status(k, results, "Protocol")
  171. local sanity = status(k, results, "URLSanity")
  172. local integrity = status(k, results, "Integrity")
  173. local updated = status(k, results, "Updated")
  174. -- DNS-RR test.
  175. if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
  176. for l, w in pairs(APT.mirrors[k].IPs) do
  177. if type(w) == "table" then
  178. for i, u in pairs(w) do
  179. if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
  180. local log = logCount("deb.devuan.org", i)
  181. if "" ~= log then
  182. if "" == dns then dns = " " else dns = dns .. " " end
  183. dns = dns .. logCount("deb.devuan.org", i)
  184. else
  185. if "" == dns then dns = " " else dns = dns .. " " end
  186. dns = dns .. i
  187. end
  188. end
  189. end
  190. else
  191. if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
  192. local log = logCount("deb.devuan.org", l)
  193. if "" ~= log then
  194. if "" == dns then dns = " " else dns = dns .. " " end
  195. dns = dns .. log
  196. else
  197. if "" == dns then dns = " " else dns = dns .. " " end
  198. dns = dns .. l
  199. end
  200. end
  201. end
  202. end
  203. if "" == dns then dns = "[no]" end
  204. dns = " DNS-RR: " .. dns
  205. end
  206. email:write( " ftp: " .. ftp .. " http: " .. http .. " https: " .. https .." rsync: " .. rsync .. "\n" ..
  207. " " .. dns .. "\n" ..
  208. " Protocol: " .. protocol .. " URL-sanity: " .. sanity .. " Integrity: " .. integrity .. "\n" ..
  209. " Updated: " .. updated .. "\n")
  210. end
  211. email:write( "\n==== faulty mirrors: ====\n" .. faulty)
  212. email:write( "\n-------------------------\n\n" ..
  213. "* This means that this protocol isn't actually supported, but the test was run ayway.\n\n" ..
  214. "Thanks for your precious help in ensuring that Devuan GNU+Linux \nremains a universal, stable, dependable, free operating system.\n\n" ..
  215. "You can get the source code from https://sledjhamr.org/cgit/apt-panopticon/about/ .\n\n" ..
  216. "Love\n\n" ..
  217. "The Dev1Devs\n\n")
  218. email:close()
  219. end
  220. local colours =
  221. {
  222. 'f0000080',
  223. '0f000080',
  224. '00f00080',
  225. '000f0080',
  226. '0000f080',
  227. '00000f80',
  228. '80000080',
  229. '08000080',
  230. '00800080',
  231. '00080080',
  232. '00008080',
  233. '00000880',
  234. 'ff000080',
  235. '0ff00080',
  236. '00ff0080',
  237. '000ff080',
  238. '0000ff80',
  239. '88000080',
  240. '08800080',
  241. '00880080',
  242. '00088080',
  243. '00008880',
  244. }
  245. local g = {}
  246. local count = 0
  247. for k, v in APT.orderedPairs(mirrors) do
  248. if 'pkgmaster.devuan.org' ~= k then count = count + 1 end
  249. end
  250. for i = 1, count do
  251. end
  252. count = 1
  253. for k, v in APT.orderedPairs(mirrors) do
  254. if 'deb.devuan.org' ~= k then
  255. local c = colours[count]
  256. if 'pkgmaster.devuan.org' == k then c = 'ffffff' end
  257. table.insert(g, 'DEF:speed' .. count .. '=rrd/' .. k .. '/HTTP/Speed.rrd:max:LAST')
  258. table.insert(g, 'VDEF:vspeed' .. count .. '=speed' .. count .. ',AVERAGE')
  259. table.insert(g, 'LINE2:speed' .. count .. '#' .. c .. ':' .. k .. '\t')
  260. table.insert(g, 'GPRINT:vspeed' .. count .. ':%5.1lf%s\\l')
  261. count = count + 1
  262. end
  263. end
  264. APT.rrd.graph('results/speed.png', '--start', 'now-1w', '--end', 'now', '-t', 'Speed', '-v', 'bytes per second', '-w', '900', '-h', '400', '-Z',
  265. '-c', 'BACK#000000', '-c', 'CANVAS#000000', '-c', 'FONT#FFFFFF', '-c', 'AXIS#FFFFFF', '-c', 'FRAME#FFFFFF', '-c', 'ARROW#FFFFFF',
  266. unpack(g))
  267. results = {}
  268. m = {}
  269. faulty = ""
  270. APT.html = true
  271. local web, e = io.open("results/Report-web.html", "w+")
  272. if nil == web then C("opening mirrors file - " .. e) else
  273. web:write( "<html><head><title>apt-panopticon results</title>\n" ..
  274. '</head><body bgcolor="black" text="white">' ..
  275. "<h1>Welcome to the apt-panopticon results page.</h1>\n" ..
  276. "<p>This is the status of the mirror servers in the Devuan package mirror network.</p>\n" ..
  277. "<p><font style='background-color:red; color:black'>EXPERIMENTAL CODE - double check all results you see here, and read the logs if it's important.</font></p>" ..
  278. "<p>The full list of Devuan package mirrors is available at the URL: " ..
  279. "<a href='https://pkgmaster.devuan.org/mirror_list.txt'>https://pkgmaster.devuan.org/mirror_list.txt</a></p>\n" ..
  280. "<p>Due to the nature of the tests, some errors or warnings will be counted several times. &nbsp; " ..
  281. "The links in the table and DNS list go to the detailed testing logs.</p>\n\n" ..
  282. "<hr>\n<h2>==== package mirror status " .. os.date("!%Y-%m-%d %H:%M") .. " GMT ====</h2>\n" ..
  283. "<p>[<font color='red'><b>FAILED</b></font>] or [<font color='lime'><b>OK</b></font>]" ..
  284. " means the tested thing is supported for that mirror.</p>\n" ..
  285. "<p>[<font color='darkred'><b>FAILED*</b></font>] or [<font color='darkgreen'><b>OK*</b></font>]" ..
  286. " means the tested thing is unsupported for that mirror, but might have been tested anyway.</p>\n" ..
  287. "<p>[<font color='blue'><b>TIMEOUT</b></font>] or [<font color='darkblue'><b>TIMEOUT</b></font>]" ..
  288. " means the server had too many timeouts, and tests where aborted, so there is no result for this test.</p>" ..
  289. "<p>The DNS round robin (DNS-RR) column shows the IPs for that mirror, or [<font color='grey'><b>no</b></font>] if it isn't part of the DNS-RR. &nbsp; " ..
  290. "The IPs link to the testing log for that IP accessed via the DNS-RR. &nbsp; " ..
  291. "deb.devuan.org is the DNS-RR itself, so it doesn't get tested directly.</p>\n" ..
  292. "<p>The time in the Updated column is how often the mirror updates itself.</p>" ..
  293. "<p>Mirrors with a <font style='background-color:dimgrey'>grey background</font> are not active (though may be usable as part of the DNS-RR).</p>\n" ..
  294. "<p>[<font color='grey'><b>skip</b></font>] means that the test hasn't been written yet.</p>\n" ..
  295. "<table>\n<tr><th></th><th>FTP</th><th>HTTP</th><th>HTTPS</th><th>RSYNC</th><th>DNS round robin</th>" ..
  296. "<th>Protocol</th><th>URL sanity</th><th>Integrity</th><th>Updated</th><th colspan='2'>Speed range</th></tr>\n"
  297. )
  298. for k, v in APT.orderedPairs(APT.mirrors) do
  299. local results = loadfile("results/" .. k .. ".lua")()
  300. local active = ""
  301. if "yes" == v.Active then
  302. web:write(" <tr><th>" .. k .. "</th> ")
  303. else
  304. if nil == v.Active then active = 'nil' else active = v.Active end
  305. web:write(" <tr style='background-color:dimgrey'><th>" .. k .. "</th> ")
  306. end
  307. local IPs = v.IPs
  308. for i, u in pairs(IPs) do
  309. if "table" == type(u) then
  310. for h, t in pairs(u) do
  311. results = APT.collate(k, h, results)
  312. end
  313. else
  314. results = APT.collate(k, i, results)
  315. end
  316. end
  317. local ftp = "[<font color='grey'><b>skip</b></font>]"
  318. local http = status(k, results, "http")
  319. local https = status(k, results, "https")
  320. local rsync = "[<font color='grey'><b>skip</b></font>]"
  321. local dns = ""
  322. local protocol = status(k, results, "Protocol")
  323. local sanity = status(k, results, "URLSanity")
  324. local integrity = status(k, results, "Integrity")
  325. local updated = status(k, results, "Updated")
  326. local rate = v.Rate
  327. if nil ~= rate then updated = updated .. ' ' .. rate end
  328. local min = tonumber(results.speed.min)
  329. local max = tonumber(results.speed.max)
  330. local spd = ''
  331. -- DNS-RR test.
  332. if ("deb.devuan.org" ~= k) and (nil ~= APT.mirrors["deb.devuan.org"]) then
  333. for l, w in pairs(APT.mirrors[k].IPs) do
  334. if type(w) == "table" then
  335. for i, u in pairs(w) do
  336. if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][i] then
  337. local log = logCount("deb.devuan.org", i)
  338. if "" ~= log then
  339. if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
  340. dns = dns .. logCount("deb.devuan.org", i)
  341. else
  342. if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
  343. dns = dns .. "<font color='maroon'><b>" .. i .. "</b></font>"
  344. end
  345. end
  346. end
  347. else
  348. if nil ~= APT.mirrors["deb.devuan.org"].IPs["deb.roundr.devuan.org"][l] then
  349. local log = logCount("deb.devuan.org", l)
  350. if "" ~= log then
  351. if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
  352. dns = dns .. log
  353. else
  354. if "" == dns then dns = " " else dns = dns .. " &nbsp; " end
  355. dns = dns .. "<font color='maroon'><b>" .. l .. "</b></font>"
  356. end
  357. end
  358. end
  359. end
  360. if "" == dns then dns = "[<font color='grey'><b>no</b></font>]" end
  361. if 0 == max then
  362. spd = '<td></td><td></td>'
  363. else
  364. spd = string.format('<td align="right">%d -></td><td align="right">%d</td>', min, max)
  365. end
  366. end
  367. web:write("<td>" .. ftp .. "&nbsp;</td><td>" .. http .. "&nbsp;</td><td>" .. https .. "&nbsp;</td><td>" .. rsync .. "&nbsp;</td><td>" .. dns ..
  368. "&nbsp;</td><td>" .. protocol .. "&nbsp;</td><td>" .. sanity ..
  369. "&nbsp;</td><td>" .. integrity .. "&nbsp;</td><td>" .. updated .. "&nbsp;</td>" .. spd .. "</tr>\n")
  370. if "" ~= active then
  371. web:write("<tr><td style='background-color:dimgrey'>" .. active .. "</td></tr>\n")
  372. end
  373. end
  374. web:write( "</table>\n<br>\n<h2>==== faulty mirrors: ====</h2>\n" .. faulty)
  375. web:write( "<br>\n<br>\n<h2>==== DNS and logs: ====</h2>\n")
  376. for k, v in pairs(APT.mirrors) do
  377. local log = k
  378. local n = {}
  379. log = logCount(k)
  380. APT.mirrors[k].Protocols = nil
  381. APT.mirrors[k].FQDN = nil
  382. APT.mirrors[k].Active = nil
  383. APT.mirrors[k].Rate = nil
  384. APT.mirrors[k].BaseURL = nil
  385. APT.mirrors[k].Country = nil
  386. APT.mirrors[k].Bandwidth = nil
  387. for l, w in pairs(APT.mirrors[k].IPs) do
  388. if type(w) == "table" then
  389. n[l] = {}
  390. for i, u in pairs(w) do
  391. local log = logCount(k, i)
  392. if "" == log then n[l][i] = u else n[l][log .. " " .. revDNS(k, i)] = u end
  393. end
  394. else
  395. local log = logCount(k, l)
  396. if "" == log then n[l] = w else n[log .. " " .. revDNS(k, l)] = w end
  397. end
  398. end
  399. m[log .. " DNS entries -"] = n
  400. end
  401. web:write( "<p>This lists each mirror, and the DNS entries for that mirror. &nbsp; " ..
  402. "The links point to the testing log files for " .. logCount("apt-panopticon") .. " for each domain name / IP combination that was tested. &nbsp; " ..
  403. "If a mirror has a CNAME, that CNAME is listed along with that CNAMEs DNS entries. &nbsp; " ..
  404. "deb.devuan.org is the DNS round robin, which points to the mirrors that are part of the DNS-RR. &nbsp; " ..
  405. "If an IP is part of the DNS-RR, it is marked with '<font color='purple'><b>DNS-RR</b></font>' &nbsp; " ..
  406. "pkgmaster.devuan.org is the master mirror, all the others sync to it. &nbsp; " ..
  407. "</p>\n"
  408. )
  409. web:write(APT.dumpTableHTML(m, "", ""))
  410. web:write( "\n<br>\n<br>\n<h2>==== Graphs: ====</h2>\n" ..
  411. "<img src='speed.png'>\n<br>\n<p><a href='../apt-panopticon_cgp/'>More graphs.</a></p><hr>\n\n" ..
  412. "<p>The <a href='Report-email.txt'>email report</a>. &nbsp; " ..
  413. "All <a href='../results'>the logs and other output</a>. &nbsp; " ..
  414. "You can get the <a href='https://sledjhamr.org/cgit/apt-panopticon/about/'>source code here</a>.</p>" ..
  415. "</body></html>\n")
  416. web:close()
  417. end