A Lua script to check the health of Devuan Linux package mirrors. The master repo is at https://sledjhamr.org/cgit/apt-panopticon/ and the master issues tracker is at https://sledjhamr.org/mantisbt/project_page.php?project_id=13 https://sledjhamr.org/cgit/apt-panopticon/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

1284 lines
56 KiB

  1. #!/usr/bin/env luajit
  2. local now = os.time()
  3. local APT = require 'apt-panopticommon'
  4. local D = APT.D
  5. local I = APT.I
  6. local T = APT.T
  7. local W = APT.W
  8. local E = APT.E
  9. local C = APT.C
  10. local arg, sendArgs = APT.parseArgs({...})
  11. APT.html = true
  12. local defaultURL = {scheme = "http"}
  13. local releases = {"jessie", "ascii", "beowulf", "ceres"}
  14. local releaseFiles =
  15. {
  16. -- Release file.
  17. "Release", -- 3.7 MB
  18. "Release.gpg", --
  19. -- "InRelease", -- 3.7 MB
  20. -- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently.
  21. -- Contents files. -- 3.3 GB
  22. -- "main/Contents-all.xz",
  23. -- "main/Contents-amd64.xz",
  24. -- "main/Contents-arm64.xz",
  25. -- "-security/main/Contents-all.xz",
  26. -- "-security/main/Contents-amd64.xz",
  27. -- "-security/main/Contents-arm64.xz",
  28. }
  29. local notExist =
  30. {
  31. "ceres-security" -- This will never exist, it's our code name for the testing suite.
  32. }
  33. local referenceDebs =
  34. {
  35. -- Debian package.
  36. "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2019.02.25_all.deb",
  37. -- Debian security package. NOTE this one should always be redirected?
  38. "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.10_amd64.deb",
  39. }
  40. local referenceDevs =
  41. {
  42. -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error.
  43. "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb",
  44. }
  45. local curlStatus =
  46. {
  47. [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.",
  48. [2 ] = "Failed to initialize.",
  49. [3 ] = "URL malformed. The syntax was not correct.",
  50. [4 ] = "A feature or option that was needed to perform the desired request was not enabled or was explicitly disabled at build-time. To make curl able to do this, you probably need another build of libcurl!",
  51. [5 ] = "Couldn't resolve proxy. The given proxy host could not be resolved.",
  52. [6 ] = "Couldn't resolve host. The given remote host was not resolved.",
  53. [7 ] = "Failed to connect to host.",
  54. [8 ] = "Weird server reply. The server sent data curl couldn't parse.",
  55. [9 ] = "FTP access denied. The server denied login or denied access to the particular resource or directory you wanted to reach. Most often you tried to change to a directory that doesn't exist on the server.",
  56. [10] = "While waiting for the server to connect back when an active FTP session is used, an error code was sent over the control connection or similar.",
  57. [11] = "FTP weird PASS reply. Curl couldn't parse the reply sent to the PASS request.",
  58. [12] = "During an active FTP session while waiting for the server to connect, the CURLOPT_ACCEPTTIMEOUT_MS (or the internal default) timeout expired.",
  59. [13] = "FTP weird PASV reply, Curl couldn't parse the reply sent to the PASV request.",
  60. [14] = "FTP weird 227 format. Curl couldn't parse the 227-line the server sent.",
  61. [15] = "FTP can't get host. Couldn't resolve the host IP we got in the 227-line.",
  62. [16] = "A problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems, see the error buffer for details.",
  63. [17] = "FTP couldn't set binary. Couldn't change transfer method to binary.",
  64. [18] = "Partial file. Only a part of the file was transferred.",
  65. [19] = "FTP couldn't download/access the given file, the RETR (or similar) command failed.",
  66. [21] = "FTP quote error. A quote command returned error from the server.",
  67. [22] = "HTTP page not retrieved. The requested url was not found or returned another error with the HTTP error code being 400 or above. This return code only appears if -f, --fail is used.",
  68. [23] = "Write error. Curl couldn't write data to a local filesystem or similar.",
  69. [25] = "FTP couldn't STOR file. The server denied the STOR operation, used for FTP uploading.",
  70. [26] = "Read error. Various reading problems.",
  71. [27] = "Out of memory. A memory allocation request failed.",
  72. [28] = "Operation timeout. The specified time-out period was reached according to the conditions.",
  73. [30] = "FTP PORT failed. The PORT command failed. Not all FTP servers support the PORT command, try doing a transfer using PASV instead!",
  74. [31] = "FTP couldn't use REST. The REST command failed. This command is used for resumed FTP transfers.",
  75. [33] = "HTTP range error. The range \"command\" didn't work.",
  76. [34] = "HTTP post error. Internal post-request generation error.",
  77. [35] = "SSL connect error. The SSL handshaking failed.",
  78. [36] = "FTP bad download resume. Couldn't continue an earlier aborted download.",
  79. [37] = "FILE couldn't read file. Failed to open the file. Permissions?",
  80. [38] = "LDAP cannot bind. LDAP bind operation failed.",
  81. [39] = "LDAP search failed.",
  82. [41] = "Function not found. A required LDAP function was not found.",
  83. [42] = "Aborted by callback. An application told curl to abort the operation.",
  84. [43] = "Internal error. A function was called with a bad parameter.",
  85. [45] = "Interface error. A specified outgoing interface could not be used.",
  86. [47] = "Too many redirects. When following redirects, curl hit the maximum amount.",
  87. [48] = "Unknown option specified to libcurl. This indicates that you passed a weird option to curl that was passed on to libcurl and rejected. Read up in the manual!",
  88. [49] = "Malformed telnet option.",
  89. [51] = "The peer's SSL certificate or SSH MD5 fingerprint was not OK.",
  90. [52] = "The server didn't reply anything, which here is considered an error.",
  91. [53] = "SSL crypto engine not found.",
  92. [54] = "Cannot set SSL crypto engine as default.",
  93. [55] = "Failed sending network data.",
  94. [56] = "Failure in receiving network data.",
  95. [58] = "Problem with the local certificate.",
  96. [59] = "Couldn't use specified SSL cipher.",
  97. [60] = "Peer certificate cannot be authenticated with known CA certificates.",
  98. [61] = "Unrecognized transfer encoding.",
  99. [62] = "Invalid LDAP URL.",
  100. [63] = "Maximum file size exceeded.",
  101. [64] = "Requested FTP SSL level failed.",
  102. [65] = "Sending the data requires a rewind that failed.",
  103. [66] = "Failed to initialise SSL Engine.",
  104. [67] = "The user name, password, or similar was not accepted and curl failed to log in.",
  105. [68] = "File not found on TFTP server.",
  106. [69] = "Permission problem on TFTP server.",
  107. [70] = "Out of disk space on TFTP server.",
  108. [71] = "Illegal TFTP operation.",
  109. [72] = "Unknown TFTP transfer ID.",
  110. [73] = "File already exists (TFTP).",
  111. [74] = "No such user (TFTP).",
  112. [75] = "Character conversion failed.",
  113. [76] = "Character conversion functions required.",
  114. [77] = "Problem with reading the SSL CA cert (path? access rights?).",
  115. [78] = "The resource referenced in the URL does not exist.",
  116. [79] = "An unspecified error occurred during the SSH session.",
  117. [80] = "Failed to shut down the SSL connection.",
  118. [81] = "Socket is not ready for send/recv wait till it's ready and try again. This return code is only returned from curl_easy_recv and curl_easy_send.",
  119. [82] = "Could not load CRL file, missing or wrong format (added in 7.19.0).",
  120. [83] = "Issuer check failed (added in 7.19.0).",
  121. [84] = "The FTP PRET command failed",
  122. [85] = "RTSP: mismatch of CSeq numbers",
  123. [86] = "RTSP: mismatch of Session Identifiers",
  124. [87] = "unable to parse FTP file list",
  125. [88] = "FTP chunk callback reported error",
  126. [89] = "No connection available, the session will be queued",
  127. [90] = "SSL public key does not matched pinned public key",
  128. [91] = "Status returned failure when asked with CURLOPT_SSL_VERIFYSTATUS.",
  129. [92] = "Stream error in the HTTP/2 framing layer.",
  130. [93] = "An API function was called from inside a callback.",
  131. [94] = "An authentication function returned an error.",
  132. [95] = "A problem was detected in the HTTP/3 layer. This is somewhat generic and can be one out of several problems, see the error buffer for details.",
  133. }
  134. local socket = require 'socket'
  135. local ftp = require 'socket.ftp'
  136. local http = require 'socket.http'
  137. local url = require 'socket.url'
  138. local ip = ""
  139. local cor = nil
  140. local Updating = false
  141. local downloadLock = "flock -n results/curl-"
  142. local arw = ' &nbsp; <font color="magenta"><b>-></b></font> &nbsp; '
  143. local repoExists = function (r)
  144. r = r:match("([%a-]*)")
  145. if nil == r then return false end
  146. for k, v in pairs(notExist) do
  147. if v == r then return false end
  148. end
  149. return true
  150. end
  151. local IP = {}
  152. gatherIPs = function (host)
  153. if nil == IP[host] then
  154. local IPs
  155. -- Takes about 30 seconds to look up the lot.
  156. -- I tested using dig's -f option, it didn't seem much faster.
  157. -- The sort -r assumes that deb.devuan.org is the first alphabetically.
  158. local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
  159. repeat
  160. IPs = dig:read("*l")
  161. if nil ~= IPs then
  162. for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
  163. if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
  164. if nil == IP[k] then IP[k] = {} end
  165. IP[k][v] = t
  166. D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
  167. if t == "CNAME" then
  168. gatherIPs(v)
  169. IP[k][v] = IP[v]
  170. elseif v == "SRV" then
  171. print("SVR record found, now what do we do?")
  172. end
  173. end
  174. end
  175. until nil == IPs
  176. end
  177. -- If this is the DNS-RR domain name, gather the IPs for the mirrors that mirror_list.txt says should be in it.
  178. if host == APT.options.roundRobin.value then
  179. for k, m in pairs(APT.mirrors) do
  180. if "yes" == m.DNSRR then
  181. gatherIPs(m.FQDN)
  182. IP[host][m.FQDN] = IP[m.FQDN]
  183. end
  184. end
  185. end
  186. return IP[host]
  187. end
  188. -- Returns FTP directory listing
  189. local nlst = function (u)
  190. local t = {}
  191. local p = url.parse(u)
  192. p.command = "nlst"
  193. p.sink = ltn12.sink.table(t)
  194. local r, e = ftp.get(p)
  195. return r and table.concat(t), e
  196. end
  197. local timeouts = 0;
  198. local totalTimeouts = 0
  199. local spcd = ' &nbsp; '
  200. checkHEAD = function (host, URL, r, retry, sanity)
  201. if nil == r then r = 0 end
  202. if nil == retry then retry = 0 end
  203. if true == sanity then sanity = 'URLSanity' else sanity = '' end
  204. local check = "HEAD testing file"
  205. local PU = url.parse(URL, defaultURL)
  206. local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
  207. local fname = host .. "_" .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".log.txt"
  208. local hdr = ""
  209. local IP = ""
  210. if pu.host ~= PU.host then
  211. if "http" == PU.scheme then
  212. hdr = '-H "Host: ' .. host .. '"'
  213. end
  214. IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
  215. fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt"
  216. end
  217. os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null; touch results/STATUS_' .. fname)
  218. if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested. &nbsp; " .. URL, host); return end
  219. if 0 < r then
  220. check = "Redirecting to"
  221. end
  222. if 0 < retry then
  223. os.execute("sleep " .. math.random(1, 3))
  224. check = "Retry " .. retry .. " " .. check
  225. end
  226. if 2 <= timeouts then
  227. E(spcd .. spcd .. "too many timeouts! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
  228. return
  229. end
  230. if APT.options.timeouts.value <= (totalTimeouts) then
  231. E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host)
  232. return
  233. end
  234. if 20 <= r then
  235. E(spcd .. spcd .. "too many redirects! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
  236. return
  237. end
  238. if APT.options.retries.value <= retry then
  239. E(spcd .. spcd .. "too many retries! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
  240. return
  241. end
  242. if "https" == PU.scheme and APT.options.roundRobin.value == host then
  243. I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host)
  244. return
  245. else
  246. I(spcd .. check .. " " .. APT.lnk(URL), host)
  247. end
  248. --[[ Using curl command line -
  249. -I - HEAD
  250. --connect-to domain:port:IP:port - connect to IP, but use SNI from URL.
  251. -header "" - add extra headers.
  252. -L - DO follow redirects.
  253. --max-redirs n - set maximum redirects, default is 50, -1 = unlimited.
  254. -4 or -6 - Only use IPv4 or IPv6
  255. --retry n - maximum retries, default is 0, no retries.
  256. -o file - write to file instead of stdout.
  257. --path-as-is - https://curl.haxx.se/libcurl/c/CURLOPT_PATH_AS_IS.html might be useful for URLSanity.
  258. -s silent - don't output progress or error messages.
  259. --connect-timeout n - timeout in seconds.
  260. Should return with error code 28 on a timeout?
  261. -D file - write the received headers to a file. This includes the status code and string.
  262. ]]
  263. local status = APT.exe(
  264. 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
  265. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
  266. hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
  267. ):Nice():log():Do().status
  268. if 0 < r then
  269. APT.tested(PU.scheme, 'Redirects', host)
  270. else
  271. APT.tested(PU.scheme, '', host)
  272. end
  273. local code = "???"
  274. local cstr = ""
  275. local location = nil
  276. local tmot = 1
  277. while not APT.checkFile('results/STATUS_' .. fname) do
  278. D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.')
  279. os.execute('sleep ' .. tmot)
  280. tmot = tmot * 2
  281. if 8 < tmot then
  282. T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
  283. timeouts = timeouts + 1
  284. checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
  285. os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
  286. return
  287. end
  288. end
  289. os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
  290. if 0 ~= status then
  291. local msg = curlStatus[status]
  292. if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
  293. if (28 == status) or (7 == status) then
  294. T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
  295. timeouts = timeouts + 1
  296. else
  297. E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host)
  298. end
  299. if 60 == status then return end -- Certificate is invalid, don't bother retrying.
  300. checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
  301. return
  302. end
  303. local rfile, e = io.open("results/STATUS_" .. fname, "r")
  304. if nil == rfile then W("opening results/STATUS_" .. fname .. " file - " .. e) else
  305. for line in rfile:lines("*l") do
  306. if "#" == line:sub(1, 1) then
  307. code = line:sub(2, 4)
  308. if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then
  309. os.execute('cp results/STATUS_' .. fname .. ' results/STATUS_' .. fname .. '_SAVED')
  310. if '' ~= sanity then
  311. E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host)
  312. else
  313. E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host)
  314. end
  315. end
  316. elseif "http" == line:sub(1, 4):lower() then
  317. -- -2 coz the headers file gets a \r at the end.
  318. cstr = line:sub(14, -2)
  319. elseif "location" == line:sub(1, 8):lower() then
  320. location = line:sub(11, -2)
  321. end
  322. end
  323. if '???' == code then
  324. W(spcd .. spcd .. 'Could not find response code. ' .. APT.lnk(URL), PU.scheme, sanity, host)
  325. end
  326. end
  327. os.execute('cat results/STATUS_' .. fname .. ' >> results/curl_HEAD_' .. fname .. '; rm -f results/STATUS_' .. fname .. ' 2>/dev/null')
  328. if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
  329. E(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host)
  330. else
  331. if not APT.testing(PU.scheme, host) then
  332. I(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host)
  333. end
  334. I(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), host)
  335. -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
  336. if nil ~= location then
  337. pu = url.parse(location, defaultURL)
  338. if (pu.host == APT.options.roundRobin.value) and (nil ~= PU.path:find('merged/pool/DEVUAN/')) then
  339. E('DEVUAN packages must not be redirected to ' .. APT.options.roundRobin.value .. ' - ' .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, 'Redirects', host)
  340. end
  341. if APT.testing("Protocol") then
  342. if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change.
  343. if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin.
  344. E(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
  345. end
  346. W(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
  347. else
  348. end
  349. APT.tested(PU.scheme, 'Protocol', host)
  350. end
  351. if location == URL then
  352. E(spcd .. spcd .. "Redirect loop! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host)
  353. elseif nil == pu.host then
  354. I(spcd .. spcd .. "Relative redirect. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  355. if 1 <= APT.options.bandwidth.value then checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity) end
  356. elseif (PU.host == pu.host) or (host == pu.host) then
  357. if PU.host ~= host then
  358. local t = pu.host
  359. pu.host = PU.host
  360. location = url.build(pu)
  361. pu.host = t
  362. end
  363. I(spcd .. spcd .. "Redirect to same host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  364. if 1 <= APT.options.bandwidth.value then checkHEAD(host, location, r + 1, retry, '' ~= sanity) end
  365. else
  366. I(spcd .. spcd .. "Redirect to different host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  367. if 1 <= APT.options.bandwidth.value then
  368. --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
  369. Not good for deb.debian.org, which we should only be throwing .debs at.
  370. What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
  371. ]]
  372. local u = pu.host .. "/" .. pu.path
  373. local file = pu.path:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  374. local path = pu.path:sub(2, -1 -(#file))
  375. local check = u:gsub("/", "_")
  376. local extraArgs = sendArgs .. ' -o -r '
  377. if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
  378. if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
  379. local pth = path:match('^(.*/pool/).*$')
  380. if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
  381. I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  382. APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
  383. D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
  384. APT.tested(PU.scheme, 'Redirects', host)
  385. end
  386. end
  387. elseif nil ~= PU.path:find('merged/pool/DEBIAN-SECURITY/') then
  388. W('DEBIAN-SECURITY packages must be redirected to a Debian mirror - ' .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, 'Redirects', host)
  389. end
  390. end
  391. end
  392. local checkTimeouts = function(host, scheme, URL)
  393. totalTimeouts = totalTimeouts + timeouts; timeouts = 0
  394. checkHEAD(host, scheme .. "://" .. URL)
  395. if (1 <= APT.options.bandwidth.value) and APT.testing("URLSanity") then
  396. URL = URL:gsub("/", "///")
  397. URL = URL:gsub("///", "/", 1)
  398. checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
  399. APT.tested(scheme, 'URLSanity', host)
  400. end
  401. if nil ~= cor then
  402. D('*&gt;* About to resume coroutine after checkHEAD(' .. host .. ' , ' .. scheme .. ' :// ' .. URL .. ')')
  403. local ok, message = coroutine.resume(cor)
  404. if not ok then cor = nil; print(message) end
  405. end
  406. if APT.options.timeouts.value <= (totalTimeouts) then
  407. E("Way too many timeouts!", scheme, "URLSanity", host)
  408. return true
  409. end
  410. return false
  411. end
  412. local checkFiles = function (host, ip, path, file)
  413. timeouts = 0
  414. if nil == path then path = "" end
  415. if nil ~= file then
  416. if "redir" == ip then ip = host end
  417. if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end
  418. if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
  419. else
  420. I(" HEAD testing files for " .. host .. arw .. ip .. " " .. path, host)
  421. if 1 <= APT.options.bandwidth.value then
  422. -- Do these first, coz they are likely to fork off a different server.
  423. for i, s in pairs(referenceDebs) do
  424. if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
  425. if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
  426. end
  427. end
  428. for i, s in pairs(releases) do
  429. for j, k in pairs(releaseFiles) do
  430. if repoExists(s .. k) then
  431. if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
  432. if 1 <= APT.options.bandwidth.value then
  433. if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
  434. else
  435. break
  436. end
  437. end
  438. if 2 >= APT.options.bandwidth.value then break end
  439. end
  440. if 2 >= APT.options.bandwidth.value then break end
  441. end
  442. if 1 <= APT.options.bandwidth.value then
  443. for i, s in pairs(referenceDevs) do
  444. if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
  445. if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
  446. end
  447. end
  448. end
  449. end
  450. checkHost = function (orig, host, path, ip, file)
  451. if nil == host then host = orig end
  452. if nil == path then path = "" end
  453. if nil == file then file = "" end
  454. local ph = url.parse("http://" .. host)
  455. if (nil ~= ip) and ("redir" ~= ip) then
  456. local po = url.parse("http://" .. orig)
  457. if "" ~= file then
  458. D("checking redirected file " .. po.host .. " " .. file)
  459. checkFiles(po.host, ip, path, file)
  460. else
  461. checkFiles(po.host, ip, path)
  462. end
  463. else
  464. if orig == host then
  465. I("Testing mirror " .. orig .. "" .. file)
  466. APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
  467. D('logging to ' .. APT.logName(ph.host, nil, file)[2])
  468. else D("checkHost " .. orig .. arw .. host) end
  469. end
  470. end
  471. local addDownload = function(host, URL, f, r, k)
  472. local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  473. if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. '/' .. k) then
  474. -- Curls "check timestamp and overwrite file" stuff sucks.
  475. -- -R means the destination file gets the timestamp of the remote file.
  476. -- Can only do ONE timestamp check per command.
  477. -- This doesn't work either. All downloads get all these headers. Pffft
  478. -- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. '/' .. k .. ' | cut -d " " -f 6-11')
  479. -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
  480. -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first,
  481. -- which wont work with multiple files in multiple directories. WTF?
  482. --TODO - change tactic, do a HEAD if-modified test first before adding the file to the list to download.
  483. os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. '/' .. k ..
  484. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old")
  485. end
  486. D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k)
  487. f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
  488. f:write('output "results/' .. host .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
  489. end
  490. local postDownload = function(host, r, k)
  491. local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  492. if nil == file then file = k end
  493. os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old ]" ..
  494. " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " ]; then cp -a" ..
  495. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" ..
  496. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi")
  497. if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then
  498. if ".gz" == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
  499. if ".xz" == k:sub(-3, -1) then APT.exe("xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
  500. end
  501. end
  502. local download = "curl" ..
  503. " --connect-timeout " .. APT.options.timeout.value ..
  504. " --create-dirs -f -L" ..
  505. " --fail-early" ..
  506. " --max-time " .. APT.options.maxtime.value ..
  507. APT.IPv46 .. ' ' ..
  508. " --retry " .. APT.options.retries.value ..
  509. " -R -v -z 'results/stamp.old' --stderr results/"
  510. local downloads = function(host, URL, meta, release, list)
  511. if nil == URL then URL = "" end
  512. local files = 'curl-' .. meta .. '-' .. host .. '.files.txt'
  513. local lock = meta .. "-" .. host .. ".log.txt"
  514. local log = "curl-" .. meta .. "-" .. host .. ".log.txt"
  515. local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
  516. if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
  517. if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
  518. f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+")
  519. if nil == f then C("opening curl downloads list file - " .. e); return end
  520. if nil ~= list then
  521. if "" ~= list then
  522. if nil ~= release then
  523. for l in list:gmatch("\n*([^\n]+)\n*") do
  524. addDownload(host, URL, f, release, "/" .. l)
  525. end
  526. else
  527. I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list))
  528. f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n')
  529. f:write('output "results/' .. host .. '/merged/' .. list .. '"\n')
  530. end
  531. f:close()
  532. return
  533. end
  534. else
  535. for i, s in pairs(releases) do
  536. for j, k in pairs(releaseFiles) do
  537. if repoExists(s .. k) then
  538. addDownload(host, URL, f, s, k)
  539. end
  540. end
  541. end
  542. end
  543. f:close()
  544. APT.exe(cm):Nice():log():fork()
  545. D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
  546. end
  547. local validateURL = function(m)
  548. if " " == m.BaseURL:sub(-1, -1) then
  549. W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
  550. m.BaseURL = m.BaseURL:sub(1, -2)
  551. end
  552. if "/" == m.BaseURL:sub(-1, -1) then
  553. W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
  554. m.BaseURL = m.BaseURL:sub(1, -2)
  555. end
  556. local p = url.parse("http://" .. m.BaseURL)
  557. if nil == p.path then p.path = '' end
  558. if nil ~= p.port then p.authority = authority .. ':' .. p.port end
  559. if nil == m.FDQN then W("Something wrong in FDQN from mirror_list.txt! nil", "", "", p.authority) else
  560. if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt! " .. m.FDQN, "", "", p.authority) end
  561. end
  562. if nil == m.BaseURL then W("Something wrong in BaseURL from mirror_list.txt! nil", "", "", p.authority) else
  563. if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end
  564. end
  565. if (nil ~= p.query) or (nil ~= p.fragment) or (nil ~= p.params) then W("Something wrong in BaseURL from mirror_list.txt, should be nothing after the path! " .. m.BaseURL, "", "", p.authority) end
  566. if (nil ~= p.user) or (nil ~= p.userinfo) or (nil ~= p.password) then W("Something wrong in BaseURL from mirror_list.txt, should be no credentials! " .. m.BaseURL, "", "", p.authority) end
  567. m.FQDN = p.authority
  568. m.BaseURL = p.authority .. p.path
  569. return m
  570. end
  571. local getMirrors = function ()
  572. local mirrors = {}
  573. local host = ""
  574. local m = {}
  575. local active = true
  576. local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt'
  577. I('Downloading and parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt')
  578. local p, c, h = http.request(URL)
  579. if nil == p then E(c .. " fetching " .. URL) else
  580. for l in p:gmatch("\n*([^\n]+)\n*") do
  581. local t, d = l:match("(%a*):%s*(.*)")
  582. d = string.lower(d)
  583. if "FQDN" == t then
  584. if "" ~= host then
  585. mirrors[host] = validateURL(m)
  586. m = {}
  587. active = true
  588. end
  589. host = d
  590. m[t] = d
  591. elseif "Protocols" == t then
  592. local prot = {}
  593. for w in d:gmatch("(%w+)") do
  594. if APT.search(APT.protocols, w:lower()) then prot[w] = true end
  595. end
  596. m[t] = prot
  597. elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then
  598. W("Mirror " .. host .. " is not active - " .. d, "", "", host)
  599. active = false
  600. m[t] = d
  601. -- TODO - Should do some more input validation on everything.
  602. elseif "Rate" == t then
  603. local time, unit = d:match('(%d+) *(%a+)')
  604. time = tonumber(time)
  605. unit = unit:sub(1, 1)
  606. m[t] = time .. ' ' .. unit
  607. if 'm' == unit then
  608. time = time * 60
  609. elseif 'h' == unit then
  610. time = time * 60 * 60
  611. else
  612. C('Unknown Rate for mirror ' .. host)
  613. end
  614. m['Updated'] = time
  615. else
  616. m[t] = d
  617. end
  618. end
  619. if "" ~= host --[[and active]] then
  620. mirrors[host] = validateURL(m)
  621. end
  622. end
  623. if APT.testing("DNSRR") then
  624. mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; }; ['Updated'] = 300; ['DNSRR'] = false;
  625. ["FQDN"] = APT.options.roundRobin.value; ["Active"] = 'yes'; ["BaseURL"] = APT.options.roundRobin.value; }
  626. end
  627. local file, e = io.open("results/mirrors.lua", "w+")
  628. if nil == file then C("opening mirrors file - " .. e) else
  629. file:write(APT.dumpTable(mirrors, "mirrors") .. "\nreturn mirrors\n")
  630. file:close()
  631. end
  632. if 42 < #mirrors then print(#mirrors .. ' is too many mirrors!') ; os.exit(false) end
  633. return mirrors
  634. end
  635. local postParse = function(host, list)
  636. if APT.options.referenceSite.value == host then
  637. if nil ~= list then
  638. local sem = 'results/NEW_' .. list.out .. '_%s.txt'
  639. for i, n in pairs(releases) do
  640. local f = sem:format(n)
  641. if APT.checkFile(f .. '.tmp') then
  642. os.execute('mv ' .. f .. '.tmp ' .. f)
  643. else
  644. os.execute('touch ' .. f)
  645. end
  646. end
  647. end
  648. end
  649. end
  650. local parseDebs = function(host)
  651. for i, n in pairs(releases) do
  652. local inFile = 'results/NEW_debs_' .. n .. '.txt'
  653. local nfile, e = io.open(inFile, "r")
  654. if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
  655. for l in nfile:lines() do
  656. local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
  657. if nil ~= p then
  658. if APT.checkFile('results/' .. host .. "/merged/" .. p) then
  659. local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result
  660. if APT.testing("Integrity") then
  661. if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
  662. E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host)
  663. else
  664. local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):log():Do().result
  665. if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end
  666. -- TODO - maybe check the PGP key, though packages are mostly not signed.
  667. end
  668. APT.tested('http', 'Integrity', host)
  669. end
  670. if APT.testing("Updated") then
  671. if sz ~= fsz:sub(2, -2) then
  672. E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
  673. end
  674. APT.tested('http', 'Updated', host)
  675. end
  676. os.execute('rm -f results/' .. host .. "/merged/" .. p)
  677. else
  678. if Updating then
  679. W('Not yet able to download, awaiting update for ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
  680. else
  681. E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
  682. end
  683. end
  684. end
  685. end
  686. end
  687. end
  688. return nil
  689. end
  690. local parsePackages = function(host)
  691. local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
  692. for i, n in pairs(releases) do
  693. local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt'
  694. local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
  695. if APT.options.referenceSite.value == host then
  696. outFile = outFile .. '.tmp'
  697. end
  698. local dFile, e = io.open(inFile, "r")
  699. if nil == dFile then W("opening " .. inFile .. " file - " .. e) else
  700. for l in dFile:lines() do
  701. postDownload(host, n, l)
  702. l = '/' .. l
  703. local file = l:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  704. local dir = l:sub(1, 0 - (#file + 1))
  705. if "Packages." == file:sub(1, 9) then
  706. -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
  707. -- Also note that this might get only a partial download due to maxtime.
  708. if APT.options.referenceSite.value == host then
  709. local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
  710. if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
  711. local pp = {}
  712. for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
  713. if "Package: " == l:sub(1, 9) then
  714. if 0 ~= #pp then
  715. for i = 1, 5 do
  716. if nil == pp[i] then print(host .. " " .. n .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end
  717. end
  718. Pp:write("\n")
  719. end
  720. pp = {}
  721. pp[1] = l:sub(10, -1)
  722. elseif "Version: " == l:sub(1, 9) then
  723. pp[2] = l:sub(10, -1)
  724. elseif "Filename: " == l:sub(1, 10) then
  725. pp[3] = l:sub(11, -1)
  726. elseif "Size: " == l:sub(1, 6) then
  727. pp[4] = l:sub(7, -1)
  728. elseif "SHA256: " == l:sub(1, 8) then
  729. pp[5] = l:sub(9, -1)
  730. end
  731. end
  732. Pp:close()
  733. os.execute('sort results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages_parsed-sorted')
  734. if APT.checkFile('Packages/' .. n .. dir .. 'Packages_parsed-sorted') then
  735. os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  736. 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  737. ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_' .. list.out .. '_' .. n .. '.txt')
  738. os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  739. 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  740. ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
  741. for i, s in pairs(referenceDebs) do
  742. if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then
  743. print('Reference package is out of date (' .. n .. ') - ' .. s)
  744. end
  745. end
  746. for i, s in pairs(referenceDevs) do
  747. if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then
  748. print('Reference package is out of date (' .. n .. ') - ' .. s)
  749. end
  750. end
  751. else
  752. W("Can't find file Packages/" .. n .. dir .. "Packages_parsed-sorted")
  753. end
  754. os.execute('mkdir -p Packages/' .. n .. dir)
  755. os.execute('mv -f results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted Packages/' .. n .. dir .. 'Packages_parsed-sorted')
  756. end
  757. else
  758. end
  759. os.execute('rm -fr results/' .. host .. '/merged/dists/' .. n .. dir .. ' 2>/dev/null')
  760. end
  761. end
  762. if APT.checkFile('results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') then
  763. -- Sort by size.
  764. os.execute('sort -b -k 9,9 -n results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt >results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt')
  765. os.execute('grep -s " | pool/DEVUAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
  766. os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
  767. os.execute('grep -s " | pool/DEBIAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >' .. outFile)
  768. os.execute('rm -f results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
  769. end
  770. end
  771. local nfile, e = io.open(outFile, "r")
  772. if nil ~= nfile then
  773. -- for l in nfile:lines() do
  774. local l = nfile:read('*l')
  775. if nil ~= l then
  776. local p = l:match('(pool/.*%.deb)')
  777. if nil ~= p then
  778. table.insert(list.files, p)
  779. end
  780. end
  781. -- end
  782. end
  783. end
  784. postParse(host, list)
  785. return list
  786. end
  787. local parseRelease = function(host)
  788. local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
  789. local updated = false
  790. local now = tonumber(os.date('%s'))
  791. for i, n in pairs(releases) do
  792. for l, o in pairs(releaseFiles) do
  793. if repoExists(i .. o) then
  794. postDownload(host, n, o)
  795. if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
  796. if APT.testing("Integrity") then
  797. local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
  798. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
  799. if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
  800. -- TODO - should check the PGP sig of InRelease as well.
  801. APT.tested('http', 'Integrity', host)
  802. end
  803. os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
  804. end
  805. end
  806. end
  807. local fR = 'results/' .. host .. '/merged/dists/' .. n .. '/Release'
  808. local fRp = APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED'
  809. if APT.checkFile(fR) then
  810. os.execute('sort -k 3 ' .. fR .. ' >' .. fR .. '.SORTED')
  811. local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
  812. if APT.checkFile('results_old/' .. fRp) then
  813. if APT.options.referenceSite.value == host then
  814. outFile = outFile .. '.tmp'
  815. os.execute('diff -U 0 results_old/' .. fRp .. ' ' ..
  816. 'results/' .. fRp .. ' ' ..
  817. '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >' .. outFile)
  818. -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
  819. -- Also check if that date is in the future, apt recently got a check for that, though not sure why.
  820. os.execute('rm -f ' .. fR .. ' 2>/dev/null; ')
  821. else
  822. -- TODO - compare to the pkgmaster copy.
  823. if APT.testing('Updated') then
  824. while not APT.checkFile('results_old/' .. fRp) do
  825. D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(results_old/' .. fRp .. ')')
  826. coroutine.yield()
  827. D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(results_old/' .. fRp .. ')')
  828. end
  829. local pkt = tonumber(APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. fRp .. [[ | cut -d ' ' -f 2-)" '+%s']]):Do().result:sub(2, -2))
  830. local new = tonumber(APT.exe([[TZ="GMT" date -d "$(grep '^Date:' ]] .. fR .. [[.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result:sub(2, -2))
  831. local upd = pkt + APT.mirrors[host].Updated
  832. local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update.
  833. if pkt > new then
  834. D( 'pkt is ' .. os.date('!%F %T', pkt) .. ' new is ' .. os.date('!%F %T', new) ..
  835. ' upd is ' .. os.date('!%F %T', upd) .. ' updd is ' .. os.date('!%F %T', updd) ..
  836. ' now is ' .. os.date('!%F %T', now) .. ' Updated is ' .. APT.mirrors[host].Updated)
  837. if updd >= now then
  838. W('Release ' .. n .. ' not updated yet, should update @ ' .. os.date('!%F %T', upd) .. ', and was last updated @ ' .. os.date('!%F %T', new), 'http', 'Updated', host)
  839. Updating = true
  840. else
  841. E('Release ' .. n .. ' not updated, should have updated @ ' .. os.date('!%F %T', upd) .. ', but was last updated @ ' .. os.date('!%F %T', new), 'http', 'Updated', host)
  842. end
  843. updated = false
  844. else
  845. updated = true
  846. end
  847. APT.tested('http', 'Updated', host)
  848. end
  849. end
  850. -- TODO - if it's not Integrity and not reference, then just do a HEAD check and compare file times?
  851. -- TODO - like we do with debs, pick just the smallest Packages.xz that has changed.
  852. -- Though we are sorting Release by name, so we can do the diff with the one from results_old, so we'll need to sort by size to.
  853. -- pkgmaster still needs to download all the changed Packages.xz files though.
  854. if (2 <= APT.options.bandwidth.value) and (updated or APT.testing("Integrity") or (APT.options.referenceSite.value == host)) then
  855. local dfile, e = io.open(outFile, "r")
  856. if nil == dfile then W("opening " .. outFile .. " file - " .. e) else
  857. for l in dfile:lines() do
  858. table.insert(list.files, 'dists/' .. n .. '/' .. l)
  859. end
  860. end
  861. end
  862. end
  863. end
  864. end
  865. postParse(host, list)
  866. return list
  867. end
  868. local parseStart = function(host)
  869. local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
  870. for i, n in pairs(releases) do
  871. local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
  872. for l, o in pairs(releaseFiles) do
  873. if repoExists(n .. o) then
  874. if APT.options.referenceSite.value == host then
  875. local dfile, e = io.open(outFile .. '.tmp', "a+")
  876. if nil == dfile then W("opening " .. outFile .. ".tmp file - " .. e) else
  877. dfile:write(o .. '\n')
  878. end
  879. end
  880. table.insert(list.files, 'dists/' .. n .. '/' .. o)
  881. end
  882. end
  883. end
  884. postParse(host, list)
  885. return list
  886. end
  887. local doDownloads = function(host, path, list)
  888. while nil ~= list do
  889. if 0 ~= #(list.files) then
  890. for j, f in pairs(list.files) do
  891. downloads(host, path, list.out, nil, f)
  892. end
  893. downloads(host, path, list.out, nil, '')
  894. --[[ I've seen flock & curl act oddly. Perhaps flock didn't have time to start up?
  895. /var/www/html/apt-panopticon/apt-panopticon/results_2019-12-22-15-00
  896. Mon Dec 23 01:02:54 2019 DEBUG : forking
  897. flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
  898. Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running.
  899. Mon Dec 23 01:02:54 2019 DEBUG : *>* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(flock -n results/curl-debs-pkgmaster.devuan.org.log
  900. Mon Dec 23 01:02:54 2019 DEBUG : *** Doing list.parser() for debs
  901. Mon Dec 23 01:02:54 2019 ERROR (http Updated pkgmaster.devuan.org): Failed to download - results/pkgmaster.devuan.org/merged/pool/DEBIAN/main/a/aptly/aptly_1.3.0+ds1-4_amd64.deb
  902. drwxr-x--- 2 www-data www-data 4096 2019-12-23 01:02:57.000000000 +1000 aptly
  903. -rw-r--r-- 1 www-data www-data 7129 2019-12-23 01:03:54.000000000 +1000 curl-debs-pkgmaster.devuan.org.log
  904. ]]
  905. os.execute('sleep 1') -- Wait for things to start up before checking for them.
  906. while 0 < APT.checkExes(downloadLock .. list.out .. "-" .. host .. ".log.txt") do
  907. D('*&lt;* About to yield coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log.txt')
  908. coroutine.yield()
  909. D('*&gt;* Resumed coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log.txt')
  910. end
  911. D('*&gt;* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log.txt')
  912. local f = 'results/curl-' .. list.out .. "-" .. host .. ".log.txt"
  913. local f = 'results/curl-' .. list.out .. "-" .. host .. ".log.txt"
  914. -- Should not be needed, but maybe this is why sometimes I don't see the speeds, though the file is there aand valid when I look later.
  915. while not APT.checkFile(f) do
  916. D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  917. coroutine.yield()
  918. D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  919. end
  920. --[[ TODO - should try to figure out which server the file actually got downloaded from, and attribute the speed and errors to that server.
  921. Which means parsing the curl logs, not just a simple match().
  922. Watch out for misplaced ^M, they don't all come at the end of the line.
  923. Also note curl-Release-mirror.devuan.de.log.txt, timeouts don't always show the "Connected to" string.
  924. * Immediate connect fail for 2001:4ca0:4300::1:19: Network is unreachable
  925. * Connected to debian.ipacct.com (2a01:9e40::251) port 80 (#1)
  926. 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0)
  927. curl: (22) The requested URL returned error: 404 Not Found
  928. ]]
  929. local min, max, spd = 999999999999, 0
  930. local num = '[%d%.]+[kM]?'
  931. if APT.checkFile(f) then
  932. for l in io.lines(f) do
  933. local speed, crrnt = l:match('^%c *'..num..' +'..num..' +%d+ +'..num..' +%d+ +%d+ +('..num..') +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +('..num..')')
  934. if nil ~= speed then
  935. if 'k' == speed:sub(-1, -1) then speed = tonumber(speed:sub(1, -2)) * 1024
  936. elseif 'M' == speed:sub(-1, -1) then speed = tonumber(speed:sub(1, -2)) * 1024 * 1024
  937. end
  938. speed = tonumber(speed)
  939. if 'k' == crrnt:sub(-1, -1) then crrnt = tonumber(crrnt:sub(1, -2)) * 1024
  940. elseif 'M' == crrnt:sub(-1, -1) then crrnt = tonumber(crrnt:sub(1, -2)) * 1024 * 1024
  941. end
  942. crrnt = tonumber(crrnt)
  943. if speed < min and speed ~= 0 then min = speed end
  944. if speed > max then max = speed end
  945. if crrnt < min and crrnt ~= 0 then min = crrnt end
  946. if crrnt > max then max = crrnt end
  947. end
  948. if l:find('timed out') ~= nil then
  949. E(" TIMEOUT " .. timeouts + 1 .. ', details in <a href="curl-' .. list.out .. '-' .. host .. '.log.txt">curl-' .. list.out .. '-' .. host .. '.log.txt</a>', 'http', '', host)
  950. timeouts = timeouts + 1
  951. APT.results["timeout"] = true
  952. end
  953. end
  954. end
  955. APT.results["speed"] = {["min"] = min, ["max"] = max}
  956. end
  957. if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
  958. local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
  959. for i, n in pairs(releases) do
  960. local f = sem:format(n)
  961. while not APT.checkFile(f) do
  962. D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  963. coroutine.yield()
  964. D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  965. end
  966. end
  967. end
  968. D('*** Doing list.parser() for ' .. list.out)
  969. list = list.parser(host)
  970. if APT.options.timeouts.value <= (totalTimeouts) then break end
  971. end
  972. D('*&lt;&lt;* About to end coroutine.')
  973. cor = nil
  974. end
  975. if 0 < #arg then
  976. if "/" == arg[1]:sub(-1, -1) then
  977. W("slash at end of path! " .. arg[1])
  978. arg[1] = arg[1]:sub(1, -2)
  979. end
  980. if " " == arg[1]:sub(-1, -1) then
  981. W("space at end of path! " .. arg[1])
  982. arg[1] = arg[1]:sub(1, -2)
  983. end
  984. local pu = url.parse("http://" .. arg[1])
  985. if APT.redir and (nil == arg[3])then
  986. arg[3] = arg[2]
  987. arg[2] = nil
  988. end
  989. if APT.testing("Integrity") or APT.testing("Updated") then
  990. if APT.origin and APT.options.referenceSite.value == pu.host then
  991. -- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
  992. end
  993. end
  994. if not APT.logOpen(pu.host, arg[2], arg[3]) then return end
  995. I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
  996. APT.mirrors = loadfile("results/mirrors.lua")()
  997. APT.results = APT.padResults(APT.results)
  998. if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
  999. if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
  1000. if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
  1001. if APT.origin then
  1002. local file = arg[3]
  1003. if nil == file then file = '' end
  1004. local path = pu.path
  1005. if nil == path then path = '' end
  1006. if APT.origin then
  1007. local ips = APT.results["IPs"]
  1008. if nil ~= ips then
  1009. APT.allpairs(ips,
  1010. function(k, v)
  1011. if v == "A" then
  1012. if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
  1013. elseif v == "AAAA" then
  1014. if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
  1015. end
  1016. D('logging to ' .. APT.logName(pu.host, k, file)[2])
  1017. end
  1018. )
  1019. else
  1020. E("no IPs for " .. pu.host)
  1021. end
  1022. end
  1023. if not APT.redir then
  1024. if (1 <= APT.options.bandwidth.value) and (APT.testing("Integrity") or APT.testing("Updated")) then
  1025. if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
  1026. I("Starting file downloads for " .. pu.host)
  1027. D('*&gt;* About to create coroutine.')
  1028. cor = coroutine.create(doDownloads)
  1029. local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host))
  1030. if not ok then cor = nil; print(message) end
  1031. end
  1032. end
  1033. checkFiles(pu.host, pu.host, pu.path);
  1034. else
  1035. checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file);
  1036. end
  1037. else
  1038. checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
  1039. end
  1040. while nil ~= cor do
  1041. os.execute('sleep 10')
  1042. D('*&gt;* About to resume coroutine before writing results.')
  1043. local ok, message = coroutine.resume(cor)
  1044. if not ok then cor = nil; print(message); break end
  1045. end
  1046. local f = pu.host
  1047. if "" ~= ip then f = f .. "_" .. ip end
  1048. -- TODO - perhaps number them if there's more than one?
  1049. if APT.redir then f = f .. '_' .. 'R' end
  1050. local rfile, e = io.open("results/" .. f .. ".lua", "w+")
  1051. if nil == rfile then C("opening results file - " .. e) else
  1052. rfile:write(APT.dumpTable(APT.results, "results") .. "\nreturn results\n")
  1053. rfile:close()
  1054. end
  1055. if APT.origin and (not APT.redir) and (APT.options.referenceSite.value ~= pu.host) then
  1056. os.execute('sleep 1') -- Wait for things to start up before checking for them.
  1057. while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
  1058. while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
  1059. while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
  1060. os.execute("sleep 5")
  1061. if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
  1062. end
  1063. APT.logPost()
  1064. else
  1065. local adt = APT.exe("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2"):Do().result:sub(2, -2)
  1066. if (nil ~= adt) and APT.checkFile(adt) then
  1067. APT.exe('mkdir -p ' .. adt:sub(1, 18))
  1068. :And():Nice('tar -c --xz ' .. adt .. ' -f ' .. adt:sub(1, 18) .. '/' .. adt .. '.tar.xz')
  1069. :And('rm -fr ' .. adt):noErr():fork()
  1070. end
  1071. local dt = os.date('!%F-%H-%M')
  1072. local odt = APT.exe('TZ="GMT" date -r results/stamp +%F-%H-%M 2>/dev/null'):Do().result:sub(2, -2)
  1073. if nil ~= odt then os.execute(' rm -f results_old; ln -s results_' .. odt .. ' results_old 2>/dev/null') end
  1074. if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end
  1075. os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
  1076. if not APT.keep then
  1077. os.execute("rm -f results/*.html 2>/dev/null")
  1078. os.execute("rm -f results/*.txt 2>/dev/null")
  1079. end
  1080. if not APT.logOpen('apt-panopticon') then return end
  1081. I("Starting tests " .. table.concat(APT.options.tests.value, ", "))
  1082. os.execute("mkdir -p results")
  1083. APT.mirrors = getMirrors()
  1084. checkHost(APT.options.referenceSite.value)
  1085. for k, m in pairs(APT.mirrors) do
  1086. local pu = url.parse("http://" .. m.BaseURL)
  1087. if APT.options.referenceSite.value ~= pu.host then
  1088. checkHost(m.BaseURL)
  1089. end
  1090. end
  1091. while not APT.checkFile('results/LOG_' .. APT.options.referenceSite.value .. '.html') do -- Wait for things to start up before checking for them.
  1092. D('Waiting for results/LOG_' .. APT.options.referenceSite.value .. '.html');
  1093. os.execute("sleep 5")
  1094. -- TODO - count these, and abort if it takes too long.
  1095. -- Try something similar for the other "Wait for things to start up before checking for them.", maybe fold it into APT.exe.
  1096. end
  1097. while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end
  1098. local APT_args = APT.args
  1099. local APT_logFile = APT.logFile
  1100. local debians = {}
  1101. local srvs = io.popen('ls -1 results/*.lua')
  1102. for l in srvs:lines() do
  1103. local hst = l:sub(9, -5)
  1104. if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end
  1105. if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then
  1106. local ips = loadfile(l)().IPs
  1107. if nil ~= ips then
  1108. debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
  1109. local baseFiles = {}
  1110. local IPfiles = {}
  1111. for i, a in pairs(ips) do
  1112. IPfiles[i] = {}
  1113. if type(a) == 'table' then
  1114. for j, b in pairs(a) do
  1115. IPfiles[i][j] = {}
  1116. end
  1117. end
  1118. end
  1119. local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
  1120. for ll in files:lines() do
  1121. local dn = false
  1122. for i, a in pairs(ips) do
  1123. if type(a) == 'table' then
  1124. for j, b in pairs(a) do
  1125. if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. j .. '_.*)') then
  1126. table.insert(IPfiles[i][j], ll)
  1127. dn = true
  1128. end
  1129. end
  1130. else
  1131. if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. i .. '_.*)') then
  1132. table.insert(IPfiles[i], ll)
  1133. dn = true
  1134. end
  1135. end
  1136. end
  1137. if not dn then table.insert(baseFiles, ll) end
  1138. end
  1139. local combine = function(ip, a)
  1140. if not APT.logOpen(hst, ip) then
  1141. print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip)
  1142. else
  1143. APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
  1144. for i, f in pairs(a) do
  1145. f = f:sub(9, -1)
  1146. APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
  1147. for ln in io.lines('results/' .. f) do
  1148. if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'.
  1149. end
  1150. end
  1151. end
  1152. APT.logPost()
  1153. APT.args = APT_args
  1154. APT.logFile = APT_logFile
  1155. end
  1156. combine('', baseFiles)
  1157. for ip, a in pairs(IPfiles) do
  1158. if nil == a[1] then
  1159. for i, f in pairs(a) do
  1160. combine(i, f)
  1161. end
  1162. else
  1163. combine(ip, a)
  1164. end
  1165. end
  1166. end
  1167. end
  1168. end
  1169. local file, e = io.open("results/debians.lua", "w+")
  1170. if nil == file then C("opening debians file - " .. e) else
  1171. file:write(APT.dumpTable(debians, "debians") .. "\nreturn debians\n")
  1172. file:close()
  1173. end
  1174. for k, v in pairs(APT.mirrors) do
  1175. local f = 'results/' .. k .. '.lua'
  1176. if APT.checkFile(f) then
  1177. results = loadfile(f)()
  1178. APT.mirrors[k]['IPs'] = results.IPs
  1179. end
  1180. end
  1181. local file, e = io.open("results/mirrors.lua", "w+")
  1182. if nil == file then C("opening mirrors file - " .. e) else
  1183. file:write(APT.dumpTable(APT.mirrors, "mirrors") .. "\nreturn mirrors\n")
  1184. file:close()
  1185. end
  1186. -- Create the reports.
  1187. for n, r in pairs(APT.options.reports.value) do
  1188. if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then
  1189. I("Creating " .. r .. " report.")
  1190. APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do()
  1191. end
  1192. end
  1193. I('Total run time was ' .. (os.time() - now) .. ' seconds.')
  1194. APT.logPost()
  1195. end