A Lua script to check the health of Devuan Linux package mirrors. The master repo is at https://sledjhamr.org/cgit/apt-panopticon/ and the master issues tracker is at https://sledjhamr.org/mantisbt/project_page.php?project_id=13 https://sledjhamr.org/cgit/apt-panopticon/
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 
 

1255 lines
55 KiB

  1. #!/usr/bin/env luajit
  2. local now = os.time()
  3. local APT = require 'apt-panopticommon'
  4. local D = APT.D
  5. local I = APT.I
  6. local T = APT.T
  7. local W = APT.W
  8. local E = APT.E
  9. local C = APT.C
  10. local arg, sendArgs = APT.parseArgs({...})
  11. APT.html = true
  12. local defaultURL = {scheme = "http"}
  13. local releases = {"jessie", "ascii", "beowulf", "ceres"}
  14. local releaseFiles =
  15. {
  16. -- Release file.
  17. "Release", -- 3.7 MB
  18. "Release.gpg", --
  19. -- "InRelease", -- 3.7 MB
  20. -- "main/binary-all/Packages.xz", -- 2.6 GB for all that changed recently.
  21. -- Contents files. -- 3.3 GB
  22. -- "main/Contents-all.xz",
  23. -- "main/Contents-amd64.xz",
  24. -- "main/Contents-arm64.xz",
  25. -- "-security/main/Contents-all.xz",
  26. -- "-security/main/Contents-amd64.xz",
  27. -- "-security/main/Contents-arm64.xz",
  28. }
  29. local notExist =
  30. {
  31. "ceres-security" -- This will never exist, it's our code name for the testing suite.
  32. }
  33. local referenceDebs =
  34. {
  35. -- Debian package.
  36. "merged/pool/DEBIAN/main/d/debian-keyring/debian-keyring_2020.02.02_all.deb",
  37. -- Debian security package. NOTE this one should always be redirected?
  38. "merged/pool/DEBIAN-SECURITY/updates/main/a/apt/apt-transport-https_1.4.9_amd64.deb",
  39. }
  40. local referenceDevs =
  41. {
  42. -- Devuan package. NOTE this one should not get redirected, but that's more a warning than an error.
  43. "merged/pool/DEVUAN/main/d/devuan-keyring/devuan-keyring_2017.10.03_all.deb",
  44. }
  45. local curlStatus =
  46. {
  47. [1 ] = "Unsupported protocol. This build of curl has no support for this protocol.",
  48. [2 ] = "Failed to initialize.",
  49. [3 ] = "URL malformed. The syntax was not correct.",
  50. [4 ] = "A feature or option that was needed to perform the desired request was not enabled or was explicitly disabled at build-time. To make curl able to do this, you probably need another build of libcurl!",
  51. [5 ] = "Couldn't resolve proxy. The given proxy host could not be resolved.",
  52. [6 ] = "Couldn't resolve host. The given remote host was not resolved.",
  53. [7 ] = "Failed to connect to host.",
  54. [8 ] = "Weird server reply. The server sent data curl couldn't parse.",
  55. [9 ] = "FTP access denied. The server denied login or denied access to the particular resource or directory you wanted to reach. Most often you tried to change to a directory that doesn't exist on the server.",
  56. [10] = "While waiting for the server to connect back when an active FTP session is used, an error code was sent over the control connection or similar.",
  57. [11] = "FTP weird PASS reply. Curl couldn't parse the reply sent to the PASS request.",
  58. [12] = "During an active FTP session while waiting for the server to connect, the CURLOPT_ACCEPTTIMEOUT_MS (or the internal default) timeout expired.",
  59. [13] = "FTP weird PASV reply, Curl couldn't parse the reply sent to the PASV request.",
  60. [14] = "FTP weird 227 format. Curl couldn't parse the 227-line the server sent.",
  61. [15] = "FTP can't get host. Couldn't resolve the host IP we got in the 227-line.",
  62. [16] = "A problem was detected in the HTTP2 framing layer. This is somewhat generic and can be one out of several problems, see the error buffer for details.",
  63. [17] = "FTP couldn't set binary. Couldn't change transfer method to binary.",
  64. [18] = "Partial file. Only a part of the file was transferred.",
  65. [19] = "FTP couldn't download/access the given file, the RETR (or similar) command failed.",
  66. [21] = "FTP quote error. A quote command returned error from the server.",
  67. [22] = "HTTP page not retrieved. The requested url was not found or returned another error with the HTTP error code being 400 or above. This return code only appears if -f, --fail is used.",
  68. [23] = "Write error. Curl couldn't write data to a local filesystem or similar.",
  69. [25] = "FTP couldn't STOR file. The server denied the STOR operation, used for FTP uploading.",
  70. [26] = "Read error. Various reading problems.",
  71. [27] = "Out of memory. A memory allocation request failed.",
  72. [28] = "Operation timeout. The specified time-out period was reached according to the conditions.",
  73. [30] = "FTP PORT failed. The PORT command failed. Not all FTP servers support the PORT command, try doing a transfer using PASV instead!",
  74. [31] = "FTP couldn't use REST. The REST command failed. This command is used for resumed FTP transfers.",
  75. [33] = "HTTP range error. The range \"command\" didn't work.",
  76. [34] = "HTTP post error. Internal post-request generation error.",
  77. [35] = "SSL connect error. The SSL handshaking failed.",
  78. [36] = "FTP bad download resume. Couldn't continue an earlier aborted download.",
  79. [37] = "FILE couldn't read file. Failed to open the file. Permissions?",
  80. [38] = "LDAP cannot bind. LDAP bind operation failed.",
  81. [39] = "LDAP search failed.",
  82. [41] = "Function not found. A required LDAP function was not found.",
  83. [42] = "Aborted by callback. An application told curl to abort the operation.",
  84. [43] = "Internal error. A function was called with a bad parameter.",
  85. [45] = "Interface error. A specified outgoing interface could not be used.",
  86. [47] = "Too many redirects. When following redirects, curl hit the maximum amount.",
  87. [48] = "Unknown option specified to libcurl. This indicates that you passed a weird option to curl that was passed on to libcurl and rejected. Read up in the manual!",
  88. [49] = "Malformed telnet option.",
  89. [51] = "The peer's SSL certificate or SSH MD5 fingerprint was not OK.",
  90. [52] = "The server didn't reply anything, which here is considered an error.",
  91. [53] = "SSL crypto engine not found.",
  92. [54] = "Cannot set SSL crypto engine as default.",
  93. [55] = "Failed sending network data.",
  94. [56] = "Failure in receiving network data.",
  95. [58] = "Problem with the local certificate.",
  96. [59] = "Couldn't use specified SSL cipher.",
  97. [60] = "Peer certificate cannot be authenticated with known CA certificates.",
  98. [61] = "Unrecognized transfer encoding.",
  99. [62] = "Invalid LDAP URL.",
  100. [63] = "Maximum file size exceeded.",
  101. [64] = "Requested FTP SSL level failed.",
  102. [65] = "Sending the data requires a rewind that failed.",
  103. [66] = "Failed to initialise SSL Engine.",
  104. [67] = "The user name, password, or similar was not accepted and curl failed to log in.",
  105. [68] = "File not found on TFTP server.",
  106. [69] = "Permission problem on TFTP server.",
  107. [70] = "Out of disk space on TFTP server.",
  108. [71] = "Illegal TFTP operation.",
  109. [72] = "Unknown TFTP transfer ID.",
  110. [73] = "File already exists (TFTP).",
  111. [74] = "No such user (TFTP).",
  112. [75] = "Character conversion failed.",
  113. [76] = "Character conversion functions required.",
  114. [77] = "Problem with reading the SSL CA cert (path? access rights?).",
  115. [78] = "The resource referenced in the URL does not exist.",
  116. [79] = "An unspecified error occurred during the SSH session.",
  117. [80] = "Failed to shut down the SSL connection.",
  118. [81] = "Socket is not ready for send/recv wait till it's ready and try again. This return code is only returned from curl_easy_recv and curl_easy_send.",
  119. [82] = "Could not load CRL file, missing or wrong format (added in 7.19.0).",
  120. [83] = "Issuer check failed (added in 7.19.0).",
  121. [84] = "The FTP PRET command failed",
  122. [85] = "RTSP: mismatch of CSeq numbers",
  123. [86] = "RTSP: mismatch of Session Identifiers",
  124. [87] = "unable to parse FTP file list",
  125. [88] = "FTP chunk callback reported error",
  126. [89] = "No connection available, the session will be queued",
  127. [90] = "SSL public key does not matched pinned public key",
  128. [91] = "Status returned failure when asked with CURLOPT_SSL_VERIFYSTATUS.",
  129. [92] = "Stream error in the HTTP/2 framing layer.",
  130. [93] = "An API function was called from inside a callback.",
  131. [94] = "An authentication function returned an error.",
  132. [95] = "A problem was detected in the HTTP/3 layer. This is somewhat generic and can be one out of several problems, see the error buffer for details.",
  133. }
  134. local socket = require 'socket'
  135. local ftp = require 'socket.ftp'
  136. local http = require 'socket.http'
  137. local url = require 'socket.url'
  138. local ip = ""
  139. local cor = nil
  140. local Updating = false
  141. local downloadLock = "flock -n results/curl-"
  142. local arw = ' &nbsp; <font color="magenta"><b>-></b></font> &nbsp; '
  143. local repoExists = function (r)
  144. r = r:match("([%a-]*)")
  145. if nil == r then return false end
  146. for k, v in pairs(notExist) do
  147. if v == r then return false end
  148. end
  149. return true
  150. end
  151. local IP = {}
  152. gatherIPs = function (host)
  153. if nil == IP[host] then
  154. local IPs
  155. -- Takes about 30 seconds to look up the lot.
  156. -- I tested using dig's -f option, it didn't seem much faster.
  157. -- The sort -r assumes that deb.devuan.org is the first alphabetically.
  158. local dig = io.popen('dig +keepopen +noall +nottlid +answer ' .. host .. ' A ' .. host .. ' AAAA ' .. host .. ' CNAME ' .. host .. ' SRV | sort -r | uniq')
  159. repeat
  160. IPs = dig:read("*l")
  161. if nil ~= IPs then
  162. for k, t, v in IPs:gmatch("([%w_%-%.]*)%.%s*IN%s*(%a*)%s*(.*)") do
  163. if "." == v:sub(-1, -1) then v = v:sub(1, -2) end
  164. if nil == IP[k] then IP[k] = {} end
  165. IP[k][v] = t
  166. D(" DNS record " .. host .. " == " .. k .. " type " .. t .. " -> " .. v)
  167. if t == "CNAME" then
  168. gatherIPs(v)
  169. IP[k][v] = IP[v]
  170. elseif v == "SRV" then
  171. print("SVR record found, now what do we do?")
  172. end
  173. end
  174. end
  175. until nil == IPs
  176. end
  177. return IP[host]
  178. end
  179. -- Returns FTP directory listing
  180. local nlst = function (u)
  181. local t = {}
  182. local p = url.parse(u)
  183. p.command = "nlst"
  184. p.sink = ltn12.sink.table(t)
  185. local r, e = ftp.get(p)
  186. return r and table.concat(t), e
  187. end
  188. local timeouts = 0;
  189. local totalTimeouts = 0
  190. local spcd = ' &nbsp; '
  191. checkHEAD = function (host, URL, r, retry, sanity)
  192. if nil == r then r = 0 end
  193. if nil == retry then retry = 0 end
  194. if true == sanity then sanity = 'URLSanity' else sanity = '' end
  195. local check = "HEAD testing file"
  196. local PU = url.parse(URL, defaultURL)
  197. local pu = url.parse(PU.scheme .. "://" .. host, defaultURL)
  198. local fname = host .. "_" .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".log.txt"
  199. local hdr = ""
  200. local IP = ""
  201. if pu.host ~= PU.host then
  202. if "http" == PU.scheme then
  203. hdr = '-H "Host: ' .. host .. '"'
  204. end
  205. IP = '--connect-to "' .. pu.host .. '::' .. PU.host .. ':"'
  206. fname = host .. "_" .. pu.host .. '_' .. PU.host .. "_" .. PU.path:gsub("/", "_") .. ".txt"
  207. end
  208. os.execute('rm -f results/HEADERS_' .. fname .. ' 2>/dev/null; rm -f results/STATUS_' .. fname .. ' 2>/dev/null')
  209. if not APT.testing(PU.scheme, host) and APT.redir then I(spcd .. string.upper(PU.scheme) .. " not supported, not tested. &nbsp; " .. URL, host); return end
  210. if 0 < r then
  211. check = "Redirecting to"
  212. end
  213. if 0 < retry then
  214. os.execute("sleep " .. math.random(1, 3))
  215. check = "Retry " .. retry .. " " .. check
  216. end
  217. if 2 <= timeouts then
  218. E(spcd .. spcd .. "too many timeouts! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
  219. return
  220. end
  221. if APT.options.timeouts.value <= (totalTimeouts) then
  222. E(spcd .. spcd .. "Way too many timeouts!", PU.scheme, "", host)
  223. return
  224. end
  225. if 20 <= r then
  226. E(spcd .. spcd .. "too many redirects! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
  227. return
  228. end
  229. if APT.options.retries.value <= retry then
  230. E(spcd .. spcd .. "too many retries! " .. check .. " " .. host .. arw .. URL, PU.scheme, "", host)
  231. return
  232. end
  233. if "https" == PU.scheme and APT.options.roundRobin.value == host then
  234. I(spcd .. "Not testing " .. APT.lnk(URL) .. " mirrors wont have the correct HTTPS certificate for the round robin.", host)
  235. return
  236. else
  237. I(spcd .. check .. " " .. APT.lnk(URL), host)
  238. end
  239. --[[ Using curl command line -
  240. -I - HEAD
  241. --connect-to domain:port:IP:port - connect to IP, but use SNI from URL.
  242. -header "" - add extra headers.
  243. -L - DO follow redirects.
  244. --max-redirs n - set maximum redirects, default is 50, -1 = unlimited.
  245. -4 or -6 - Only use IPv4 or IPv6
  246. --retry n - maximum retries, default is 0, no retries.
  247. -o file - write to file instead of stdout.
  248. --path-as-is - https://curl.haxx.se/libcurl/c/CURLOPT_PATH_AS_IS.html might be useful for URLSanity.
  249. -s silent - don't output progress or error messages.
  250. --connect-timeout n - timeout in seconds.
  251. Should return with error code 28 on a timeout?
  252. -D file - write the received headers to a file. This includes the status code and string.
  253. ]]
  254. local status = APT.exe(
  255. 'curl -I --retry 0 -s --path-as-is --connect-timeout ' .. APT.options.timeout.value .. ' --max-redirs 0 ' .. APT.IPv46 .. ' ' ..
  256. IP .. ' ' .. '-o /dev/null -D results/"HEADERS_' .. fname .. '" ' ..
  257. hdr .. ' -w "#%{http_code} %{ssl_verify_result} %{url_effective}\\n" ' .. PU.scheme .. '://' .. host .. PU.path .. ' >>results/"STATUS_' .. fname .. '"'
  258. ):Nice():log():Do().status
  259. local code = "???"
  260. local cstr = ""
  261. local location = nil
  262. local tmot = 1
  263. while not APT.checkFile('results/STATUS_' .. fname) do
  264. D(spcd .. spcd .. 'Waiting for results/STATUS_' .. fname .. ' file.')
  265. os.execute('sleep ' .. tmot)
  266. tmot = tmot * 2
  267. if 8 < tmot then
  268. T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
  269. timeouts = timeouts + 1
  270. checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
  271. os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
  272. return
  273. end
  274. end
  275. os.execute('cat results/"HEADERS_' .. fname .. '" >>results/"STATUS_' .. fname .. '" 2>/dev/null; rm -f results/"HEADERS_' .. fname .. '" 2>/dev/null')
  276. if 0 ~= status then
  277. local msg = curlStatus[status]
  278. if nil == msg then msg = "UNKNOWN CURL STATUS CODE!" end
  279. if (28 == status) or (7 == status) then
  280. T(spcd .. spcd .. "TIMEOUT " .. timeouts + 1 .. ", retry " .. retry + 1 .. ' ' .. APT.lnk(URL), PU.scheme, sanity, host)
  281. timeouts = timeouts + 1
  282. else
  283. E(spcd .. spcd .. "The curl command return an error code of " .. status .. " - " .. msg .. ' for '.. APT.lnk(URL), PU.scheme, sanity, host)
  284. end
  285. if 60 == status then return end -- Certificate is invalid, don't bother retrying.
  286. checkHEAD(host, URL, r, retry + 1, '' ~= sanity)
  287. return
  288. end
  289. local rfile, e = io.open("results/STATUS_" .. fname, "r")
  290. if nil == rfile then W("opening results/STATUS_" .. fname .. " file - " .. e) else
  291. for line in rfile:lines("*l") do
  292. if "#" == line:sub(1, 1) then
  293. code = line:sub(2, 4)
  294. if ("https" == PU.scheme) and ("0" ~= line:sub(6, 6)) then
  295. os.execute('cp results/STATUS_' .. fname .. ' results/STATUS_' .. fname .. '_SAVED')
  296. if '' ~= sanity then
  297. E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, sanity, host)
  298. else
  299. E(spcd .. spcd .. "The certificate is invalid.", PU.scheme, "https", host)
  300. end
  301. end
  302. elseif "http" == line:sub(1, 4):lower() then
  303. -- -2 coz the headers file gets a \r at the end.
  304. cstr = line:sub(14, -2)
  305. elseif "location" == line:sub(1, 8):lower() then
  306. location = line:sub(11, -2)
  307. end
  308. end
  309. if '???' == code then
  310. W(spcd .. spcd .. 'Could not find response code. ' .. APT.lnk(URL), PU.scheme, sanity, host)
  311. end
  312. end
  313. os.execute('cat results/STATUS_' .. fname .. ' >> results/curl_HEAD_' .. fname .. '; rm -f results/STATUS_' .. fname .. ' 2>/dev/null')
  314. if ("4" == tostring(code):sub(1, 1)) or ("5" == tostring(code):sub(1, 1)) then
  315. E(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), PU.scheme, sanity, host)
  316. else
  317. if not APT.testing(PU.scheme, host) then
  318. I(spcd .. spcd .. "Not supported, but works " .. PU.scheme .. " " .. APT.lnk(URL), PU.scheme, "", host)
  319. end
  320. I(spcd .. spcd .. code .. " " .. cstr .. ". " .. check .. " " .. APT.lnk(URL), host)
  321. -- timeouts = timeouts - 1 -- Backoff the timeouts count if we managed to get through.
  322. if nil ~= location then
  323. pu = url.parse(location, defaultURL)
  324. if (pu.host == APT.options.roundRobin.value) and (nil ~= PU.path:find('merged/pool/DEVUAN/')) then
  325. E('DEVUAN packages must not be redirected to ' .. APT.options.roundRobin.value .. ' - ' .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, 'Redirects', host)
  326. end
  327. if ('http' == location:sub(1, 4)) and (pu.scheme ~= PU.scheme) then -- Sometimes a location sans scheme is returned, this is not a protocol change.
  328. if APT.testing("Protocol") then
  329. if APT.options.roundRobin.value == host then -- Coz HTTPS shouldn't happen via the round robin.
  330. E(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
  331. end
  332. W(spcd .. spcd .. "Protocol changed during redirect! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "Protocol", host)
  333. else
  334. end
  335. end
  336. if location == URL then
  337. E(spcd .. spcd .. "Redirect loop! " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, "", host)
  338. elseif nil == pu.host then
  339. I(spcd .. spcd .. "Relative redirect. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  340. if 1 <= APT.options.bandwidth.value then checkHEAD(host, PU.scheme .. "://" .. PU.host .. location, r + 1, retry, '' ~= sanity) end
  341. elseif (PU.host == pu.host) or (host == pu.host) then
  342. if PU.host ~= host then
  343. local t = pu.host
  344. pu.host = PU.host
  345. location = url.build(pu)
  346. pu.host = t
  347. end
  348. I(spcd .. spcd .. "Redirect to same host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  349. if 1 <= APT.options.bandwidth.value then checkHEAD(host, location, r + 1, retry, '' ~= sanity) end
  350. else
  351. I(spcd .. spcd .. "Redirect to different host. " .. check .. " " .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  352. if 1 <= APT.options.bandwidth.value then
  353. --[[ The hard part here is that we end up throwing ALL of the test files at the redirected location.
  354. Not good for deb.debian.org, which we should only be throwing .debs at.
  355. What we do is loop through the DNS entries, and only test the specific protocol & file being tested here.
  356. ]]
  357. local u = pu.host .. "/" .. pu.path
  358. local file = pu.path:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  359. local path = pu.path:sub(2, -1 -(#file))
  360. local check = u:gsub("/", "_")
  361. local extraArgs = sendArgs .. ' -o -r '
  362. if 'https' == pu.scheme then extraArgs = extraArgs .. ' --tests=-http' end
  363. if 'http' == pu.scheme then extraArgs = extraArgs .. ' --tests=-https' end
  364. local pth = path:match('^(.*/pool/).*$')
  365. if nil ~= pth then table.insert(APT.results[PU.scheme].redirects, pu.host .. "/" .. pth) else E(spcd .. spcd .. 'Odd redirect path ' .. path) end
  366. I(spcd .. spcd .. "Now checking redirected host " .. u .. ' &nbsp; for &nbsp; ' .. APT.lnk(URL) .. arw .. APT.lnk(location), host)
  367. APT.exe(downloadLock .. "REDIR-" .. check .. ".log.txt" .. " ./apt-panopticon.lua " .. extraArgs .. ' ' .. pu.host .. "/" .. path .. " " .. file):Nice():log():fork()
  368. D(spcd .. 'logging to ' .. APT.logName(pu.host, nil, file)[2])
  369. end
  370. end
  371. elseif nil ~= PU.path:find('merged/pool/DEBIAN-SECURITY/') then
  372. W('DEBIAN-SECURITY packages must be redirected to a Debian mirror - ' .. APT.lnk(URL) .. arw .. APT.lnk(location), PU.scheme, 'Redirects', host)
  373. end
  374. end
  375. end
  376. local checkTimeouts = function(host, scheme, URL)
  377. totalTimeouts = totalTimeouts + timeouts; timeouts = 0
  378. checkHEAD(host, scheme .. "://" .. URL)
  379. if (1 <= APT.options.bandwidth.value) and APT.testing("URLSanity") then
  380. URL = URL:gsub("/", "///")
  381. URL = URL:gsub("///", "/", 1)
  382. checkHEAD(host, scheme .. "://" .. URL, 0, 0, true)
  383. end
  384. if nil ~= cor then
  385. D('*&gt;* About to resume coroutine after checkHEAD(' .. host .. ' , ' .. scheme .. ' :// ' .. URL .. ')')
  386. local ok, message = coroutine.resume(cor)
  387. if not ok then cor = nil; print(message) end
  388. end
  389. if APT.options.timeouts.value <= (totalTimeouts) then
  390. E("Way too many timeouts!", scheme, "URLSanity", host)
  391. return true
  392. end
  393. return false
  394. end
  395. local checkFiles = function (host, ip, path, file)
  396. timeouts = 0
  397. if nil == path then path = "" end
  398. if nil ~= file then
  399. if "redir" == ip then ip = host end
  400. if checkTimeouts(host, "http", ip .. path .. "/" .. file) then return end
  401. if checkTimeouts(host, "https", ip .. path .. "/" .. file) then return end
  402. else
  403. I(" HEAD testing files for " .. host .. arw .. ip .. " " .. path, host)
  404. if 1 <= APT.options.bandwidth.value then
  405. -- Do these first, coz they are likely to fork off a different server.
  406. for i, s in pairs(referenceDebs) do
  407. if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
  408. if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
  409. end
  410. end
  411. for i, s in pairs(releases) do
  412. for j, k in pairs(releaseFiles) do
  413. if repoExists(s .. k) then
  414. if checkTimeouts(host, "http", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
  415. if 1 <= APT.options.bandwidth.value then
  416. if checkTimeouts(host, "https", ip .. path .. "/merged/dists/" .. s .. '/' .. k) then return end
  417. else
  418. break
  419. end
  420. end
  421. if 2 >= APT.options.bandwidth.value then break end
  422. end
  423. if 2 >= APT.options.bandwidth.value then break end
  424. end
  425. if 1 <= APT.options.bandwidth.value then
  426. for i, s in pairs(referenceDevs) do
  427. if checkTimeouts(host, "http", ip .. path .. "/" .. s) then return end
  428. if checkTimeouts(host, "https", ip .. path .. "/" .. s) then return end
  429. end
  430. end
  431. end
  432. end
  433. checkHost = function (orig, host, path, ip, file)
  434. if nil == host then host = orig end
  435. if nil == path then path = "" end
  436. if nil == file then file = "" end
  437. local ph = url.parse("http://" .. host)
  438. if (nil ~= ip) and ("redir" ~= ip) then
  439. local po = url.parse("http://" .. orig)
  440. if "" ~= file then
  441. D("checking redirected file " .. po.host .. " " .. file)
  442. checkFiles(po.host, ip, path, file)
  443. else
  444. checkFiles(po.host, ip, path)
  445. end
  446. else
  447. if orig == host then
  448. I("Testing mirror " .. orig .. "" .. file)
  449. APT.exe("./apt-panopticon.lua " .. sendArgs .. " -o " .. orig .. path .. " " .. file):Nice():log():fork()
  450. D('logging to ' .. APT.logName(ph.host, nil, file)[2])
  451. else D("checkHost " .. orig .. arw .. host) end
  452. end
  453. end
  454. local addDownload = function(host, URL, f, r, k)
  455. local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  456. if APT.checkFile("results/" .. host .. "/merged/dists/" .. r .. '/' .. k) then
  457. -- Curls "check timestamp and overwrite file" stuff sucks.
  458. -- -R means the destination file gets the timestamp of the remote file.
  459. -- Can only do ONE timestamp check per command.
  460. -- This doesn't work either. All downloads get all these headers. Pffft
  461. -- local status, ts = APT.execute('TZ="GMT" ls -l --time-style="+%a, %d %b %Y %T %Z" results/' .. host .. "/merged/dists/" .. r .. '/' .. k .. ' | cut -d " " -f 6-11')
  462. -- f:write('header "If-Modified-Since: ' .. ts:sub(2, -2) .. '"\n')
  463. -- Curl will DELETE the existing file if the timestamp fails to download a new one, unless we change directory first,
  464. -- which wont work with multiple files in multiple directories. WTF?
  465. --TODO - change tactic, do a HEAD if-modified test first before adding the file to the list to download.
  466. os.execute(" mv results/" .. host .. "/merged/dists/" .. r .. '/' .. k ..
  467. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old")
  468. end
  469. D('Downloading http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k)
  470. f:write('url "' .. 'http://' .. host .. URL .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
  471. f:write('output "results/' .. host .. '/merged/dists/' .. r .. '/' .. k .. '"\n')
  472. end
  473. local postDownload = function(host, r, k)
  474. local file = k:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  475. if nil == file then file = k end
  476. os.execute("if [ -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old ]" ..
  477. " && [ ! -f results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. " ]; then cp -a" ..
  478. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. ".old" ..
  479. " results/" .. host .. "/merged/dists/" .. r .. '/' .. k .. "; fi")
  480. if APT.checkFile('results/' .. host .. '/merged/dists/' .. r .. '/' .. k) then
  481. if ".gz" == k:sub(-3, -1) then APT.exe("gzip -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
  482. if ".xz" == k:sub(-3, -1) then APT.exe("xz -dfk results/" .. host .. "/merged/dists/" .. r .. '/' .. k):Nice():noErr():Do() end
  483. end
  484. end
  485. local download = "curl" ..
  486. " --connect-timeout " .. APT.options.timeout.value ..
  487. " --create-dirs -f -L" ..
  488. " --fail-early" ..
  489. " --max-time " .. APT.options.maxtime.value ..
  490. APT.IPv46 .. ' ' ..
  491. " --retry " .. APT.options.retries.value ..
  492. " -R -v -z 'results/stamp.old' --stderr results/"
  493. local downloads = function(host, URL, meta, release, list)
  494. if nil == URL then URL = "" end
  495. local files = 'curl-' .. meta .. '-' .. host .. '.files.txt'
  496. local lock = meta .. "-" .. host .. ".log.txt"
  497. local log = "curl-" .. meta .. "-" .. host .. ".log.txt"
  498. local cm = downloadLock .. lock .. " " .. download .. log .. " -K results/" .. files
  499. if APT.testing("IPv4") and (not APT.testing("IPv6")) then cm = cm .. ' -4' end
  500. if (not APT.testing("IPv4")) and APT.testing("IPv6") then cm = cm .. ' -6' end
  501. f, e = io.open("results/curl-" .. meta .. '-' .. host .. ".files.txt", "a+")
  502. if nil == f then C("opening curl downloads list file - " .. e); return end
  503. if nil ~= list then
  504. if "" ~= list then
  505. if nil ~= release then
  506. for l in list:gmatch("\n*([^\n]+)\n*") do
  507. addDownload(host, URL, f, release, "/" .. l)
  508. end
  509. else
  510. I('Downloading ' .. APT.lnk('http://' .. host .. URL .. '/merged/' .. list))
  511. f:write('url "' .. 'http://' .. host .. URL .. '/merged/' .. list .. '"\n')
  512. f:write('output "results/' .. host .. '/merged/' .. list .. '"\n')
  513. end
  514. f:close()
  515. return
  516. end
  517. else
  518. for i, s in pairs(releases) do
  519. for j, k in pairs(releaseFiles) do
  520. if repoExists(s .. k) then
  521. addDownload(host, URL, f, s, k)
  522. end
  523. end
  524. end
  525. end
  526. f:close()
  527. APT.exe(cm):Nice():log():fork()
  528. D('logging to <a href="' .. log .. '">' .. log .. '</a>, with <a href="' .. files .. '">these files</a>')
  529. end
  530. local validateURL = function(m)
  531. if " " == m.BaseURL:sub(-1, -1) then
  532. W("space at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
  533. m.BaseURL = m.BaseURL:sub(1, -2)
  534. end
  535. if "/" == m.BaseURL:sub(-1, -1) then
  536. W("slash at end of BaseURL in mirror_list.txt! " .. m.BaseURL, "", "", m.FQDN)
  537. m.BaseURL = m.BaseURL:sub(1, -2)
  538. end
  539. local p = url.parse("http://" .. m.BaseURL)
  540. if nil == p.path then p.path = '' end
  541. if nil ~= p.port then p.authority = authority .. ':' .. p.port end
  542. if m.FQDN ~= p.authority then W("Something wrong in FDQN from mirror_list.txt! " .. m.FDQN, "", "", p.authority) end
  543. if m.BaseURL ~= (p.authority .. p.path) then W("Something wrong in BaseURL from mirror_list.txt! " .. m.BaseURL, "", "", p.authority) end
  544. if (nil ~= p.query) or (nil ~= p.fragment) or (nil ~= p.params) then W("Something wrong in BaseURL from mirror_list.txt, should be nothing after the path! " .. m.BaseURL, "", "", p.authority) end
  545. if (nil ~= p.user) or (nil ~= p.userinfo) or (nil ~= p.password) then W("Something wrong in BaseURL from mirror_list.txt, should be no credentials! " .. m.BaseURL, "", "", p.authority) end
  546. m.FQDN = p.authority
  547. m.BaseURL = p.authority .. p.path
  548. return m
  549. end
  550. local getMirrors = function ()
  551. local mirrors = {}
  552. local host = ""
  553. local m = {}
  554. local active = true
  555. local URL = 'http://' .. APT.options.referenceSite.value .. '/mirror_list.txt'
  556. I('Downloadin gand parsing http://' .. APT.options.referenceSite.value .. '/mirror_list.txt')
  557. local p, c, h = http.request(URL)
  558. if nil == p then E(c .. " fetching " .. URL) else
  559. for l in p:gmatch("\n*([^\n]+)\n*") do
  560. local t, d = l:match("(%a*):%s*(.*)")
  561. d = string.lower(d)
  562. if "FQDN" == t then
  563. if "" ~= host then
  564. mirrors[host] = validateURL(m)
  565. m = {}
  566. active = true
  567. end
  568. host = d
  569. m[t] = d
  570. elseif "Protocols" == t then
  571. local prot = {}
  572. for w in d:gmatch("(%w+)") do
  573. if APT.search(APT.protocols, w:lower()) then prot[w] = true end
  574. end
  575. m[t] = prot
  576. elseif "Active" == t and nil == d:sub(1, 3):find("yes", 1, true) then
  577. W("Mirror " .. host .. " is not active - " .. d, "", "", host)
  578. active = false
  579. m[t] = d
  580. -- TODO - Should do some input validation on BaseURL, and everything else.
  581. elseif "Rate" == t then
  582. local time, unit = d:match('(%d+) *(%a+)')
  583. time = tonumber(time)
  584. unit = unit:sub(1, 1)
  585. m[t] = time .. ' ' .. unit
  586. if 'm' == unit then
  587. time = time * 60
  588. elseif 'h' == unit then
  589. time = time * 60 * 60
  590. else
  591. C('Unknown Rate for mirror ' .. host)
  592. end
  593. m['Updated'] = time
  594. else
  595. m[t] = d
  596. end
  597. end
  598. if "" ~= host --[[and active]] then
  599. mirrors[host] = validateURL(m)
  600. end
  601. end
  602. if APT.testing("DNSRR") then
  603. mirrors[APT.options.roundRobin.value] = { ["Protocols"] = { ["http"] = true; }; ['Updated'] = 300;
  604. ["FQDN"] = APT.options.roundRobin.value; ["Active"] = 'yes'; ["BaseURL"] = APT.options.roundRobin.value; }
  605. end
  606. local file, e = io.open("results/mirrors.lua", "w+")
  607. if nil == file then C("opening mirrors file - " .. e) else
  608. file:write(APT.dumpTable(mirrors, "mirrors") .. "\nreturn mirrors\n")
  609. file:close()
  610. end
  611. if 42 < #mirrors then print(#mirrors .. ' is too many mirrors!') ; os.exit(false) end
  612. return mirrors
  613. end
  614. local postParse = function(host, list)
  615. if APT.options.referenceSite.value == host then
  616. if nil ~= list then
  617. local sem = 'results/NEW_' .. list.out .. '_%s.txt'
  618. for i, n in pairs(releases) do
  619. local f = sem:format(n)
  620. if APT.checkFile(f .. '.tmp') then
  621. os.execute('mv ' .. f .. '.tmp ' .. f)
  622. else
  623. os.execute('touch ' .. f)
  624. end
  625. end
  626. end
  627. end
  628. end
  629. local parseDebs = function(host)
  630. for i, n in pairs(releases) do
  631. local inFile = 'results/NEW_debs_' .. n .. '.txt'
  632. local nfile, e = io.open(inFile, "r")
  633. if nil == nfile then W("opening " .. inFile .. " file - " .. e) else
  634. for l in nfile:lines() do
  635. local v, p, sz, sha = l:match(' | (.+) | (pool/.+%.deb) | (%d.+) | (%x.+) |')
  636. if nil ~= p then
  637. if APT.checkFile('results/' .. host .. "/merged/" .. p) then
  638. local fsz = APT.exe('ls -l results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 5-5'):Do().result
  639. if APT.testing("Integrity") then
  640. if sz ~= fsz:sub(2, -2) then -- The sub bit is to slice off the EOLs at each end.
  641. E('Package size mismatch - results/' .. host .. "/merged/" .. p .. ' should be ' .. sz .. ', but is ' .. fsz:sub(2, -2) .. '.', 'http', 'Integrity', host)
  642. else
  643. local fsha = APT.exe('sha256sum results/' .. host .. "/merged/" .. p .. ' | cut -d " " -f 1'):log():Do().result
  644. if sha ~= fsha:sub(2, -2) then E('Package SHA256 sum mismatch - results/' .. host .. "/merged/" .. p, 'http', 'Integrity', host) end
  645. -- TODO - maybe check the PGP key, though packages are mostly not signed.
  646. end
  647. end
  648. if APT.testing("Updated") then
  649. if sz ~= fsz:sub(2, -2) then
  650. E('Package size mismatch for ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
  651. end
  652. end
  653. os.execute('rm -f results/' .. host .. "/merged/" .. p)
  654. else
  655. if Updating then
  656. W('Not yet able to download, awaiting update for ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
  657. else
  658. E('Failed to download ' .. host .. "/merged/" .. p, 'http', 'Updated', host)
  659. end
  660. end
  661. end
  662. end
  663. end
  664. end
  665. return nil
  666. end
  667. local parsePackages = function(host)
  668. local list = {inf = 'Packages', parser = parseDebs, out = 'debs', files = {}, nextf = ''}
  669. for i, n in pairs(releases) do
  670. local inFile = 'results/NEW_' .. list.inf .. '_' .. n .. '.txt'
  671. local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
  672. if APT.options.referenceSite.value == host then
  673. outFile = outFile .. '.tmp'
  674. end
  675. local dFile, e = io.open(inFile, "r")
  676. if nil == dFile then W("opening " .. inFile .. " file - " .. e) else
  677. for l in dFile:lines() do
  678. postDownload(host, n, l)
  679. l = '/' .. l
  680. local file = l:match(".*/([%w%.%+%-_]*)$") -- Get the filename.
  681. local dir = l:sub(1, 0 - (#file + 1))
  682. if "Packages." == file:sub(1, 9) then
  683. -- TODO - compare the SHA256 sums in pkgmaster's Release for both the packed and unpacked versions.
  684. -- Also note that this might get only a partial download due to maxtime.
  685. if APT.options.referenceSite.value == host then
  686. local Pp, e = io.open('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed', "w+")
  687. if nil == Pp then W('opening results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed' .. ' file - ' .. e) else
  688. local pp = {}
  689. for l in io.lines('results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages') do
  690. if "Package: " == l:sub(1, 9) then
  691. if 0 ~= #pp then
  692. for i = 1, 5 do
  693. if nil == pp[i] then print(host .. " " .. n .. " " .. dir .. " " .. i) else Pp:write(pp[i] .. " | ") end
  694. end
  695. Pp:write("\n")
  696. end
  697. pp = {}
  698. pp[1] = l:sub(10, -1)
  699. elseif "Version: " == l:sub(1, 9) then
  700. pp[2] = l:sub(10, -1)
  701. elseif "Filename: " == l:sub(1, 10) then
  702. pp[3] = l:sub(11, -1)
  703. elseif "Size: " == l:sub(1, 6) then
  704. pp[4] = l:sub(7, -1)
  705. elseif "SHA256: " == l:sub(1, 8) then
  706. pp[5] = l:sub(9, -1)
  707. end
  708. end
  709. Pp:close()
  710. os.execute('sort results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages.parsed >results/' .. host .. '/merged/dists/'.. n .. dir .. 'Packages_parsed-sorted')
  711. if APT.checkFile('Packages/' .. n .. dir .. 'Packages_parsed-sorted') then
  712. os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  713. 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  714. ' | grep -E "^-" | grep -Ev "^\\+\\+\\+|^---" >>results/OLD_' .. list.out .. '_' .. n .. '.txt')
  715. os.execute('diff -U 0 Packages/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  716. 'results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted ' ..
  717. ' | grep -E "^\\+" | grep -Ev "^\\+\\+\\+|^---" >>results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
  718. for i, s in pairs(referenceDebs) do
  719. if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then
  720. print('Reference package is out of date (' .. n .. ') - ' .. s)
  721. end
  722. end
  723. for i, s in pairs(referenceDevs) do
  724. if 0 == APT.exe('grep -q "' .. s:sub(8, -1) .. '" results/OLD_' .. list.out .. '_' .. n .. '.txt'):log():Do().status then
  725. print('Reference package is out of date (' .. n .. ') - ' .. s)
  726. end
  727. end
  728. else
  729. W("Can't find file Packages/" .. n .. dir .. "Packages_parsed-sorted")
  730. end
  731. os.execute('mkdir -p Packages/' .. n .. dir)
  732. os.execute('mv -f results/' .. APT.options.referenceSite.value .. '/merged/dists/' .. n .. dir .. 'Packages_parsed-sorted Packages/' .. n .. dir .. 'Packages_parsed-sorted')
  733. end
  734. else
  735. end
  736. os.execute('rm -fr results/' .. host .. '/merged/dists/' .. n .. dir .. ' 2>/dev/null')
  737. end
  738. end
  739. if APT.checkFile('results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt') then
  740. -- Sort by size.
  741. os.execute('sort -b -k 9,9 -n results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt >results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt')
  742. os.execute('grep -s " | pool/DEVUAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
  743. os.execute('grep -s " | pool/DEBIAN-SECURITY/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >>' .. outFile)
  744. os.execute('grep -s " | pool/DEBIAN/" results/NEW_' .. list.out .. '_' .. n .. '.sorted.txt 2>/dev/null | head -n 1 >' .. outFile)
  745. os.execute('rm -f results/NEW_' .. list.out .. '_TMP_' .. n .. '.txt')
  746. end
  747. end
  748. local nfile, e = io.open(outFile, "r")
  749. if nil ~= nfile then
  750. -- for l in nfile:lines() do
  751. local l = nfile:read('*l')
  752. if nil ~= l then
  753. local p = l:match('(pool/.*%.deb)')
  754. if nil ~= p then
  755. table.insert(list.files, p)
  756. end
  757. end
  758. -- end
  759. end
  760. end
  761. postParse(host, list)
  762. return list
  763. end
  764. local parseRelease = function(host)
  765. local list = {inf = 'Release', parser = parsePackages, out = 'Packages', files = {}, nextf = 'debs'}
  766. local updated = false
  767. local now = tonumber(os.date('%s'))
  768. for i, n in pairs(releases) do
  769. for l, o in pairs(releaseFiles) do
  770. if repoExists(i .. o) then
  771. postDownload(host, n, o)
  772. if (".gpg" == o:sub(-4, -1)) and (APT.checkFile('results/' .. host .. '/merged/dists/' .. n .. '/' .. o)) then
  773. if APT.testing("Integrity") then
  774. local status = APT.exe( "gpgv --keyring /usr/share/keyrings/devuan-keyring.gpg results/" .. host .. "/merged/dists/" .. n .. '/' .. o ..
  775. " results/" .. host .. "/merged/dists/" .. n .. '/' .. o:sub(1, -5)):Nice():noErr():log():Do().status
  776. if 0 ~= status then E("GPG check failed for " .. host .. "/merged/dists/" .. n .. '/' .. o, "http", "Integrity", host) end
  777. -- TODO - should check the PGP sig of InRelease as well.
  778. end
  779. os.execute('rm results/' .. host .. '/merged/dists/' .. n .. '/' .. o)
  780. end
  781. end
  782. end
  783. local fR = 'results/' .. host .. '/merged/dists/' .. n .. '/Release'
  784. local fRp = APT.options.referenceSite.value .. '/merged/dists/' .. n .. '/Release.SORTED'
  785. if APT.checkFile(fR) then
  786. os.execute('sort -k 3 ' .. fR .. ' >' .. fR .. '.SORTED')
  787. local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
  788. if APT.checkFile('results_old/' .. fRp) then
  789. if APT.options.referenceSite.value == host then
  790. outFile = outFile .. '.tmp'
  791. os.execute('diff -U 0 results_old/' .. fRp .. ' ' ..
  792. 'results/' .. fRp .. ' ' ..
  793. '| grep -v "@@" | grep "^+" | grep "Packages.xz$" | cut -c 77- >' .. outFile)
  794. -- TODO - Maybe check the date in Release, though since they are updated daily, is there any point? Perhaps it's for checking amprolla got run?
  795. -- Also check if that date is in the future, apt recently got a check for that, though not sure why.
  796. os.execute('rm -f ' .. fR .. ' 2>/dev/null; ')
  797. else
  798. -- TODO - compare to the pkgmaster copy.
  799. if APT.testing('Updated') then
  800. while not APT.checkFile('results_old/' .. fRp) do
  801. D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(results_old/' .. fRp .. ')')
  802. coroutine.yield()
  803. D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(results_old/' .. fRp .. ')')
  804. end
  805. local pkt = tonumber(APT.exe([[TZ="GMT" date -d "$(grep '^Date:' results/]] .. fRp .. [[ | cut -d ' ' -f 2-)" '+%s']]):Do().result:sub(2, -2))
  806. local new = tonumber(APT.exe([[TZ="GMT" date -d "$(grep '^Date:' ]] .. fR .. [[.SORTED | cut -d ' ' -f 2-)" '+%s']]):Do().result:sub(2, -2))
  807. local upd = pkt + APT.mirrors[host].Updated
  808. local updd = pkt + (APT.mirrors[host].Updated * 1.5) -- Give the mirror time to actually do the update.
  809. if pkt > new then
  810. D( 'pkt is ' .. os.date('!%F %T', pkt) .. ' new is ' .. os.date('!%F %T', new) ..
  811. ' upd is ' .. os.date('!%F %T', upd) .. ' updd is ' .. os.date('!%F %T', updd) ..
  812. ' now is ' .. os.date('!%F %T', now) .. ' Updated is ' .. APT.mirrors[host].Updated)
  813. if updd >= now then
  814. W('Release ' .. n .. ' not updated yet, should update @ ' .. os.date('!%F %T', upd) .. ', and was last updated @ ' .. os.date('!%F %T', new), 'http', 'Updated', host)
  815. Updating = true
  816. else
  817. E('Release ' .. n .. ' not updated, should have updated @ ' .. os.date('!%F %T', upd) .. ', but was last updated @ ' .. os.date('!%F %T', new), 'http', 'Updated', host)
  818. end
  819. updated = false
  820. else
  821. updated = true
  822. end
  823. end
  824. end
  825. -- TODO - if it's not Integrity and not reference, then just do a HEAD check and compare file times?
  826. -- TODO - like we do with debs, pick just the smallest Packages.xz that has changed.
  827. -- Though we are sorting Release by name, so we can do the diff with the one from results_old, so we'll need to sort by size to.
  828. -- pkgmaster still needs to download all the changed Packages.xz files though.
  829. if (2 <= APT.options.bandwidth.value) and (updated or APT.testing("Integrity") or (APT.options.referenceSite.value == host)) then
  830. local dfile, e = io.open(outFile, "r")
  831. if nil == dfile then W("opening " .. outFile .. " file - " .. e) else
  832. for l in dfile:lines() do
  833. table.insert(list.files, 'dists/' .. n .. '/' .. l)
  834. end
  835. end
  836. end
  837. end
  838. end
  839. end
  840. postParse(host, list)
  841. return list
  842. end
  843. local parseStart = function(host)
  844. local list = {inf = '', parser = parseRelease, out = 'Release', files = {}, nextf = 'Packages'}
  845. for i, n in pairs(releases) do
  846. local outFile = 'results/NEW_' .. list.out .. '_' .. n .. '.txt'
  847. for l, o in pairs(releaseFiles) do
  848. if repoExists(n .. o) then
  849. if APT.options.referenceSite.value == host then
  850. local dfile, e = io.open(outFile .. '.tmp', "a+")
  851. if nil == dfile then W("opening " .. outFile .. ".tmp file - " .. e) else
  852. dfile:write(o .. '\n')
  853. end
  854. end
  855. table.insert(list.files, 'dists/' .. n .. '/' .. o)
  856. end
  857. end
  858. end
  859. postParse(host, list)
  860. return list
  861. end
  862. local doDownloads = function(host, path, list)
  863. while nil ~= list do
  864. if 0 ~= #(list.files) then
  865. for j, f in pairs(list.files) do
  866. downloads(host, path, list.out, nil, f)
  867. end
  868. downloads(host, path, list.out, nil, '')
  869. --[[ I've seen flock & curl act oddly. Perhaps flock didn't have time to start up?
  870. /var/www/html/apt-panopticon/apt-panopticon/results_2019-12-22-15-00
  871. Mon Dec 23 01:02:54 2019 DEBUG : forking
  872. flock -n results/curl-debs-pkgmaster.devuan.org.log curl --connect-timeout 5 --create-dirs -f -L --fail-early --max-time 300 --retry 3 -R -v -z 'results/stamp.old' --stderr results/curl-debs-pkgmaster.devuan.org.log -K results/curl-debs-pkgmaster.devuan.org.files
  873. Mon Dec 23 01:02:54 2019 DEBUG : 0 flock -n results/curl-debs-pkgmaster.devuan.org.log commands still running.
  874. Mon Dec 23 01:02:54 2019 DEBUG : *>* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(flock -n results/curl-debs-pkgmaster.devuan.org.log
  875. Mon Dec 23 01:02:54 2019 DEBUG : *** Doing list.parser() for debs
  876. Mon Dec 23 01:02:54 2019 ERROR (http Updated pkgmaster.devuan.org): Failed to download - results/pkgmaster.devuan.org/merged/pool/DEBIAN/main/a/aptly/aptly_1.3.0+ds1-4_amd64.deb
  877. drwxr-x--- 2 www-data www-data 4096 2019-12-23 01:02:57.000000000 +1000 aptly
  878. -rw-r--r-- 1 www-data www-data 7129 2019-12-23 01:03:54.000000000 +1000 curl-debs-pkgmaster.devuan.org.log
  879. ]]
  880. os.execute('sleep 1') -- Wait for things to start up before checking for them.
  881. while 0 < APT.checkExes(downloadLock .. list.out .. "-" .. host .. ".log.txt") do
  882. D('*&lt;* About to yield coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log.txt')
  883. coroutine.yield()
  884. D('*&gt;* Resumed coroutine while waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log.txt')
  885. end
  886. D('*&gt;* Resumed coroutine NO LONGER waiting on - 0 < APT.checkExes(' .. downloadLock .. list.out .. '-' .. host .. '.log.txt')
  887. local f = 'results/curl-' .. list.out .. "-" .. host .. ".log.txt"
  888. local f = 'results/curl-' .. list.out .. "-" .. host .. ".log.txt"
  889. -- Should not be needed, but maybe this is why sometimes I don't see the speeds, though the file is there aand valid when I look later.
  890. while not APT.checkFile(f) do
  891. D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  892. coroutine.yield()
  893. D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  894. end
  895. --[[ TODO - should try to figure out which server the file actually got downloaded from, and attribute the speed and errors to that server.
  896. Which means parsing the curl logs, not just a simple match().
  897. Watch out for misplaced ^M, they don't all come at the end of the line.
  898. Also note curl-Release-mirror.devuan.de.log.txt, timeouts don't always show the "Connected to" string.
  899. * Immediate connect fail for 2001:4ca0:4300::1:19: Network is unreachable
  900. * Connected to debian.ipacct.com (2a01:9e40::251) port 80 (#1)
  901. 0 0 0 0 0 0 0 0 --:--:-- --:--:-- --:--:-- 0* Connected to devuan.bio.lmu.de (141.84.43.19) port 80 (#0)
  902. curl: (22) The requested URL returned error: 404 Not Found
  903. ]]
  904. local min, max, spd = 999999999999, 0
  905. local num = '[%d%.]+[kM]?'
  906. if APT.checkFile(f) then
  907. for l in io.lines(f) do
  908. local speed, crrnt = l:match('^%c *'..num..' +'..num..' +%d+ +'..num..' +%d+ +%d+ +('..num..') +%d+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +[%d%-]+:[%d%-]+:[%d%-]+ +('..num..')')
  909. if nil ~= speed then
  910. if 'k' == speed:sub(-1, -1) then speed = tonumber(speed:sub(1, -2)) * 1024
  911. elseif 'M' == speed:sub(-1, -1) then speed = tonumber(speed:sub(1, -2)) * 1024 * 1024
  912. end
  913. speed = tonumber(speed)
  914. if 'k' == crrnt:sub(-1, -1) then crrnt = tonumber(crrnt:sub(1, -2)) * 1024
  915. elseif 'M' == crrnt:sub(-1, -1) then crrnt = tonumber(crrnt:sub(1, -2)) * 1024 * 1024
  916. end
  917. crrnt = tonumber(crrnt)
  918. if speed < min and speed ~= 0 then min = speed end
  919. if speed > max then max = speed end
  920. if crrnt < min and crrnt ~= 0 then min = crrnt end
  921. if crrnt > max then max = crrnt end
  922. end
  923. if l:find('timed out') ~= nil then
  924. E(" TIMEOUT " .. timeouts + 1 .. ', details in <a href="curl-' .. list.out .. '-' .. host .. '.log.txt">curl-' .. list.out .. '-' .. host .. '.log.txt</a>', 'http', '', host)
  925. timeouts = timeouts + 1
  926. APT.results["timeout"] = true
  927. end
  928. end
  929. end
  930. APT.results["speed"] = {["min"] = min, ["max"] = max}
  931. end
  932. if (APT.options.referenceSite.value ~= host) and ('' ~= list.nextf) then
  933. local sem = 'results/NEW_' .. list.nextf .. '_%s.txt'
  934. for i, n in pairs(releases) do
  935. local f = sem:format(n)
  936. while not APT.checkFile(f) do
  937. D('*&lt;* About to yield coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  938. coroutine.yield()
  939. D('*&gt;* Resumed coroutine while waiting on - not APT.checkFile(' .. f .. ')')
  940. end
  941. end
  942. end
  943. D('*** Doing list.parser() for ' .. list.out)
  944. list = list.parser(host)
  945. if APT.options.timeouts.value <= (totalTimeouts) then break end
  946. end
  947. D('*&lt;&lt;* About to end coroutine.')
  948. cor = nil
  949. end
  950. if 0 < #arg then
  951. if "/" == arg[1]:sub(-1, -1) then
  952. W("slash at end of path! " .. arg[1])
  953. arg[1] = arg[1]:sub(1, -2)
  954. end
  955. if " " == arg[1]:sub(-1, -1) then
  956. W("space at end of path! " .. arg[1])
  957. arg[1] = arg[1]:sub(1, -2)
  958. end
  959. local pu = url.parse("http://" .. arg[1])
  960. if APT.redir and (nil == arg[3])then
  961. arg[3] = arg[2]
  962. arg[2] = nil
  963. end
  964. if APT.testing("Integrity") or APT.testing("Updated") then
  965. if APT.origin and APT.options.referenceSite.value == pu.host then
  966. -- if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
  967. end
  968. end
  969. if not APT.logOpen(pu.host, arg[2], arg[3]) then return end
  970. I("Starting tests for " .. arg[1] .. " with these tests - " .. table.concat(APT.options.tests.value, ", "))
  971. if APT.origin or APT.redir then APT.results["IPs"] = gatherIPs(pu.host) end
  972. if nil ~= arg[2] then I(" &nbsp; Using IP " .. arg[2]); ip = arg[2] end
  973. if nil ~= arg[3] then I(" &nbsp; Using file " .. arg[3]); end
  974. APT.mirrors = loadfile("results/mirrors.lua")()
  975. APT.results = APT.padResults(APT.results)
  976. if APT.origin then
  977. local file = arg[3]
  978. if nil == file then file = '' end
  979. local path = pu.path
  980. if nil == path then path = '' end
  981. if APT.origin then
  982. local ips = APT.results["IPs"]
  983. if nil ~= ips then
  984. APT.allpairs(ips,
  985. function(k, v)
  986. if v == "A" then
  987. if APT.testing("IPv4") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -4 ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
  988. elseif v == "AAAA" then
  989. if APT.testing("IPv6") then APT.exe('./apt-panopticon.lua ' .. sendArgs .. ' -6 ' .. APT.IPv46 .. ' ' .. pu.host .. path .. ' ' .. k .. ' ' .. file):Nice():log():fork() end
  990. end
  991. D('logging to ' .. APT.logName(pu.host, k, file)[2])
  992. end
  993. )
  994. else
  995. E("no IPs for " .. pu.host)
  996. end
  997. end
  998. if not APT.redir then
  999. if (1 <= APT.options.bandwidth.value) and (APT.testing("Integrity") or APT.testing("Updated")) then
  1000. if APT.origin and (APT.options.roundRobin.value ~= pu.host) then
  1001. I("Starting file downloads for " .. pu.host)
  1002. D('*&gt;* About to create coroutine.')
  1003. cor = coroutine.create(doDownloads)
  1004. local ok, message = coroutine.resume(cor, pu.host, pu.path, parseStart(pu.host))
  1005. if not ok then cor = nil; print(message) end
  1006. end
  1007. end
  1008. checkFiles(pu.host, pu.host, pu.path);
  1009. else
  1010. checkFiles(pu.host, pu.host, pu.path:sub(1, -1), file);
  1011. end
  1012. else
  1013. checkHost(pu.host, pu.host, pu.path, arg[2], arg[3])
  1014. end
  1015. while nil ~= cor do
  1016. os.execute('sleep 10')
  1017. D('*&gt;* About to resume coroutine before writing results.')
  1018. local ok, message = coroutine.resume(cor)
  1019. if not ok then cor = nil; print(message); break end
  1020. end
  1021. local f = pu.host
  1022. if "" ~= ip then f = f .. "_" .. ip end
  1023. -- TODO - perhaps number them if there's more than one?
  1024. if APT.redir then f = f .. '_' .. 'R' end
  1025. local rfile, e = io.open("results/" .. f .. ".lua", "w+")
  1026. if nil == rfile then C("opening results file - " .. e) else
  1027. rfile:write(APT.dumpTable(APT.results, "results") .. "\nreturn results\n")
  1028. rfile:close()
  1029. end
  1030. if APT.origin and (not APT.redir) and (APT.options.referenceSite.value ~= pu.host) then
  1031. os.execute('sleep 1') -- Wait for things to start up before checking for them.
  1032. while 0 < APT.checkExes(downloadLock .. "Release-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
  1033. while 0 < APT.checkExes(downloadLock .. "Packages-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
  1034. while 0 < APT.checkExes(downloadLock .. "package-" .. pu.host .. ".log.txt") do os.execute("sleep 10") end
  1035. os.execute("sleep 5")
  1036. if not APT.keep then os.execute("rm -fr results/" .. pu.host .. " 2>/dev/null") end
  1037. end
  1038. APT.logPost()
  1039. else
  1040. local adt = APT.exe("ls -dl results_old 2>/dev/null | cut -d '>' -f 2 | cut -d ' ' -f 2"):Do().result:sub(2, -2)
  1041. if (nil ~= adt) and APT.checkFile(adt) then
  1042. APT.exe('mkdir -p ' .. adt:sub(1, 18))
  1043. :And():Nice('tar -c --xz ' .. adt .. ' -f ' .. adt:sub(1, 18) .. '/' .. adt .. '.tar.xz')
  1044. :And('rm -fr ' .. adt):noErr():fork()
  1045. end
  1046. local dt = os.date('!%F-%H-%M')
  1047. local odt = APT.exe('TZ="GMT" date -r results/stamp +%F-%H-%M 2>/dev/null'):Do().result:sub(2, -2)
  1048. if nil ~= odt then os.execute(' rm -f results_old; ln -s results_' .. odt .. ' results_old 2>/dev/null') end
  1049. if nil ~= dt then os.execute('mkdir -p results_' .. dt .. '; rm -f results; ln -s results_' .. dt .. ' results 2>/dev/null') end
  1050. os.execute('if [ -f results/stamp ]; then mv results/stamp results/stamp.old; else touch results/stamp.old -t 199901010000; fi; touch results/stamp')
  1051. if not APT.keep then
  1052. os.execute("rm -f results/*.html 2>/dev/null")
  1053. os.execute("rm -f results/*.txt 2>/dev/null")
  1054. end
  1055. if not APT.logOpen('apt-panopticon') then return end
  1056. I("Starting tests " .. table.concat(APT.options.tests.value, ", "))
  1057. os.execute("mkdir -p results")
  1058. APT.mirrors = getMirrors()
  1059. checkHost(APT.options.referenceSite.value)
  1060. for k, m in pairs(APT.mirrors) do
  1061. local pu = url.parse("http://" .. m.BaseURL)
  1062. if APT.options.referenceSite.value ~= pu.host then
  1063. checkHost(m.BaseURL)
  1064. end
  1065. end
  1066. while not APT.checkFile('results/LOG_' .. APT.options.referenceSite.value .. '.html') do -- Wait for things to start up before checking for them.
  1067. D('Waiting for results/LOG_' .. APT.options.referenceSite.value .. '.html');
  1068. os.execute("sleep 5")
  1069. -- TODO - count these, and abort if it takes too long.
  1070. -- Try something similar for the other "Wait for things to start up before checking for them.", maybe fold it into APT.exe.
  1071. end
  1072. while 1 <= APT.checkExes("apt-panopticon.lua " .. sendArgs) do os.execute("sleep 5") end
  1073. local APT_args = APT.args
  1074. local APT_logFile = APT.logFile
  1075. local debians = {}
  1076. local srvs = io.popen('ls -1 results/*.lua')
  1077. for l in srvs:lines() do
  1078. local hst = l:sub(9, -5)
  1079. if nil ~= l:find('_R%.lua') then hst = hst:sub(1, -3) end
  1080. if (hst:find('_') == nil) and (nil == APT.mirrors[hst]) then
  1081. local ips = loadfile(l)().IPs
  1082. if nil ~= ips then
  1083. debians[hst] = {Country = '', FDQN = hst, Active = 'yes', Rate = '', BaseURL = hst, Protocols = {http = true, https = true}, Bandwidth = '', IPs = ips}
  1084. local baseFiles = {}
  1085. local IPfiles = {}
  1086. for i, a in pairs(ips) do
  1087. IPfiles[i] = {}
  1088. if type(a) == 'table' then
  1089. for j, b in pairs(a) do
  1090. IPfiles[i][j] = {}
  1091. end
  1092. end
  1093. end
  1094. local files = io.popen('ls -1 results/LOG_' .. hst .. '_*.html')
  1095. for ll in files:lines() do
  1096. local dn = false
  1097. for i, a in pairs(ips) do
  1098. if type(a) == 'table' then
  1099. for j, b in pairs(a) do
  1100. if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. j .. '_.*)') then
  1101. table.insert(IPfiles[i][j], ll)
  1102. dn = true
  1103. end
  1104. end
  1105. else
  1106. if nil ~= ll:match('(results/LOG_' .. hst .. '_' .. i .. '_.*)') then
  1107. table.insert(IPfiles[i], ll)
  1108. dn = true
  1109. end
  1110. end
  1111. end
  1112. if not dn then table.insert(baseFiles, ll) end
  1113. end
  1114. local combine = function(ip, a)
  1115. if not APT.logOpen(hst, ip) then print('PROBLEM OPENING LOG FILE ' .. hst .. ' ' .. ip) end
  1116. APT.logFile:write('<h1>Note log lines will be out of order, this is a bunch of other log files combined.</h1>\n')
  1117. for i, f in pairs(a) do
  1118. f = f:sub(9, -1)
  1119. APT.logFile:write('<hr>\n<hr>\n<h2><a href="' .. f .. '">' .. f .. '</a></h2>\n')
  1120. for ln in io.lines('results/' .. f) do
  1121. if ln:match('^' .. os.date('!%Y%-%m%-%d ') .. '.*$') then APT.logFile:write(ln .. '\n') end -- %F isn't good enough, coz we have to escape the '-'.
  1122. end
  1123. end
  1124. APT.logPost()
  1125. APT.args = APT_args
  1126. APT.logFile = APT_logFile
  1127. end
  1128. combine('', baseFiles)
  1129. for ip, a in pairs(IPfiles) do
  1130. if nil == a[1] then
  1131. for i, f in pairs(a) do
  1132. combine(i, f)
  1133. end
  1134. else
  1135. combine(ip, a)
  1136. end
  1137. end
  1138. end
  1139. end
  1140. end
  1141. local file, e = io.open("results/debians.lua", "w+")
  1142. if nil == file then C("opening debians file - " .. e) else
  1143. file:write(APT.dumpTable(debians, "debians") .. "\nreturn debians\n")
  1144. file:close()
  1145. end
  1146. for k, v in pairs(APT.mirrors) do
  1147. local f = 'results/' .. k .. '.lua'
  1148. if APT.checkFile(f) then
  1149. results = loadfile(f)()
  1150. APT.mirrors[k]['IPs'] = results.IPs
  1151. end
  1152. end
  1153. local file, e = io.open("results/mirrors.lua", "w+")
  1154. if nil == file then C("opening mirrors file - " .. e) else
  1155. file:write(APT.dumpTable(APT.mirrors, "mirrors") .. "\nreturn mirrors\n")
  1156. file:close()
  1157. end
  1158. -- Create the reports.
  1159. for n, r in pairs(APT.options.reports.value) do
  1160. if APT.checkFile("apt-panopticon-report-" .. r .. ".lua") then
  1161. I("Creating " .. r .. " report.")
  1162. APT.exe("./apt-panopticon-report-" .. r .. ".lua " .. sendArgs):log():Do()
  1163. end
  1164. end
  1165. I('Total run time was ' .. (os.time() - now) .. ' seconds.')
  1166. APT.logPost()
  1167. end