archive.php 27 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741
  1. <?php
  2. /**
  3. * Gallery ZIP archives: building them, and keeping them up to date.
  4. *
  5. * Why it works this way
  6. * --------------------
  7. * A gallery can hold hundreds of 8 MB originals, and the host caps
  8. * max_execution_time at 60 s. Streaming a multi-gigabyte ZIP through PHP would
  9. * need a request that stays alive for the whole download, so instead the archive
  10. * is *built once into S3* and visitors are redirected to a presigned URL for it.
  11. * The download then never touches the webhost at all — it is a plain S3 GET,
  12. * resumable and immune to any server timeout.
  13. *
  14. * Building it is split into slices. archive_run_slice() copies as many photos as
  15. * fit in a time budget and returns; progress is committed to a state file after
  16. * every photo, so "build the archive" is just "run slices until finished". No
  17. * single request ever approaches the 60 s cap, and nothing depends on
  18. * set_time_limit() being allowed.
  19. *
  20. * Each photo is streamed S3 → buffer file → S3: the ZIP needs a local header
  21. * immediately before the file's bytes, and multipart parts are atomic, so the
  22. * bytes cannot be copied server-side with UploadPartCopy. The buffer file
  23. * accumulates until it passes the 5 MB multipart minimum, then becomes one part.
  24. *
  25. * Interruptions
  26. * -------------
  27. * State is written with json_write() (tmp file + rename), so it is never half
  28. * written — at worst it is one slice old and that slice replays. The rest is
  29. * handled by ordering:
  30. *
  31. * - mid photo → every slice starts by truncating the buffer back to the last
  32. * committed length, so a partial tail is dropped without
  33. * needing the failure to have been caught.
  34. * - mid part → the ETag is committed to state only after S3 accepts the
  35. * part, and the buffer is truncated only after that. A crash
  36. * anywhere in between re-uploads the same part number, which
  37. * S3 accepts until the upload is completed.
  38. * - mid finish → a retried CompleteMultipartUpload returns NoSuchUpload once
  39. * it has already succeeded; that is treated as done after a
  40. * HEAD confirms the object exists.
  41. * - abandoned → the queue entry survives on disk, and a build with no
  42. * progress for archive.abandon_hours is aborted and restarted.
  43. *
  44. * Staying current
  45. * ---------------
  46. * Every stored or deleted image marks its gallery dirty (archive_mark_dirty),
  47. * as does anything that moves a photo between the ZIP's folders — assigning a
  48. * topic, renaming one, reordering them.
  49. * A dirty gallery's download button is disabled until the rebuild lands, rather
  50. * than handing out a ZIP that is missing the newest photos. Rebuilds run in the
  51. * background: archive_kick() on a page render dispatches worker.php, which runs
  52. * a slice and then dispatches its own successor, so one upload starts a chain
  53. * that finishes unattended. See docs/ARCHITECTURE.md.
  54. */
  55. declare(strict_types=1);
  56. /** Queue of galleries whose archive no longer matches their contents. */
  57. function archive_queue_file(): string
  58. {
  59. return DATA_DIR . '/archive-queue.json';
  60. }
  61. /** Site-wide lock: exactly one archive worker runs at a time. */
  62. function archive_lock_file(): string
  63. {
  64. return DATA_DIR . '/archive.lock';
  65. }
  66. /** Throttle marker so a busy site does not dispatch a worker per page view. */
  67. function archive_tick_file(): string
  68. {
  69. return DATA_DIR . '/archive.tick';
  70. }
  71. /**
  72. * A topic's name as a ZIP folder: ASCII, no separators, nothing a desktop
  73. * unzipper could refuse. Deliberately the same character set safe_filename()
  74. * uses for the files inside it, so a path is uniform end to end.
  75. */
  76. function archive_folder_name(string $name): string
  77. {
  78. $folder = ascii_transliterate($name);
  79. // Runs collapse to one dash: "Day 1 – Reykjavík" is a perfectly ordinary
  80. // topic name, and transliterating it would otherwise leave "Day-1---…".
  81. $folder = preg_replace('/[^A-Za-z0-9._-]+/', '-', $folder) ?? '';
  82. // Collapse afterwards, not in the same pass: transliteration turns the dash
  83. // in "Day 1 – Reykjavík" into one of its own, and the spaces around it into
  84. // two more, which would otherwise leave "Day-1---Reykjavik".
  85. $folder = preg_replace('/-{2,}/', '-', $folder) ?? '';
  86. $folder = trim(substr($folder, 0, 60), '-.');
  87. return $folder !== '' ? $folder : 'topic';
  88. }
  89. /**
  90. * Every photo the archive contains, in build order, with the path it gets
  91. * inside the ZIP: untopiced photos in the root, each topic its own folder.
  92. *
  93. * Order and grouping come from gallery_groups(), the same function the gallery
  94. * page renders from, so the ZIP's structure always matches what the client saw.
  95. * Two topics that reduce to the same folder name are kept apart by the same
  96. * dedupe the filenames use.
  97. *
  98. * Returns [ ['image' => <image record>, 'path' => 'Day-1/DSC_0001.jpg'], … ].
  99. */
  100. function archive_entries(array $gallery): array
  101. {
  102. $entries = [];
  103. $folders = [];
  104. foreach (gallery_groups($gallery) as $group) {
  105. $prefix = '';
  106. if ($group['topic'] !== null) {
  107. $prefix = zip_dedupe_name(archive_folder_name($group['topic']['name']), $folders) . '/';
  108. }
  109. foreach ($group['images'] as $image) {
  110. $entries[] = [
  111. 'image' => $image,
  112. 'path' => $prefix . safe_filename((string)($image['name'] ?? 'photo.jpg'), 'photo'),
  113. ];
  114. }
  115. }
  116. return $entries;
  117. }
  118. /**
  119. * Identity of a gallery's image set. A build records the hash it was made from;
  120. * when the gallery's current hash differs, the archive is out of date.
  121. *
  122. * A gallery that uses no topics hashes exactly as it did before topics existed,
  123. * so installing this version does not invalidate a single archive already built.
  124. * Once topics are in play the hash is taken over the ZIP paths instead, which
  125. * covers key, order, topic membership, topic order and topic names at once:
  126. * anything that would change the archive's layout changes the hash.
  127. */
  128. function archive_source_hash(array $gallery): string
  129. {
  130. if (!gallery_uses_topics($gallery)) {
  131. return sha1(implode("\n", array_column($gallery['images'] ?? [], 'key')));
  132. }
  133. $lines = [];
  134. foreach (archive_entries($gallery) as $entry) {
  135. $lines[] = $entry['path'] . "\t" . (string)($entry['image']['key'] ?? '');
  136. }
  137. return sha1(implode("\n", $lines));
  138. }
  139. /** Whether a gallery's archive is missing or no longer matches its images. */
  140. function archive_is_stale(array $gallery): bool
  141. {
  142. $archive = $gallery['archive'] ?? null;
  143. if (empty($archive['key'])) {
  144. return true;
  145. }
  146. return ($archive['source_hash'] ?? '') !== archive_source_hash($gallery);
  147. }
  148. // ---------------------------------------------------------------------------
  149. // The queue
  150. // ---------------------------------------------------------------------------
  151. /**
  152. * Flag a gallery for rebuilding. Called from gallery_append_image(), from image
  153. * deletion and from the topic mutations in app/storage.php — everywhere the
  154. * gallery's contents or their arrangement can change.
  155. *
  156. * Uploads run in parallel (uploads.concurrency), so the queue is written through
  157. * json_update()'s exclusive lock — three uploads finishing together must not
  158. * drop each other's entries. Galleries without downloads enabled are skipped:
  159. * queueing them would give the worker chain work it can never finish.
  160. */
  161. function archive_mark_dirty(string $slug, ?array $gallery = null): void
  162. {
  163. $gallery ??= gallery_load($slug);
  164. if ($gallery === null || empty($gallery['downloads_enabled'])) {
  165. return;
  166. }
  167. json_update(archive_queue_file(), function (array $queue) use ($slug): array {
  168. $queue['galleries'][$slug] = time();
  169. return $queue;
  170. });
  171. }
  172. /** Drop a gallery from the queue (rebuilt, deleted, or downloads turned off). */
  173. function archive_unqueue(string $slug): void
  174. {
  175. json_update(archive_queue_file(), function (array $queue) use ($slug): ?array {
  176. if (!isset($queue['galleries'][$slug])) {
  177. return null; // nothing to do; leave the file untouched
  178. }
  179. unset($queue['galleries'][$slug]);
  180. return $queue;
  181. });
  182. }
  183. /**
  184. * The next gallery due for a rebuild, or null if none has settled yet.
  185. *
  186. * A gallery is only rebuilt once it has been quiet for archive.settle_seconds.
  187. * Thirty wedding guests uploading over an hour must not restart the build thirty
  188. * times; the delay batches an upload burst into a single rebuild.
  189. */
  190. function archive_next_due(): ?string
  191. {
  192. $queue = json_read(archive_queue_file());
  193. $settle = (int)config('archive.settle_seconds', 300);
  194. foreach ($queue['galleries'] ?? [] as $slug => $dirtyAt) {
  195. if (time() - (int)$dirtyAt >= $settle) {
  196. return (string)$slug;
  197. }
  198. }
  199. return null;
  200. }
  201. /**
  202. * Seconds until the earliest queued gallery settles: 0 if one is due now, null
  203. * if the queue is empty. The worker uses this to decide between doing a slice
  204. * and waiting out the settle window.
  205. */
  206. function archive_queue_wait(): ?int
  207. {
  208. $queue = json_read(archive_queue_file());
  209. $settle = (int)config('archive.settle_seconds', 300);
  210. $wait = null;
  211. foreach ($queue['galleries'] ?? [] as $dirtyAt) {
  212. $due = max(0, $settle - (time() - (int)$dirtyAt));
  213. $wait = $wait === null ? $due : min($wait, $due);
  214. }
  215. return $wait;
  216. }
  217. // ---------------------------------------------------------------------------
  218. // Build state
  219. // ---------------------------------------------------------------------------
  220. /**
  221. * Start a fresh build: abandon anything in flight, open a new multipart upload,
  222. * and write the initial state. Returns the state, or null if it cannot start.
  223. *
  224. * Content-Type and Content-Disposition are set on the multipart create, so S3
  225. * stores them as the finished object's metadata and the presigned URL downloads
  226. * as a properly named .zip with no extra signing work.
  227. */
  228. function archive_start(string $slug, array $gallery): ?array
  229. {
  230. if (empty($gallery['images'])) {
  231. return null; // nothing to archive
  232. }
  233. archive_abort($slug);
  234. $filename = safe_filename(($gallery['title'] ?: $slug) . '.zip', $slug);
  235. $key = s3_gallery_prefix($slug) . '/archive/' . random_token(6) . '-' . $filename;
  236. $uploadId = s3_mpu_create($key, 'application/zip', 'attachment; filename="' . $filename . '"');
  237. if ($uploadId === null) {
  238. return null;
  239. }
  240. $state = [
  241. 'key' => $key,
  242. 'upload_id' => $uploadId,
  243. 'source_hash' => archive_source_hash($gallery),
  244. // One entry per photo, so this still counts photos — but it counts them
  245. // in the order and grouping the ZIP will actually use.
  246. 'total' => count(archive_entries($gallery)),
  247. 'next_index' => 0,
  248. // Archive length so far, and how much of it S3 already has. The
  249. // difference is exactly what the buffer file holds.
  250. 'offset' => 0,
  251. 'uploaded' => 0,
  252. 'part_number' => 1,
  253. 'parts' => [],
  254. 'entries' => [],
  255. 'names' => [], // filename dedupe map, carried across slices
  256. 'slowest' => 5.0, // seconds; grows to the slowest photo seen
  257. 'started_at' => time(),
  258. ];
  259. json_write(gallery_archive_file($slug), $state);
  260. @unlink(gallery_archive_buffer($slug));
  261. return $state;
  262. }
  263. /**
  264. * Abandon an in-flight build. Aborting the multipart upload matters: S3 keeps
  265. * (and bills for) the parts of an incomplete upload indefinitely.
  266. */
  267. function archive_abort(string $slug): void
  268. {
  269. $state = json_read(gallery_archive_file($slug));
  270. if (!empty($state['upload_id']) && !empty($state['key'])) {
  271. s3_mpu_abort((string)$state['key'], (string)$state['upload_id']);
  272. }
  273. @unlink(gallery_archive_file($slug));
  274. @unlink(gallery_archive_buffer($slug));
  275. }
  276. /** Delete a gallery's finished archive from S3 and forget it. */
  277. function archive_delete(string $slug): void
  278. {
  279. archive_abort($slug);
  280. $gallery = gallery_load($slug);
  281. if ($gallery !== null && !empty($gallery['archive']['key'])) {
  282. s3_delete((string)$gallery['archive']['key']);
  283. }
  284. json_update(gallery_file($slug), function (array $g): ?array {
  285. if ($g === [] || !isset($g['archive'])) {
  286. return null;
  287. }
  288. unset($g['archive']);
  289. return $g;
  290. });
  291. archive_unqueue($slug);
  292. }
  293. // ---------------------------------------------------------------------------
  294. // The slice
  295. // ---------------------------------------------------------------------------
  296. /**
  297. * Copy as many of a gallery's photos into its archive as fit in $budget seconds.
  298. *
  299. * Returns ['done' => int, 'total' => int, 'finished' => bool, 'size' => ?int,
  300. * 'error' => ?string]. Call again to continue; all progress is on disk.
  301. */
  302. function archive_run_slice(string $slug, ?float $budget = null): array
  303. {
  304. $started = microtime(true);
  305. $budget ??= (float)config('archive.step_seconds', 25);
  306. $gallery = gallery_load($slug);
  307. if ($gallery === null) {
  308. archive_unqueue($slug);
  309. return archive_result(0, 0, false, null, 'Gallery no longer exists');
  310. }
  311. $stateFile = gallery_archive_file($slug);
  312. $state = json_read($stateFile);
  313. $abandoned = $state !== []
  314. && time() - (int)($state['started_at'] ?? 0) > (int)config('archive.abandon_hours', 24) * 3600;
  315. // Restart whenever the gallery has changed under an in-flight build, or the
  316. // build has been stalled long enough to be considered dead.
  317. if ($state === [] || $abandoned || ($state['source_hash'] ?? '') !== archive_source_hash($gallery)) {
  318. $state = archive_start($slug, $gallery);
  319. if ($state === null) {
  320. archive_unqueue($slug);
  321. return archive_result(0, 0, false, null, 'Cannot start archive (empty gallery or S3 refused)');
  322. }
  323. }
  324. // Rebuilt from the gallery on every slice rather than carried in the state
  325. // file: it is a pure function of the gallery, and any change to the gallery
  326. // has already restarted the build through the source hash above.
  327. $entries = archive_entries($gallery);
  328. $bufferPath = gallery_archive_buffer($slug);
  329. $fh = fopen($bufferPath, 'c+b');
  330. if ($fh === false) {
  331. return archive_result((int)$state['next_index'], (int)$state['total'], false, null, 'Cannot open archive buffer');
  332. }
  333. flock($fh, LOCK_EX);
  334. // Recovery: drop anything written past the last committed position. Doing
  335. // this unconditionally means a hard kill needs no cleanup of its own.
  336. ftruncate($fh, (int)$state['offset'] - (int)$state['uploaded']);
  337. fseek($fh, 0, SEEK_END);
  338. $partMin = (int)config('archive.part_min_bytes', 5 * 1024 * 1024);
  339. $mtime = strtotime((string)($gallery['created_at'] ?? '')) ?: time();
  340. $error = null;
  341. $processed = 0;
  342. while ($state['next_index'] < $state['total']) {
  343. // Check the clock before starting a photo, never during one, and leave
  344. // room for one that runs as long as the slowest seen so far.
  345. //
  346. // Always do at least one photo, whatever the estimate says. A gallery
  347. // whose photos each take longer than the whole budget must still creep
  348. // forward one photo per slice; refusing to start would leave the build
  349. // stuck for ever, which is far worse than a slice that overruns.
  350. if ($processed > 0 && microtime(true) - $started + $state['slowest'] * 1.5 >= $budget) {
  351. break;
  352. }
  353. $photoStart = microtime(true);
  354. $entry = $entries[$state['next_index']];
  355. $image = $entry['image'];
  356. // Reserve the name in a copy: a photo that fails below is retried by the
  357. // next slice, and a name left registered by the failed attempt would
  358. // make the retry rename itself to "… (2)".
  359. //
  360. // Deduping the whole path rather than the bare filename makes it
  361. // per-folder for free: the same DSC_0001.jpg may appear once in every
  362. // topic, and only a genuine clash inside one folder gets renamed.
  363. $names = $state['names'];
  364. $name = zip_dedupe_name($entry['path'], $names);
  365. $entryOffset = (int)$state['offset'];
  366. $header = zip_local_header($name, $mtime);
  367. fwrite($fh, $header);
  368. // One pass: bytes go to the buffer and through the CRC at the same time,
  369. // so an original never has to be held in memory or read back.
  370. $crcContext = hash_init('crc32b');
  371. [$status, $bytes] = s3_get_stream((string)$image['key'], function (string $chunk) use ($fh, $crcContext): void {
  372. hash_update($crcContext, $chunk);
  373. fwrite($fh, $chunk);
  374. });
  375. if ($status < 200 || $status >= 300) {
  376. // Undo this photo entirely and stop; the next slice retries it.
  377. ftruncate($fh, $entryOffset - (int)$state['uploaded']);
  378. fseek($fh, 0, SEEK_END);
  379. $error = 'S3 returned HTTP ' . $status . ' for ' . ($image['name'] ?? $image['key']);
  380. break;
  381. }
  382. // The real size and CRC are only known now, so patch them into the
  383. // header written above. Using the streamed byte count rather than the
  384. // recorded one also heals a gallery whose stored size was ever wrong.
  385. $crc = (int)hexdec(hash_final($crcContext));
  386. zip_patch_local_header($fh, $entryOffset - (int)$state['uploaded'], $name, $crc, $bytes);
  387. $state['names'] = $names;
  388. $state['entries'][] = [
  389. 'name' => $name,
  390. 'crc' => $crc,
  391. 'size' => $bytes,
  392. 'offset' => $entryOffset,
  393. 'mtime' => $mtime,
  394. ];
  395. $state['offset'] = $entryOffset + strlen($header) + $bytes;
  396. $state['next_index']++;
  397. $state['slowest'] = max((float)$state['slowest'], microtime(true) - $photoStart);
  398. $processed++;
  399. if ((int)$state['offset'] - (int)$state['uploaded'] >= $partMin) {
  400. $error = archive_flush_part($state, $fh, $bufferPath, false);
  401. if ($error !== null) {
  402. break;
  403. }
  404. }
  405. json_write($stateFile, $state);
  406. }
  407. // Everything copied: append the central directory and close the upload.
  408. $finished = false;
  409. if ($error === null && $state['next_index'] >= $state['total']) {
  410. [$finished, $error] = archive_finish($slug, $state, $fh, $bufferPath);
  411. }
  412. json_write($stateFile, $state);
  413. flock($fh, LOCK_UN);
  414. fclose($fh);
  415. if ($finished) {
  416. @unlink($stateFile);
  417. @unlink($bufferPath);
  418. }
  419. return archive_result(
  420. (int)$state['next_index'],
  421. (int)$state['total'],
  422. $finished,
  423. $finished ? (int)$state['offset'] : null,
  424. $error
  425. );
  426. }
  427. /**
  428. * Send the buffered bytes to S3 as the next multipart part.
  429. *
  430. * The order here is what makes an interrupted build safe: S3 accepts the part,
  431. * then the ETag is committed to state, and only then is the buffer cleared. A
  432. * crash before the commit re-uploads the same part number with the same bytes,
  433. * which S3 allows until the upload is completed.
  434. *
  435. * Returns an error message, or null on success.
  436. *
  437. * @param resource $fh
  438. */
  439. function archive_flush_part(array &$state, $fh, string $bufferPath, bool $isLast): ?string
  440. {
  441. fflush($fh);
  442. if (!$isLast && (int)$state['offset'] - (int)$state['uploaded'] === 0) {
  443. return null; // nothing pending
  444. }
  445. $etag = s3_mpu_upload_part(
  446. (string)$state['key'],
  447. (string)$state['upload_id'],
  448. (int)$state['part_number'],
  449. $bufferPath
  450. );
  451. if ($etag === null) {
  452. return 'S3 rejected part ' . $state['part_number'];
  453. }
  454. $state['parts'][] = ['n' => (int)$state['part_number'], 'etag' => $etag];
  455. $state['part_number']++;
  456. $state['uploaded'] = (int)$state['offset'];
  457. ftruncate($fh, 0);
  458. fseek($fh, 0, SEEK_END);
  459. return null;
  460. }
  461. /**
  462. * Write the central directory, upload the last part and complete the multipart
  463. * upload, then record the archive on the gallery.
  464. *
  465. * Returns [finished, error].
  466. *
  467. * @param resource $fh
  468. */
  469. function archive_finish(string $slug, array &$state, $fh, string $bufferPath): array
  470. {
  471. // The central directory is built now, from the real sizes and offsets, so
  472. // it uses ZIP64 fields only where a value actually overflows 32 bits.
  473. $directory = '';
  474. foreach ($state['entries'] as $entry) {
  475. $directory .= zip_central_entry($entry);
  476. }
  477. $trailer = $directory . zip_end_of_central_directory(
  478. count($state['entries']),
  479. strlen($directory),
  480. (int)$state['offset']
  481. );
  482. fwrite($fh, $trailer);
  483. $state['offset'] = (int)$state['offset'] + strlen($trailer);
  484. // The 5 MB minimum does not apply to the final part, which is what lets a
  485. // gallery of small files work with the same buffering scheme.
  486. $error = archive_flush_part($state, $fh, $bufferPath, true);
  487. if ($error !== null) {
  488. return [false, $error];
  489. }
  490. [$ok, $body] = s3_mpu_complete((string)$state['key'], (string)$state['upload_id'], $state['parts']);
  491. if (!$ok) {
  492. // A completed upload no longer exists; if the object is there, an
  493. // earlier attempt succeeded and only the state write was lost.
  494. if (str_contains($body, 'NoSuchUpload') && s3_head((string)$state['key']) !== null) {
  495. $ok = true;
  496. }
  497. }
  498. if (!$ok) {
  499. return [false, 'S3 could not complete the archive upload'];
  500. }
  501. $summary = [
  502. 'key' => (string)$state['key'],
  503. 'size' => (int)$state['offset'],
  504. 'count' => count($state['entries']),
  505. 'built_at' => date('Y-m-d H:i:s'),
  506. 'source_hash' => (string)$state['source_hash'],
  507. ];
  508. // json_update, not gallery_save: an upload finishing right now must not be
  509. // overwritten by a gallery this function read minutes ago.
  510. $previousKey = null;
  511. json_update(gallery_file($slug), function (array $g) use ($summary, &$previousKey): ?array {
  512. if ($g === []) {
  513. return null; // deleted mid-build
  514. }
  515. $previousKey = $g['archive']['key'] ?? null;
  516. $g['archive'] = $summary;
  517. return $g;
  518. });
  519. if ($previousKey !== null && $previousKey !== $summary['key']) {
  520. s3_delete((string)$previousKey);
  521. }
  522. // Leave the gallery queued if it changed while this build was running — the
  523. // archive just written is already out of date and needs another pass.
  524. $current = gallery_load($slug);
  525. if ($current === null || !archive_is_stale($current)) {
  526. archive_unqueue($slug);
  527. }
  528. return [true, null];
  529. }
  530. /** Uniform slice/step result shape, shared by the worker and the admin API. */
  531. function archive_result(int $done, int $total, bool $finished, ?int $size, ?string $error): array
  532. {
  533. return [
  534. 'done' => $done,
  535. 'total' => $total,
  536. 'finished' => $finished,
  537. 'size' => $size,
  538. 'error' => $error,
  539. ];
  540. }
  541. /**
  542. * What the admin page shows for a gallery: whether an archive exists, whether it
  543. * is current, and how far any in-flight build has got.
  544. */
  545. function archive_status(array $gallery): array
  546. {
  547. $slug = (string)$gallery['slug'];
  548. $archive = $gallery['archive'] ?? null;
  549. $state = json_read(gallery_archive_file($slug));
  550. $queue = json_read(archive_queue_file());
  551. return [
  552. 'archive' => $archive,
  553. 'stale' => archive_is_stale($gallery),
  554. 'building' => $state !== [],
  555. 'done' => (int)($state['next_index'] ?? 0),
  556. 'total' => (int)($state['total'] ?? count($gallery['images'] ?? [])),
  557. 'queued' => isset($queue['galleries'][$slug]),
  558. 'due_in' => isset($queue['galleries'][$slug])
  559. ? max(0, (int)config('archive.settle_seconds', 300) - (time() - (int)$queue['galleries'][$slug]))
  560. : null,
  561. ];
  562. }
  563. // ---------------------------------------------------------------------------
  564. // Background execution
  565. // ---------------------------------------------------------------------------
  566. /** Shared secret authenticating the self-dispatched worker requests. */
  567. function archive_worker_key(): string
  568. {
  569. $file = DATA_DIR . '/worker-key.json';
  570. $data = json_read($file);
  571. if (empty($data['key'])) {
  572. $data = ['key' => random_token(32)];
  573. json_write($file, $data);
  574. }
  575. return (string)$data['key'];
  576. }
  577. /** URL of worker.php on this installation. See self_url() in app/bootstrap.php. */
  578. function archive_worker_url(): ?string
  579. {
  580. return self_url('/worker.php?key=' . rawurlencode(archive_worker_key()));
  581. }
  582. /**
  583. * Fire a request at worker.php and hang up without waiting for it. The worker
  584. * sets ignore_user_abort(), so it runs its slice regardless.
  585. * The mechanics are shared with manage-worker.php: see self_dispatch().
  586. */
  587. function archive_dispatch(int $timeoutMs = 1000): bool
  588. {
  589. return self_dispatch(archive_worker_url(), $timeoutMs);
  590. }
  591. /**
  592. * Take the site-wide worker lock, or report that someone else holds it.
  593. * Returns the lock handle (which must stay open for the lock to hold) or null.
  594. *
  595. * $waitSeconds polls rather than blocking outright, so an admin asking for a
  596. * rebuild can wait out a background worker's short settle-sleep instead of
  597. * failing the moment it finds the lock taken — while a background worker, which
  598. * has nothing to wait for, passes 0 and steps aside immediately.
  599. *
  600. * @return resource|null
  601. */
  602. function archive_lock(int $waitSeconds = 0)
  603. {
  604. $fh = fopen(archive_lock_file(), 'c');
  605. if ($fh === false) {
  606. return null;
  607. }
  608. $deadline = time() + $waitSeconds;
  609. do {
  610. if (flock($fh, LOCK_EX | LOCK_NB)) {
  611. return $fh;
  612. }
  613. if (time() < $deadline) {
  614. sleep(1);
  615. }
  616. } while (time() < $deadline);
  617. fclose($fh);
  618. return null;
  619. }
  620. /**
  621. * Run one slice for whichever gallery is due, under the worker lock.
  622. * Returns the slice result, or null if nothing was due or another worker holds
  623. * the lock.
  624. */
  625. function archive_run_due(): ?array
  626. {
  627. $lock = archive_lock();
  628. if ($lock === null) {
  629. return null;
  630. }
  631. try {
  632. $slug = archive_next_due();
  633. return $slug === null ? null : archive_run_slice($slug);
  634. } finally {
  635. flock($lock, LOCK_UN);
  636. fclose($lock);
  637. }
  638. }
  639. /**
  640. * Called at the end of every public page render. Decides, as cheaply as
  641. * possible, whether any background work is pending and gets it moving.
  642. *
  643. * The page is flushed to the visitor before anything slow happens, so neither
  644. * the dispatch nor the inline fallback can delay it. The fallback matters on
  645. * hosts that cannot make an HTTP request to themselves: there, progress needs
  646. * one page view per slice instead of running on its own.
  647. */
  648. function archive_kick(): void
  649. {
  650. $tick = archive_tick_file();
  651. if (is_file($tick) && time() - (int)filemtime($tick) < 30) {
  652. return; // dispatched recently; do not spend anything on this request
  653. }
  654. $queue = json_read(archive_queue_file());
  655. if (empty($queue['galleries'])) {
  656. return;
  657. }
  658. @touch($tick);
  659. if (!function_exists('fastcgi_finish_request')) {
  660. // Cannot detach: keep the delay to the visitor as short as possible.
  661. archive_dispatch(200);
  662. return;
  663. }
  664. @fastcgi_finish_request();
  665. if (archive_dispatch(2000)) {
  666. return;
  667. }
  668. // No self-dispatch on this host. The visitor already has the page, so this
  669. // process can do the work itself.
  670. if (session_status() === PHP_SESSION_ACTIVE) {
  671. session_write_close();
  672. }
  673. archive_run_due();
  674. }