|
|
@@ -0,0 +1,705 @@
|
|
|
+<?php
|
|
|
+/**
|
|
|
+ * Gallery ZIP archives: building them, and keeping them up to date.
|
|
|
+ *
|
|
|
+ * Why it works this way
|
|
|
+ * --------------------
|
|
|
+ * A gallery can hold hundreds of 8 MB originals, and the host caps
|
|
|
+ * max_execution_time at 60 s. Streaming a multi-gigabyte ZIP through PHP would
|
|
|
+ * need a request that stays alive for the whole download, so instead the archive
|
|
|
+ * is *built once into S3* and visitors are redirected to a presigned URL for it.
|
|
|
+ * The download then never touches the webhost at all — it is a plain S3 GET,
|
|
|
+ * resumable and immune to any server timeout.
|
|
|
+ *
|
|
|
+ * Building it is split into slices. archive_run_slice() copies as many photos as
|
|
|
+ * fit in a time budget and returns; progress is committed to a state file after
|
|
|
+ * every photo, so "build the archive" is just "run slices until finished". No
|
|
|
+ * single request ever approaches the 60 s cap, and nothing depends on
|
|
|
+ * set_time_limit() being allowed.
|
|
|
+ *
|
|
|
+ * Each photo is streamed S3 → buffer file → S3: the ZIP needs a local header
|
|
|
+ * immediately before the file's bytes, and multipart parts are atomic, so the
|
|
|
+ * bytes cannot be copied server-side with UploadPartCopy. The buffer file
|
|
|
+ * accumulates until it passes the 5 MB multipart minimum, then becomes one part.
|
|
|
+ *
|
|
|
+ * Interruptions
|
|
|
+ * -------------
|
|
|
+ * State is written with json_write() (tmp file + rename), so it is never half
|
|
|
+ * written — at worst it is one slice old and that slice replays. The rest is
|
|
|
+ * handled by ordering:
|
|
|
+ *
|
|
|
+ * - mid photo → every slice starts by truncating the buffer back to the last
|
|
|
+ * committed length, so a partial tail is dropped without
|
|
|
+ * needing the failure to have been caught.
|
|
|
+ * - mid part → the ETag is committed to state only after S3 accepts the
|
|
|
+ * part, and the buffer is truncated only after that. A crash
|
|
|
+ * anywhere in between re-uploads the same part number, which
|
|
|
+ * S3 accepts until the upload is completed.
|
|
|
+ * - mid finish → a retried CompleteMultipartUpload returns NoSuchUpload once
|
|
|
+ * it has already succeeded; that is treated as done after a
|
|
|
+ * HEAD confirms the object exists.
|
|
|
+ * - abandoned → the queue entry survives on disk, and a build with no
|
|
|
+ * progress for archive.abandon_hours is aborted and restarted.
|
|
|
+ *
|
|
|
+ * Staying current
|
|
|
+ * ---------------
|
|
|
+ * Every stored or deleted image marks its gallery dirty (archive_mark_dirty).
|
|
|
+ * A dirty gallery's download button is disabled until the rebuild lands, rather
|
|
|
+ * than handing out a ZIP that is missing the newest photos. Rebuilds run in the
|
|
|
+ * background: archive_kick() on a page render dispatches worker.php, which runs
|
|
|
+ * a slice and then dispatches its own successor, so one upload starts a chain
|
|
|
+ * that finishes unattended. See docs/ARCHITECTURE.md.
|
|
|
+ */
|
|
|
+
|
|
|
+declare(strict_types=1);
|
|
|
+
|
|
|
+/** Queue of galleries whose archive no longer matches their contents. */
|
|
|
+function archive_queue_file(): string
|
|
|
+{
|
|
|
+ return DATA_DIR . '/archive-queue.json';
|
|
|
+}
|
|
|
+
|
|
|
+/** Site-wide lock: exactly one archive worker runs at a time. */
|
|
|
+function archive_lock_file(): string
|
|
|
+{
|
|
|
+ return DATA_DIR . '/archive.lock';
|
|
|
+}
|
|
|
+
|
|
|
+/** Throttle marker so a busy site does not dispatch a worker per page view. */
|
|
|
+function archive_tick_file(): string
|
|
|
+{
|
|
|
+ return DATA_DIR . '/archive.tick';
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Identity of a gallery's image set. A build records the hash it was made from;
|
|
|
+ * when the gallery's current hash differs, the archive is out of date.
|
|
|
+ */
|
|
|
+function archive_source_hash(array $gallery): string
|
|
|
+{
|
|
|
+ return sha1(implode("\n", array_column($gallery['images'] ?? [], 'key')));
|
|
|
+}
|
|
|
+
|
|
|
+/** Whether a gallery's archive is missing or no longer matches its images. */
|
|
|
+function archive_is_stale(array $gallery): bool
|
|
|
+{
|
|
|
+ $archive = $gallery['archive'] ?? null;
|
|
|
+ if (empty($archive['key'])) {
|
|
|
+ return true;
|
|
|
+ }
|
|
|
+ return ($archive['source_hash'] ?? '') !== archive_source_hash($gallery);
|
|
|
+}
|
|
|
+
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+// The queue
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+
|
|
|
+/**
|
|
|
+ * Flag a gallery for rebuilding. Called from gallery_append_image() and from
|
|
|
+ * image deletion, i.e. everywhere a gallery's contents can change.
|
|
|
+ *
|
|
|
+ * Uploads run in parallel (uploads.concurrency), so the queue is written through
|
|
|
+ * json_update()'s exclusive lock — three uploads finishing together must not
|
|
|
+ * drop each other's entries. Galleries without downloads enabled are skipped:
|
|
|
+ * queueing them would give the worker chain work it can never finish.
|
|
|
+ */
|
|
|
+function archive_mark_dirty(string $slug, ?array $gallery = null): void
|
|
|
+{
|
|
|
+ $gallery ??= gallery_load($slug);
|
|
|
+ if ($gallery === null || empty($gallery['downloads_enabled'])) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ json_update(archive_queue_file(), function (array $queue) use ($slug): array {
|
|
|
+ $queue['galleries'][$slug] = time();
|
|
|
+ return $queue;
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+/** Drop a gallery from the queue (rebuilt, deleted, or downloads turned off). */
|
|
|
+function archive_unqueue(string $slug): void
|
|
|
+{
|
|
|
+ json_update(archive_queue_file(), function (array $queue) use ($slug): ?array {
|
|
|
+ if (!isset($queue['galleries'][$slug])) {
|
|
|
+ return null; // nothing to do; leave the file untouched
|
|
|
+ }
|
|
|
+ unset($queue['galleries'][$slug]);
|
|
|
+ return $queue;
|
|
|
+ });
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * The next gallery due for a rebuild, or null if none has settled yet.
|
|
|
+ *
|
|
|
+ * A gallery is only rebuilt once it has been quiet for archive.settle_seconds.
|
|
|
+ * Thirty wedding guests uploading over an hour must not restart the build thirty
|
|
|
+ * times; the delay batches an upload burst into a single rebuild.
|
|
|
+ */
|
|
|
+function archive_next_due(): ?string
|
|
|
+{
|
|
|
+ $queue = json_read(archive_queue_file());
|
|
|
+ $settle = (int)config('archive.settle_seconds', 300);
|
|
|
+ foreach ($queue['galleries'] ?? [] as $slug => $dirtyAt) {
|
|
|
+ if (time() - (int)$dirtyAt >= $settle) {
|
|
|
+ return (string)$slug;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ return null;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Seconds until the earliest queued gallery settles: 0 if one is due now, null
|
|
|
+ * if the queue is empty. The worker uses this to decide between doing a slice
|
|
|
+ * and waiting out the settle window.
|
|
|
+ */
|
|
|
+function archive_queue_wait(): ?int
|
|
|
+{
|
|
|
+ $queue = json_read(archive_queue_file());
|
|
|
+ $settle = (int)config('archive.settle_seconds', 300);
|
|
|
+ $wait = null;
|
|
|
+ foreach ($queue['galleries'] ?? [] as $dirtyAt) {
|
|
|
+ $due = max(0, $settle - (time() - (int)$dirtyAt));
|
|
|
+ $wait = $wait === null ? $due : min($wait, $due);
|
|
|
+ }
|
|
|
+ return $wait;
|
|
|
+}
|
|
|
+
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+// Build state
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+
|
|
|
+/**
|
|
|
+ * Start a fresh build: abandon anything in flight, open a new multipart upload,
|
|
|
+ * and write the initial state. Returns the state, or null if it cannot start.
|
|
|
+ *
|
|
|
+ * Content-Type and Content-Disposition are set on the multipart create, so S3
|
|
|
+ * stores them as the finished object's metadata and the presigned URL downloads
|
|
|
+ * as a properly named .zip with no extra signing work.
|
|
|
+ */
|
|
|
+function archive_start(string $slug, array $gallery): ?array
|
|
|
+{
|
|
|
+ if (empty($gallery['images'])) {
|
|
|
+ return null; // nothing to archive
|
|
|
+ }
|
|
|
+ archive_abort($slug);
|
|
|
+
|
|
|
+ $filename = safe_filename(($gallery['title'] ?: $slug) . '.zip', $slug);
|
|
|
+ $key = s3_gallery_prefix($slug) . '/archive/' . random_token(6) . '-' . $filename;
|
|
|
+
|
|
|
+ $uploadId = s3_mpu_create($key, 'application/zip', 'attachment; filename="' . $filename . '"');
|
|
|
+ if ($uploadId === null) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+
|
|
|
+ $state = [
|
|
|
+ 'key' => $key,
|
|
|
+ 'upload_id' => $uploadId,
|
|
|
+ 'source_hash' => archive_source_hash($gallery),
|
|
|
+ 'total' => count($gallery['images']),
|
|
|
+ 'next_index' => 0,
|
|
|
+ // Archive length so far, and how much of it S3 already has. The
|
|
|
+ // difference is exactly what the buffer file holds.
|
|
|
+ 'offset' => 0,
|
|
|
+ 'uploaded' => 0,
|
|
|
+ 'part_number' => 1,
|
|
|
+ 'parts' => [],
|
|
|
+ 'entries' => [],
|
|
|
+ 'names' => [], // filename dedupe map, carried across slices
|
|
|
+ 'slowest' => 5.0, // seconds; grows to the slowest photo seen
|
|
|
+ 'started_at' => time(),
|
|
|
+ ];
|
|
|
+ json_write(gallery_archive_file($slug), $state);
|
|
|
+ @unlink(gallery_archive_buffer($slug));
|
|
|
+ return $state;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Abandon an in-flight build. Aborting the multipart upload matters: S3 keeps
|
|
|
+ * (and bills for) the parts of an incomplete upload indefinitely.
|
|
|
+ */
|
|
|
+function archive_abort(string $slug): void
|
|
|
+{
|
|
|
+ $state = json_read(gallery_archive_file($slug));
|
|
|
+ if (!empty($state['upload_id']) && !empty($state['key'])) {
|
|
|
+ s3_mpu_abort((string)$state['key'], (string)$state['upload_id']);
|
|
|
+ }
|
|
|
+ @unlink(gallery_archive_file($slug));
|
|
|
+ @unlink(gallery_archive_buffer($slug));
|
|
|
+}
|
|
|
+
|
|
|
+/** Delete a gallery's finished archive from S3 and forget it. */
|
|
|
+function archive_delete(string $slug): void
|
|
|
+{
|
|
|
+ archive_abort($slug);
|
|
|
+ $gallery = gallery_load($slug);
|
|
|
+ if ($gallery !== null && !empty($gallery['archive']['key'])) {
|
|
|
+ s3_delete((string)$gallery['archive']['key']);
|
|
|
+ }
|
|
|
+ json_update(gallery_file($slug), function (array $g): ?array {
|
|
|
+ if ($g === [] || !isset($g['archive'])) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ unset($g['archive']);
|
|
|
+ return $g;
|
|
|
+ });
|
|
|
+ archive_unqueue($slug);
|
|
|
+}
|
|
|
+
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+// The slice
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+
|
|
|
+/**
|
|
|
+ * Copy as many of a gallery's photos into its archive as fit in $budget seconds.
|
|
|
+ *
|
|
|
+ * Returns ['done' => int, 'total' => int, 'finished' => bool, 'size' => ?int,
|
|
|
+ * 'error' => ?string]. Call again to continue; all progress is on disk.
|
|
|
+ */
|
|
|
+function archive_run_slice(string $slug, ?float $budget = null): array
|
|
|
+{
|
|
|
+ $started = microtime(true);
|
|
|
+ $budget ??= (float)config('archive.step_seconds', 25);
|
|
|
+
|
|
|
+ $gallery = gallery_load($slug);
|
|
|
+ if ($gallery === null) {
|
|
|
+ archive_unqueue($slug);
|
|
|
+ return archive_result(0, 0, false, null, 'Gallery no longer exists');
|
|
|
+ }
|
|
|
+
|
|
|
+ $stateFile = gallery_archive_file($slug);
|
|
|
+ $state = json_read($stateFile);
|
|
|
+ $abandoned = $state !== []
|
|
|
+ && time() - (int)($state['started_at'] ?? 0) > (int)config('archive.abandon_hours', 24) * 3600;
|
|
|
+
|
|
|
+ // Restart whenever the gallery has changed under an in-flight build, or the
|
|
|
+ // build has been stalled long enough to be considered dead.
|
|
|
+ if ($state === [] || $abandoned || ($state['source_hash'] ?? '') !== archive_source_hash($gallery)) {
|
|
|
+ $state = archive_start($slug, $gallery);
|
|
|
+ if ($state === null) {
|
|
|
+ archive_unqueue($slug);
|
|
|
+ return archive_result(0, 0, false, null, 'Cannot start archive (empty gallery or S3 refused)');
|
|
|
+ }
|
|
|
+ }
|
|
|
+
|
|
|
+ $images = $gallery['images'];
|
|
|
+ $bufferPath = gallery_archive_buffer($slug);
|
|
|
+ $fh = fopen($bufferPath, 'c+b');
|
|
|
+ if ($fh === false) {
|
|
|
+ return archive_result((int)$state['next_index'], (int)$state['total'], false, null, 'Cannot open archive buffer');
|
|
|
+ }
|
|
|
+ flock($fh, LOCK_EX);
|
|
|
+
|
|
|
+ // Recovery: drop anything written past the last committed position. Doing
|
|
|
+ // this unconditionally means a hard kill needs no cleanup of its own.
|
|
|
+ ftruncate($fh, (int)$state['offset'] - (int)$state['uploaded']);
|
|
|
+ fseek($fh, 0, SEEK_END);
|
|
|
+
|
|
|
+ $partMin = (int)config('archive.part_min_bytes', 5 * 1024 * 1024);
|
|
|
+ $mtime = strtotime((string)($gallery['created_at'] ?? '')) ?: time();
|
|
|
+ $error = null;
|
|
|
+ $processed = 0;
|
|
|
+
|
|
|
+ while ($state['next_index'] < $state['total']) {
|
|
|
+ // Check the clock before starting a photo, never during one, and leave
|
|
|
+ // room for one that runs as long as the slowest seen so far.
|
|
|
+ //
|
|
|
+ // Always do at least one photo, whatever the estimate says. A gallery
|
|
|
+ // whose photos each take longer than the whole budget must still creep
|
|
|
+ // forward one photo per slice; refusing to start would leave the build
|
|
|
+ // stuck for ever, which is far worse than a slice that overruns.
|
|
|
+ if ($processed > 0 && microtime(true) - $started + $state['slowest'] * 1.5 >= $budget) {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ $photoStart = microtime(true);
|
|
|
+ $image = $images[$state['next_index']];
|
|
|
+
|
|
|
+ // Reserve the name in a copy: a photo that fails below is retried by the
|
|
|
+ // next slice, and a name left registered by the failed attempt would
|
|
|
+ // make the retry rename itself to "… (2)".
|
|
|
+ $names = $state['names'];
|
|
|
+ $name = zip_dedupe_name(
|
|
|
+ safe_filename((string)($image['name'] ?? 'photo.jpg'), 'photo'),
|
|
|
+ $names
|
|
|
+ );
|
|
|
+
|
|
|
+ $entryOffset = (int)$state['offset'];
|
|
|
+ $header = zip_local_header($name, $mtime);
|
|
|
+ fwrite($fh, $header);
|
|
|
+
|
|
|
+ // One pass: bytes go to the buffer and through the CRC at the same time,
|
|
|
+ // so an original never has to be held in memory or read back.
|
|
|
+ $crcContext = hash_init('crc32b');
|
|
|
+ [$status, $bytes] = s3_get_stream((string)$image['key'], function (string $chunk) use ($fh, $crcContext): void {
|
|
|
+ hash_update($crcContext, $chunk);
|
|
|
+ fwrite($fh, $chunk);
|
|
|
+ });
|
|
|
+
|
|
|
+ if ($status < 200 || $status >= 300) {
|
|
|
+ // Undo this photo entirely and stop; the next slice retries it.
|
|
|
+ ftruncate($fh, $entryOffset - (int)$state['uploaded']);
|
|
|
+ fseek($fh, 0, SEEK_END);
|
|
|
+ $error = 'S3 returned HTTP ' . $status . ' for ' . ($image['name'] ?? $image['key']);
|
|
|
+ break;
|
|
|
+ }
|
|
|
+
|
|
|
+ // The real size and CRC are only known now, so patch them into the
|
|
|
+ // header written above. Using the streamed byte count rather than the
|
|
|
+ // recorded one also heals a gallery whose stored size was ever wrong.
|
|
|
+ $crc = (int)hexdec(hash_final($crcContext));
|
|
|
+ zip_patch_local_header($fh, $entryOffset - (int)$state['uploaded'], $name, $crc, $bytes);
|
|
|
+
|
|
|
+ $state['names'] = $names;
|
|
|
+ $state['entries'][] = [
|
|
|
+ 'name' => $name,
|
|
|
+ 'crc' => $crc,
|
|
|
+ 'size' => $bytes,
|
|
|
+ 'offset' => $entryOffset,
|
|
|
+ 'mtime' => $mtime,
|
|
|
+ ];
|
|
|
+ $state['offset'] = $entryOffset + strlen($header) + $bytes;
|
|
|
+ $state['next_index']++;
|
|
|
+ $state['slowest'] = max((float)$state['slowest'], microtime(true) - $photoStart);
|
|
|
+ $processed++;
|
|
|
+
|
|
|
+ if ((int)$state['offset'] - (int)$state['uploaded'] >= $partMin) {
|
|
|
+ $error = archive_flush_part($state, $fh, $bufferPath, false);
|
|
|
+ if ($error !== null) {
|
|
|
+ break;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ json_write($stateFile, $state);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Everything copied: append the central directory and close the upload.
|
|
|
+ $finished = false;
|
|
|
+ if ($error === null && $state['next_index'] >= $state['total']) {
|
|
|
+ [$finished, $error] = archive_finish($slug, $state, $fh, $bufferPath);
|
|
|
+ }
|
|
|
+
|
|
|
+ json_write($stateFile, $state);
|
|
|
+ flock($fh, LOCK_UN);
|
|
|
+ fclose($fh);
|
|
|
+
|
|
|
+ if ($finished) {
|
|
|
+ @unlink($stateFile);
|
|
|
+ @unlink($bufferPath);
|
|
|
+ }
|
|
|
+
|
|
|
+ return archive_result(
|
|
|
+ (int)$state['next_index'],
|
|
|
+ (int)$state['total'],
|
|
|
+ $finished,
|
|
|
+ $finished ? (int)$state['offset'] : null,
|
|
|
+ $error
|
|
|
+ );
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Send the buffered bytes to S3 as the next multipart part.
|
|
|
+ *
|
|
|
+ * The order here is what makes an interrupted build safe: S3 accepts the part,
|
|
|
+ * then the ETag is committed to state, and only then is the buffer cleared. A
|
|
|
+ * crash before the commit re-uploads the same part number with the same bytes,
|
|
|
+ * which S3 allows until the upload is completed.
|
|
|
+ *
|
|
|
+ * Returns an error message, or null on success.
|
|
|
+ *
|
|
|
+ * @param resource $fh
|
|
|
+ */
|
|
|
+function archive_flush_part(array &$state, $fh, string $bufferPath, bool $isLast): ?string
|
|
|
+{
|
|
|
+ fflush($fh);
|
|
|
+ if (!$isLast && (int)$state['offset'] - (int)$state['uploaded'] === 0) {
|
|
|
+ return null; // nothing pending
|
|
|
+ }
|
|
|
+
|
|
|
+ $etag = s3_mpu_upload_part(
|
|
|
+ (string)$state['key'],
|
|
|
+ (string)$state['upload_id'],
|
|
|
+ (int)$state['part_number'],
|
|
|
+ $bufferPath
|
|
|
+ );
|
|
|
+ if ($etag === null) {
|
|
|
+ return 'S3 rejected part ' . $state['part_number'];
|
|
|
+ }
|
|
|
+
|
|
|
+ $state['parts'][] = ['n' => (int)$state['part_number'], 'etag' => $etag];
|
|
|
+ $state['part_number']++;
|
|
|
+ $state['uploaded'] = (int)$state['offset'];
|
|
|
+
|
|
|
+ ftruncate($fh, 0);
|
|
|
+ fseek($fh, 0, SEEK_END);
|
|
|
+ return null;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Write the central directory, upload the last part and complete the multipart
|
|
|
+ * upload, then record the archive on the gallery.
|
|
|
+ *
|
|
|
+ * Returns [finished, error].
|
|
|
+ *
|
|
|
+ * @param resource $fh
|
|
|
+ */
|
|
|
+function archive_finish(string $slug, array &$state, $fh, string $bufferPath): array
|
|
|
+{
|
|
|
+ // The central directory is built now, from the real sizes and offsets, so
|
|
|
+ // it uses ZIP64 fields only where a value actually overflows 32 bits.
|
|
|
+ $directory = '';
|
|
|
+ foreach ($state['entries'] as $entry) {
|
|
|
+ $directory .= zip_central_entry($entry);
|
|
|
+ }
|
|
|
+ $trailer = $directory . zip_end_of_central_directory(
|
|
|
+ count($state['entries']),
|
|
|
+ strlen($directory),
|
|
|
+ (int)$state['offset']
|
|
|
+ );
|
|
|
+ fwrite($fh, $trailer);
|
|
|
+ $state['offset'] = (int)$state['offset'] + strlen($trailer);
|
|
|
+
|
|
|
+ // The 5 MB minimum does not apply to the final part, which is what lets a
|
|
|
+ // gallery of small files work with the same buffering scheme.
|
|
|
+ $error = archive_flush_part($state, $fh, $bufferPath, true);
|
|
|
+ if ($error !== null) {
|
|
|
+ return [false, $error];
|
|
|
+ }
|
|
|
+
|
|
|
+ [$ok, $body] = s3_mpu_complete((string)$state['key'], (string)$state['upload_id'], $state['parts']);
|
|
|
+ if (!$ok) {
|
|
|
+ // A completed upload no longer exists; if the object is there, an
|
|
|
+ // earlier attempt succeeded and only the state write was lost.
|
|
|
+ if (str_contains($body, 'NoSuchUpload') && s3_head((string)$state['key']) !== null) {
|
|
|
+ $ok = true;
|
|
|
+ }
|
|
|
+ }
|
|
|
+ if (!$ok) {
|
|
|
+ return [false, 'S3 could not complete the archive upload'];
|
|
|
+ }
|
|
|
+
|
|
|
+ $summary = [
|
|
|
+ 'key' => (string)$state['key'],
|
|
|
+ 'size' => (int)$state['offset'],
|
|
|
+ 'count' => count($state['entries']),
|
|
|
+ 'built_at' => date('Y-m-d H:i:s'),
|
|
|
+ 'source_hash' => (string)$state['source_hash'],
|
|
|
+ ];
|
|
|
+
|
|
|
+ // json_update, not gallery_save: an upload finishing right now must not be
|
|
|
+ // overwritten by a gallery this function read minutes ago.
|
|
|
+ $previousKey = null;
|
|
|
+ json_update(gallery_file($slug), function (array $g) use ($summary, &$previousKey): ?array {
|
|
|
+ if ($g === []) {
|
|
|
+ return null; // deleted mid-build
|
|
|
+ }
|
|
|
+ $previousKey = $g['archive']['key'] ?? null;
|
|
|
+ $g['archive'] = $summary;
|
|
|
+ return $g;
|
|
|
+ });
|
|
|
+
|
|
|
+ if ($previousKey !== null && $previousKey !== $summary['key']) {
|
|
|
+ s3_delete((string)$previousKey);
|
|
|
+ }
|
|
|
+
|
|
|
+ // Leave the gallery queued if it changed while this build was running — the
|
|
|
+ // archive just written is already out of date and needs another pass.
|
|
|
+ $current = gallery_load($slug);
|
|
|
+ if ($current === null || !archive_is_stale($current)) {
|
|
|
+ archive_unqueue($slug);
|
|
|
+ }
|
|
|
+
|
|
|
+ return [true, null];
|
|
|
+}
|
|
|
+
|
|
|
+/** Uniform slice/step result shape, shared by the worker and the admin API. */
|
|
|
+function archive_result(int $done, int $total, bool $finished, ?int $size, ?string $error): array
|
|
|
+{
|
|
|
+ return [
|
|
|
+ 'done' => $done,
|
|
|
+ 'total' => $total,
|
|
|
+ 'finished' => $finished,
|
|
|
+ 'size' => $size,
|
|
|
+ 'error' => $error,
|
|
|
+ ];
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * What the admin page shows for a gallery: whether an archive exists, whether it
|
|
|
+ * is current, and how far any in-flight build has got.
|
|
|
+ */
|
|
|
+function archive_status(array $gallery): array
|
|
|
+{
|
|
|
+ $slug = (string)$gallery['slug'];
|
|
|
+ $archive = $gallery['archive'] ?? null;
|
|
|
+ $state = json_read(gallery_archive_file($slug));
|
|
|
+ $queue = json_read(archive_queue_file());
|
|
|
+
|
|
|
+ return [
|
|
|
+ 'archive' => $archive,
|
|
|
+ 'stale' => archive_is_stale($gallery),
|
|
|
+ 'building' => $state !== [],
|
|
|
+ 'done' => (int)($state['next_index'] ?? 0),
|
|
|
+ 'total' => (int)($state['total'] ?? count($gallery['images'] ?? [])),
|
|
|
+ 'queued' => isset($queue['galleries'][$slug]),
|
|
|
+ 'due_in' => isset($queue['galleries'][$slug])
|
|
|
+ ? max(0, (int)config('archive.settle_seconds', 300) - (time() - (int)$queue['galleries'][$slug]))
|
|
|
+ : null,
|
|
|
+ ];
|
|
|
+}
|
|
|
+
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+// Background execution
|
|
|
+// ---------------------------------------------------------------------------
|
|
|
+
|
|
|
+/** Shared secret authenticating the self-dispatched worker requests. */
|
|
|
+function archive_worker_key(): string
|
|
|
+{
|
|
|
+ $file = DATA_DIR . '/worker-key.json';
|
|
|
+ $data = json_read($file);
|
|
|
+ if (empty($data['key'])) {
|
|
|
+ $data = ['key' => random_token(32)];
|
|
|
+ json_write($file, $data);
|
|
|
+ }
|
|
|
+ return (string)$data['key'];
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * URL of worker.php on this installation.
|
|
|
+ *
|
|
|
+ * site.base_url is preferred when it has been filled in, because it does not
|
|
|
+ * depend on the request's Host header. Otherwise the URL is derived from the
|
|
|
+ * current request, mapping APP_ROOT against DOCUMENT_ROOT so an app installed in
|
|
|
+ * a subdirectory still resolves.
|
|
|
+ */
|
|
|
+function archive_worker_url(): ?string
|
|
|
+{
|
|
|
+ $query = '/worker.php?key=' . rawurlencode(archive_worker_key());
|
|
|
+
|
|
|
+ $configured = rtrim((string)config('site.base_url', ''), '/');
|
|
|
+ if ($configured !== '' && !str_contains($configured, 'example.com')) {
|
|
|
+ return $configured . $query;
|
|
|
+ }
|
|
|
+
|
|
|
+ $host = (string)($_SERVER['HTTP_HOST'] ?? '');
|
|
|
+ $root = rtrim(str_replace('\\', '/', (string)($_SERVER['DOCUMENT_ROOT'] ?? '')), '/');
|
|
|
+ $app = str_replace('\\', '/', APP_ROOT);
|
|
|
+ if ($host === '' || $root === '' || !str_starts_with($app, $root)) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
|
|
|
+ return $scheme . '://' . $host . rtrim(substr($app, strlen($root)), '/') . $query;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Fire a request at worker.php and hang up without waiting for it. The worker
|
|
|
+ * sets ignore_user_abort(), so it runs its slice regardless.
|
|
|
+ *
|
|
|
+ * A timeout is the expected, successful outcome here — it means the request was
|
|
|
+ * delivered and the worker is busy with it.
|
|
|
+ */
|
|
|
+function archive_dispatch(int $timeoutMs = 1000): bool
|
|
|
+{
|
|
|
+ $url = archive_worker_url();
|
|
|
+ if ($url === null || !function_exists('curl_init')) {
|
|
|
+ return false;
|
|
|
+ }
|
|
|
+ $ch = curl_init($url);
|
|
|
+ curl_setopt_array($ch, [
|
|
|
+ CURLOPT_RETURNTRANSFER => true,
|
|
|
+ CURLOPT_NOSIGNAL => true,
|
|
|
+ CURLOPT_CONNECTTIMEOUT_MS => $timeoutMs,
|
|
|
+ CURLOPT_TIMEOUT_MS => $timeoutMs,
|
|
|
+ ]);
|
|
|
+ curl_exec($ch);
|
|
|
+ $errno = curl_errno($ch);
|
|
|
+ return $errno === 0 || $errno === CURLE_OPERATION_TIMEOUTED;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Take the site-wide worker lock, or report that someone else holds it.
|
|
|
+ * Returns the lock handle (which must stay open for the lock to hold) or null.
|
|
|
+ *
|
|
|
+ * $waitSeconds polls rather than blocking outright, so an admin asking for a
|
|
|
+ * rebuild can wait out a background worker's short settle-sleep instead of
|
|
|
+ * failing the moment it finds the lock taken — while a background worker, which
|
|
|
+ * has nothing to wait for, passes 0 and steps aside immediately.
|
|
|
+ *
|
|
|
+ * @return resource|null
|
|
|
+ */
|
|
|
+function archive_lock(int $waitSeconds = 0)
|
|
|
+{
|
|
|
+ $fh = fopen(archive_lock_file(), 'c');
|
|
|
+ if ($fh === false) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ $deadline = time() + $waitSeconds;
|
|
|
+ do {
|
|
|
+ if (flock($fh, LOCK_EX | LOCK_NB)) {
|
|
|
+ return $fh;
|
|
|
+ }
|
|
|
+ if (time() < $deadline) {
|
|
|
+ sleep(1);
|
|
|
+ }
|
|
|
+ } while (time() < $deadline);
|
|
|
+
|
|
|
+ fclose($fh);
|
|
|
+ return null;
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Run one slice for whichever gallery is due, under the worker lock.
|
|
|
+ * Returns the slice result, or null if nothing was due or another worker holds
|
|
|
+ * the lock.
|
|
|
+ */
|
|
|
+function archive_run_due(): ?array
|
|
|
+{
|
|
|
+ $lock = archive_lock();
|
|
|
+ if ($lock === null) {
|
|
|
+ return null;
|
|
|
+ }
|
|
|
+ try {
|
|
|
+ $slug = archive_next_due();
|
|
|
+ return $slug === null ? null : archive_run_slice($slug);
|
|
|
+ } finally {
|
|
|
+ flock($lock, LOCK_UN);
|
|
|
+ fclose($lock);
|
|
|
+ }
|
|
|
+}
|
|
|
+
|
|
|
+/**
|
|
|
+ * Called at the end of every public page render. Decides, as cheaply as
|
|
|
+ * possible, whether any background work is pending and gets it moving.
|
|
|
+ *
|
|
|
+ * The page is flushed to the visitor before anything slow happens, so neither
|
|
|
+ * the dispatch nor the inline fallback can delay it. The fallback matters on
|
|
|
+ * hosts that cannot make an HTTP request to themselves: there, progress needs
|
|
|
+ * one page view per slice instead of running on its own.
|
|
|
+ */
|
|
|
+function archive_kick(): void
|
|
|
+{
|
|
|
+ $tick = archive_tick_file();
|
|
|
+ if (is_file($tick) && time() - (int)filemtime($tick) < 30) {
|
|
|
+ return; // dispatched recently; do not spend anything on this request
|
|
|
+ }
|
|
|
+
|
|
|
+ $queue = json_read(archive_queue_file());
|
|
|
+ if (empty($queue['galleries'])) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ @touch($tick);
|
|
|
+
|
|
|
+ if (!function_exists('fastcgi_finish_request')) {
|
|
|
+ // Cannot detach: keep the delay to the visitor as short as possible.
|
|
|
+ archive_dispatch(200);
|
|
|
+ return;
|
|
|
+ }
|
|
|
+
|
|
|
+ @fastcgi_finish_request();
|
|
|
+ if (archive_dispatch(2000)) {
|
|
|
+ return;
|
|
|
+ }
|
|
|
+ // No self-dispatch on this host. The visitor already has the page, so this
|
|
|
+ // process can do the work itself.
|
|
|
+ if (session_status() === PHP_SESSION_ACTIVE) {
|
|
|
+ session_write_close();
|
|
|
+ }
|
|
|
+ archive_run_due();
|
|
|
+}
|
