foto-portfolio/app/archive.php

<?php
/**
 * Gallery ZIP archives: building them, and keeping them up to date.
 *
 * Why it works this way
 * --------------------
 * A gallery can hold hundreds of 8 MB originals, and the host caps
 * max_execution_time at 60 s. Streaming a multi-gigabyte ZIP through PHP would
 * need a request that stays alive for the whole download, so instead the archive
 * is *built once into S3* and visitors are redirected to a presigned URL for it.
 * The download then never touches the webhost at all — it is a plain S3 GET,
 * resumable and immune to any server timeout.
 *
 * Building it is split into slices. archive_run_slice() copies as many photos as
 * fit in a time budget and returns; progress is committed to a state file after
 * every photo, so "build the archive" is just "run slices until finished". No
 * single request ever approaches the 60 s cap, and nothing depends on
 * set_time_limit() being allowed.
 *
 * Each photo is streamed S3 → buffer file → S3: the ZIP needs a local header
 * immediately before the file's bytes, and multipart parts are atomic, so the
 * bytes cannot be copied server-side with UploadPartCopy. The buffer file
 * accumulates until it passes the 5 MB multipart minimum, then becomes one part.
 *
 * Interruptions
 * -------------
 * State is written with json_write() (tmp file + rename), so it is never half
 * written — at worst it is one slice old and that slice replays. The rest is
 * handled by ordering:
 *
 *   - mid photo   → every slice starts by truncating the buffer back to the last
 *                   committed length, so a partial tail is dropped without
 *                   needing the failure to have been caught.
 *   - mid part    → the ETag is committed to state only after S3 accepts the
 *                   part, and the buffer is truncated only after that. A crash
 *                   anywhere in between re-uploads the same part number, which
 *                   S3 accepts until the upload is completed.
 *   - mid finish  → a retried CompleteMultipartUpload returns NoSuchUpload once
 *                   it has already succeeded; that is treated as done after a
 *                   HEAD confirms the object exists.
 *   - abandoned   → the queue entry survives on disk, and a build with no
 *                   progress for archive.abandon_hours is aborted and restarted.
 *
 * Staying current
 * ---------------
 * Every stored or deleted image marks its gallery dirty (archive_mark_dirty).
 * A dirty gallery's download button is disabled until the rebuild lands, rather
 * than handing out a ZIP that is missing the newest photos. Rebuilds run in the
 * background: archive_kick() on a page render dispatches worker.php, which runs
 * a slice and then dispatches its own successor, so one upload starts a chain
 * that finishes unattended. See docs/ARCHITECTURE.md.
 */

declare(strict_types=1);

/** Queue of galleries whose archive no longer matches their contents. */
function archive_queue_file(): string
{
    return DATA_DIR . '/archive-queue.json';
}

/** Site-wide lock: exactly one archive worker runs at a time. */
function archive_lock_file(): string
{
    return DATA_DIR . '/archive.lock';
}

/** Throttle marker so a busy site does not dispatch a worker per page view. */
function archive_tick_file(): string
{
    return DATA_DIR . '/archive.tick';
}

/**
 * Identity of a gallery's image set. A build records the hash it was made from;
 * when the gallery's current hash differs, the archive is out of date.
 */
function archive_source_hash(array $gallery): string
{
    return sha1(implode("\n", array_column($gallery['images'] ?? [], 'key')));
}

/** Whether a gallery's archive is missing or no longer matches its images. */
function archive_is_stale(array $gallery): bool
{
    $archive = $gallery['archive'] ?? null;
    if (empty($archive['key'])) {
        return true;
    }
    return ($archive['source_hash'] ?? '') !== archive_source_hash($gallery);
}

// ---------------------------------------------------------------------------
// The queue
// ---------------------------------------------------------------------------

/**
 * Flag a gallery for rebuilding. Called from gallery_append_image() and from
 * image deletion, i.e. everywhere a gallery's contents can change.
 *
 * Uploads run in parallel (uploads.concurrency), so the queue is written through
 * json_update()'s exclusive lock — three uploads finishing together must not
 * drop each other's entries. Galleries without downloads enabled are skipped:
 * queueing them would give the worker chain work it can never finish.
 */
function archive_mark_dirty(string $slug, ?array $gallery = null): void
{
    $gallery ??= gallery_load($slug);
    if ($gallery === null || empty($gallery['downloads_enabled'])) {
        return;
    }
    json_update(archive_queue_file(), function (array $queue) use ($slug): array {
        $queue['galleries'][$slug] = time();
        return $queue;
    });
}

/** Drop a gallery from the queue (rebuilt, deleted, or downloads turned off). */
function archive_unqueue(string $slug): void
{
    json_update(archive_queue_file(), function (array $queue) use ($slug): ?array {
        if (!isset($queue['galleries'][$slug])) {
            return null;   // nothing to do; leave the file untouched
        }
        unset($queue['galleries'][$slug]);
        return $queue;
    });
}

/**
 * The next gallery due for a rebuild, or null if none has settled yet.
 *
 * A gallery is only rebuilt once it has been quiet for archive.settle_seconds.
 * Thirty wedding guests uploading over an hour must not restart the build thirty
 * times; the delay batches an upload burst into a single rebuild.
 */
function archive_next_due(): ?string
{
    $queue = json_read(archive_queue_file());
    $settle = (int)config('archive.settle_seconds', 300);
    foreach ($queue['galleries'] ?? [] as $slug => $dirtyAt) {
        if (time() - (int)$dirtyAt >= $settle) {
            return (string)$slug;
        }
    }
    return null;
}

/**
 * Seconds until the earliest queued gallery settles: 0 if one is due now, null
 * if the queue is empty. The worker uses this to decide between doing a slice
 * and waiting out the settle window.
 */
function archive_queue_wait(): ?int
{
    $queue = json_read(archive_queue_file());
    $settle = (int)config('archive.settle_seconds', 300);
    $wait = null;
    foreach ($queue['galleries'] ?? [] as $dirtyAt) {
        $due = max(0, $settle - (time() - (int)$dirtyAt));
        $wait = $wait === null ? $due : min($wait, $due);
    }
    return $wait;
}

// ---------------------------------------------------------------------------
// Build state
// ---------------------------------------------------------------------------

/**
 * Start a fresh build: abandon anything in flight, open a new multipart upload,
 * and write the initial state. Returns the state, or null if it cannot start.
 *
 * Content-Type and Content-Disposition are set on the multipart create, so S3
 * stores them as the finished object's metadata and the presigned URL downloads
 * as a properly named .zip with no extra signing work.
 */
function archive_start(string $slug, array $gallery): ?array
{
    if (empty($gallery['images'])) {
        return null;   // nothing to archive
    }
    archive_abort($slug);

    $filename = safe_filename(($gallery['title'] ?: $slug) . '.zip', $slug);
    $key = s3_gallery_prefix($slug) . '/archive/' . random_token(6) . '-' . $filename;

    $uploadId = s3_mpu_create($key, 'application/zip', 'attachment; filename="' . $filename . '"');
    if ($uploadId === null) {
        return null;
    }

    $state = [
        'key'         => $key,
        'upload_id'   => $uploadId,
        'source_hash' => archive_source_hash($gallery),
        'total'       => count($gallery['images']),
        'next_index'  => 0,
        // Archive length so far, and how much of it S3 already has. The
        // difference is exactly what the buffer file holds.
        'offset'      => 0,
        'uploaded'    => 0,
        'part_number' => 1,
        'parts'       => [],
        'entries'     => [],
        'names'       => [],   // filename dedupe map, carried across slices
        'slowest'     => 5.0,  // seconds; grows to the slowest photo seen
        'started_at'  => time(),
    ];
    json_write(gallery_archive_file($slug), $state);
    @unlink(gallery_archive_buffer($slug));
    return $state;
}

/**
 * Abandon an in-flight build. Aborting the multipart upload matters: S3 keeps
 * (and bills for) the parts of an incomplete upload indefinitely.
 */
function archive_abort(string $slug): void
{
    $state = json_read(gallery_archive_file($slug));
    if (!empty($state['upload_id']) && !empty($state['key'])) {
        s3_mpu_abort((string)$state['key'], (string)$state['upload_id']);
    }
    @unlink(gallery_archive_file($slug));
    @unlink(gallery_archive_buffer($slug));
}

/** Delete a gallery's finished archive from S3 and forget it. */
function archive_delete(string $slug): void
{
    archive_abort($slug);
    $gallery = gallery_load($slug);
    if ($gallery !== null && !empty($gallery['archive']['key'])) {
        s3_delete((string)$gallery['archive']['key']);
    }
    json_update(gallery_file($slug), function (array $g): ?array {
        if ($g === [] || !isset($g['archive'])) {
            return null;
        }
        unset($g['archive']);
        return $g;
    });
    archive_unqueue($slug);
}

// ---------------------------------------------------------------------------
// The slice
// ---------------------------------------------------------------------------

/**
 * Copy as many of a gallery's photos into its archive as fit in $budget seconds.
 *
 * Returns ['done' => int, 'total' => int, 'finished' => bool, 'size' => ?int,
 *          'error' => ?string]. Call again to continue; all progress is on disk.
 */
function archive_run_slice(string $slug, ?float $budget = null): array
{
    $started = microtime(true);
    $budget ??= (float)config('archive.step_seconds', 25);

    $gallery = gallery_load($slug);
    if ($gallery === null) {
        archive_unqueue($slug);
        return archive_result(0, 0, false, null, 'Gallery no longer exists');
    }

    $stateFile = gallery_archive_file($slug);
    $state = json_read($stateFile);
    $abandoned = $state !== []
        && time() - (int)($state['started_at'] ?? 0) > (int)config('archive.abandon_hours', 24) * 3600;

    // Restart whenever the gallery has changed under an in-flight build, or the
    // build has been stalled long enough to be considered dead.
    if ($state === [] || $abandoned || ($state['source_hash'] ?? '') !== archive_source_hash($gallery)) {
        $state = archive_start($slug, $gallery);
        if ($state === null) {
            archive_unqueue($slug);
            return archive_result(0, 0, false, null, 'Cannot start archive (empty gallery or S3 refused)');
        }
    }

    $images = $gallery['images'];
    $bufferPath = gallery_archive_buffer($slug);
    $fh = fopen($bufferPath, 'c+b');
    if ($fh === false) {
        return archive_result((int)$state['next_index'], (int)$state['total'], false, null, 'Cannot open archive buffer');
    }
    flock($fh, LOCK_EX);

    // Recovery: drop anything written past the last committed position. Doing
    // this unconditionally means a hard kill needs no cleanup of its own.
    ftruncate($fh, (int)$state['offset'] - (int)$state['uploaded']);
    fseek($fh, 0, SEEK_END);

    $partMin = (int)config('archive.part_min_bytes', 5 * 1024 * 1024);
    $mtime = strtotime((string)($gallery['created_at'] ?? '')) ?: time();
    $error = null;
    $processed = 0;

    while ($state['next_index'] < $state['total']) {
        // Check the clock before starting a photo, never during one, and leave
        // room for one that runs as long as the slowest seen so far.
        //
        // Always do at least one photo, whatever the estimate says. A gallery
        // whose photos each take longer than the whole budget must still creep
        // forward one photo per slice; refusing to start would leave the build
        // stuck for ever, which is far worse than a slice that overruns.
        if ($processed > 0 && microtime(true) - $started + $state['slowest'] * 1.5 >= $budget) {
            break;
        }

        $photoStart = microtime(true);
        $image = $images[$state['next_index']];

        // Reserve the name in a copy: a photo that fails below is retried by the
        // next slice, and a name left registered by the failed attempt would
        // make the retry rename itself to "… (2)".
        $names = $state['names'];
        $name = zip_dedupe_name(
            safe_filename((string)($image['name'] ?? 'photo.jpg'), 'photo'),
            $names
        );

        $entryOffset = (int)$state['offset'];
        $header = zip_local_header($name, $mtime);
        fwrite($fh, $header);

        // One pass: bytes go to the buffer and through the CRC at the same time,
        // so an original never has to be held in memory or read back.
        $crcContext = hash_init('crc32b');
        [$status, $bytes] = s3_get_stream((string)$image['key'], function (string $chunk) use ($fh, $crcContext): void {
            hash_update($crcContext, $chunk);
            fwrite($fh, $chunk);
        });

        if ($status < 200 || $status >= 300) {
            // Undo this photo entirely and stop; the next slice retries it.
            ftruncate($fh, $entryOffset - (int)$state['uploaded']);
            fseek($fh, 0, SEEK_END);
            $error = 'S3 returned HTTP ' . $status . ' for ' . ($image['name'] ?? $image['key']);
            break;
        }

        // The real size and CRC are only known now, so patch them into the
        // header written above. Using the streamed byte count rather than the
        // recorded one also heals a gallery whose stored size was ever wrong.
        $crc = (int)hexdec(hash_final($crcContext));
        zip_patch_local_header($fh, $entryOffset - (int)$state['uploaded'], $name, $crc, $bytes);

        $state['names'] = $names;
        $state['entries'][] = [
            'name'   => $name,
            'crc'    => $crc,
            'size'   => $bytes,
            'offset' => $entryOffset,
            'mtime'  => $mtime,
        ];
        $state['offset'] = $entryOffset + strlen($header) + $bytes;
        $state['next_index']++;
        $state['slowest'] = max((float)$state['slowest'], microtime(true) - $photoStart);
        $processed++;

        if ((int)$state['offset'] - (int)$state['uploaded'] >= $partMin) {
            $error = archive_flush_part($state, $fh, $bufferPath, false);
            if ($error !== null) {
                break;
            }
        }
        json_write($stateFile, $state);
    }

    // Everything copied: append the central directory and close the upload.
    $finished = false;
    if ($error === null && $state['next_index'] >= $state['total']) {
        [$finished, $error] = archive_finish($slug, $state, $fh, $bufferPath);
    }

    json_write($stateFile, $state);
    flock($fh, LOCK_UN);
    fclose($fh);

    if ($finished) {
        @unlink($stateFile);
        @unlink($bufferPath);
    }

    return archive_result(
        (int)$state['next_index'],
        (int)$state['total'],
        $finished,
        $finished ? (int)$state['offset'] : null,
        $error
    );
}

/**
 * Send the buffered bytes to S3 as the next multipart part.
 *
 * The order here is what makes an interrupted build safe: S3 accepts the part,
 * then the ETag is committed to state, and only then is the buffer cleared. A
 * crash before the commit re-uploads the same part number with the same bytes,
 * which S3 allows until the upload is completed.
 *
 * Returns an error message, or null on success.
 *
 * @param resource $fh
 */
function archive_flush_part(array &$state, $fh, string $bufferPath, bool $isLast): ?string
{
    fflush($fh);
    if (!$isLast && (int)$state['offset'] - (int)$state['uploaded'] === 0) {
        return null;   // nothing pending
    }

    $etag = s3_mpu_upload_part(
        (string)$state['key'],
        (string)$state['upload_id'],
        (int)$state['part_number'],
        $bufferPath
    );
    if ($etag === null) {
        return 'S3 rejected part ' . $state['part_number'];
    }

    $state['parts'][] = ['n' => (int)$state['part_number'], 'etag' => $etag];
    $state['part_number']++;
    $state['uploaded'] = (int)$state['offset'];

    ftruncate($fh, 0);
    fseek($fh, 0, SEEK_END);
    return null;
}

/**
 * Write the central directory, upload the last part and complete the multipart
 * upload, then record the archive on the gallery.
 *
 * Returns [finished, error].
 *
 * @param resource $fh
 */
function archive_finish(string $slug, array &$state, $fh, string $bufferPath): array
{
    // The central directory is built now, from the real sizes and offsets, so
    // it uses ZIP64 fields only where a value actually overflows 32 bits.
    $directory = '';
    foreach ($state['entries'] as $entry) {
        $directory .= zip_central_entry($entry);
    }
    $trailer = $directory . zip_end_of_central_directory(
        count($state['entries']),
        strlen($directory),
        (int)$state['offset']
    );
    fwrite($fh, $trailer);
    $state['offset'] = (int)$state['offset'] + strlen($trailer);

    // The 5 MB minimum does not apply to the final part, which is what lets a
    // gallery of small files work with the same buffering scheme.
    $error = archive_flush_part($state, $fh, $bufferPath, true);
    if ($error !== null) {
        return [false, $error];
    }

    [$ok, $body] = s3_mpu_complete((string)$state['key'], (string)$state['upload_id'], $state['parts']);
    if (!$ok) {
        // A completed upload no longer exists; if the object is there, an
        // earlier attempt succeeded and only the state write was lost.
        if (str_contains($body, 'NoSuchUpload') && s3_head((string)$state['key']) !== null) {
            $ok = true;
        }
    }
    if (!$ok) {
        return [false, 'S3 could not complete the archive upload'];
    }

    $summary = [
        'key'         => (string)$state['key'],
        'size'        => (int)$state['offset'],
        'count'       => count($state['entries']),
        'built_at'    => date('Y-m-d H:i:s'),
        'source_hash' => (string)$state['source_hash'],
    ];

    // json_update, not gallery_save: an upload finishing right now must not be
    // overwritten by a gallery this function read minutes ago.
    $previousKey = null;
    json_update(gallery_file($slug), function (array $g) use ($summary, &$previousKey): ?array {
        if ($g === []) {
            return null;   // deleted mid-build
        }
        $previousKey = $g['archive']['key'] ?? null;
        $g['archive'] = $summary;
        return $g;
    });

    if ($previousKey !== null && $previousKey !== $summary['key']) {
        s3_delete((string)$previousKey);
    }

    // Leave the gallery queued if it changed while this build was running — the
    // archive just written is already out of date and needs another pass.
    $current = gallery_load($slug);
    if ($current === null || !archive_is_stale($current)) {
        archive_unqueue($slug);
    }

    return [true, null];
}

/** Uniform slice/step result shape, shared by the worker and the admin API. */
function archive_result(int $done, int $total, bool $finished, ?int $size, ?string $error): array
{
    return [
        'done'     => $done,
        'total'    => $total,
        'finished' => $finished,
        'size'     => $size,
        'error'    => $error,
    ];
}

/**
 * What the admin page shows for a gallery: whether an archive exists, whether it
 * is current, and how far any in-flight build has got.
 */
function archive_status(array $gallery): array
{
    $slug = (string)$gallery['slug'];
    $archive = $gallery['archive'] ?? null;
    $state = json_read(gallery_archive_file($slug));
    $queue = json_read(archive_queue_file());

    return [
        'archive'  => $archive,
        'stale'    => archive_is_stale($gallery),
        'building' => $state !== [],
        'done'     => (int)($state['next_index'] ?? 0),
        'total'    => (int)($state['total'] ?? count($gallery['images'] ?? [])),
        'queued'   => isset($queue['galleries'][$slug]),
        'due_in'   => isset($queue['galleries'][$slug])
            ? max(0, (int)config('archive.settle_seconds', 300) - (time() - (int)$queue['galleries'][$slug]))
            : null,
    ];
}

// ---------------------------------------------------------------------------
// Background execution
// ---------------------------------------------------------------------------

/** Shared secret authenticating the self-dispatched worker requests. */
function archive_worker_key(): string
{
    $file = DATA_DIR . '/worker-key.json';
    $data = json_read($file);
    if (empty($data['key'])) {
        $data = ['key' => random_token(32)];
        json_write($file, $data);
    }
    return (string)$data['key'];
}

/**
 * URL of worker.php on this installation.
 *
 * site.base_url is preferred when it has been filled in, because it does not
 * depend on the request's Host header. Otherwise the URL is derived from the
 * current request, mapping APP_ROOT against DOCUMENT_ROOT so an app installed in
 * a subdirectory still resolves.
 */
function archive_worker_url(): ?string
{
    $query = '/worker.php?key=' . rawurlencode(archive_worker_key());

    $configured = rtrim((string)config('site.base_url', ''), '/');
    if ($configured !== '' && !str_contains($configured, 'example.com')) {
        return $configured . $query;
    }

    $host = (string)($_SERVER['HTTP_HOST'] ?? '');
    $root = rtrim(str_replace('\\', '/', (string)($_SERVER['DOCUMENT_ROOT'] ?? '')), '/');
    $app  = str_replace('\\', '/', APP_ROOT);
    if ($host === '' || $root === '' || !str_starts_with($app, $root)) {
        return null;
    }
    $scheme = (!empty($_SERVER['HTTPS']) && $_SERVER['HTTPS'] !== 'off') ? 'https' : 'http';
    return $scheme . '://' . $host . rtrim(substr($app, strlen($root)), '/') . $query;
}

/**
 * Fire a request at worker.php and hang up without waiting for it. The worker
 * sets ignore_user_abort(), so it runs its slice regardless.
 *
 * A timeout is the expected, successful outcome here — it means the request was
 * delivered and the worker is busy with it. A *completed* response is only a
 * success if it is the worker's own 204; anything else (a 404 from a wrong key
 * or a misconfigured base URL) means nothing is running, and saying so lets the
 * caller fall back to doing the work inline instead of silently stalling.
 */
function archive_dispatch(int $timeoutMs = 1000): bool
{
    $url = archive_worker_url();
    if ($url === null || !function_exists('curl_init')) {
        return false;
    }
    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_RETURNTRANSFER    => true,
        CURLOPT_NOSIGNAL          => true,
        CURLOPT_CONNECTTIMEOUT_MS => $timeoutMs,
        CURLOPT_TIMEOUT_MS        => $timeoutMs,
    ]);
    curl_exec($ch);
    $errno = curl_errno($ch);
    $status = (int)curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
    return $errno === CURLE_OPERATION_TIMEOUTED || ($errno === 0 && $status === 204);
}

/**
 * Take the site-wide worker lock, or report that someone else holds it.
 * Returns the lock handle (which must stay open for the lock to hold) or null.
 *
 * $waitSeconds polls rather than blocking outright, so an admin asking for a
 * rebuild can wait out a background worker's short settle-sleep instead of
 * failing the moment it finds the lock taken — while a background worker, which
 * has nothing to wait for, passes 0 and steps aside immediately.
 *
 * @return resource|null
 */
function archive_lock(int $waitSeconds = 0)
{
    $fh = fopen(archive_lock_file(), 'c');
    if ($fh === false) {
        return null;
    }
    $deadline = time() + $waitSeconds;
    do {
        if (flock($fh, LOCK_EX | LOCK_NB)) {
            return $fh;
        }
        if (time() < $deadline) {
            sleep(1);
        }
    } while (time() < $deadline);

    fclose($fh);
    return null;
}

/**
 * Run one slice for whichever gallery is due, under the worker lock.
 * Returns the slice result, or null if nothing was due or another worker holds
 * the lock.
 */
function archive_run_due(): ?array
{
    $lock = archive_lock();
    if ($lock === null) {
        return null;
    }
    try {
        $slug = archive_next_due();
        return $slug === null ? null : archive_run_slice($slug);
    } finally {
        flock($lock, LOCK_UN);
        fclose($lock);
    }
}

/**
 * Called at the end of every public page render. Decides, as cheaply as
 * possible, whether any background work is pending and gets it moving.
 *
 * The page is flushed to the visitor before anything slow happens, so neither
 * the dispatch nor the inline fallback can delay it. The fallback matters on
 * hosts that cannot make an HTTP request to themselves: there, progress needs
 * one page view per slice instead of running on its own.
 */
function archive_kick(): void
{
    $tick = archive_tick_file();
    if (is_file($tick) && time() - (int)filemtime($tick) < 30) {
        return;   // dispatched recently; do not spend anything on this request
    }

    $queue = json_read(archive_queue_file());
    if (empty($queue['galleries'])) {
        return;
    }
    @touch($tick);

    if (!function_exists('fastcgi_finish_request')) {
        // Cannot detach: keep the delay to the visitor as short as possible.
        archive_dispatch(200);
        return;
    }

    @fastcgi_finish_request();
    if (archive_dispatch(2000)) {
        return;
    }
    // No self-dispatch on this host. The visitor already has the page, so this
    // process can do the work itself.
    if (session_status() === PHP_SESSION_ACTIVE) {
        session_write_close();
    }
    archive_run_due();
}