adding parallel uploads

admin/api.php

@@ -25,6 +25,12 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
 }
 csrf_verify();
 
+// Release the session file before the slow S3 leg. PHP holds an exclusive lock
+// on it for the whole request, so without this every parallel upload from the
+// browser would queue behind the previous one and the uploader would be
+// serial again no matter how many requests it starts.
+session_write_close();
+
 // One image per request; a single file may still be large, so lift the time cap.
 @set_time_limit(0);
 

admin/gallery-edit.php

@@ -95,9 +95,10 @@ flash_render();
          data-slug="<?= e($slug) ?>"
          data-csrf="<?= e(csrf_token()) ?>"
          data-thumb-size="<?= (int)config('uploads.thumb_size', 600) ?>"
-         data-thumb-quality="<?= e((string)config('uploads.thumb_quality', 0.8)) ?>">
+         data-thumb-quality="<?= e((string)config('uploads.thumb_quality', 0.8)) ?>"
+         data-concurrency="<?= (int)config('uploads.concurrency', 3) ?>">
         Drop images here or click to select.<br>
-        <small>Uploaded through the site to S3, in full resolution, unmodified. One file at a time.</small>
+        <small>Uploaded through the site to S3, in full resolution, unmodified.</small>
     </div>
     <input type="file" id="file-input" accept="image/*" multiple style="display:none">
     <div class="upload-list" id="upload-list"></div>

app/s3.php

@@ -176,14 +176,74 @@ function s3_presign_get(string $key, ?int $ttl = null): string
     return s3_presign('GET', $key, $ttl);
 }
 
+/**
+ * One curl handle per PHP process, reused across requests to the same endpoint.
+ * curl_reset() clears the options but keeps the handle's live connection, DNS
+ * and TLS-session caches, so the second PUT of a request (the thumbnail) and
+ * any retry skip a full TCP + TLS handshake.
+ */
+function s3_curl(): CurlHandle
+{
+    static $ch = null;
+    if ($ch === null) {
+        $ch = curl_init();
+    } else {
+        curl_reset($ch);
+    }
+    return $ch;
+}
+
+/**
+ * Whether an S3 attempt failed in a way that is worth repeating: a curl-level
+ * failure (status 0), throttling, or a server-side error. 4xx is a real
+ * rejection (bad key, bad signature) and must not be retried.
+ */
+function s3_is_transient(int $status): bool
+{
+    return $status === 0 || $status === 408 || $status === 429 || $status >= 500;
+}
+
 /**
  * Stream a local file to S3 with a signed PUT (header auth). The payload is sent
  * as UNSIGNED-PAYLOAD so the body is never hashed or buffered into memory — curl
  * streams it straight from the file handle, letting the webhost proxy originals
  * far larger than memory_limit. Content-Type is sent but not signed.
- * Returns [httpStatus, responseBody].
+ *
+ * Transient failures are retried up to $attempts times with a short backoff; the
+ * file handle is rewound and the request re-signed for each try, so a dropped
+ * connection costs one repeat instead of a failed image.
+ * Returns [httpStatus, responseBody] of the last attempt.
+ */
+function s3_put_file(string $key, string $filePath, string $contentType = 'application/octet-stream', int $attempts = 3): array
+{
+    // Suppressed: a warning printed here would land in front of the JSON body
+    // the API endpoints emit, and the failure is reported through the return.
+    $fh = @fopen($filePath, 'rb');
+    if ($fh === false) {
+        return [0, 'Cannot open upload for reading'];
+    }
+    $size = (int)filesize($filePath);
+
+    $status = 0;
+    $body = '';
+    for ($try = 1; $try <= $attempts; $try++) {
+        rewind($fh);
+        [$status, $body] = s3_put_stream($key, $fh, $size, $contentType);
+        if (!s3_is_transient($status) || $try === $attempts) {
+            break;
+        }
+        usleep(250000 * $try); // 0.25s, then 0.5s
+    }
+    fclose($fh);
+    return [$status, $body];
+}
+
+/**
+ * One signed PUT attempt streaming from an open, positioned file handle.
+ *
+ * @param resource $fh
  */
-function s3_put_file(string $key, string $filePath, string $contentType = 'application/octet-stream'): array
+function s3_put_stream(string $key, $fh, int $size, string $contentType): array
 {
     $host = s3_host();
     $amzDate = gmdate('Ymd\THis\Z');
@@ -216,18 +276,21 @@ function s3_put_file(string $key, string $filePath, string $contentType = 'appli
         . ', SignedHeaders=host;x-amz-content-sha256;x-amz-date'
         . ', Signature=' . $signature;
 
-    $fh = fopen($filePath, 'rb');
-    if ($fh === false) {
-        return [0, 'Cannot open upload for reading'];
-    }
-    $ch = curl_init(s3_base_url() . $canonicalUri);
+    $ch = s3_curl();
     curl_setopt_array($ch, [
+        CURLOPT_URL            => s3_base_url() . $canonicalUri,
         CURLOPT_UPLOAD         => true,   // sets method to PUT and streams CURLOPT_INFILE
         CURLOPT_INFILE         => $fh,
-        CURLOPT_INFILESIZE     => filesize($filePath),
+        CURLOPT_INFILESIZE     => $size,
         CURLOPT_RETURNTRANSFER => true,
         CURLOPT_CONNECTTIMEOUT => 30,
         CURLOPT_TIMEOUT        => 0,       // no cap: originals can be large
+        // Abort a connection that has stalled below 1 KB/s for two minutes,
+        // instead of pinning a PHP worker on a dead socket until the web
+        // server kills it. A retry then gets a fresh connection.
+        CURLOPT_LOW_SPEED_LIMIT => 1024,
+        CURLOPT_LOW_SPEED_TIME  => 120,
+        CURLOPT_TCP_NODELAY    => true,
         CURLOPT_HTTPHEADER     => [
             'Authorization: ' . $authorization,
             'x-amz-content-sha256: ' . $payloadHash,
@@ -238,7 +301,6 @@ function s3_put_file(string $key, string $filePath, string $contentType = 'appli
     ]);
     $body = curl_exec($ch);
     $status = (int)curl_getinfo($ch, CURLINFO_RESPONSE_CODE);
-    fclose($fh);
     return [$status, (string)$body];
 }
 
@@ -279,8 +341,9 @@ function s3_request(string $method, string $key): array
         . ', SignedHeaders=host;x-amz-content-sha256;x-amz-date'
         . ', Signature=' . $signature;
 
-    $ch = curl_init(s3_base_url() . $canonicalUri);
+    $ch = s3_curl();
     curl_setopt_array($ch, [
+        CURLOPT_URL            => s3_base_url() . $canonicalUri,
         CURLOPT_CUSTOMREQUEST  => strtoupper($method),
         CURLOPT_RETURNTRANSFER => true,
         CURLOPT_TIMEOUT        => 30,
@@ -332,9 +395,11 @@ function upload_error_message(int $code): string
  * browser-generated thumbnail) to S3, then append it to the gallery's JSON file.
  *
  * Shared by admin/api.php (trusted admin) and upload-api.php (public guest link).
- * The gallery is re-loaded under a fresh read before appending to reduce lost
- * updates between concurrent uploads. Object keys are generated server-side
- * under the gallery's own prefix — never taken from the client.
+ * The browser uploads several images at once, so the gallery entry is appended
+ * through gallery_append_image(), which re-reads and rewrites the JSON file
+ * under an exclusive lock — two uploads finishing together cannot drop one
+ * another's entry. Object keys are generated server-side under the gallery's
+ * own prefix — never taken from the client.
  *
  * $original / $thumb are $_FILES entries (or null). When $imagesOnly is true the
  * original must have a recognised image extension and decode via getimagesize(),
@@ -389,15 +454,23 @@ function gallery_store_s3_upload(array $gallery, ?array $original, ?array $thumb
         }
     }
 
-    // Append under a fresh load to reduce lost updates between concurrent uploads.
-    $gallery = gallery_load($slug);
-    $gallery['images'][] = [
+    // Locked read-modify-write: concurrent uploads append without clobbering.
+    $count = gallery_append_image($slug, [
         'key'   => $key,
         'thumb' => $thumbKey,
         'name'  => substr((string)($original['name'] ?? basename($key)), 0, 200),
         'size'  => (int)($original['size'] ?? 0),
-    ];
-    gallery_save($gallery);
+    ]);
+
+    // The gallery was deleted while this image was in flight: drop the objects
+    // we just wrote rather than leaving them unreferenced in the bucket.
+    if ($count === null) {
+        s3_delete($key);
+        if ($thumbKey !== null) {
+            s3_delete($thumbKey);
+        }
+        return [404, ['error' => 'Gallery no longer exists']];
+    }
 
-    return [200, ['ok' => true, 'key' => $key, 'thumb' => $thumbKey, 'count' => count($gallery['images'])]];
+    return [200, ['ok' => true, 'key' => $key, 'thumb' => $thumbKey, 'count' => $count]];
 }

app/storage.php

@@ -41,6 +41,43 @@ function json_write(string $file, array $data): void
     }
 }
 
+/**
+ * Read-modify-write a JSON file with an exclusive lock held across the whole
+ * cycle, so concurrent writers cannot lose each other's changes.
+ *
+ * json_write() replaces the file by rename(), so the target inode changes on
+ * every write and cannot itself carry the lock — a sidecar "<file>.lock" does.
+ * $mutate receives the current contents and returns the array to store, or
+ * null to leave the file untouched. Returns the current (or stored) array.
+ */
+function json_update(string $file, callable $mutate, array $default = []): array
+{
+    $dir = dirname($file);
+    if (!is_dir($dir)) {
+        mkdir($dir, 0755, true);
+    }
+    // Cannot lock (read-only dir, exotic host): still perform the update rather
+    // than dropping it — degrades to the previous last-writer-wins behaviour.
+    $lock = fopen($file . '.lock', 'c');
+    if ($lock !== false) {
+        flock($lock, LOCK_EX);
+    }
+    try {
+        $current = json_read($file, $default);
+        $data = $mutate($current);
+        if ($data === null) {
+            return $current;
+        }
+        json_write($file, $data);
+        return $data;
+    } finally {
+        if ($lock !== false) {
+            flock($lock, LOCK_UN);
+            fclose($lock);
+        }
+    }
+}
+
 /** URL-safe random token. */
 function random_token(int $chars = 8): string
 {
@@ -202,6 +239,28 @@ function gallery_delete(string $slug): void
     if (is_file($file)) {
         unlink($file);
     }
+    @unlink($file . '.lock');
+}
+
+/**
+ * Append one image to a gallery under an exclusive lock, so parallel uploads
+ * into the same gallery cannot overwrite each other's entries.
+ *
+ * Returns the new image count, or null if the gallery no longer exists — an
+ * absent gallery must not be resurrected as a stub by a late upload.
+ */
+function gallery_append_image(string $slug, array $image): ?int
+{
+    $missing = false;
+    $gallery = json_update(gallery_file($slug), function (array $g) use ($image, &$missing) {
+        if ($g === []) {
+            $missing = true;
+            return null; // deleted mid-upload — do not write a stub file back
+        }
+        $g['images'][] = $image;
+        return $g;
+    });
+    return $missing ? null : count($gallery['images'] ?? []);
 }
 
 /** All galleries, newest first. */

assets/admin.js

@@ -9,6 +9,13 @@
  * One file per request keeps each PHP process small, so the size of the whole
  * gallery upload never matters — only the largest single image. The browser
  * never sees an S3 URL or credential for writing.
+ *
+ * Several files are in flight at once (data-concurrency, default 3). Each file
+ * is store-and-forward — the webhost receives the whole body before it starts
+ * the S3 PUT — so a single-file queue leaves the uplink idle for the entire
+ * webhost→S3 leg and for every thumbnail decode. Overlapping requests keeps it
+ * saturated; the server appends to the gallery JSON under a lock, so parallel
+ * completions cannot lose entries.
  */
 (function () {
     'use strict';
@@ -27,9 +34,15 @@
     var uploadKey = zone.dataset.key || '';
     var thumbSize = parseInt(zone.dataset.thumbSize, 10) || 600;
     var thumbQuality = parseFloat(zone.dataset.thumbQuality) || 0.8;
+    // How many uploads may be in flight at once. Small on purpose: each one
+    // occupies a PHP worker on the webhost for its whole S3 round trip.
+    var maxParallel = Math.max(1, parseInt(zone.dataset.concurrency, 10) || 3);
+    // Transient failures (network drop, 5xx, throttling) are retried
+    // automatically before the row is marked failed.
+    var maxAttempts = 3;
 
     var queue = [];
-    var busy = false;
+    var active = 0;
 
     zone.addEventListener('click', function () { input.click(); });
     input.addEventListener('change', function () { enqueue(input.files); input.value = ''; });
@@ -43,7 +56,7 @@
     zone.addEventListener('drop', function (e) { enqueue(e.dataTransfer.files); });
 
     window.addEventListener('beforeunload', function (e) {
-        if (busy || queue.length) { e.preventDefault(); e.returnValue = ''; }
+        if (active || queue.length) { e.preventDefault(); e.returnValue = ''; }
     });
 
     function enqueue(files) {
@@ -58,10 +71,15 @@
         pump();
     }
 
+    /* Start jobs until the parallel slots are full; called again as each ends. */
     function pump() {
-        if (busy || !queue.length) return;
-        busy = true;
-        var job = queue.shift();
+        while (active < maxParallel && queue.length) {
+            run(queue.shift());
+        }
+    }
+
+    function run(job) {
+        active++;
         uploadOne(job.file, job.row)
             .then(function () { setState(job.row, 'done', 'done'); bumpCount(); })
             .catch(function (err) {
@@ -79,7 +97,7 @@
                 });
                 job.row.appendChild(retry);
             })
-            .finally(function () { busy = false; pump(); });
+            .finally(function () { active--; pump(); });
     }
 
     function setState(row, text, cls) {
@@ -92,6 +110,15 @@
         if (countEl) countEl.textContent = String(parseInt(countEl.textContent, 10) + 1);
     }
 
+    /* An error worth repeating: the request never landed, or the server said it
+       was a temporary condition. A 4xx is a real rejection (bad CSRF token,
+       wrong file type, gone gallery) and must not be retried. */
+    function failure(message, status) {
+        var err = new Error(message);
+        err.transient = status === 0 || status === 408 || status === 429 || status >= 500;
+        return err;
+    }
+
     /* POST multipart to the webhost with upload progress (fetch has none → XHR). */
     function sendForm(form, onProgress) {
         return new Promise(function (resolve, reject) {
@@ -106,18 +133,35 @@
                 var data = {};
                 try { data = JSON.parse(xhr.responseText); } catch (err) {}
                 if (xhr.status >= 200 && xhr.status < 300 && data.ok) resolve(data);
-                else reject(new Error((data && data.error) || ('Upload failed (' + xhr.status + ')')));
+                else reject(failure((data && data.error) || ('Upload failed (' + xhr.status + ')'), xhr.status));
             });
-            xhr.addEventListener('error', function () { reject(new Error('Network error during upload')); });
+            xhr.addEventListener('error', function () { reject(failure('Network error during upload', 0)); });
             xhr.send(form);
         });
     }
 
+    function delay(ms) {
+        return new Promise(function (resolve) { setTimeout(resolve, ms); });
+    }
+
     /* Thumbnail as JPEG blob; null when the browser cannot decode the file
-       (e.g. RAW) — the original still uploads untouched. */
+       (e.g. RAW) — the original still uploads untouched.
+
+       createImageBitmap gets a resize hint so it can downsample while decoding
+       (a JPEG decoder scales by DCT factors) instead of materialising a
+       full-resolution bitmap — much faster and far less memory on 40MP files.
+       Only the width is given, so the aspect ratio is preserved; the canvas
+       step below still fixes the exact long edge. The hint is skipped for small
+       files, where it could upscale before we downscale again, and browsers
+       that ignore resizeWidth simply return the full-size bitmap. */
     function makeThumb(file) {
+        var options = { imageOrientation: 'from-image' };
+        if (file.size > 2 * 1024 * 1024) {
+            options.resizeWidth = thumbSize;
+            options.resizeQuality = 'high';
+        }
         var decode = window.createImageBitmap
-            ? createImageBitmap(file, { imageOrientation: 'from-image' })
+            ? createImageBitmap(file, options)
             : new Promise(function (resolve, reject) {
                 var img = new Image();
                 img.onload = function () { resolve(img); };
@@ -144,15 +188,27 @@
         setState(row, 'thumbnail');
 
         return makeThumb(file).then(function (thumbBlob) {
-            setState(row, 'uploading');
+            // The same FormData is re-sent on retry: it reads from the File on
+            // disk each time, so nothing is buffered between attempts.
             var form = new FormData();
             form.append('slug', slug);
             if (uploadKey) form.append('key', uploadKey);
             form.append('original', file, file.name);
             if (thumbBlob) form.append('thumb', thumbBlob, 'thumb.jpg');
-            return sendForm(form, function (f) {
-                bar.style.width = Math.round(f * 100) + '%';
-            });
+
+            function attempt(n) {
+                setState(row, n === 1 ? 'uploading' : 'retrying ' + n + '/' + maxAttempts);
+                bar.style.width = '0%';
+                return sendForm(form, function (f) {
+                    bar.style.width = Math.round(f * 100) + '%';
+                }).catch(function (err) {
+                    if (!err.transient || n >= maxAttempts) throw err;
+                    // Back off so a briefly overloaded host is not hammered by
+                    // every parallel slot at once.
+                    return delay(1000 * n).then(function () { return attempt(n + 1); });
+                });
+            }
+            return attempt(1);
         });
     }
 })();

config/config.sample.php

@@ -44,5 +44,9 @@ return [
         'thumb_size'    => 600,
         // JPEG quality for thumbnails (0.0 - 1.0, used by the browser canvas).
         'thumb_quality' => 0.8,
+        // How many images the browser uploads at once. Each one holds a PHP
+        // worker on the webhost for its whole S3 round trip, so keep this
+        // below the host's per-site process limit. 1 restores serial uploads.
+        'concurrency'   => 3,
     ],
 ];

docs/ARCHITECTURE.md

@@ -98,14 +98,39 @@ admin.js                         api.php                    Hetzner S3
    │ ◀──────────────────────── { ok, key, thumb, count }
 ```
 
-Files are uploaded **one request per file**, sequentially, with progress;
-failures get a per-file retry. The thumbnail is drawn client-side
-(`createImageBitmap` + `imageOrientation: 'from-image'` for EXIF rotation) and
-sent alongside the original; undecodable files (RAW, video) upload without a
-thumbnail and the grid falls back to the original key. A random 6-char token per
-file prevents same-filename collisions. Object keys are laid out as
-`<prefix>/<slug>/{originals,thumbs}/…`, where `<prefix>` comes from `s3.prefix`
-(default `galleries`, `''` = bucket root).
+Files are uploaded **one request per file**, with `uploads.concurrency` (default
+3) files in flight at once and per-file progress. The thumbnail is drawn
+client-side (`createImageBitmap` + `imageOrientation: 'from-image'` for EXIF
+rotation, with a `resizeWidth` hint so large JPEGs downsample during decode
+instead of being decoded at full resolution) and sent alongside the original;
+undecodable files (RAW, video) upload without a thumbnail and the grid falls
+back to the original key. A random 6-char token per file prevents same-filename
+collisions. Object keys are laid out as `<prefix>/<slug>/{originals,thumbs}/…`,
+where `<prefix>` comes from `s3.prefix` (default `galleries`, `''` = bucket
+root).
+
+**Why parallel.** Each request is store-and-forward: PHP buffers the whole body
+to a temp file before `s3_put_file()` starts, so during the webhost→S3 leg (and
+during every thumbnail decode) the browser's uplink sits idle. Overlapping a few
+requests keeps it saturated. Three things make that safe rather than merely
+faster:
+
+- Both endpoints call `session_write_close()` right after authenticating. PHP
+  holds an exclusive lock on the session file for the whole request, so without
+  it every parallel upload would queue behind the previous one and the uploader
+  would be serial again regardless of how many requests it starts.
+- The gallery entry is appended via `gallery_append_image()` →`json_update()`,
+  which holds `flock(LOCK_EX)` on a sidecar `<file>.lock` across the whole
+  read-modify-write. (The lock cannot live on the JSON file itself: `json_write()`
+  replaces it by `rename()`, so the inode changes on every write.) Unlocked,
+  eight simultaneous appends lose about five of them.
+- Transient failures are retried on both sides — up to 3 attempts with backoff
+  in `s3_put_file()` (re-signed and rewound per attempt) and in `admin.js` for
+  network errors, 408, 429 and 5xx. 4xx is a real rejection and is never
+  retried. The manual per-file *retry* link remains for permanent failures.
+
+Uploads reuse one curl handle per PHP process (`s3_curl()`), so the thumbnail
+PUT and any retry skip a fresh TCP + TLS handshake.
 
 Proxying uploads through the webhost keeps them same-origin (no bucket CORS) and
 means no S3 write credential ever reaches the browser. The one-file-per-request
@@ -141,8 +166,8 @@ irrelevant — only the **largest single image** must fit within the host's
 
 - One admin account, one shared session store — fine for a single
   photographer, not a multi-user CMS.
-- Gallery JSON writes are last-writer-wins; the uploader registers files
-  sequentially, so this only matters if two admin tabs edit the same gallery
-  simultaneously.
+- Gallery JSON writes are last-writer-wins, *except* image appends during
+  upload, which take an exclusive lock (`json_update()`). So this only matters
+  if two admin tabs edit the same gallery's settings simultaneously.
 - Presigned URLs mean gallery pages must be re-rendered after `s3.url_ttl`;
   a visitor who keeps a tab open >1 h reloads to see images again.

docs/SETUP.md

@@ -36,6 +36,18 @@ post_max_size       = 200M
 it at the web-server level (e.g. Apache/FPM request timeout), raise that too for
 large files.
 
+The browser uploads several images at once (`uploads.concurrency`, default 3),
+which keeps the uplink busy while the webhost forwards earlier files to S3. Each
+one occupies a PHP worker for its whole S3 round trip, so on shared hosting with
+a tight per-site process limit, lower it:
+
+```php
+'concurrency' => 2,   // or 1 to restore strictly serial uploads
+```
+
+If uploads start failing with 503s under load, that limit is the first thing to
+check.
+
 ## 3. Configuration
 
 ```bash

upload-api.php

@@ -20,6 +20,13 @@ if ($_SERVER['REQUEST_METHOD'] !== 'POST') {
 }
 csrf_verify();
 
+// Take the session-held facts now, then release the session file before the slow
+// S3 leg. PHP locks it exclusively for the whole request, so without this every
+// parallel upload from the browser would queue behind the previous one and the
+// uploader would be serial again no matter how many requests it starts.
+$unlockedGalleries = (array)($_SESSION['gallery_unlocked'] ?? []);
+session_write_close();
+
 // One image per request; a single file may still be large, so lift the time cap.
 @set_time_limit(0);
 
@@ -37,7 +44,7 @@ $authorized = $gallery !== null
     && !empty($gallery['upload_key'])
     && hash_equals((string)$gallery['upload_key'], (string)($_POST['key'] ?? ''))
     && !gallery_is_expired($gallery)
-    && (empty($gallery['password_hash']) || !empty($_SESSION['gallery_unlocked'][$gallery['slug']]));
+    && (empty($gallery['password_hash']) || !empty($unlockedGalleries[$gallery['slug']]));
 
 if (!$authorized) {
     json_response(['error' => 'Not authorized'], 403);

upload.php

@@ -71,9 +71,10 @@ public_header(e($gallery['title']));
          data-csrf="<?= e(csrf_token()) ?>"
          data-key="<?= e((string)$gallery['upload_key']) ?>"
          data-thumb-size="<?= (int)config('uploads.thumb_size', 600) ?>"
-         data-thumb-quality="<?= e((string)config('uploads.thumb_quality', 0.8)) ?>">
+         data-thumb-quality="<?= e((string)config('uploads.thumb_quality', 0.8)) ?>"
+         data-concurrency="<?= (int)config('uploads.concurrency', 3) ?>">
         Drop images here or click to select.<br>
-        <small>Full resolution, unmodified. One file at a time.</small>
+        <small>Full resolution, unmodified.</small>
     </div>
     <input type="file" id="file-input" accept="image/*" multiple style="display:none">
     <div class="upload-list" id="upload-list"></div>