diff --git a/app/Devnull/Client.php b/app/Devnull/Client.php index 19313d0c..753a2e4c 100644 --- a/app/Devnull/Client.php +++ b/app/Devnull/Client.php @@ -410,6 +410,7 @@ private function get(string $path, array $query = []): array { $json = $this->json($this->send('GET', $path, $query)); $this->cache->put($path, $query, $json); + ImageCache::prefetch($json); return $json; } diff --git a/app/Devnull/DownloadImages.php b/app/Devnull/DownloadImages.php new file mode 100644 index 00000000..b213d024 --- /dev/null +++ b/app/Devnull/DownloadImages.php @@ -0,0 +1,70 @@ + $urls + */ + public function handle(array $urls): int + { + File::ensureDirectoryExists(ImageCache::directory()); + self::prune(); + + $saved = 0; + + foreach (array_chunk($urls, self::AtOnce) as $batch) { + $responses = Http::pool(fn (Pool $pool): array => array_map( + fn (string $url) => $pool->timeout(15)->get($url), + $batch, + )); + + foreach ($batch as $index => $url) { + $response = $responses[$index] ?? null; + + if ($response instanceof Response && $response->successful() && str_starts_with((string) $response->header('Content-Type'), 'image/')) { + $path = ImageCache::path($url); + File::put($path.'.part', $response->body()); + File::move($path.'.part', $path); + $saved++; + } + } + } + + return $saved; + } + + /** + * Deletes images older than the cache keeps them, and any half-written ones. + */ + private static function prune(): void + { + $oldest = time() - ImageCache::hours() * 3600; + + foreach (File::files(ImageCache::directory()) as $file) { + try { + if (str_ends_with($file->getFilename(), '.part') || $file->getMTime() <= $oldest) { + File::delete($file->getPathname()); + } + } catch (Throwable) { + // Another download may have just replaced it. + } + } + } +} diff --git a/app/Devnull/ImageCache.php b/app/Devnull/ImageCache.php new file mode 100644 index 00000000..a364913c --- /dev/null +++ b/app/Devnull/ImageCache.php @@ -0,0 +1,119 @@ + $json + */ + public static function prefetch(array $json): void + { + if (self::hours() <= 0) { + return; + } + + $missing = array_values(array_filter( + array_unique(self::urlsIn($json)), + fn (string $url): bool => self::freshPath($url) === null, + )); + + if ($missing !== []) { + DownloadImages::dispatch($missing)->timeout(120); + } + } + + /** + * Every image URL in an answer, wherever it's nested: people's avatars, + * commits' images and link previews' images. + * + * @param array $json + * @return list + */ + public static function urlsIn(array $json): array + { + $urls = []; + + foreach ($json as $key => $value) { + if (in_array($key, ['avatar_url', 'image_url'], true) && self::isRemote($value)) { + $urls[] = $value; + } elseif ($key === 'images' && is_array($value)) { + foreach ($value as $image) { + if (is_array($image) && self::isRemote($image['url'] ?? null)) { + $urls[] = $image['url']; + } + } + } elseif (is_array($value)) { + $urls = [...$urls, ...self::urlsIn($value)]; + } + } + + return $urls; + } + + /** + * Where a URL's image is kept, named for the URL. + */ + public static function path(string $url): string + { + return self::directory().'/'.hash('sha256', $url); + } + + public static function directory(): string + { + return storage_path('app/'.self::Directory); + } + + public static function hours(): int + { + return (int) config('devnull.image_cache_hours'); + } + + /** + * The file a URL was downloaded to, while it's younger than the cache + * keeps images, or null. + */ + private static function freshPath(?string $url): ?string + { + if (self::hours() <= 0 || ! self::isRemote($url)) { + return null; + } + + $path = self::path($url); + $modified = is_file($path) ? filemtime($path) : false; + + return $modified !== false && $modified > time() - self::hours() * 3600 ? $path : null; + } + + /** + * @phpstan-assert-if-true string $value + */ + private static function isRemote(mixed $value): bool + { + return is_string($value) && preg_match('#^https?://#i', $value) === 1; + } +} diff --git a/app/NativeComponents/Concerns/RefreshesInBackground.php b/app/NativeComponents/Concerns/RefreshesInBackground.php index e9cedee0..ed6fd106 100644 --- a/app/NativeComponents/Concerns/RefreshesInBackground.php +++ b/app/NativeComponents/Concerns/RefreshesInBackground.php @@ -9,6 +9,7 @@ use App\Devnull\Exceptions\KeychainUnavailable; use App\Devnull\Exceptions\Unauthenticated; use App\Devnull\FetchInBackground; +use App\Devnull\ImageCache; use App\Devnull\ResponseCache; use Closure; use Throwable; @@ -177,6 +178,7 @@ private function settleRefresh(array $result, string $path, array $query, Closur $json = is_array($result['json'] ?? null) ? $result['json'] : []; app(ResponseCache::class)->put($path, $query, $json); + ImageCache::prefetch($json); $this->refreshFailed = false; $this->error = null; $apply($json); diff --git a/config/devnull.php b/config/devnull.php index b270b6ac..b58ea401 100644 --- a/config/devnull.php +++ b/config/devnull.php @@ -30,6 +30,20 @@ 'key_store' => env('DEVNULL_KEY_STORE'), + /* + |-------------------------------------------------------------------------- + | Image cache + |-------------------------------------------------------------------------- + | + | Avatars, commit images and link preview images are downloaded in the + | background the first time an answer from devnull carries them, then drawn + | from the device for this many hours instead of being fetched again each + | time they scroll into view. 0 turns the cache off. + | + */ + + 'image_cache_hours' => (int) env('DEVNULL_IMAGE_CACHE_HOURS', 24), + /* |-------------------------------------------------------------------------- | Sign in with Apple diff --git a/phpunit.xml b/phpunit.xml index 2b38ff0a..82d34840 100644 --- a/phpunit.xml +++ b/phpunit.xml @@ -23,6 +23,7 @@ + diff --git a/resources/views/native/partials/avatar.blade.php b/resources/views/native/partials/avatar.blade.php index 330388cd..cbb31e43 100644 --- a/resources/views/native/partials/avatar.blade.php +++ b/resources/views/native/partials/avatar.blade.php @@ -1,8 +1,8 @@ -{{-- A person as a circle: their GitHub avatar on the wash, or their initial while it loads. --}} +{{-- A person as a circle: their GitHub avatar on the wash, kept on the device for a day (see ImageCache), or their initial while it loads. --}} @php($size ??= 40) {{ mb_strtoupper(mb_substr($user->name, 0, 1)) }} @if ($user->avatarUrl) - + @endif diff --git a/resources/views/native/partials/commit-media.blade.php b/resources/views/native/partials/commit-media.blade.php index 68f3574f..eb06d11c 100644 --- a/resources/views/native/partials/commit-media.blade.php +++ b/resources/views/native/partials/commit-media.blade.php @@ -2,13 +2,14 @@ What hangs under a commit's body on the web: its images, its poll, the commit it quotes, then the link preview. A reply in a thread shows only its images and link preview ($withEmbeds false), as the web's thread does. Images hold their space at their aspect ratio on a wash, so one that fails to load leaves the row as it was. Each is - read out by the description its author gave it, as the web's alt text. + read out by the description its author gave it, as the web's alt text. Images come from the device once ImageCache has + them, so they don't load again each time they scroll into view. --}} @php($images = $commit->images) @if (count($images) === 1) - + @elseif (count($images) > 1) @@ -19,7 +20,7 @@ @foreach ($column as $image) @php($index = array_search($image, $images, true)) - + @endforeach @@ -42,7 +43,7 @@ @if ($preview->imageUrl) - + @endif diff --git a/tests/Feature/ImageCacheTest.php b/tests/Feature/ImageCacheTest.php new file mode 100644 index 00000000..34e86263 --- /dev/null +++ b/tests/Feature/ImageCacheTest.php @@ -0,0 +1,170 @@ + 'http://devnull.test', 'devnull.image_cache_hours' => 24]); + app()->useStoragePath(sys_get_temp_dir().'/devnull-image-cache-'.bin2hex(random_bytes(4))); + File::ensureDirectoryExists(ImageCache::directory()); +}); + +afterEach(function () { + File::deleteDirectory(storage_path()); + AsyncTask::clearFake(); +}); + +/** + * Puts an image on the device for a URL, downloaded this many hours ago. + */ +function cachedImage(string $url, int $hoursAgo = 0): string +{ + $path = ImageCache::path($url); + File::put($path, 'image bytes'); + touch($path, time() - $hoursAgo * 3600); + + return $path; +} + +/** + * The URLs a dispatched DownloadImages task was given, across every dispatch. + * + * @return list + */ +function imagesQueued(): array +{ + return collect(AsyncTask::fakeInstance()->dispatched ?? []) + ->filter(fn (array $dispatch): bool => ($dispatch['work']['task'] ?? null) === DownloadImages::class) + ->flatMap(fn (array $dispatch): array => $dispatch['work']['args'][0]) + ->values() + ->all(); +} + +it('finds the avatars, commit images and link preview images in an answer, however deep', function () { + $json = pageJson([ + commitJson([ + 'id' => 'c1', + 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/2?v=4']), + 'images' => [['id' => 'i1', 'url' => 'https://dev-null.sh/commit-images/i1', 'width' => 10, 'height' => 10]], + 'link_preview' => ['url' => 'https://example.com', 'image_url' => 'https://example.com/card.png'], + 'quoted_commit' => commitJson(['id' => 'q1', 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/3?v=4'])]), + ]), + commitJson(['id' => 'c2', 'author' => userJson(['avatar_url' => null])]), + ]); + + expect(ImageCache::urlsIn($json))->toContain( + 'https://avatars.githubusercontent.com/u/2?v=4', + 'https://dev-null.sh/commit-images/i1', + 'https://example.com/card.png', + 'https://avatars.githubusercontent.com/u/3?v=4', + )->not->toContain('https://example.com'); +}); + +it('draws an image from the device while it is younger than a day, and from its URL otherwise', function () { + $url = 'https://avatars.githubusercontent.com/u/2?v=4'; + + expect(ImageCache::src($url))->toBe($url) + ->and(ImageCache::src(null))->toBeNull() + ->and(ImageCache::src('/already/on/the/device.png'))->toBe('/already/on/the/device.png'); + + $path = cachedImage($url, hoursAgo: 23); + + expect(ImageCache::src($url))->toBe($path); + + touch($path, time() - 25 * 3600); + + expect(ImageCache::src($url))->toBe($url); + + // With the cache off, every image comes from its URL. + touch($path); + config(['devnull.image_cache_hours' => 0]); + + expect(ImageCache::src($url))->toBe($url); +}); + +it('downloads in the background only the images that are not on the device yet, once each', function () { + AsyncTask::fake(); + cachedImage('https://avatars.githubusercontent.com/u/2?v=4'); + + ImageCache::prefetch(pageJson([ + commitJson(['id' => 'c1', 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/2?v=4'])]), + commitJson(['id' => 'c2', 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/3?v=4'])]), + commitJson(['id' => 'c3', 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/3?v=4'])]), + ])); + + expect(imagesQueued())->toBe(['https://avatars.githubusercontent.com/u/3?v=4']); + + // Nothing to fetch, and nothing at all with the cache off. + AsyncTask::fake(); + ImageCache::prefetch(pageJson([commitJson(['id' => 'c1', 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/2?v=4'])])])); + config(['devnull.image_cache_hours' => 0]); + ImageCache::prefetch(pageJson([commitJson(['id' => 'c2', 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/9?v=4'])])])); + + AsyncTask::fakeInstance()->assertNotDispatched(); +}); + +it('keeps only what downloads as an image, and clears out images older than a day', function () { + $stale = cachedImage('https://avatars.githubusercontent.com/u/8?v=4', hoursAgo: 30); + File::put(ImageCache::path('https://avatars.githubusercontent.com/u/9?v=4').'.part', 'half'); + + Http::fake([ + 'avatars.githubusercontent.com/u/2*' => Http::response('png bytes', 200, ['Content-Type' => 'image/png']), + 'dev-null.sh/commit-images/i1' => Http::response('webp bytes', 200, ['Content-Type' => 'image/webp']), + 'example.com/not-an-image' => Http::response('', 200, ['Content-Type' => 'text/html']), + 'example.com/missing.png' => Http::response('', 404), + 'example.com/down.png' => fn () => throw new ConnectionException('Could not resolve host'), + ]); + + $saved = (new DownloadImages)->handle([ + 'https://avatars.githubusercontent.com/u/2?v=4', + 'https://dev-null.sh/commit-images/i1', + 'https://example.com/not-an-image', + 'https://example.com/missing.png', + 'https://example.com/down.png', + ]); + + expect($saved)->toBe(2) + ->and(File::get(ImageCache::path('https://avatars.githubusercontent.com/u/2?v=4')))->toBe('png bytes') + ->and(File::get(ImageCache::path('https://dev-null.sh/commit-images/i1')))->toBe('webp bytes') + ->and(File::exists(ImageCache::path('https://example.com/not-an-image')))->toBeFalse() + ->and(File::exists(ImageCache::path('https://example.com/missing.png')))->toBeFalse() + ->and(File::exists($stale))->toBeFalse() + ->and(File::files(ImageCache::directory()))->toHaveCount(2); +}); + +it('draws the Log\'s avatars from the device once the refresh has brought them down', function () { + signIn(); + holdRefreshes(); + $avatar = 'https://avatars.githubusercontent.com/u/2?v=4'; + // Before serveApi(), whose fake answers every other URL. + Http::fake(['avatars.githubusercontent.com/*' => Http::response('png bytes', 200, ['Content-Type' => 'image/png'])]); + serveApi(['GET /api/v1/log/following' => [pageJson([commitJson(['id' => 'c1', 'author' => userJson(['id' => 'u2', 'username' => 'ada', 'avatar_url' => $avatar])])])]]); + + $imageSources = fn ($screen): array => array_map( + fn (array $node): string => $node['props']['src'] ?? '', + findNodes($screen->tree(), fn (array $node): bool => ($node['type'] ?? null) === 'image'), + ); + + finishRefreshes(Native::visit('/log')); + + expect(File::exists(ImageCache::path($avatar)))->toBeTrue(); + + expect($imageSources(Native::visit('/log')))->toContain(ImageCache::path($avatar)) + ->not->toContain($avatar); +}); + +it('downloads the images in an answer the app waited for, too', function () { + signIn(); + AsyncTask::fake(); + serveApi(['GET /api/v1/log/following' => [pageJson([commitJson(['id' => 'c1', 'author' => userJson(['avatar_url' => 'https://avatars.githubusercontent.com/u/4?v=4'])])])]]); + + app(Client::class)->log('following'); + + expect(imagesQueued())->toBe(['https://avatars.githubusercontent.com/u/4?v=4']); +});