From 8c7a08b4e91ef3b0840179a6e0d662f886267b7a Mon Sep 17 00:00:00 2001 From: JoaquinOlivero Date: Wed, 31 Jul 2024 10:32:04 +0000 Subject: [PATCH] refactor: proxy and cache user avatar images --- next.config.js | 1 - overseerr-api.yml | 9 ++ server/index.ts | 2 + server/interfaces/api/settingsInterfaces.ts | 2 +- server/job/schedule.ts | 3 + server/lib/imageproxy.ts | 101 ++++++++---- server/routes/avatarproxy.ts | 31 ++++ server/routes/settings/index.ts | 2 + .../IssueDetails/IssueComment/index.tsx | 6 +- src/components/IssueDetails/index.tsx | 145 +++++++++--------- src/components/Layout/UserDropdown/index.tsx | 10 +- src/components/ManageSlideOver/index.tsx | 101 ++++++------ src/components/RequestCard/index.tsx | 9 +- .../RequestList/RequestItem/index.tsx | 17 +- .../RequestModal/AdvancedRequester/index.tsx | 10 +- .../Settings/SettingsJobsCache/index.tsx | 14 ++ .../UserList/JellyfinImportModal.tsx | 35 ++--- src/components/UserList/index.tsx | 54 ++++--- .../UserProfile/ProfileHeader/index.tsx | 7 +- 19 files changed, 321 insertions(+), 238 deletions(-) create mode 100644 server/routes/avatarproxy.ts diff --git a/next.config.js b/next.config.js index 35a316c6..43aa421d 100644 --- a/next.config.js +++ b/next.config.js @@ -10,7 +10,6 @@ module.exports = { remotePatterns: [ { hostname: 'gravatar.com' }, { hostname: 'image.tmdb.org' }, - { hostname: '*', protocol: 'https' }, ], }, webpack(config) { diff --git a/overseerr-api.yml b/overseerr-api.yml index f5e1d162..96a4520a 100644 --- a/overseerr-api.yml +++ b/overseerr-api.yml @@ -2790,6 +2790,15 @@ paths: imageCount: type: number example: 123 + avatar: + type: object + properties: + size: + type: number + example: 123456 + imageCount: + type: number + example: 123 apiCaches: type: array items: diff --git a/server/index.ts b/server/index.ts index ef20674d..4ccc6fed 100644 --- a/server/index.ts +++ b/server/index.ts @@ -19,6 +19,7 @@ import { getSettings } from '@server/lib/settings'; import logger from '@server/logger'; import clearCookies from '@server/middleware/clearcookies'; import routes from '@server/routes'; +import avatarproxy from '@server/routes/avatarproxy'; import imageproxy from '@server/routes/imageproxy'; import { getAppVersion } from '@server/utils/appVersion'; import restartFlag from '@server/utils/restartFlag'; @@ -202,6 +203,7 @@ app // Do not set cookies so CDNs can cache them server.use('/imageproxy', clearCookies, imageproxy); + server.use('/avatarproxy', clearCookies, avatarproxy); server.get('*', (req, res) => handle(req, res)); server.use( diff --git a/server/interfaces/api/settingsInterfaces.ts b/server/interfaces/api/settingsInterfaces.ts index 1bf40cdb..579f1109 100644 --- a/server/interfaces/api/settingsInterfaces.ts +++ b/server/interfaces/api/settingsInterfaces.ts @@ -58,7 +58,7 @@ export interface CacheItem { export interface CacheResponse { apiCaches: CacheItem[]; - imageCache: Record<'tmdb', { size: number; imageCount: number }>; + imageCache: Record<'tmdb' | 'avatar', { size: number; imageCount: number }>; } export interface StatusResponse { diff --git a/server/job/schedule.ts b/server/job/schedule.ts index b358130c..a210988e 100644 --- a/server/job/schedule.ts +++ b/server/job/schedule.ts @@ -227,6 +227,9 @@ export const startJobs = (): void => { }); // Clean TMDB image cache ImageProxy.clearCache('tmdb'); + + // Clean users avatar image cache + ImageProxy.clearCache('avatar'); }), }); diff --git a/server/lib/imageproxy.ts b/server/lib/imageproxy.ts index 195e96b9..cd7aa8cd 100644 --- a/server/lib/imageproxy.ts +++ b/server/lib/imageproxy.ts @@ -27,29 +27,37 @@ class ImageProxy { let deletedImages = 0; const cacheDirectory = path.join(baseCacheDirectory, key); - const files = await promises.readdir(cacheDirectory); + try { + const files = await promises.readdir(cacheDirectory); - for (const file of files) { - const filePath = path.join(cacheDirectory, file); - const stat = await promises.lstat(filePath); + for (const file of files) { + const filePath = path.join(cacheDirectory, file); + const stat = await promises.lstat(filePath); - if (stat.isDirectory()) { - const imageFiles = await promises.readdir(filePath); + if (stat.isDirectory()) { + const imageFiles = await promises.readdir(filePath); - for (const imageFile of imageFiles) { - const [, expireAtSt] = imageFile.split('.'); - const expireAt = Number(expireAtSt); - const now = Date.now(); + for (const imageFile of imageFiles) { + const [, expireAtSt] = imageFile.split('.'); + const expireAt = Number(expireAtSt); + const now = Date.now(); - if (now > expireAt) { - await promises.rm(path.join(filePath, imageFile)); - deletedImages += 1; + if (now > expireAt) { + await promises.rm(path.join(filePath), { + recursive: true, + }); + deletedImages += 1; + } } } } + } catch (e) { + if (e.code === 'ENOENT') { + logger.error(e.message); + } } - logger.info(`Cleared ${deletedImages} stale image(s) from cache`, { + logger.info(`Cleared ${deletedImages} stale image(s) from cache '${key}'`, { label: 'Image Cache', }); } @@ -69,33 +77,49 @@ class ImageProxy { } private static async getDirectorySize(dir: string): Promise { - const files = await promises.readdir(dir, { - withFileTypes: true, - }); + try { + const files = await promises.readdir(dir, { + withFileTypes: true, + }); - const paths = files.map(async (file) => { - const path = join(dir, file.name); + const paths = files.map(async (file) => { + const path = join(dir, file.name); - if (file.isDirectory()) return await ImageProxy.getDirectorySize(path); + if (file.isDirectory()) return await ImageProxy.getDirectorySize(path); - if (file.isFile()) { - const { size } = await promises.stat(path); + if (file.isFile()) { + const { size } = await promises.stat(path); - return size; + return size; + } + + return 0; + }); + + return (await Promise.all(paths)) + .flat(Infinity) + .reduce((i, size) => i + size, 0); + } catch (e) { + if (e.code === 'ENOENT') { + return 0; } + } - return 0; - }); - - return (await Promise.all(paths)) - .flat(Infinity) - .reduce((i, size) => i + size, 0); + return 0; } private static async getImageCount(dir: string) { - const files = await promises.readdir(dir); + try { + const files = await promises.readdir(dir); - return files.length; + return files.length; + } catch (e) { + if (e.code === 'ENOENT') { + return 0; + } + } + + return 0; } private fetch: typeof fetch; @@ -187,16 +211,25 @@ class ImageProxy { const directory = join(this.getCacheDirectory(), cacheKey); const href = this.baseUrl + - (this.baseUrl.endsWith('/') ? '' : '/') + + (this.baseUrl.length > 0 + ? this.baseUrl.endsWith('/') + ? '' + : '/' + : '') + (path.startsWith('/') ? path.slice(1) : path); const response = await this.fetch(href); const arrayBuffer = await response.arrayBuffer(); const buffer = Buffer.from(arrayBuffer); - const extension = path.split('.').pop() ?? ''; - const maxAge = Number( + const extension = (response.headers.get('content-type') ?? '').replace( + 'image/', + '' + ); + let maxAge = Number( (response.headers.get('cache-control') ?? '0').split('=')[1] ); + + if (!maxAge) maxAge = 604800; const expireAt = Date.now() + maxAge * 1000; const etag = (response.headers.get('etag') ?? '').replace(/"/g, ''); diff --git a/server/routes/avatarproxy.ts b/server/routes/avatarproxy.ts new file mode 100644 index 00000000..6e74cda0 --- /dev/null +++ b/server/routes/avatarproxy.ts @@ -0,0 +1,31 @@ +import ImageProxy from '@server/lib/imageproxy'; +import logger from '@server/logger'; +import { Router } from 'express'; + +const router = Router(); + +const avatarImageProxy = new ImageProxy('avatar', ''); +// Proxy avatar images +router.get('/*', async (req, res) => { + const imagePath = req.path.startsWith('/') ? req.path.slice(1) : req.path; + try { + const imageData = await avatarImageProxy.getImage(imagePath); + + res.writeHead(200, { + 'Content-Type': `image/${imageData.meta.extension}`, + 'Content-Length': imageData.imageBuffer.length, + 'Cache-Control': `public, max-age=${imageData.meta.curRevalidate}`, + 'OS-Cache-Key': imageData.meta.cacheKey, + 'OS-Cache-Status': imageData.meta.cacheMiss ? 'MISS' : 'HIT', + }); + + res.end(imageData.imageBuffer); + } catch (e) { + logger.error('Failed to proxy avatar image', { + imagePath, + errorMessage: e.message, + }); + } +}); + +export default router; diff --git a/server/routes/settings/index.ts b/server/routes/settings/index.ts index 30898d2a..30c854af 100644 --- a/server/routes/settings/index.ts +++ b/server/routes/settings/index.ts @@ -746,11 +746,13 @@ settingsRoutes.get('/cache', async (_req, res) => { })); const tmdbImageCache = await ImageProxy.getImageStats('tmdb'); + const avatarImageCache = await ImageProxy.getImageStats('avatar'); return res.status(200).json({ apiCaches, imageCache: { tmdb: tmdbImageCache, + avatar: avatarImageCache, }, }); }); diff --git a/src/components/IssueDetails/IssueComment/index.tsx b/src/components/IssueDetails/IssueComment/index.tsx index 0c36ca66..a4d6181b 100644 --- a/src/components/IssueDetails/IssueComment/index.tsx +++ b/src/components/IssueDetails/IssueComment/index.tsx @@ -1,4 +1,5 @@ import Button from '@app/components/Common/Button'; +import CachedImage from '@app/components/Common/CachedImage'; import Modal from '@app/components/Common/Modal'; import { Permission, useUser } from '@app/hooks/useUser'; import defineMessages from '@app/utils/defineMessages'; @@ -6,7 +7,6 @@ import { Menu, Transition } from '@headlessui/react'; import { EllipsisVerticalIcon } from '@heroicons/react/24/solid'; import type { default as IssueCommentType } from '@server/entity/IssueComment'; import { Field, Form, Formik } from 'formik'; -import Image from 'next/image'; import Link from 'next/link'; import { Fragment, useState } from 'react'; import { FormattedRelativeTime, useIntl } from 'react-intl'; @@ -88,8 +88,8 @@ const IssueComment = ({ - {

{title} @@ -287,10 +286,10 @@ const IssueDetails = () => { } className="group ml-1 inline-flex h-full items-center xl:ml-1.5" > - @@ -303,7 +302,7 @@ const IssueDetails = () => { { { {settings.currentSettings.mediaServerType === - MediaServerType.EMBY + MediaServerType.EMBY ? intl.formatMessage(messages.playonplex, { - mediaServerName: 'Emby', - }) + mediaServerName: 'Emby', + }) : settings.currentSettings.mediaServerType === MediaServerType.PLEX - ? intl.formatMessage(messages.playonplex, { + ? intl.formatMessage(messages.playonplex, { mediaServerName: 'Plex', }) - : intl.formatMessage(messages.playonplex, { + : intl.formatMessage(messages.playonplex, { mediaServerName: 'Jellyfin', })} @@ -437,16 +436,16 @@ const IssueDetails = () => { {settings.currentSettings.mediaServerType === - MediaServerType.EMBY + MediaServerType.EMBY ? intl.formatMessage(messages.play4konplex, { - mediaServerName: 'Emby', - }) + mediaServerName: 'Emby', + }) : settings.currentSettings.mediaServerType === MediaServerType.PLEX - ? intl.formatMessage(messages.play4konplex, { + ? intl.formatMessage(messages.play4konplex, { mediaServerName: 'Plex', }) - : intl.formatMessage(messages.play4konplex, { + : intl.formatMessage(messages.play4konplex, { mediaServerName: 'Jellyfin', })} @@ -531,52 +530,52 @@ const IssueDetails = () => {
{(hasPermission(Permission.MANAGE_ISSUES) || belongsToUser) && ( - <> - {issueData.status === IssueStatus.OPEN ? ( - - ) : ( - + ) : ( + - )} - - )} + if (values.message) { + handleSubmit(); + } + }} + > + + + {intl.formatMessage( + values.message + ? messages.reopenissueandcomment + : messages.reopenissue + )} + + + )} + + )}
diff --git a/src/components/RequestCard/index.tsx b/src/components/RequestCard/index.tsx index eb78806f..7f3f3d75 100644 --- a/src/components/RequestCard/index.tsx +++ b/src/components/RequestCard/index.tsx @@ -22,7 +22,6 @@ import type { MediaRequest } from '@server/entity/MediaRequest'; import type { NonFunctionProperties } from '@server/interfaces/api/common'; import type { MovieDetails } from '@server/models/Movie'; import type { TvDetails } from '@server/models/Tv'; -import Image from 'next/image'; import Link from 'next/link'; import { useEffect, useState } from 'react'; import { useInView } from 'react-intersection-observer'; @@ -116,8 +115,8 @@ const RequestCardError = ({ requestData }: RequestCardErrorProps) => { className="group flex items-center" > - { className="group flex items-center" > - - - { className="group flex items-center truncate" > - { className="group flex items-center truncate" > - - - {appDataPath}/cache/images.', imagecachecount: 'Images Cached', imagecachesize: 'Total Cache Size', + useravatars: 'User Avatars', } ); @@ -573,6 +574,19 @@ const SettingsJobs = () => { {formatBytes(cacheData?.imageCache.tmdb.size ?? 0)} + + + {intl.formatMessage(messages.useravatars)} (avatar) + + + {intl.formatNumber( + cacheData?.imageCache.avatar.imageCount ?? 0 + )} + + + {formatBytes(cacheData?.imageCache.avatar.size ?? 0)} + + diff --git a/src/components/UserList/JellyfinImportModal.tsx b/src/components/UserList/JellyfinImportModal.tsx index 36dbe0aa..2f909934 100644 --- a/src/components/UserList/JellyfinImportModal.tsx +++ b/src/components/UserList/JellyfinImportModal.tsx @@ -1,4 +1,5 @@ import Alert from '@app/components/Common/Alert'; +import CachedImage from '@app/components/Common/CachedImage'; import Modal from '@app/components/Common/Modal'; import useSettings from '@app/hooks/useSettings'; import globalMessages from '@app/i18n/globalMessages'; @@ -157,7 +158,7 @@ const JellyfinImportModal: React.FC = ({ title={intl.formatMessage(messages.newJellyfinsigninenabled, { mediaServerName: settings.currentSettings.mediaServerType === - MediaServerType.EMBY + MediaServerType.EMBY ? 'Emby' : 'Jellyfin', strong: (msg: React.ReactNode) => ( @@ -189,15 +190,13 @@ const JellyfinImportModal: React.FC = ({ > @@ -231,27 +230,25 @@ const JellyfinImportModal: React.FC = ({ >
- { error, mutate: revalidate, } = useSWR( - `/api/v1/user?take=${currentPageSize}&skip=${ - pageIndex * currentPageSize + `/api/v1/user?take=${currentPageSize}&skip=${pageIndex * currentPageSize }&sort=${currentSort}` ); @@ -215,9 +215,9 @@ const UserList = () => { !value ? Yup.string() : Yup.string().min( - 8, - intl.formatMessage(messages.validationpasswordminchars) - ) + 8, + intl.formatMessage(messages.validationpasswordminchars) + ) ), }); @@ -406,9 +406,8 @@ const UserList = () => {
{user.id === currentUser?.id || - currentHasPermission( - [Permission.MANAGE_REQUESTS, Permission.REQUEST_VIEW], - { type: 'or' } - ) ? ( + currentHasPermission( + [Permission.MANAGE_REQUESTS, Permission.REQUEST_VIEW], + { type: 'or' } + ) ? ( { {user.id === 1 ? intl.formatMessage(messages.owner) : hasPermission(Permission.ADMIN, user.permissions) - ? intl.formatMessage(messages.admin) - : intl.formatMessage(messages.user)} + ? intl.formatMessage(messages.admin) + : intl.formatMessage(messages.user)} {intl.formatDate(user.createdAt, { diff --git a/src/components/UserProfile/ProfileHeader/index.tsx b/src/components/UserProfile/ProfileHeader/index.tsx index a9bc0db0..9b5c080f 100644 --- a/src/components/UserProfile/ProfileHeader/index.tsx +++ b/src/components/UserProfile/ProfileHeader/index.tsx @@ -1,9 +1,10 @@ import Button from '@app/components/Common/Button'; +import CachedImage from '@app/components/Common/CachedImage'; import type { User } from '@app/hooks/useUser'; import { Permission, useUser } from '@app/hooks/useUser'; import defineMessages from '@app/utils/defineMessages'; import { CogIcon, UserIcon } from '@heroicons/react/24/solid'; -import Image from 'next/image'; +// import Image from 'next/image'; import Link from 'next/link'; import { useIntl } from 'react-intl'; @@ -42,9 +43,9 @@ const ProfileHeader = ({ user, isSettingsPage }: ProfileHeaderProps) => {
-