feat: optimize video retrieval with improved caching and fallback mechanisms

apps/box-app-api/src/feature/video/video.service.ts

@@ -313,72 +313,55 @@ export class VideoService {
   }): Promise<VideoPageDto<VideoDetailDto>> {
     const { categoryId, page, pageSize } = params;
 
-    try {
-      const key = tsCacheKeys.video.categoryList(categoryId);
-      const offset = (page - 1) * pageSize;
-      const limit = pageSize;
+    const key = tsCacheKeys.video.categoryList(categoryId);
+    const start = (page - 1) * pageSize;
+    const stop = start + pageSize - 1;
 
-      // Use helper to read video IDs from LIST
-      const videoIds = await this.cacheHelper.getVideoIdList(
-        key,
-        offset,
-        offset + limit - 1,
-      );
+    let listExists = false;
+    let videoIds: string[] = [];
 
-      if (!videoIds || videoIds.length === 0) {
-        this.logger.debug(
-          `Cache miss for category list, falling back to DB: ${key}`,
-        );
-        return this.getVideosByCategoryFromDb({ categoryId, page, pageSize });
-      }
+    try {
+      listExists = (await this.redis.exists(key)) > 0;
+      videoIds = await this.cacheHelper.getVideoIdList(key, start, stop);
+    } catch (err) {
+      this.logger.error(
+        `Error reading category list key=${key}`,
+        err instanceof Error ? err.stack : String(err),
+      );
+      listExists = false;
+      videoIds = [];
+    }
 
-      // Legacy format detection: check if first element looks like JSON
+    let usedCache = false;
+    if (listExists && videoIds.length > 0) {
       if (videoIds[0] && this.isLegacyJsonFormat(videoIds[0])) {
         this.logger.warn(
           `Detected legacy JSON format in ${key}, falling back to DB query`,
         );
-        return this.getVideosByCategoryFromDb({ categoryId, page, pageSize });
+      } else {
+        usedCache = true;
+        const details = await this.getVideoDetailsBatchFromDb(videoIds);
+        return {
+          items: details.filter((d) => d !== null) as VideoDetailDto[],
+          total: undefined,
+          page,
+          pageSize,
+        };
       }
+    }
 
-      // Fetch video details from MongoDB
-      const details = await this.getVideoDetailsBatchFromDb(videoIds);
-
-      return {
-        items: details.filter((d) => d !== null) as VideoDetailDto[],
-        total: undefined,
-        page,
-        pageSize,
-      };
-    } catch (err) {
-      this.logger.error(
-        `Error in category list fallback for categoryId=${categoryId}`,
-        err instanceof Error ? err.stack : String(err),
+    if (!usedCache) {
+      const reason = listExists ? 'empty list' : 'missing key';
+      this.logger.debug(
+        `Cache miss for category list (${reason}), falling back to DB: ${key}`,
       );
-      return {
-        items: [],
-        total: 0,
-        page,
-        pageSize,
-      };
     }
-  }
-
-  /**
-   * Fallback: Get videos from MongoDB when cache is unavailable.
-   */
-  private async getVideosByCategoryFromDb(params: {
-    categoryId: string;
-    page: number;
-    pageSize: number;
-  }): Promise<VideoPageDto<VideoDetailDto>> {
-    const { categoryId, page, pageSize } = params;
 
     try {
-      const offset = (page - 1) * pageSize;
       const videos = await this.mongoPrisma.videoMedia.findMany({
         where: { categoryIds: { has: categoryId }, listStatus: 1 },
         orderBy: [{ addedTime: 'desc' }, { createdAt: 'desc' }],
-        skip: offset,
+        skip: start,
         take: pageSize,
       });
 
@@ -392,6 +375,30 @@ export class VideoService {
         updatedAt: v.updatedAt.toISOString(),
       }));
 
+      const cachedIds = videos.map((video) => video.id);
+      try {
+        await this.cacheHelper.saveVideoIdList(key, cachedIds);
+      } catch (err) {
+        this.logger.error(
+          `Error saving video ID list for key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+      }
+
+      const entries = videos.map((video) => ({
+        key: videoCacheKeys.videoPayloadKey(video.id),
+        value: toVideoPayload(video as RawVideoPayloadRow),
+      }));
+
+      try {
+        await this.redis.pipelineSetJson(entries);
+      } catch (err) {
+        this.logger.error(
+          `Error writing payload cache for category list fallback key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+      }
+
       return {
         items,
         total: undefined,
@@ -486,78 +493,59 @@ export class VideoService {
   }): Promise<VideoPageDto<VideoDetailDto>> {
     const { categoryId, tagId, page, pageSize } = params;
 
-    try {
-      const key = tsCacheKeys.video.tagList(categoryId, tagId);
-      const offset = (page - 1) * pageSize;
-      const limit = pageSize;
+    const key = tsCacheKeys.video.tagList(categoryId, tagId);
+    const start = (page - 1) * pageSize;
+    const stop = start + pageSize - 1;
 
-      // Use helper to read video IDs from LIST
-      const videoIds = await this.cacheHelper.getVideoIdList(
-        key,
-        offset,
-        offset + limit - 1,
-      );
+    let listExists = false;
+    let videoIds: string[] = [];
 
-      if (!videoIds || videoIds.length === 0) {
-        this.logger.debug(
-          `Cache miss for tag list, falling back to DB: ${key}`,
-        );
-        return this.getVideosByTagFromDb({ categoryId, tagId, page, pageSize });
-      }
+    try {
+      listExists = (await this.redis.exists(key)) > 0;
+      videoIds = await this.cacheHelper.getVideoIdList(key, start, stop);
+    } catch (err) {
+      this.logger.error(
+        `Error reading tag list key=${key}`,
+        err instanceof Error ? err.stack : String(err),
+      );
+      listExists = false;
+      videoIds = [];
+    }
 
-      // Legacy format detection
+    let usedCache = false;
+    if (listExists && videoIds.length > 0) {
       if (videoIds[0] && this.isLegacyJsonFormat(videoIds[0])) {
         this.logger.warn(
           `Detected legacy JSON format in ${key}, falling back to DB query`,
         );
-        return this.getVideosByTagFromDb({ categoryId, tagId, page, pageSize });
+      } else {
+        usedCache = true;
+        const details = await this.getVideoDetailsBatchFromDb(videoIds);
+        return {
+          items: details.filter((d) => d !== null) as VideoDetailDto[],
+          total: undefined,
+          page,
+          pageSize,
+        };
       }
+    }
 
-      // Fetch video details from MongoDB
-      const details = await this.getVideoDetailsBatchFromDb(videoIds);
-
-      return {
-        items: details.filter((d) => d !== null) as VideoDetailDto[],
-        total: undefined,
-        page,
-        pageSize,
-      };
-    } catch (err) {
-      this.logger.error(
-        `Error in tag list fallback for categoryId=${categoryId}, tagId=${tagId}`,
-        err instanceof Error ? err.stack : String(err),
+    if (!usedCache) {
+      const reason = listExists ? 'empty list' : 'missing key';
+      this.logger.debug(
+        `Cache miss for tag list (${reason}), falling back to DB: ${key}`,
       );
-      return {
-        items: [],
-        total: 0,
-        page,
-        pageSize,
-      };
     }
-  }
-
-  /**
-   * Fallback: Get videos from MongoDB filtered by tag.
-   */
-  private async getVideosByTagFromDb(params: {
-    categoryId: string;
-    tagId: string;
-    page: number;
-    pageSize: number;
-  }): Promise<VideoPageDto<VideoDetailDto>> {
-    const { categoryId, tagId, page, pageSize } = params;
 
     try {
-      const offset = (page - 1) * pageSize;
       const videos = await this.mongoPrisma.videoMedia.findMany({
         where: {
           categoryIds: { has: categoryId },
           status: 'Completed',
-          // listStatus: 1,
           tagIds: { has: tagId },
         },
         orderBy: [{ addedTime: 'desc' }, { createdAt: 'desc' }],
-        skip: offset,
+        skip: start,
         take: pageSize,
       });
 
@@ -571,6 +559,30 @@ export class VideoService {
         updatedAt: v.updatedAt.toISOString(),
       }));
 
+      const cachedIds = videos.map((video) => video.id);
+      try {
+        await this.cacheHelper.saveVideoIdList(key, cachedIds);
+      } catch (err) {
+        this.logger.error(
+          `Error saving video ID list for key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+      }
+
+      const entries = videos.map((video) => ({
+        key: videoCacheKeys.videoPayloadKey(video.id),
+        value: toVideoPayload(video as RawVideoPayloadRow),
+      }));
+
+      try {
+        await this.redis.pipelineSetJson(entries);
+      } catch (err) {
+        this.logger.error(
+          `Error writing payload cache for tag list fallback key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+      }
+
       return {
         items,
         total: undefined,
@@ -696,6 +708,7 @@ export class VideoService {
         `Error fetching video details from DB`,
         err instanceof Error ? err.stack : String(err),
       );
+    }
     return videoIds.map(() => null);
   }
 
@@ -776,7 +789,6 @@ export class VideoService {
       return [];
     }
   }
-  }
 
   /**
    * Detect legacy JSON format in Redis cache.
@@ -842,9 +854,8 @@ export class VideoService {
         rawCategories.map(async (category) => {
           try {
             const tagKey = tsCacheKeys.tag.metadataByCategory(category.id);
-            const tagMetadata = await this.cacheHelper.getTagListForCategory(
-              tagKey,
-            );
+            const tagMetadata =
+              await this.cacheHelper.getTagListForCategory(tagKey);
             const tags = (tagMetadata ?? []).map((tag) => ({
               name: tag.name,
               seq: tag.seq,
@@ -893,7 +904,7 @@ export class VideoService {
    */
   async getVideoList(dto: VideoListRequestDto): Promise<VideoListResponseDto> {
     const { page, size, tagName } = dto;
-    let categoryId = dto.categoryId;
+    const categoryId = dto.categoryId;
     let key: string;
     let tagId: string | undefined;
 
@@ -971,13 +982,23 @@ export class VideoService {
       }
     }
 
-    // Step 2: Get total count and compute pagination
-    let total: number;
+    type VideoPayloadWithTags = RawVideoPayloadRow & { tagIds?: string[] };
+
+    // Step 2: Compute pagination indices
+    const start = (page - 1) * size;
+    const stop = start + size - 1;
+
+    let total = 0;
+    let pageVideoIds: string[] = [];
+    let fallbackRecords: VideoPayloadWithTags[] = [];
+    let videoTagMap = new Map<string, string[]>();
+
+    let listKeyExists: boolean;
     try {
-      total = await this.redis.llen(key);
+      listKeyExists = (await this.redis.exists(key)) > 0;
     } catch (err) {
       this.logger.error(
-        `Error getting list length for key=${key}`,
+        `Error checking list key existence for key=${key}`,
         err instanceof Error ? err.stack : String(err),
       );
       return {
@@ -989,55 +1010,184 @@ export class VideoService {
       };
     }
 
-    if (total === 0) {
-      this.logger.debug(`Empty video list for key=${key}`);
-      return {
-        page,
-        size,
-        total: 0,
-        tagName,
-        items: [],
-      };
-    }
+    if (listKeyExists) {
+      try {
+        total = await this.redis.llen(key);
+      } catch (err) {
+        this.logger.error(
+          `Error getting list length for key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+        return {
+          page,
+          size,
+          total: 0,
+          tagName,
+          items: [],
+        };
+      }
 
-    // Step 3: Compute pagination indices
-    const start = (page - 1) * size;
-    const stop = start + size - 1;
+      if (total === 0) {
+        this.logger.debug(`Empty video list for key=${key}`);
+        return {
+          page,
+          size,
+          total: 0,
+          tagName,
+          items: [],
+        };
+      }
 
-    // Check if page is out of range
-    if (start >= total) {
-      this.logger.debug(
-        `Page out of range: page=${page}, size=${size}, total=${total}, key=${key}`,
-      );
-      return {
-        page,
-        size,
-        total,
-        tagName,
-        items: [],
-      };
-    }
+      if (start >= total) {
+        this.logger.debug(
+          `Page out of range: page=${page}, size=${size}, total=${total}, key=${key}`,
+        );
+        return {
+          page,
+          size,
+          total,
+          tagName,
+          items: [],
+        };
+      }
 
-    // Step 4: Fetch video IDs from Redis
-    let videoIds: string[];
-    try {
-      videoIds = await this.redis.lrange(key, start, stop);
-    } catch (err) {
-      this.logger.error(
-        `Error fetching video IDs from key=${key}`,
-        err instanceof Error ? err.stack : String(err),
+      try {
+        pageVideoIds = await this.redis.lrange(key, start, stop);
+      } catch (err) {
+        this.logger.error(
+          `Error fetching video IDs from key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+        return {
+          page,
+          size,
+          total,
+          tagName,
+          items: [],
+        };
+      }
+
+      if (!pageVideoIds || pageVideoIds.length === 0) {
+        this.logger.debug(`No video IDs found for key=${key}`);
+        return {
+          page,
+          size,
+          total,
+          tagName,
+          items: [],
+        };
+      }
+    } else {
+      this.logger.debug(`Cache miss for video list key=${key}`);
+      try {
+        const where = tagId
+          ? {
+              categoryIds: { has: categoryId },
+              status: 'Completed',
+              tagIds: { has: tagId },
+            }
+          : {
+              categoryIds: { has: categoryId },
+              status: 'Completed',
+            };
+
+        fallbackRecords = (await this.mongoPrisma.videoMedia.findMany({
+          where,
+          orderBy: [{ addedTime: 'desc' }, { createdAt: 'desc' }],
+          select: {
+            id: true,
+            title: true,
+            coverImg: true,
+            coverImgNew: true,
+            videoTime: true,
+            country: true,
+            firstTag: true,
+            secondTags: true,
+            preFileName: true,
+            desc: true,
+            size: true,
+            updatedAt: true,
+            filename: true,
+            fieldNameFs: true,
+            ext: true,
+            tagIds: true,
+          },
+        })) as VideoPayloadWithTags[];
+      } catch (err) {
+        this.logger.error(
+          `Error fetching videos from MongoDB for fallback key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+        return {
+          page,
+          size,
+          total: 0,
+          tagName,
+          items: [],
+        };
+      }
+
+      const allVideoIds = fallbackRecords.map((video) => video.id);
+      total = allVideoIds.length;
+
+      try {
+        await this.cacheHelper.saveVideoIdList(key, allVideoIds);
+      } catch (err) {
+        this.logger.error(
+          `Error saving video ID list for key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+      }
+
+      const entries = fallbackRecords.map((video) => ({
+        key: videoCacheKeys.videoPayloadKey(video.id),
+        value: toVideoPayload(video),
+      }));
+
+      try {
+        await this.redis.pipelineSetJson(entries);
+      } catch (err) {
+        this.logger.error(
+          `Error writing payload cache for fallback key=${key}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+      }
+
+      if (total === 0) {
+        this.logger.debug(`No videos found for fallback key=${key}`);
+        return {
+          page,
+          size,
+          total: 0,
+          tagName,
+          items: [],
+        };
+      }
+
+      if (start >= total) {
+        this.logger.debug(
+          `Page out of range: page=${page}, size=${size}, total=${total}, key=${key}`,
+        );
+        return {
+          page,
+          size,
+          total,
+          tagName,
+          items: [],
+        };
+      }
+
+      const slicedIds = allVideoIds.slice(start, stop + 1);
+      pageVideoIds = slicedIds;
+      videoTagMap = new Map(
+        fallbackRecords.map((video) => [
+          video.id,
+          Array.isArray(video.tagIds) ? video.tagIds : [],
+        ]),
       );
-      return {
-        page,
-        size,
-        total,
-        tagName,
-        items: [],
-      };
     }
 
-    if (!videoIds || videoIds.length === 0) {
-      this.logger.debug(`No video IDs found for key=${key}`);
+    if (!pageVideoIds.length) {
       return {
         page,
         size,
@@ -1047,49 +1197,28 @@ export class VideoService {
       };
     }
 
-    // Step 5: Fetch video details from MongoDB
-    let videos: Awaited<
-      ReturnType<typeof this.mongoPrisma.videoMedia.findMany>
-    >;
-    try {
-      videos = await this.mongoPrisma.videoMedia.findMany({
-        where: {
-          id: { in: videoIds },
-        },
-      });
-    } catch (err) {
-      this.logger.error(
-        `Error fetching videos from MongoDB for ids=${videoIds.join(',')}`,
-        err instanceof Error ? err.stack : String(err),
-      );
-      return {
-        page,
-        size,
-        total,
-        tagName,
-        items: [],
-      };
-    }
+    if (!videoTagMap.size) {
+      try {
+        const tagRows = await this.mongoPrisma.videoMedia.findMany({
+          where: { id: { in: pageVideoIds } },
+          select: { id: true, tagIds: true },
+        });
 
-    // Step 6: Fetch category info
-    let category;
-    try {
-      category = await this.mongoPrisma.category.findUnique({
-        where: { id: categoryId },
-      });
-    } catch (err) {
-      this.logger.error(
-        `Error fetching category for categoryId=${categoryId}`,
-        err instanceof Error ? err.stack : String(err),
-      );
-      // Continue without category info
+        for (const row of tagRows) {
+          videoTagMap.set(row.id, Array.isArray(row.tagIds) ? row.tagIds : []);
+        }
+      } catch (err) {
+        this.logger.error(
+          `Error fetching video tag IDs for ids=${pageVideoIds.join(',')}`,
+          err instanceof Error ? err.stack : String(err),
+        );
+      }
     }
 
-    // Step 7: Fetch all tags for these videos
     const allTagIds = new Set<string>();
-    for (const video of videos) {
-      if (video.tagIds && Array.isArray(video.tagIds)) {
-        for (const tid of video.tagIds) {
+    for (const ids of videoTagMap.values()) {
+      if (Array.isArray(ids)) {
+        for (const tid of ids) {
           allTagIds.add(tid);
         }
       }
@@ -1099,40 +1228,47 @@ export class VideoService {
     if (allTagIds.size > 0) {
       try {
         const tagsList = await this.mongoPrisma.tag.findMany({
-          where: {
-            id: { in: Array.from(allTagIds) },
-          },
+          where: { id: { in: Array.from(allTagIds) } },
           select: { id: true, name: true },
         });
-
-        tagsById = new Map(tagsList.map((t) => [t.id, t.name]));
+        tagsById = new Map(tagsList.map((tag) => [tag.id, tag.name]));
       } catch (err) {
         this.logger.error(
           `Error fetching tags from MongoDB`,
           err instanceof Error ? err.stack : String(err),
         );
-        // Continue without tag names
       }
     }
 
-    // Step 8: Create a map for O(1) lookup and maintain order
-    const videoMap = new Map(videos.map((v) => [v.id, v]));
+    let category;
+    try {
+      category = await this.mongoPrisma.category.findUnique({
+        where: { id: categoryId },
+      });
+    } catch (err) {
+      this.logger.error(
+        `Error fetching category for categoryId=${categoryId}`,
+        err instanceof Error ? err.stack : String(err),
+      );
+    }
 
-    // Step 9: Map to VideoListItemDto in the order of videoIds
-    const items = videoIds
+    const payloads = await this.getVideoPayloadsByIds(pageVideoIds);
+    const payloadMap = new Map(
+      payloads.map((payload) => [payload.id, payload]),
+    );
+
+    const items = pageVideoIds
       .map((videoId) => {
-        const video = videoMap.get(videoId);
-        if (!video) {
-          this.logger.debug(
-            `Video not found in MongoDB for videoId=${videoId}`,
-          );
+        const payload = payloadMap.get(videoId);
+        if (!payload) {
+          this.logger.debug(`Video payload missing for videoId=${videoId}`);
           return null;
         }
 
-        // Map tag IDs to tag names
         const tags: string[] = [];
-        if (video.tagIds && Array.isArray(video.tagIds)) {
-          for (const tid of video.tagIds) {
+        const videoTagIds = videoTagMap.get(videoId);
+        if (videoTagIds && Array.isArray(videoTagIds)) {
+          for (const tid of videoTagIds) {
             const tagName = tagsById.get(tid);
             if (tagName) {
               tags.push(tagName);
@@ -1141,15 +1277,15 @@ export class VideoService {
         }
 
         return {
-          id: video.id,
-          title: video.title ?? '',
-          coverImg: video.coverImg ?? undefined,
-          duration: video.videoTime ?? undefined,
-          categoryId: categoryId,
+          id: payload.id,
+          title: payload.title ?? '',
+          coverImg: payload.coverImg ?? undefined,
+          duration: payload.videoTime ?? undefined,
+          categoryId,
           name: category?.name ?? '',
           subtitle: category?.subtitle ?? undefined,
           tags,
-          updateAt: video.updatedAt?.toString() ?? new Date().toISOString(),
+          updateAt: payload.updatedAt ?? new Date().toISOString(),
         };
       })
       .filter((item): item is NonNullable<typeof item> => item !== null);
@@ -1246,9 +1382,8 @@ export class VideoService {
     for (const category of categories) {
       try {
         const tagKey = tsCacheKeys.tag.metadataByCategory(category.id);
-        const tagsMetadata = await this.cacheHelper.getTagListForCategory(
-          tagKey,
-        );
+        const tagsMetadata =
+          await this.cacheHelper.getTagListForCategory(tagKey);
 
         const matchingTags = (tagsMetadata ?? []).filter(
           (t) => t.name === tagName,