feat: implement cache checklist service and controller; enhance cache sync functionality

.env.mgnt

@@ -14,8 +14,7 @@ MONGO_URL="mongodb://admin:ZXcv%21%21996@localhost:27017/box_admin?authSource=ad
 # MONGO_URL="mongodb://msAdmin:Fl1%2A29MJe%26jLvj@192.168.0.100:27017/box_admin?authSource=admin&replicaSet=rs0"
 
 # Redis Config
-# REDIS_HOST=127.0.0.1
-REDIS_HOST=192.168.0.100
+REDIS_HOST=127.0.0.1
 REDIS_PORT=6379
 REDIS_PASSWORD=
 REDIS_DB=0

apps/box-app-api/src/feature/ads/ad.service.ts

@@ -36,6 +36,11 @@ export interface GetAdForPlacementParams {
   maxTries?: number; // optional, default 3
 }
 
+/**
+ * Handles ad selection for app clients using prebuilt Redis pools.
+ * Reads the ad pool for a given (scene, slot, adType), picks a candidate,
+ * and maps the cached payload back into the public AdDto shape.
+ */
 @Injectable()
 export class AdService {
   private readonly logger = new Logger(AdService.name);
@@ -54,14 +59,13 @@ export class AdService {
     const maxTries = params.maxTries ?? 3;
 
     const poolKey = CacheKeys.appAdPool(scene, slot, adType);
+    const pool = await this.readPoolWithDiagnostics(poolKey, {
+      scene,
+      slot,
+      adType,
+    });
 
-    const pool =
-      (await this.redis.getJson<AdPoolEntry[] | null>(poolKey)) ?? null;
-
-    if (!pool || pool.length === 0) {
-      this.logger.debug(
-        `getAdForPlacement: empty or missing pool for scene=${scene}, slot=${slot}, adType=${adType}, key=${poolKey}`,
-      );
+    if (!pool) {
       return null;
     }
 
@@ -103,6 +107,46 @@ export class AdService {
   }
 
   /**
+   * Fetch and parse a pool entry list, while emitting useful diagnostics when
+   * the pool is missing/empty or contains malformed JSON. Keeps API behavior
+   * unchanged (returns null when the pool is not usable).
+   */
+  private async readPoolWithDiagnostics(
+    poolKey: string,
+    placement: Pick<GetAdForPlacementParams, 'scene' | 'slot' | 'adType'>,
+  ): Promise<AdPoolEntry[] | null> {
+    const raw = await this.redis.get(poolKey);
+
+    if (raw === null) {
+      this.logger.warn(
+        `Ad pool cache miss for scene=${placement.scene}, slot=${placement.slot}, adType=${placement.adType}, key=${poolKey}. Cache may be cold; ensure cache-sync rebuilt pools.`,
+      );
+      return null;
+    }
+
+    let parsed: unknown;
+    try {
+      parsed = JSON.parse(raw);
+    } catch (err) {
+      const message =
+        err instanceof Error ? err.message : JSON.stringify(err ?? 'Unknown');
+      this.logger.error(
+        `Failed to parse ad pool JSON for key=${poolKey}: ${message}`,
+      );
+      return null;
+    }
+
+    if (!Array.isArray(parsed) || parsed.length === 0) {
+      this.logger.warn(
+        `Ad pool empty or invalid shape for scene=${placement.scene}, slot=${placement.slot}, adType=${placement.adType}, key=${poolKey}`,
+      );
+      return null;
+    }
+
+    return parsed as AdPoolEntry[];
+  }
+
+  /**
    * Pick a random index in [0, length-1] that is not in usedIndexes.
    * Returns -1 if all indexes are already used.
    */

apps/box-mgnt-api/src/cache-sync/cache-checklist.controller.ts

@@ -0,0 +1,25 @@
+import { Controller, Get, Post } from '@nestjs/common';
+import { CacheChecklistService } from './cache-checklist.service';
+
+@Controller('mgnt/cache')
+export class CacheChecklistController {
+  constructor(private readonly checklist: CacheChecklistService) {}
+
+  /** GET /mgnt/cache/checklist - current cached checklist (null if not yet run) */
+  @Get('checklist')
+  getChecklist() {
+    return (
+      this.checklist.getChecklist() ?? {
+        ok: false,
+        message: 'Checklist not yet available',
+      }
+    );
+  }
+
+  /** POST /mgnt/cache/checklist/run - re-run checklist on demand */
+  @Post('checklist/run')
+  async runChecklist() {
+    const summary = await this.checklist.runChecklist();
+    return summary;
+  }
+}

apps/box-mgnt-api/src/cache-sync/cache-checklist.service.ts

@@ -0,0 +1,136 @@
+import { Injectable, Logger, OnApplicationBootstrap } from '@nestjs/common';
+import { RedisService } from '@box/db/redis/redis.service';
+import { CacheKeys } from '@box/common/cache/cache-keys';
+import { ADTYPE_POOLS } from '@box/common/ads/ad-pool-config';
+import type { AdType } from '@box/common/ads/ad-types';
+import { CacheSyncService } from './cache-sync.service';
+
+export interface CacheKeyCheckResult {
+  key: string;
+  exists: boolean;
+  rebuilt: boolean;
+  items?: number | null; // number of array items if JSON array, else null
+  error?: string | null;
+}
+
+export interface CacheChecklistSummary {
+  timestamp: number;
+  results: CacheKeyCheckResult[];
+  ok: boolean; // all required keys exist
+  missing: number; // count missing after rebuild attempts
+}
+
+@Injectable()
+export class CacheChecklistService implements OnApplicationBootstrap {
+  private readonly logger = new Logger(CacheChecklistService.name);
+  private checklist: CacheChecklistSummary | null = null;
+
+  constructor(
+    private readonly redis: RedisService,
+    private readonly cacheSync: CacheSyncService,
+  ) {}
+
+  async onApplicationBootstrap() {
+    try {
+      this.logger.log('Startup cache checklist: beginning warm + verify');
+      // Initial warm attempt builds channels, categories, ad pools.
+      await this.cacheSync.warmCache();
+      await this.runChecklist();
+      if (this.checklist?.ok) {
+        this.logger.log('Startup cache checklist: all required keys present');
+      } else {
+        this.logger.warn(
+          `Startup cache checklist: missing=${this.checklist?.missing} key(s) after rebuild attempts`,
+        );
+      }
+    } catch (err) {
+      this.logger.error(
+        `Startup cache checklist failed: ${err instanceof Error ? err.message : String(err)}`,
+      );
+    }
+  }
+
+  /** Execute the checklist (can be manually re-triggered). */
+  async runChecklist(): Promise<CacheChecklistSummary> {
+    const requiredKeys = this.computeRequiredKeys();
+    const results: CacheKeyCheckResult[] = [];
+
+    for (const key of requiredKeys) {
+      let exists = await this.redis.exists(key);
+      let rebuilt = false;
+      let error: string | null = null;
+      let items: number | null = null;
+
+      if (!exists) {
+        try {
+          await this.targetedRebuild(key);
+          exists = await this.redis.exists(key);
+          rebuilt = exists;
+        } catch (e) {
+          error = e instanceof Error ? e.message : String(e);
+        }
+      }
+
+      if (exists) {
+        const json = await this.redis.getJson<unknown>(key);
+        if (Array.isArray(json)) items = json.length;
+      }
+
+      results.push({ key, exists, rebuilt, items, error });
+    }
+
+    const missing = results.filter((r) => !r.exists).length;
+    const summary: CacheChecklistSummary = {
+      timestamp: Date.now(),
+      results,
+      ok: missing === 0,
+      missing,
+    };
+    this.checklist = summary;
+    return summary;
+  }
+
+  getChecklist(): CacheChecklistSummary | null {
+    return this.checklist;
+  }
+
+  private computeRequiredKeys(): string[] {
+    const keys: string[] = [CacheKeys.appChannelAll, CacheKeys.appCategoryAll];
+
+    const adTypes = Object.keys(ADTYPE_POOLS) as AdType[];
+    for (const adType of adTypes) {
+      const placements = ADTYPE_POOLS[adType] ?? [];
+      for (const { scene, slot } of placements) {
+        keys.push(CacheKeys.appAdPool(scene, slot, adType));
+      }
+    }
+    return keys;
+  }
+
+  private async targetedRebuild(key: string): Promise<void> {
+    if (key === CacheKeys.appChannelAll) {
+      await this.cacheSync.rebuildChannelsAll();
+      return;
+    }
+    if (key === CacheKeys.appCategoryAll) {
+      await this.cacheSync.rebuildCategoriesAll();
+      return;
+    }
+    if (key.startsWith('app:adpool:')) {
+      // key format: app:adpool:<scene>:<slot>:<adType>
+      const parts = key.split(':');
+      // parts = ['app','adpool',scene,slot,adType]
+      if (parts.length === 5) {
+        const [, , scene, slot, adType] = parts;
+        await this.cacheSync.rebuildAdPoolForPlacement(
+          adType as AdType,
+          scene as any,
+          slot as any,
+        );
+        return;
+      }
+    }
+    // Fallback: run full warm (covers future new key types)
+    await this.cacheSync.warmCache();
+  }
+}

apps/box-mgnt-api/src/cache-sync/cache-sync.module.ts

@@ -2,13 +2,15 @@
 import { Module } from '@nestjs/common';
 import { CacheSyncService } from './cache-sync.service';
 import { CacheSyncDebugController } from './cache-sync-debug.controller';
+import { CacheChecklistService } from './cache-checklist.service';
+import { CacheChecklistController } from './cache-checklist.controller';
 
 @Module({
   imports: [
     // RedisModule is now global, no need to import
   ],
-  controllers: [CacheSyncDebugController],
-  providers: [CacheSyncService],
-  exports: [CacheSyncService],
+  controllers: [CacheSyncDebugController, CacheChecklistController],
+  providers: [CacheSyncService, CacheChecklistService],
+  exports: [CacheSyncService, CacheChecklistService],
 })
 export class CacheSyncModule {}

apps/box-mgnt-api/src/cache-sync/cache-sync.service.ts

@@ -27,9 +27,17 @@ const CATEGORY_CACHE_TTL = 900; // 15 min
 const AD_CACHE_TTL = 300; // 5 min (more dynamic)
 const AD_POOL_TTL = 300; // 5 min
 
+/**
+ * CacheSyncService
+ *  - Writes durable CacheSyncAction records in MySQL.
+ *  - Rebuilds Redis caches for channels/categories/ads/pools consumed by app-api.
+ *  - Retries transient failures with backoff using attempts + nextAttemptAt.
+ */
 @Injectable()
 export class CacheSyncService {
   private readonly logger = new Logger(CacheSyncService.name);
+  private readonly maxAttempts = 5;
+  private readonly baseBackoffMs = 5000; // initial retry delay
 
   private readonly actionHandlers: Partial<
     Record<CacheEntityType, (action: CacheSyncAction) => Promise<void>>
@@ -56,7 +64,8 @@ export class CacheSyncService {
   }
 
   /**
-   * Core generic scheduler.
+   * Enqueue a cache-sync action with optional initial delay.
+   * Downstream processing relies on attempts/nextAttemptAt for retries.
    */
   async scheduleAction(params: {
     entityType: CacheEntityType;
@@ -169,6 +178,11 @@ export class CacheSyncService {
     await this.processPendingOnce(50);
   }
 
+  /**
+   * Pull a batch of pending actions (whose nextAttemptAt <= now) and process
+   * them with retry/backoff. Keeps PENDING actions in the queue until either
+   * success or we exhaust maxAttempts (then we mark GAVE_UP).
+   */
   async processPendingOnce(limit = 20): Promise<void> {
     const now = this.nowBigInt();
 
@@ -198,12 +212,11 @@ export class CacheSyncService {
           err instanceof Error ? err.message : String(err ?? 'Unknown error');
 
         this.logger.error(
-          `Error processing CacheSyncAction id=${action.id}: ${message}`,
+          `Error processing CacheSyncAction id=${action.id} (attempt ${action.attempts + 1}/${this.maxAttempts}): ${message}`,
         );
 
         const attempts = action.attempts + 1;
-        const maxAttempts = 5;
-        const backoffMs = Math.min(60000, 5000 * attempts); // up to 60s
+        const backoffMs = this.calculateBackoffMs(attempts);
         const updateTime = this.nowBigInt();
         const nextAttemptAt = updateTime + BigInt(backoffMs);
 
@@ -211,7 +224,7 @@ export class CacheSyncService {
           where: { id: action.id },
           data: {
             status:
-              attempts >= maxAttempts
+              attempts >= this.maxAttempts
                 ? CacheStatus.GAVE_UP
                 : CacheStatus.PENDING,
             attempts,
@@ -220,6 +233,16 @@ export class CacheSyncService {
             updatedAt: updateTime,
           },
         });
+
+        if (attempts >= this.maxAttempts) {
+          this.logger.warn(
+            `CacheSyncAction id=${action.id} reached max attempts (${this.maxAttempts}) and will not be retried.`,
+          );
+        } else {
+          this.logger.debug(
+            `CacheSyncAction id=${action.id} scheduled to retry in ${backoffMs}ms (nextAttemptAt=${nextAttemptAt}).`,
+          );
+        }
       }
     }
   }
@@ -245,6 +268,15 @@ export class CacheSyncService {
     );
   }
 
+  /**
+   * Exponential backoff with light jitter so multiple workers don't retry in
+   * lockstep. Capped at 60s to avoid unbounded delays.
+   */
+  private calculateBackoffMs(attempts: number): number {
+    const jitter = Math.floor(Math.random() * 500);
+    return Math.min(60000, this.baseBackoffMs * attempts + jitter);
+  }
+
   private async markActionSuccess(action: CacheSyncAction): Promise<void> {
     await this.mysqlPrisma.cacheSyncAction.update({
       where: { id: action.id },
@@ -287,7 +319,8 @@ export class CacheSyncService {
     }
   }
 
-  private async rebuildChannelsAll(): Promise<void> {
+  // Made public so checklist service can invoke directly when a key is missing.
+  async rebuildChannelsAll(): Promise<void> {
     const channels = await this.mongoPrisma.channel.findMany({
       where: {
         // isDeleted: false,
@@ -358,7 +391,8 @@ export class CacheSyncService {
     }
   }
 
-  private async rebuildCategoriesAll(): Promise<void> {
+  // Made public so checklist service can invoke directly when a key is missing.
+  async rebuildCategoriesAll(): Promise<void> {
     const categories = await this.mongoPrisma.category.findMany({
       where: {
         // isDeleted: false,
@@ -432,6 +466,7 @@ export class CacheSyncService {
     );
   }
 
+  // Still private, only used internally for per-ad refresh logic.
   private async rebuildSingleAdCache(
     adId: string,
     adType?: string,
@@ -541,7 +576,13 @@ export class CacheSyncService {
     }
   }
 
-  private async rebuildAdPoolForPlacement(
+  /**
+   * Rebuild a single ad pool for a placement (scene + slot).
+   * Reads active ads for the adType and atomically swaps the cache key to avoid
+   * partially-written pools being read by app-api.
+   */
+  // Made public so checklist service can invoke targeted pool rebuild.
+  async rebuildAdPoolForPlacement(
     adType: AdType,
     scene: AdScene,
     slot: AdSlot,