Browse Source

feat(stats-aggregation): update hourly aggregation interval and enhance time handling

Dave 1 month ago
parent
commit
ca9da70df3

+ 32 - 5
apps/box-stats-api/src/feature/stats-events/stats-aggregation.scheduler.ts

@@ -34,9 +34,10 @@ export class StatsAggregationScheduler implements OnModuleInit {
   // guardrails: avoid overlapping runs + spam
   private runningHourly = false;
   private runningDaily = false;
+  private lastHourlyWindowToSec?: number;
 
   // run a little after boundary so late events can land
-  private runDelaySec = 90;
+  private runDelaySec = 300;
 
   constructor(
     private readonly configService: ConfigService,
@@ -86,7 +87,7 @@ export class StatsAggregationScheduler implements OnModuleInit {
       this.logger.log(
         `📊 Stats aggregation scheduler enabled (windowDays=${
           this.windowDays ?? 'all time'
-        }, hourly=${CronExpression.EVERY_HOUR}, daily=00:10, delaySec=${this.runDelaySec})`,
+        }, hourly=${CronExpression.EVERY_5_MINUTES}, daily=00:10, delaySec=${this.runDelaySec})`,
       );
     } else {
       this.logger.warn(
@@ -99,7 +100,7 @@ export class StatsAggregationScheduler implements OnModuleInit {
    * Hourly is the unit of truth.
    * Runs slightly after the hour, but still processes the previous full hour window.
    */
-  @Cron(CronExpression.EVERY_HOUR, { name: 'stats-aggregation-hourly' })
+  @Cron(CronExpression.EVERY_5_MINUTES, { name: 'stats-aggregation-hourly' })
   async runHourly(): Promise<void> {
     if (!this.enabled) return;
 
@@ -114,9 +115,26 @@ export class StatsAggregationScheduler implements OnModuleInit {
     const startedAt = Date.now();
 
     const nowSec = Math.floor(Date.now() / 1000);
-    const effectiveNowSec = nowSec - this.runDelaySec;
+    const currentHourStartSec = this.floorToHourGmt8(nowSec);
+    const gateOpenSec = currentHourStartSec + this.runDelaySec;
 
-    const toSec = this.floorToHour(effectiveNowSec);
+    if (nowSec < gateOpenSec) {
+      this.logger.debug(
+        `⏰ Hourly aggregation gate not yet open (now=${nowSec}, openAt=${gateOpenSec}, delaySec=${this.runDelaySec}).`,
+      );
+      this.runningHourly = false;
+      return;
+    }
+
+    if (this.lastHourlyWindowToSec === currentHourStartSec) {
+      this.logger.debug(
+        `⏰ Hourly aggregation already processed window [${currentHourStartSec - 3600},${currentHourStartSec}) this hour.`,
+      );
+      this.runningHourly = false;
+      return;
+    }
+
+    const toSec = currentHourStartSec;
     const fromSec = toSec - 3600;
     const tag = `⏰ Hourly aggregation [${fromSec},${toSec})`;
 
@@ -153,6 +171,8 @@ export class StatsAggregationScheduler implements OnModuleInit {
         this.logger.log(`${tag} daily refresh done`);
       }
 
+      this.lastHourlyWindowToSec = toSec;
+
       const ms = Date.now() - startedAt;
       this.logger.log(`${tag} ✅ done in ${ms}ms`);
     } catch (err: any) {
@@ -221,6 +241,13 @@ export class StatsAggregationScheduler implements OnModuleInit {
     return sec - (sec % 3600);
   }
 
+  private floorToHourGmt8(secUtc: number): number {
+    const shift = 8 * 3600;
+    const shifted = secUtc + shift;
+    const hourStartShifted = shifted - (shifted % 3600);
+    return hourStartShifted - shift;
+  }
+
   /**
    * GMT+8 day bucket start in UTC seconds:
    * shift +8h, floor to day, shift back

+ 2 - 2
apps/box-stats-api/src/feature/stats-events/stats-aggregation.service.ts

@@ -1,7 +1,7 @@
 // box-stats-api/src/feature/stats-events/stats-aggregation.service.ts
 import { Injectable, Logger } from '@nestjs/common';
 import { PrismaMongoService } from '../../prisma/prisma-mongo.service';
-import { nowEpochMsBigInt } from '@box/common/time/time.util';
+import { nowSecBigInt } from '@box/common/time/time.util';
 
 interface AggregationOptions {
   windowDays?: number; // kept for API compatibility, but ignored per new requirements
@@ -172,7 +172,7 @@ export class StatsAggregationService {
     const lastSeenAt = lastRow?.clickedAt as bigint | undefined;
     if (!firstSeenAt || !lastSeenAt) return;
 
-    const now = nowEpochMsBigInt();
+    const now = nowSecBigInt();
     const clicksBig = BigInt(clicks);
 
     await client.adsGlobalStats.upsert({