|
|
@@ -1,399 +0,0 @@
|
|
|
-// sync-videomedia.service.ts
|
|
|
-import { Injectable, BadRequestException, Logger } from '@nestjs/common';
|
|
|
-import { MongoPrismaService } from '@box/db/prisma/mongo-prisma.service';
|
|
|
-
|
|
|
-interface RawVideoMedia {
|
|
|
- id: string;
|
|
|
- srcId?: number;
|
|
|
- title?: string;
|
|
|
- checkSum?: string;
|
|
|
- type?: string;
|
|
|
- formatType?: number;
|
|
|
- contentType?: number;
|
|
|
- coverType?: number;
|
|
|
- coverImg?: string;
|
|
|
- coverImgNew?: string;
|
|
|
- videoTime?: number;
|
|
|
- publish?: string;
|
|
|
- country?: string;
|
|
|
- firstTag?: string;
|
|
|
- secondTags?: string[] | null;
|
|
|
- mediaSet?: string | null;
|
|
|
- preFileName?: string;
|
|
|
- status?: string;
|
|
|
- desc?: string;
|
|
|
- size?: number;
|
|
|
- bango?: string;
|
|
|
- actors?: string[] | null;
|
|
|
- studio?: string;
|
|
|
- addedTime: string;
|
|
|
- appids?: number[] | null;
|
|
|
- japanNames?: string[] | null;
|
|
|
- filename?: string;
|
|
|
- fieldNameFs?: string;
|
|
|
- ext?: string;
|
|
|
- taskId?: string;
|
|
|
- width?: number;
|
|
|
- height?: number;
|
|
|
- ratio?: number;
|
|
|
- frameRate?: string;
|
|
|
- syBitRate?: string;
|
|
|
- vidBitRate?: string;
|
|
|
- createdAt: string;
|
|
|
- updatedAt: string;
|
|
|
- proxyUpload?: number | null;
|
|
|
- isAdd?: boolean;
|
|
|
- retry?: number;
|
|
|
- notifySignal?: boolean;
|
|
|
- mergeRetry?: number;
|
|
|
- compressRetry?: number;
|
|
|
- segmentRetry?: number;
|
|
|
- linodeRetry?: number;
|
|
|
- failReason?: string;
|
|
|
- deleteDisk?: boolean;
|
|
|
- infoTsName?: string;
|
|
|
-}
|
|
|
-
|
|
|
-@Injectable()
|
|
|
-export class SyncVideomediaService {
|
|
|
- private readonly logger = new Logger(SyncVideomediaService.name);
|
|
|
-
|
|
|
- constructor(private readonly mongo: MongoPrismaService) {}
|
|
|
-
|
|
|
- async list() {
|
|
|
- return this.mongo.videoMedia.findMany();
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Sync video media from uploaded JSON.
|
|
|
- * Supports:
|
|
|
- * - { code, data: { list: [...] } }
|
|
|
- * - { list: [...] }
|
|
|
- * - [ ... ]
|
|
|
- */
|
|
|
- async syncFromJson(jsonData: unknown) {
|
|
|
- this.logger.log('[syncFromJson] syncFromJson called');
|
|
|
- const list = this.extractList(jsonData);
|
|
|
-
|
|
|
- this.logger.log(`[syncFromJson] Extracted ${list.length} items from JSON`);
|
|
|
-
|
|
|
- if (!list.length) {
|
|
|
- // No data to import; treat as client error for now
|
|
|
- throw new BadRequestException('JSON contains no video media records');
|
|
|
- }
|
|
|
-
|
|
|
- const normalized = list.map((item) => this.normalizeItem(item));
|
|
|
-
|
|
|
- this.logger.log(
|
|
|
- `[syncFromJson] Ready to import ${normalized.length} records`,
|
|
|
- );
|
|
|
- this.logger.debug('[syncFromJson] First record sample:', normalized[0]);
|
|
|
-
|
|
|
- const hasSecondTags = normalized.some(
|
|
|
- (v) => Array.isArray(v.secondTags) && v.secondTags.length > 0,
|
|
|
- );
|
|
|
-
|
|
|
- if (hasSecondTags) {
|
|
|
- // this.logger.log(
|
|
|
- // `[syncFromJson] Extracted secondTags from ${normalized.length} records`,
|
|
|
- // );
|
|
|
- await this.upsertSecondTagsFromVideos_NoUniqueName(normalized);
|
|
|
- }
|
|
|
-
|
|
|
- // Batch processing - try to create each record individually and catch duplicate errors
|
|
|
- const BATCH_SIZE = 100;
|
|
|
- let created = 0;
|
|
|
- let updated = 0;
|
|
|
- let skipped = 0;
|
|
|
- const errors: any[] = [];
|
|
|
-
|
|
|
- for (let i = 0; i < normalized.length; i += BATCH_SIZE) {
|
|
|
- const batch = normalized.slice(i, i + BATCH_SIZE);
|
|
|
- this.logger.debug(
|
|
|
- `[syncFromJson] Processing batch ${i / BATCH_SIZE + 1}, size: ${batch.length}`,
|
|
|
- );
|
|
|
-
|
|
|
- await Promise.all(
|
|
|
- batch.map(async (record) => {
|
|
|
- try {
|
|
|
- const exists = await this.mongo.videoMedia.findUnique({
|
|
|
- where: { id: record.id },
|
|
|
- select: { id: true },
|
|
|
- });
|
|
|
-
|
|
|
- if (exists?.id) {
|
|
|
- const { id, ...updateData } = record;
|
|
|
- await this.mongo.videoMedia.update({
|
|
|
- where: { id },
|
|
|
- data: updateData,
|
|
|
- });
|
|
|
- updated++;
|
|
|
- this.logger.debug(`[syncFromJson] Updated record: ${id}`);
|
|
|
- return;
|
|
|
- }
|
|
|
-
|
|
|
- await this.mongo.videoMedia.create({ data: record });
|
|
|
- created++;
|
|
|
- this.logger.debug(`[syncFromJson] Created record: ${record.id}`);
|
|
|
- } catch (error: any) {
|
|
|
- this.logger.error(
|
|
|
- `[syncFromJson] Failed for ${record.id}: ${error.message}`,
|
|
|
- );
|
|
|
- skipped++;
|
|
|
- errors.push({ id: record.id, error: error.message });
|
|
|
- }
|
|
|
- }),
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- this.logger.log(
|
|
|
- `[syncFromJson] Import complete: ${created} created, ${updated} updated, ${skipped} skipped`,
|
|
|
- );
|
|
|
- if (errors.length > 0) {
|
|
|
- this.logger.log(`[syncFromJson] Errors:`, errors.slice(0, 5));
|
|
|
- }
|
|
|
-
|
|
|
- return {
|
|
|
- imported: normalized.length,
|
|
|
- created,
|
|
|
- updated,
|
|
|
- skipped,
|
|
|
- errors: errors.length > 0 ? errors.slice(0, 10) : undefined,
|
|
|
- };
|
|
|
- }
|
|
|
-
|
|
|
- private async upsertSecondTagsFromVideos_NoUniqueName(
|
|
|
- normalizedVideos: Array<{ secondTags?: string[] }>,
|
|
|
- ): Promise<any> {
|
|
|
- try {
|
|
|
- const set = new Set<string>();
|
|
|
-
|
|
|
- for (const v of normalizedVideos) {
|
|
|
- const tags = v.secondTags ?? [];
|
|
|
- for (const t of tags) {
|
|
|
- if (typeof t !== 'string') continue;
|
|
|
- const name = t.trim();
|
|
|
- if (!name) continue;
|
|
|
- set.add(name);
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- // this.logger.log(
|
|
|
- // `[upsertSecondTagsFromVideos] secondTags found in: ${normalizedVideos}`,
|
|
|
- // );
|
|
|
-
|
|
|
- const names = Array.from(set);
|
|
|
- if (!names.length)
|
|
|
- return { unique: 0, upserted: 0, skipped: 0, errors: [] };
|
|
|
-
|
|
|
- // Concurrency limit to reduce race collisions and DB pressure
|
|
|
- const CONCURRENCY = 20;
|
|
|
- let idx = 0;
|
|
|
-
|
|
|
- let upserted = 0;
|
|
|
- let skipped = 0;
|
|
|
- const errors: Array<{ name: string; error: string }> = [];
|
|
|
-
|
|
|
- const worker = async () => {
|
|
|
- while (true) {
|
|
|
- const current = idx;
|
|
|
- idx += 1;
|
|
|
- if (current >= names.length) return;
|
|
|
-
|
|
|
- const name = names[current];
|
|
|
-
|
|
|
- try {
|
|
|
- // 1) check existence by name (NOT unique)
|
|
|
- const exists = await this.mongo.tag.findFirst({
|
|
|
- where: { name },
|
|
|
- select: { id: true },
|
|
|
- });
|
|
|
-
|
|
|
- if (exists?.id) {
|
|
|
- // already exists
|
|
|
- continue;
|
|
|
- }
|
|
|
-
|
|
|
- // 2) create if not exists
|
|
|
- await this.mongo.tag.create({
|
|
|
- data: {
|
|
|
- name,
|
|
|
- // If your Tag schema requires seconds fields:
|
|
|
- // createdAt: Math.floor(Date.now() / 1000),
|
|
|
- // updatedAt: Math.floor(Date.now() / 1000),
|
|
|
- },
|
|
|
- });
|
|
|
-
|
|
|
- upserted += 1;
|
|
|
- } catch (e: any) {
|
|
|
- // If another worker created it after our check, create may fail (duplicate on some index)
|
|
|
- // We treat that as skipped (safe).
|
|
|
- const msg = e?.message ?? 'Tag create failed';
|
|
|
- skipped += 1;
|
|
|
- errors.push({ name, error: msg });
|
|
|
- }
|
|
|
- }
|
|
|
- };
|
|
|
-
|
|
|
- await Promise.all(Array.from({ length: CONCURRENCY }, () => worker()));
|
|
|
-
|
|
|
- if (errors.length) {
|
|
|
- this.logger.warn(
|
|
|
- `[upsertSecondTagsFromVideos] errors=${errors.length}, sample=${JSON.stringify(
|
|
|
- errors.slice(0, 3),
|
|
|
- )}`,
|
|
|
- );
|
|
|
- } else {
|
|
|
- this.logger.log(
|
|
|
- `[upsertSecondTagsFromVideos] unique=${names.length} created=${upserted}`,
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- return { unique: names.length, upserted, skipped, errors };
|
|
|
- } catch (error: any) {
|
|
|
- const message = error?.message ?? 'Unhandled tag upsert error';
|
|
|
- const trace = error?.stack ?? undefined;
|
|
|
- this.logger.error(
|
|
|
- `[upsertSecondTagsFromVideos_NoUniqueName] ${message}`,
|
|
|
- trace,
|
|
|
- );
|
|
|
- return {
|
|
|
- unique: 0,
|
|
|
- upserted: 0,
|
|
|
- skipped: 0,
|
|
|
- errors: [{ name: 'global', error: message }],
|
|
|
- };
|
|
|
- }
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Extracts the list of items from different possible JSON shapes.
|
|
|
- */
|
|
|
- private extractList(jsonData: unknown): RawVideoMedia[] {
|
|
|
- const data = jsonData as any;
|
|
|
-
|
|
|
- if (Array.isArray(data)) {
|
|
|
- return data as RawVideoMedia[];
|
|
|
- }
|
|
|
-
|
|
|
- if (data?.data?.list && Array.isArray(data.data.list)) {
|
|
|
- return data.data.list as RawVideoMedia[];
|
|
|
- }
|
|
|
-
|
|
|
- if (data?.list && Array.isArray(data.list)) {
|
|
|
- return data.list as RawVideoMedia[];
|
|
|
- }
|
|
|
-
|
|
|
- throw new BadRequestException(
|
|
|
- 'Invalid JSON structure: expected array, data.list, or list',
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- /**
|
|
|
- * Maps RawVideoMedia to Prisma videoMedia create/update input.
|
|
|
- * Applies defaults and type conversions to match the Prisma model.
|
|
|
- */
|
|
|
- private normalizeItem(item: RawVideoMedia) {
|
|
|
- // Basic validation
|
|
|
- if (!item.id) {
|
|
|
- throw new BadRequestException('Each item must have an id');
|
|
|
- }
|
|
|
- if (!item.addedTime || !item.createdAt || !item.updatedAt) {
|
|
|
- throw new BadRequestException(
|
|
|
- `Item ${item.id} is missing required datetime fields`,
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- const addedTime = new Date(item.addedTime);
|
|
|
- const createdAt = new Date(item.createdAt);
|
|
|
- const updatedAt = new Date(item.updatedAt);
|
|
|
-
|
|
|
- if (
|
|
|
- isNaN(addedTime.getTime()) ||
|
|
|
- isNaN(createdAt.getTime()) ||
|
|
|
- isNaN(updatedAt.getTime())
|
|
|
- ) {
|
|
|
- throw new BadRequestException(
|
|
|
- `Item ${item.id} has invalid datetime format`,
|
|
|
- );
|
|
|
- }
|
|
|
-
|
|
|
- return {
|
|
|
- id: item.id, // String mapped to Mongo ObjectId via @db.ObjectId
|
|
|
-
|
|
|
- srcId: item.srcId ?? 0,
|
|
|
- title: item.title ?? '',
|
|
|
- checkSum: item.checkSum ?? '',
|
|
|
- type: item.type ?? '',
|
|
|
-
|
|
|
- formatType: item.formatType ?? 0,
|
|
|
- contentType: item.contentType ?? 0,
|
|
|
-
|
|
|
- coverType: item.coverType ?? 0,
|
|
|
- coverImg: item.coverImg ?? '',
|
|
|
- coverImgNew: item.coverImgNew ?? '',
|
|
|
-
|
|
|
- videoTime: item.videoTime ?? 0,
|
|
|
-
|
|
|
- publish: item.publish ?? '',
|
|
|
- country: item.country ?? '',
|
|
|
- firstTag: item.firstTag ?? '',
|
|
|
-
|
|
|
- // null → []
|
|
|
- secondTags: item.secondTags ?? [],
|
|
|
-
|
|
|
- // null → ""
|
|
|
- mediaSet: item.mediaSet ?? '',
|
|
|
-
|
|
|
- preFileName: item.preFileName ?? '',
|
|
|
-
|
|
|
- status: item.status ?? '',
|
|
|
- desc: item.desc ?? '',
|
|
|
-
|
|
|
- // number → BigInt
|
|
|
- size: BigInt(item.size ?? 0),
|
|
|
-
|
|
|
- bango: item.bango ?? '',
|
|
|
-
|
|
|
- // null → []
|
|
|
- actors: item.actors ?? [],
|
|
|
-
|
|
|
- studio: item.studio ?? '',
|
|
|
-
|
|
|
- addedTime,
|
|
|
- appids: item.appids ?? [],
|
|
|
-
|
|
|
- japanNames: item.japanNames ?? [],
|
|
|
-
|
|
|
- filename: item.filename ?? '',
|
|
|
- fieldNameFs: item.fieldNameFs ?? '',
|
|
|
- ext: item.ext ?? '',
|
|
|
- taskId: item.taskId ?? '',
|
|
|
-
|
|
|
- width: item.width ?? 0,
|
|
|
- height: item.height ?? 0,
|
|
|
- ratio: item.ratio ?? 0,
|
|
|
-
|
|
|
- frameRate: item.frameRate ?? '',
|
|
|
- syBitRate: item.syBitRate ?? '',
|
|
|
- vidBitRate: item.vidBitRate ?? '',
|
|
|
-
|
|
|
- proxyUpload: item.proxyUpload ?? 0,
|
|
|
- isAdd: item.isAdd ?? false,
|
|
|
- retry: item.retry ?? 0,
|
|
|
- notifySignal: item.notifySignal ?? false,
|
|
|
-
|
|
|
- mergeRetry: item.mergeRetry ?? 0,
|
|
|
- compressRetry: item.compressRetry ?? 0,
|
|
|
- segmentRetry: item.segmentRetry ?? 0,
|
|
|
- linodeRetry: item.linodeRetry ?? 0,
|
|
|
-
|
|
|
- failReason: item.failReason ?? '',
|
|
|
- deleteDisk: item.deleteDisk ?? false,
|
|
|
- infoTsName: item.infoTsName ?? '',
|
|
|
-
|
|
|
- createdAt,
|
|
|
- updatedAt,
|
|
|
- };
|
|
|
- }
|
|
|
-}
|