chore: 清理归档文件和文档模板

删除不再需要的归档文件和过时的文档模板,包括多个README、安全策略、前端集成蓝图等文件,同时移除了未使用的业务文档和项目结构文件。

优化项目结构,移除冗余文件,保持代码库整洁。主要删除archive/handover目录下的多个文件及doc目录下的部分文档模板。
This commit is contained in:
2026-03-18 01:21:15 +08:00
parent 56b8a2e2f8
commit 72cd7f6f45
147 changed files with 5982 additions and 16716 deletions

View File

@@ -1,115 +0,0 @@
import { Job } from 'bullmq';
import { WorkerHub } from './WorkerHub';
import { CrawlerService } from '../services/CrawlerService';
import { AIService } from '../services/AIService';
import { FingerprintEngine } from '../core/ai/FingerprintEngine';
import { ProductService } from '../services/ProductService';
import { AuditService } from '../services/AuditService';
import { logger } from '../utils/logger';
/**
* [CORE_WORK_01] 采集 Worker (Crawler Worker)
* @description 异步执行产品抓取、多模态解析、指纹生成并入库,支持任务追踪与审计
*/
export class CrawlerWorker {
private static QUEUE_NAME = 'crawler-tasks';
/**
* 初始化并注册 Worker
*/
static init() {
WorkerHub.registerWorker(this.QUEUE_NAME, async (job: Job) => {
const { url, sandbox, traceContext } = job.data;
const { tenantId, shopId, taskId, traceId, userId } = traceContext;
logger.info(`[CrawlerWorker] Starting task ${job.id} for URL: ${url}`);
try {
// 1. 抓取
let productData = await CrawlerService.crawlProduct(url, { useSandbox: sandbox });
// 2. 多模态优化
const optimized = await AIService.analyzeMultiModalProduct({
title: productData.title || '',
description: productData.description,
attributes: productData.attributes || {},
imageUrls: productData.images || []
});
productData.title = optimized.optimizedTitle;
productData.description = optimized.optimizedDescription;
productData.attributes = { ...productData.attributes, ...optimized.validatedAttributes };
// 3. 指纹生成
const fingerprint = await FingerprintEngine.generateCompositeFingerprint({
title: productData.title,
description: productData.description,
mainImage: productData.mainImage || ''
});
// 4. 入库
const id = await ProductService.create({
...productData,
phash: fingerprint.phash,
semanticHash: fingerprint.semanticHash,
vectorEmbedding: JSON.stringify(fingerprint.vectorEmbedding),
status: 'draft'
});
// 5. 审计日志
await AuditService.log({
tenantId,
shopId,
taskId,
traceId,
userId,
module: 'SYNC',
action: 'CRAWLER_ASYNC_COMPLETE',
resourceType: 'product',
resourceId: String(id),
afterSnapshot: { url, id },
result: 'success',
source: 'node'
});
return { id, url, status: 'completed' };
} catch (err: any) {
logger.error(`[CrawlerWorker] Task ${job.id} failed: ${err.message}`);
// 错误审计
await AuditService.log({
tenantId,
shopId,
taskId,
traceId,
userId,
module: 'SYNC',
action: 'CRAWLER_ASYNC_FAILED',
resourceType: 'product',
resourceId: url,
result: 'failed',
errorCode: 'CRAWLER_WORKER_ERROR',
errorMessage: err.message,
source: 'node'
});
throw err;
}
}, 10); // 并发数限制为 10
}
/**
* 提交采集任务到队列
*/
static async submit(data: {
url: string;
sandbox?: boolean;
traceContext: any;
}) {
const queue = WorkerHub.getQueue(this.QUEUE_NAME);
return await queue.add(`crawl-${Date.now()}`, data, {
attempts: 3,
backoff: { type: 'exponential', delay: 1000 }
});
}
}