(김준회) nonsap sync 배치 처리(크론잡 임시 주석처리), 메뉴 스크롤 처리, 벤더풀 임포트 복호화 처리,

1 files changed, 64 insertions, 13 deletions

diff --git a/lib/nonsap-sync/enhanced-sync-service.ts b/lib/nonsap-sync/enhanced-sync-service.ts
index 5c07cafc..5d45d3ad 100644
--- a/lib/nonsap-sync/enhanced-sync-service.ts
+++ b/lib/nonsap-sync/enhanced-sync-service.ts
@@ -10,6 +10,10 @@ import * as nonsapSchema from '@/db/schema/NONSAP/nonsap';
 // 동기화할 테이블 목록 (단순화)
 const TARGET_TABLES: TableName[] = ['CMCTB_CDNM', 'CMCTB_CD'];
 
+// 배치 처리 설정
+const BATCH_SIZE = 1000; // 한 번에 처리할 레코드 수
+const BATCH_DELAY = 100; // 배치 간 대기 시간 (ms)
+
 // 간단한 로거
 const logger = {
   info: (message: string, ...args: unknown[]) => console.log(`[NONSAP-SYNC] ${message}`, ...args),
@@ -19,19 +23,51 @@ const logger = {
 };
 
 /**
- * Oracle에서 테이블 데이터 조회 (단순화)
+ * Oracle에서 테이블 데이터 조회 (배치 처리)
  */
 async function fetchOracleData<T extends TableName>(tableName: T): Promise<DatabaseSchema[T][]> {
   try {
-    const query = `SELECT * FROM ${tableName}`;
-    const result = await oracleKnex.raw(query);
-
-    // Oracle knex raw 결과에서 실제 데이터 추출
-    const rows = Array.isArray(result) ? result : result.rows || [];
-    const cleanResults = rows.map((row: any) => row);
+    // 먼저 총 레코드 수 확인
+    const countQuery = `SELECT COUNT(*) as total FROM ${tableName}`;
+    const countResult = await oracleKnex.raw(countQuery);
+    const totalCount = countResult[0]?.TOTAL || 0;
+    
+    logger.info(`Total records in ${tableName}: ${totalCount}`);
+    
+    if (totalCount === 0) {
+      return [];
+    }
+    
+    // 배치로 데이터 조회
+    const allResults: DatabaseSchema[T][] = [];
+    const totalBatches = Math.ceil(totalCount / BATCH_SIZE);
+    
+    for (let i = 0; i < totalBatches; i++) {
+      const offset = i * BATCH_SIZE;
+      const query = `
+        SELECT * FROM (
+          SELECT a.*, ROWNUM rnum FROM (
+            SELECT * FROM ${tableName}
+          ) a WHERE ROWNUM <= ${offset + BATCH_SIZE}
+        ) WHERE rnum > ${offset}
+      `;
+      
+      const result = await oracleKnex.raw(query);
+      const rows = Array.isArray(result) ? result : result.rows || [];
+      const cleanResults = rows.map((row: any) => row);
+      
+      allResults.push(...cleanResults);
+      
+      logger.info(`Fetched batch ${i + 1}/${totalBatches} (${cleanResults.length} records) from ${tableName}`);
+      
+      // 배치 간 짧은 대기 (메모리 해제 시간)
+      if (i < totalBatches - 1) {
+        await new Promise(resolve => setTimeout(resolve, BATCH_DELAY));
+      }
+    }
 
-    logger.info(`Fetched ${cleanResults.length} records from ${tableName}`);
-    return cleanResults as DatabaseSchema[T][];
+    logger.info(`Total fetched ${allResults.length} records from ${tableName}`);
+    return allResults as DatabaseSchema[T][];
   } catch (error) {
     logger.error(`Error fetching data from ${tableName}:`, error);
     throw error;
@@ -63,7 +99,7 @@ function normalizeRecord(record: any, tableSchema: any): any {
 }
 
 /**
- * PostgreSQL에 데이터 삽입 (삭제 후 재삽입)
+ * PostgreSQL에 데이터 삽입 (배치 처리)
  */
 async function syncToPostgres<T extends TableName>(
   tableName: T,
@@ -102,9 +138,24 @@ async function syncToPostgres<T extends TableName>(
       DELETE FROM ${sql.identifier('nonsap')}.${sql.identifier(tableNameLower)}
     `);
 
-    // 2. 새 데이터 모두 삽입
-    logger.info(`${tableName} - Inserting ${cleanData.length} new records`);
-    await db.insert(tableSchema as any).values(cleanData);
+    // 2. 배치로 새 데이터 삽입
+    const totalBatches = Math.ceil(cleanData.length / BATCH_SIZE);
+    logger.info(`${tableName} - Inserting ${cleanData.length} new records in ${totalBatches} batches`);
+
+    for (let i = 0; i < totalBatches; i++) {
+      const start = i * BATCH_SIZE;
+      const end = Math.min(start + BATCH_SIZE, cleanData.length);
+      const batch = cleanData.slice(start, end);
+
+      await db.insert(tableSchema as any).values(batch);
+      
+      logger.info(`${tableName} - Inserted batch ${i + 1}/${totalBatches} (${batch.length} records)`);
+      
+      // 배치 간 짧은 대기 (메모리 해제 시간)
+      if (i < totalBatches - 1) {
+        await new Promise(resolve => setTimeout(resolve, BATCH_DELAY));
+      }
+    }
 
     logger.success(`Successfully synced ${cleanData.length} records for ${tableName}`);
   } catch (error) {