diff options
Diffstat (limited to 'lib')
| -rw-r--r-- | lib/nonsap-sync/enhanced-sync-service.ts | 77 | ||||
| -rw-r--r-- | lib/vendor-pool/table/vendor-pool-excel-import-button.tsx | 22 |
2 files changed, 82 insertions, 17 deletions
diff --git a/lib/nonsap-sync/enhanced-sync-service.ts b/lib/nonsap-sync/enhanced-sync-service.ts index 5c07cafc..5d45d3ad 100644 --- a/lib/nonsap-sync/enhanced-sync-service.ts +++ b/lib/nonsap-sync/enhanced-sync-service.ts @@ -10,6 +10,10 @@ import * as nonsapSchema from '@/db/schema/NONSAP/nonsap'; // 동기화할 테이블 목록 (단순화) const TARGET_TABLES: TableName[] = ['CMCTB_CDNM', 'CMCTB_CD']; +// 배치 처리 설정 +const BATCH_SIZE = 1000; // 한 번에 처리할 레코드 수 +const BATCH_DELAY = 100; // 배치 간 대기 시간 (ms) + // 간단한 로거 const logger = { info: (message: string, ...args: unknown[]) => console.log(`[NONSAP-SYNC] ${message}`, ...args), @@ -19,19 +23,51 @@ const logger = { }; /** - * Oracle에서 테이블 데이터 조회 (단순화) + * Oracle에서 테이블 데이터 조회 (배치 처리) */ async function fetchOracleData<T extends TableName>(tableName: T): Promise<DatabaseSchema[T][]> { try { - const query = `SELECT * FROM ${tableName}`; - const result = await oracleKnex.raw(query); - - // Oracle knex raw 결과에서 실제 데이터 추출 - const rows = Array.isArray(result) ? result : result.rows || []; - const cleanResults = rows.map((row: any) => row); + // 먼저 총 레코드 수 확인 + const countQuery = `SELECT COUNT(*) as total FROM ${tableName}`; + const countResult = await oracleKnex.raw(countQuery); + const totalCount = countResult[0]?.TOTAL || 0; + + logger.info(`Total records in ${tableName}: ${totalCount}`); + + if (totalCount === 0) { + return []; + } + + // 배치로 데이터 조회 + const allResults: DatabaseSchema[T][] = []; + const totalBatches = Math.ceil(totalCount / BATCH_SIZE); + + for (let i = 0; i < totalBatches; i++) { + const offset = i * BATCH_SIZE; + const query = ` + SELECT * FROM ( + SELECT a.*, ROWNUM rnum FROM ( + SELECT * FROM ${tableName} + ) a WHERE ROWNUM <= ${offset + BATCH_SIZE} + ) WHERE rnum > ${offset} + `; + + const result = await oracleKnex.raw(query); + const rows = Array.isArray(result) ? result : result.rows || []; + const cleanResults = rows.map((row: any) => row); + + allResults.push(...cleanResults); + + logger.info(`Fetched batch ${i + 1}/${totalBatches} (${cleanResults.length} records) from ${tableName}`); + + // 배치 간 짧은 대기 (메모리 해제 시간) + if (i < totalBatches - 1) { + await new Promise(resolve => setTimeout(resolve, BATCH_DELAY)); + } + } - logger.info(`Fetched ${cleanResults.length} records from ${tableName}`); - return cleanResults as DatabaseSchema[T][]; + logger.info(`Total fetched ${allResults.length} records from ${tableName}`); + return allResults as DatabaseSchema[T][]; } catch (error) { logger.error(`Error fetching data from ${tableName}:`, error); throw error; @@ -63,7 +99,7 @@ function normalizeRecord(record: any, tableSchema: any): any { } /** - * PostgreSQL에 데이터 삽입 (삭제 후 재삽입) + * PostgreSQL에 데이터 삽입 (배치 처리) */ async function syncToPostgres<T extends TableName>( tableName: T, @@ -102,9 +138,24 @@ async function syncToPostgres<T extends TableName>( DELETE FROM ${sql.identifier('nonsap')}.${sql.identifier(tableNameLower)} `); - // 2. 새 데이터 모두 삽입 - logger.info(`${tableName} - Inserting ${cleanData.length} new records`); - await db.insert(tableSchema as any).values(cleanData); + // 2. 배치로 새 데이터 삽입 + const totalBatches = Math.ceil(cleanData.length / BATCH_SIZE); + logger.info(`${tableName} - Inserting ${cleanData.length} new records in ${totalBatches} batches`); + + for (let i = 0; i < totalBatches; i++) { + const start = i * BATCH_SIZE; + const end = Math.min(start + BATCH_SIZE, cleanData.length); + const batch = cleanData.slice(start, end); + + await db.insert(tableSchema as any).values(batch); + + logger.info(`${tableName} - Inserted batch ${i + 1}/${totalBatches} (${batch.length} records)`); + + // 배치 간 짧은 대기 (메모리 해제 시간) + if (i < totalBatches - 1) { + await new Promise(resolve => setTimeout(resolve, BATCH_DELAY)); + } + } logger.success(`Successfully synced ${cleanData.length} records for ${tableName}`); } catch (error) { diff --git a/lib/vendor-pool/table/vendor-pool-excel-import-button.tsx b/lib/vendor-pool/table/vendor-pool-excel-import-button.tsx index 704d4aff..0f51170f 100644 --- a/lib/vendor-pool/table/vendor-pool-excel-import-button.tsx +++ b/lib/vendor-pool/table/vendor-pool-excel-import-button.tsx @@ -18,6 +18,7 @@ import { getAccessorKeyByHeader, vendorPoolExcelColumns } from '../excel-utils' +import { decryptWithServerAction } from '@/components/drm/drmUtils' interface ImportExcelProps { onSuccess?: () => void @@ -26,7 +27,7 @@ interface ImportExcelProps { export function ImportVendorPoolButton({ onSuccess }: ImportExcelProps) { const fileInputRef = useRef<HTMLInputElement>(null) const [isImporting, setIsImporting] = React.useState(false) - const { data: session, status } = useSession() + const { data: session } = useSession() // 헬퍼 함수들은 excel-utils에서 import @@ -37,10 +38,23 @@ export function ImportVendorPoolButton({ onSuccess }: ImportExcelProps) { setIsImporting(true) try { - // Read the Excel file using ExcelJS - const data = await file.arrayBuffer() + // DRM 복호화 처리 + toast.info("파일을 복호화하고 있습니다...") + let decryptedData: ArrayBuffer + + try { + decryptedData = await decryptWithServerAction(file) + toast.success("파일 복호화가 완료되었습니다.") + } catch (drmError) { + console.warn("DRM 복호화 실패, 원본 파일로 진행합니다:", drmError) + toast.warning("DRM 복호화에 실패했습니다. 원본 파일로 진행합니다.") + decryptedData = await file.arrayBuffer() + } + + // 복호화된 데이터로 ExcelJS 워크북 로드 + toast.info("엑셀 파일을 분석하고 있습니다...") const workbook = new ExcelJS.Workbook() - await workbook.xlsx.load(data) + await workbook.xlsx.load(decryptedData) // Get the first worksheet const worksheet = workbook.getWorksheet(1) |
