Spring 14: 스케줄링

@Configuration
@EnableScheduling
@EnableSchedulerLock(defaultLockAtMostFor = "10m")
public class ShedLockConfig {
    
    @Bean
    public LockProvider lockProvider(DataSource dataSource) {
        return new JdbcTemplateLockProvider(
            JdbcTemplateLockProvider.Configuration.builder()
                .withJdbcTemplate(new JdbcTemplate(dataSource))
                .usingDbTime() // DB 시간 사용
                .build()
        );
    }
}

-- 테이블 생성 (MySQL)
CREATE TABLE shedlock (
    name VARCHAR(64) NOT NULL,
    lock_until TIMESTAMP(3) NOT NULL,
    locked_at TIMESTAMP(3) NOT NULL DEFAULT CURRENT_TIMESTAMP(3),
    locked_by VARCHAR(255) NOT NULL,
    PRIMARY KEY (name)
);

@Component
public class DistributedScheduledTasks {
    
    private static final Logger log = LoggerFactory.getLogger(DistributedScheduledTasks.class);
    
    @Scheduled(cron = "0 0 2 * * ?")
    @SchedulerLock(name = "dailyDataCleanup", 
                   lockAtMostFor = "PT30M",    // 최대 30분 락 유지
                   lockAtLeastFor = "PT5M")    // 최소 5분 락 유지
    public void dailyDataCleanup() {
        log.info("Starting daily data cleanup - Instance: {}", getInstanceId());
        
        try {
            // 데이터 정리 로직
            cleanupOldLogs();
            cleanupTempFiles();
            
            log.info("Daily data cleanup completed successfully");
        } catch (Exception e) {
            log.error("Daily data cleanup failed", e);
            throw e; // 예외 발생 시 락 즉시 해제
        }
    }
    
    @Scheduled(fixedRate = 300000) // 5분마다
    @SchedulerLock(name = "systemHealthCheck", 
                   lockAtMostFor = "PT2M",
                   lockAtLeastFor = "PT30S")
    public void systemHealthCheck() {
        log.info("Performing system health check - Instance: {}", getInstanceId());
        
        // 시스템 상태 점검 로직
        checkDatabaseConnection();
        checkExternalServices();
        checkDiskSpace();
    }
    
    @Scheduled(cron = "0 0 0 1 * ?") // 매월 1일
    @SchedulerLock(name = "monthlyReport", 
                   lockAtMostFor = "PT2H",
                   lockAtLeastFor = "PT10M")
    public void generateMonthlyReport() {
        log.info("Generating monthly report - Instance: {}", getInstanceId());
        
        // 월간 리포트 생성 로직
        generateSalesReport();
        generateUserActivityReport();
        sendReportToManagement();
    }
    
    private String getInstanceId() {
        return System.getProperty("instance.id", "unknown");
    }
}

@Component
public class LeaderElectionScheduler {
    
    private final LeaderElectionService leaderElectionService;
    private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1);
    
    @PostConstruct
    public void startLeaderElection() {
        // 리더 선출 프로세스 시작
        scheduler.scheduleAtFixedRate(this::electLeader, 0, 30, TimeUnit.SECONDS);
    }
    
    private void electLeader() {
        try {
            if (leaderElectionService.tryBecomeLeader()) {
                log.info("This instance is now the leader");
                executeLeaderTasks();
            } else {
                log.debug("This instance is a follower");
            }
        } catch (Exception e) {
            log.error("Leader election failed", e);
        }
    }
    
    private void executeLeaderTasks() {
        // 리더만 실행하는 스케줄 작업들
        if (leaderElectionService.isLeader()) {
            performCriticalBatchJob();
            sendSystemNotifications();
            updateGlobalConfiguration();
        }
    }
}

@Service
public class DatabaseLeaderElectionService implements LeaderElectionService {
    
    private final JdbcTemplate jdbcTemplate;
    private final String instanceId;
    private volatile boolean isLeader = false;
    
    public DatabaseLeaderElectionService(JdbcTemplate jdbcTemplate) {
        this.jdbcTemplate = jdbcTemplate;
        this.instanceId = generateInstanceId();
    }
    
    @Override
    public boolean tryBecomeLeader() {
        try {
            // 현재 리더 확인
            String currentLeader = getCurrentLeader();
            
            if (currentLeader == null || isLeaderExpired(currentLeader)) {
                // 리더가 없거나 만료된 경우 리더 등록 시도
                int updated = jdbcTemplate.update(
                    "INSERT INTO leader_election (id, instance_id, last_heartbeat) " +
                    "VALUES ('SCHEDULER_LEADER', ?, NOW()) " +
                    "ON DUPLICATE KEY UPDATE instance_id = ?, last_heartbeat = NOW()",
                    instanceId, instanceId
                );
                
                isLeader = updated > 0;
                return isLeader;
            } else if (currentLeader.equals(instanceId)) {
                // 이미 리더인 경우 하트비트 업데이트
                jdbcTemplate.update(
                    "UPDATE leader_election SET last_heartbeat = NOW() WHERE id = 'SCHEDULER_LEADER'");
                isLeader = true;
                return true;
            }
            
            isLeader = false;
            return false;
        } catch (Exception e) {
            log.error("Leader election failed", e);
            isLeader = false;
            return false;
        }
    }
    
    private String getCurrentLeader() {
        try {
            return jdbcTemplate.queryForObject(
                "SELECT instance_id FROM leader_election WHERE id = 'SCHEDULER_LEADER'",
                String.class
            );
        } catch (EmptyResultDataAccessException e) {
            return null;
        }
    }
    
    private boolean isLeaderExpired(String leaderId) {
        try {
            Integer count = jdbcTemplate.queryForObject(
                "SELECT COUNT(*) FROM leader_election " +
                "WHERE id = 'SCHEDULER_LEADER' AND last_heartbeat > DATE_SUB(NOW(), INTERVAL 60 SECOND)",
                Integer.class
            );
            return count == 0;
        } catch (Exception e) {
            return true; // 오류 시 만료된 것으로 간주
        }
    }
}

@Configuration
public class RedisLockConfig {
    
    @Bean
    public LockProvider lockProvider(RedisConnectionFactory connectionFactory) {
        return new RedisLockProvider(connectionFactory);
    }
}

@Component
public class RedisDistributedScheduler {
    
    private final RedisTemplate<String, String> redisTemplate;
    private final String instanceId;
    
    public RedisDistributedScheduler(RedisTemplate<String, String> redisTemplate) {
        this.redisTemplate = redisTemplate;
        this.instanceId = InetAddress.getLocalHost().getHostName() + "-" + UUID.randomUUID();
    }
    
    @Scheduled(fixedRate = 60000) // 1분마다
    public void distributedTask() {
        String lockKey = "scheduled:task:lock";
        String lockValue = instanceId + ":" + System.currentTimeMillis();
        
        try {
            // Redis SET NX EX 명령으로 분산 락 획득
            Boolean lockAcquired = redisTemplate.opsForValue()
                .setIfAbsent(lockKey, lockValue, Duration.ofMinutes(5));
            
            if (Boolean.TRUE.equals(lockAcquired)) {
                log.info("Lock acquired by instance: {}", instanceId);
                
                try {
                    // 실제 작업 실행
                    executeScheduledTask();
                } finally {
                    // 락 해제 (Lua 스크립트로 안전하게)
                    releaseLock(lockKey, lockValue);
                }
            } else {
                log.debug("Lock not acquired, another instance is running the task");
            }
        } catch (Exception e) {
            log.error("Distributed task execution failed", e);
        }
    }
    
    private void releaseLock(String lockKey, String lockValue) {
        String luaScript = 
            "if redis.call('get', KEYS[1]) == ARGV[1] then " +
            "    return redis.call('del', KEYS[1]) " +
            "else " +
            "    return 0 " +
            "end";
        
        redisTemplate.execute(
            RedisScript.of(luaScript, Long.class),
            Collections.singletonList(lockKey),
            lockValue
        );
    }
    
    private void executeScheduledTask() {
        // 실제 스케줄 작업 로직
        log.info("Executing distributed scheduled task");
        
        // 예: 데이터 처리, 리포트 생성 등
        processData();
        generateReports();
        cleanupResources();
    }
}

@Component
public class RobustScheduledTasks {
    
    private static final Logger log = LoggerFactory.getLogger(RobustScheduledTasks.class);
    private final EmailService emailService;
    private final MetricsService metricsService;
    
    @Scheduled(fixedRate = 300000) // 5분마다
    public void robustDataProcessing() {
        String taskName = "dataProcessing";
        long startTime = System.currentTimeMillis();
        
        try {
            log.info("Starting {} task", taskName);
            
            // 실제 작업 실행
            processData();
            
            // 성공 메트릭 기록
            metricsService.recordTaskSuccess(taskName, System.currentTimeMillis() - startTime);
            log.info("Task {} completed successfully", taskName);
            
        } catch (RetryableException e) {
            // 재시도 가능한 예외 (네트워크 오류, 일시적 DB 오류 등)
            log.warn("Retryable error in task {}: {}", taskName, e.getMessage());
            handleRetryableException(taskName, e);
            
        } catch (NonRetryableException e) {
            // 재시도 불가능한 예외 (설정 오류, 데이터 형식 오류 등)
            log.error("Non-retryable error in task {}: {}", taskName, e.getMessage(), e);
            handleNonRetryableException(taskName, e);
            
        } catch (Exception e) {
            // 예상치 못한 예외
            log.error("Unexpected error in task {}: {}", taskName, e.getMessage(), e);
            handleUnexpectedException(taskName, e);
            
        } finally {
            // 정리 작업
            cleanupResources();
            metricsService.recordTaskExecution(taskName);
        }
    }
    
    private void handleRetryableException(String taskName, Exception e) {
        // 재시도 로직 (별도 스레드에서 실행)
        CompletableFuture.runAsync(() -> {
            try {
                Thread.sleep(60000); // 1분 대기
                retryTask(taskName);
            } catch (InterruptedException ie) {
                Thread.currentThread().interrupt();
            }
        });
        
        metricsService.recordTaskRetry(taskName);
    }
    
    private void handleNonRetryableException(String taskName, Exception e) {
        // 즉시 알림 발송
        String alertMessage = String.format(
            "Task %s failed with non-retryable error: %s", 
            taskName, e.getMessage()
        );
        emailService.sendAlert("Critical Task Failure", alertMessage);
        metricsService.recordTaskFailure(taskName, "non_retryable");
    }
    
    private void handleUnexpectedException(String taskName, Exception e) {
        // 상세 정보와 함께 알림 발송
        String alertMessage = String.format(
            "Task %s failed with unexpected error: %s\nStack trace: %s", 
            taskName, e.getMessage(), getStackTrace(e)
        );
        emailService.sendAlert("Unexpected Task Failure", alertMessage);
        metricsService.recordTaskFailure(taskName, "unexpected");
    }
}

@Component
public class RetryableScheduledTasks {
    
    private final RetryTemplate retryTemplate;
    
    public RetryableScheduledTasks() {
        this.retryTemplate = RetryTemplate.builder()
            .maxAttempts(3)
            .exponentialBackoff(1000, 2, 10000) // 1초부터 시작, 2배씩 증가, 최대 10초
            .retryOn(RetryableException.class)
            .build();
    }
    
    @Scheduled(cron = "0 */10 * * * ?") // 10분마다
    public void externalApiCall() {
        try {
            retryTemplate.execute(context -> {
                log.info("Attempting external API call (attempt: {})", 
                    context.getRetryCount() + 1);
                
                // 외부 API 호출
                callExternalService();
                return null;
            });
        } catch (Exception e) {
            log.error("All retry attempts failed for external API call", e);
            handleFinalFailure("externalApiCall", e);
        }
    }
    
    @Scheduled(fixedDelay = 60000) // 1분 간격
    public void databaseOperation() {
        RetryTemplate dbRetryTemplate = RetryTemplate.builder()
            .maxAttempts(5)
            .fixedBackoff(2000) // 2초 고정 대기
            .retryOn(DataAccessException.class, SQLException.class)
            .build();
        
        try {
            dbRetryTemplate.execute(context -> {
                if (context.getRetryCount() > 0) {
                    log.warn("Retrying database operation (attempt: {})", 
                        context.getRetryCount() + 1);
                }
                
                performDatabaseOperation();
                return null;
            });
        } catch (Exception e) {
            log.error("Database operation failed after all retries", e);
            // 데이터베이스 연결 문제 알림
            notifyDatabaseIssue(e);
        }
    }
    
    // 커스텀 재시도 로직
    @Scheduled(fixedRate = 120000) // 2분마다
    public void customRetryLogic() {
        int maxRetries = 3;
        int retryCount = 0;
        boolean success = false;
        
        while (retryCount < maxRetries && !success) {
            try {
                performCriticalTask();
                success = true;
                log.info("Critical task completed successfully");
                
            } catch (Exception e) {
                retryCount++;
                log.warn("Critical task failed (attempt {}/{}): {}", 
                    retryCount, maxRetries, e.getMessage());
                
                if (retryCount < maxRetries) {
                    try {
                        // 지수 백오프
                        long delay = (long) Math.pow(2, retryCount) * 1000;
                        Thread.sleep(delay);
                    } catch (InterruptedException ie) {
                        Thread.currentThread().interrupt();
                        break;
                    }
                } else {
                    log.error("Critical task failed after {} attempts", maxRetries, e);
                    handleCriticalTaskFailure(e);
                }
            }
        }
    }
}

@Entity
@Table(name = "scheduled_task_execution_log")
public class ScheduledTaskExecutionLog {
    
    @Id
    @GeneratedValue(strategy = GenerationType.IDENTITY)
    private Long id;
    
    @Column(nullable = false)
    private String taskName;
    
    @Column(nullable = false)
    private String instanceId;
    
    @Column(nullable = false)
    private LocalDateTime startTime;
    
    private LocalDateTime endTime;
    
    @Enumerated(EnumType.STRING)
    private TaskStatus status;
    
    @Column(columnDefinition = "TEXT")
    private String errorMessage;
    
    private Long executionTimeMs;
    
    @Column(columnDefinition = "TEXT")
    private String additionalInfo;
    
    // constructors, getters, setters
}

@Service
@Transactional
public class TaskExecutionLogService {
    
    private final ScheduledTaskExecutionLogRepository logRepository;
    
    public Long logTaskStart(String taskName, String instanceId) {
        ScheduledTaskExecutionLog log = new ScheduledTaskExecutionLog();
        log.setTaskName(taskName);
        log.setInstanceId(instanceId);
        log.setStartTime(LocalDateTime.now());
        log.setStatus(TaskStatus.RUNNING);
        
        ScheduledTaskExecutionLog saved = logRepository.save(log);
        return saved.getId();
    }
    
    public void logTaskSuccess(Long logId, String additionalInfo) {
        logRepository.findById(logId).ifPresent(log -> {
            log.setEndTime(LocalDateTime.now());
            log.setStatus(TaskStatus.SUCCESS);
            log.setExecutionTimeMs(
                Duration.between(log.getStartTime(), log.getEndTime()).toMillis()
            );
            log.setAdditionalInfo(additionalInfo);
            logRepository.save(log);
        });
    }
    
    public void logTaskFailure(Long logId, Exception e) {
        logRepository.findById(logId).ifPresent(log -> {
            log.setEndTime(LocalDateTime.now());
            log.setStatus(TaskStatus.FAILED);
            log.setExecutionTimeMs(
                Duration.between(log.getStartTime(), log.getEndTime()).toMillis()
            );
            log.setErrorMessage(e.getMessage());
            log.setAdditionalInfo(getStackTrace(e));
            logRepository.save(log);
        });
    }
    
    // 실행 이력 조회 메서드들
    public List<ScheduledTaskExecutionLog> getRecentExecutions(String taskName, int limit) {
        return logRepository.findByTaskNameOrderByStartTimeDesc(taskName, 
            PageRequest.of(0, limit));
    }
    
    public TaskExecutionStatistics getTaskStatistics(String taskName, LocalDateTime since) {
        List<ScheduledTaskExecutionLog> logs = logRepository
            .findByTaskNameAndStartTimeAfter(taskName, since);
        
        return TaskExecutionStatistics.builder()
            .totalExecutions(logs.size())
            .successfulExecutions(countByStatus(logs, TaskStatus.SUCCESS))
            .failedExecutions(countByStatus(logs, TaskStatus.FAILED))
            .averageExecutionTime(calculateAverageExecutionTime(logs))
            .build();
    }
}

@Component
public class SchedulingMonitor {
    
    private final MeterRegistry meterRegistry;
    private final NotificationService notificationService;
    private final TaskExecutionLogService logService;
    
    // 메트릭 정의
    private final Counter taskExecutionCounter;
    private final Timer taskExecutionTimer;
    private final Gauge activeTasksGauge;
    
    public SchedulingMonitor(MeterRegistry meterRegistry, 
                           NotificationService notificationService,
                           TaskExecutionLogService logService) {
        this.meterRegistry = meterRegistry;
        this.notificationService = notificationService;
        this.logService = logService;
        
        this.taskExecutionCounter = Counter.builder("scheduled.tasks.total")
            .description("Total number of scheduled task executions")
            .register(meterRegistry);
            
        this.taskExecutionTimer = Timer.builder("scheduled.tasks.duration")
            .description("Duration of scheduled task executions")
            .register(meterRegistry);
            
        this.activeTasksGauge = Gauge.builder("scheduled.tasks.active")
            .description("Number of currently active scheduled tasks")
            .register(meterRegistry, this, SchedulingMonitor::getActiveTaskCount);
    }
    
    @EventListener
    public void handleTaskExecution(TaskExecutionEvent event) {
        // 메트릭 업데이트
        taskExecutionCounter.increment(
            Tags.of(
                "task.name", event.getTaskName(),
                "status", event.getStatus().toString(),
                "instance.id", event.getInstanceId()
            )
        );
        
        if (event.getExecutionTime() != null) {
            taskExecutionTimer.record(
                event.getExecutionTime(), 
                TimeUnit.MILLISECONDS,
                Tags.of("task.name", event.getTaskName())
            );
        }
        
        // 실패 시 알림 처리
        if (event.getStatus() == TaskStatus.FAILED) {
            handleTaskFailure(event);
        }
        
        // 실행 시간이 임계값을 초과한 경우
        if (event.getExecutionTime() != null && 
            event.getExecutionTime() > getThreshold(event.getTaskName())) {
            handleSlowExecution(event);
        }
    }
    
    private void handleTaskFailure(TaskExecutionEvent event) {
        // 연속 실패 횟수 확인
        int consecutiveFailures = getConsecutiveFailures(event.getTaskName());
        
        if (consecutiveFailures >= 3) {
            // 3회 연속 실패 시 긴급 알림
            notificationService.sendUrgentAlert(
                "Critical Task Failure",
                String.format("Task %s has failed %d times consecutively", 
                    event.getTaskName(), consecutiveFailures)
            );
        } else if (consecutiveFailures >= 1) {
            // 일반 실패 알림
            notificationService.sendAlert(
                "Task Failure",
                String.format("Task %s failed: %s", 
                    event.getTaskName(), event.getErrorMessage())
            );
        }
    }
    
    private void handleSlowExecution(TaskExecutionEvent event) {
        notificationService.sendWarning(
            "Slow Task Execution",
            String.format("Task %s took %d ms to execute (threshold: %d ms)",
                event.getTaskName(), 
                event.getExecutionTime(),
                getThreshold(event.getTaskName()))
        );
    }
    
    @Scheduled(fixedRate = 300000) // 5분마다
    public void performHealthCheck() {
        LocalDateTime fiveMinutesAgo = LocalDateTime.now().minusMinutes(5);
        
        // 중요한 작업들이 정상적으로 실행되고 있는지 확인
        List<String> criticalTasks = Arrays.asList(
            "dailyBackup", "systemHealthCheck", "dataSync"
        );
        
        for (String taskName : criticalTasks) {
            TaskExecutionStatistics stats = logService.getTaskStatistics(taskName, fiveMinutesAgo);
            
            if (stats.getTotalExecutions() == 0) {
                notificationService.sendWarning(
                    "Missing Task Execution",
                    String.format("Critical task %s has not executed in the last 5 minutes", taskName)
                );
            }
            
            double failureRate = (double) stats.getFailedExecutions() / stats.getTotalExecutions();
            if (failureRate > 0.5) { // 50% 이상 실패율
                notificationService.sendAlert(
                    "High Task Failure Rate",
                    String.format("Task %s has a failure rate of %.2f%%", taskName, failureRate * 100)
                );
            }
        }
    }
    
    private double getActiveTaskCount() {
        // 현재 실행 중인 작업 수 반환
        return logService.getActiveTaskCount();
    }
}

@Service
public class DataCleanupService {
    
    private final JdbcTemplate jdbcTemplate;
    private final FileSystemService fileSystemService;
    private final CacheManager cacheManager;
    
    @Scheduled(cron = "0 0 2 * * ?") // 매일 새벽 2시
    @SchedulerLock(name = "dailyDataCleanup", lockAtMostFor = "PT2H")
    public void performDailyCleanup() {
        log.info("Starting daily data cleanup process");
        
        try {
            // 1. 오래된 로그 데이터 삭제 (30일 이상)
            cleanupOldLogs();
            
            // 2. 임시 파일 정리
            cleanupTempFiles();
            
            // 3. 만료된 세션 데이터 삭제
            cleanupExpiredSessions();
            
            // 4. 캐시 정리
            cleanupExpiredCache();
            
            // 5. 데이터베이스 통계 업데이트
            updateDatabaseStatistics();
            
            log.info("Daily data cleanup completed successfully");
            
        } catch (Exception e) {
            log.error("Daily data cleanup failed", e);
            throw e;
        }
    }
    
    private void cleanupOldLogs() {
        LocalDateTime cutoffDate = LocalDateTime.now().minusDays(30);
        
        // 애플리케이션 로그 삭제
        int deletedAppLogs = jdbcTemplate.update(
            "DELETE FROM application_logs WHERE created_at < ?", 
            cutoffDate
        );
        
        // 액세스 로그 삭제
        int deletedAccessLogs = jdbcTemplate.update(
            "DELETE FROM access_logs WHERE created_at < ?", 
            cutoffDate
        );
        
        // 에러 로그는 90일 보관
        LocalDateTime errorLogCutoff = LocalDateTime.now().minusDays(90);
        int deletedErrorLogs = jdbcTemplate.update(
            "DELETE FROM error_logs WHERE created_at < ?", 
            errorLogCutoff
        );
        
        log.info("Cleaned up logs - App: {}, Access: {}, Error: {}", 
            deletedAppLogs, deletedAccessLogs, deletedErrorLogs);
    }
    
    private void cleanupTempFiles() {
        try {
            Path tempDir = Paths.get(System.getProperty("java.io.tmpdir"), "app-temp");
            if (Files.exists(tempDir)) {
                long deletedFiles = Files.walk(tempDir)
                    .filter(Files::isRegularFile)
                    .filter(path -> {
                        try {
                            FileTime lastModified = Files.getLastModifiedTime(path);
                            return lastModified.toInstant()
                                .isBefore(Instant.now().minus(1, ChronoUnit.DAYS));
                        } catch (IOException e) {
                            return false;
                        }
                    })
                    .peek(path -> {
                        try {
                            Files.delete(path);
                        } catch (IOException e) {
                            log.warn("Failed to delete temp file: {}", path, e);
                        }
                    })
                    .count();
                
                log.info("Cleaned up {} temporary files", deletedFiles);
            }
        } catch (IOException e) {
            log.error("Failed to cleanup temporary files", e);
        }
    }
    
    private void cleanupExpiredSessions() {
        // Redis 기반 세션 정리
        LocalDateTime expiredBefore = LocalDateTime.now().minusHours(24);
        
        int deletedSessions = jdbcTemplate.update(
            "DELETE FROM user_sessions WHERE last_accessed < ? AND active = false",
            expiredBefore
        );
        
        log.info("Cleaned up {} expired sessions", deletedSessions);
    }
    
    private void cleanupExpiredCache() {
        // 수동으로 관리되는 캐시 정리
        cacheManager.getCacheNames().forEach(cacheName -> {
            Cache cache = cacheManager.getCache(cacheName);
            if (cache != null) {
                cache.clear();
                log.debug("Cleared cache: {}", cacheName);
            }
        });
    }
    
    @Scheduled(cron = "0 0 3 1 * ?") // 매월 1일 새벽 3시
    @SchedulerLock(name = "monthlyArchiving", lockAtMostFor = "PT4H")
    public void performMonthlyArchiving() {
        log.info("Starting monthly data archiving");
        
        try {
            LocalDateTime archiveCutoff = LocalDateTime.now().minusMonths(6);
            
            // 오래된 주문 데이터 아카이빙
            archiveOldOrders(archiveCutoff);
            
            // 사용자 활동 로그 아카이빙
            archiveUserActivityLogs(archiveCutoff);
            
            // 시스템 메트릭 데이터 압축
            compressOldMetrics(archiveCutoff);
            
            log.info("Monthly archiving completed successfully");
            
        } catch (Exception e) {
            log.error("Monthly archiving failed", e);
            throw e;
        }
    }
    
    private void archiveOldOrders(LocalDateTime cutoffDate) {
        // 완료된 주문을 아카이브 테이블로 이동
        jdbcTemplate.update("""
            INSERT INTO archived_orders 
            SELECT *, NOW() as archived_at 
            FROM orders 
            WHERE status = 'COMPLETED' AND completed_at < ?
            """, cutoffDate);
        
        int archivedCount = jdbcTemplate.update(
            "DELETE FROM orders WHERE status = 'COMPLETED' AND completed_at < ?",
            cutoffDate
        );
        
        log.info("Archived {} old orders", archivedCount);
    }
}

@Service
public class ReportGenerationService {
    
    private final ReportRepository reportRepository;
    private final EmailService emailService;
    private final ExcelService excelService;
    private final PdfService pdfService;
    
    @Scheduled(cron = "0 0 8 * * MON") // 매주 월요일 오전 8시
    @SchedulerLock(name = "weeklyReport", lockAtMostFor = "PT1H")
    public void generateWeeklyReport() {
        log.info("Starting weekly report generation");
        
        try {
            LocalDate endDate = LocalDate.now().minusDays(1); // 어제까지
            LocalDate startDate = endDate.minusDays(6); // 지난 주
            
            // 주간 매출 리포트
            WeeklySalesReport salesReport = generateWeeklySalesReport(startDate, endDate);
            
            // 사용자 활동 리포트
            WeeklyUserActivityReport activityReport = generateWeeklyActivityReport(startDate, endDate);
            
            // 시스템 성능 리포트
            WeeklyPerformanceReport performanceReport = generateWeeklyPerformanceReport(startDate, endDate);
            
            // 리포트 파일 생성
            String reportFileName = String.format("weekly_report_%s_to_%s.xlsx", startDate, endDate);
            byte[] excelData = excelService.createWeeklyReport(salesReport, activityReport, performanceReport);
            
            // 이메일 발송
            List<String> recipients = getReportRecipients("WEEKLY_REPORT");
            emailService.sendReportEmail(
                recipients,
                "주간 운영 리포트 - " + startDate + " ~ " + endDate,
                createReportEmailContent(salesReport, activityReport, performanceReport),
                reportFileName,
                excelData
            );
            
            // 리포트 이력 저장
            saveReportHistory("WEEKLY", startDate, endDate, reportFileName);
            
            log.info("Weekly report generated and sent successfully");
            
        } catch (Exception e) {
            log.error("Weekly report generation failed", e);
            throw e;
        }
    }
    
    @Scheduled(cron = "0 0 7 1 * ?") // 매월 1일 오전 7시
    @SchedulerLock(name = "monthlyReport", lockAtMostFor = "PT2H")
    public void generateMonthlyReport() {
        log.info("Starting monthly report generation");
        
        try {
            LocalDate lastMonth = LocalDate.now().minusMonths(1);
            LocalDate startDate = lastMonth.withDayOfMonth(1);
            LocalDate endDate = lastMonth.withDayOfMonth(lastMonth.lengthOfMonth());
            
            // 월간 종합 리포트 생성
            MonthlyComprehensiveReport report = MonthlyComprehensiveReport.builder()
                .period(startDate + " ~ " + endDate)
                .salesSummary(generateMonthlySalesSummary(startDate, endDate))
                .userGrowth(calculateUserGrowth(startDate, endDate))
                .systemMetrics(aggregateSystemMetrics(startDate, endDate))
                .topProducts(getTopProducts(startDate, endDate))
                .customerInsights(analyzeCustomerBehavior(startDate, endDate))
                .build();
            
            // PDF 리포트 생성
            byte[] pdfData = pdfService.createMonthlyReport(report);
            String pdfFileName = String.format("monthly_report_%s.pdf", 
                lastMonth.format(DateTimeFormatter.ofPattern("yyyy_MM")));
            
            // 경영진에게 발송
            List<String> executiveRecipients = getReportRecipients("MONTHLY_EXECUTIVE");
            emailService.sendReportEmail(
                executiveRecipients,
                String.format("%s월 월간 운영 리포트", lastMonth.getMonthValue()),
                createExecutiveReportContent(report),
                pdfFileName,
                pdfData
            );
            
            // 상세 Excel 리포트도 생성
            byte[] detailedExcelData = excelService.createDetailedMonthlyReport(report);
            String excelFileName = String.format("monthly_detailed_report_%s.xlsx", 
                lastMonth.format(DateTimeFormatter.ofPattern("yyyy_MM")));
            
            // 운영팀에게 발송
            List<String> operationRecipients = getReportRecipients("MONTHLY_OPERATION");
            emailService.sendReportEmail(
                operationRecipients,
                String.format("%s월 상세 운영 리포트", lastMonth.getMonthValue()),
                createOperationReportContent(report),
                excelFileName,
                detailedExcelData
            );
            
            log.info("Monthly report generated and sent successfully");
            
        } catch (Exception e) {
            log.error("Monthly report generation failed", e);
            throw e;
        }
    }
    
    @Scheduled(cron = "0 30 23 * * ?") // 매일 오후 11시 30분
    @SchedulerLock(name = "dailyDashboard", lockAtMostFor = "PT30M")
    public void updateDailyDashboard() {
        log.info("Updating daily dashboard data");
        
        try {
            LocalDate today = LocalDate.now();
            
            // 일일 핵심 지표 계산
            DailyMetrics metrics = DailyMetrics.builder()
                .date(today)
                .totalSales(calculateDailySales(today))
                .newUsers(countNewUsers(today))
                .activeUsers(countActiveUsers(today))
                .orderCount(countOrders(today))
                .averageOrderValue(calculateAverageOrderValue(today))
                .conversionRate(calculateConversionRate(today))
                .systemUptime(calculateSystemUptime(today))
                .errorRate(calculateErrorRate(today))
                .build();
            
            // 대시보드 데이터 업데이트
            updateDashboardCache(metrics);
            
            // 실시간 알림이 필요한 지표 확인
            checkCriticalMetrics(metrics);
            
            log.info("Daily dashboard updated successfully");
            
        } catch (Exception e) {
            log.error("Daily dashboard update failed", e);
            throw e;
        }
    }
    
    private void checkCriticalMetrics(DailyMetrics metrics) {
        // 매출이 전일 대비 20% 이상 감소한 경우
        BigDecimal yesterdaySales = calculateDailySales(LocalDate.now().minusDays(1));
        if (metrics.getTotalSales().compareTo(yesterdaySales.multiply(BigDecimal.valueOf(0.8))) < 0) {
            emailService.sendAlert(
                "매출 급감 알림",
                String.format("오늘 매출이 전일 대비 %.2f%% 감소했습니다.", 
                    calculatePercentageChange(yesterdaySales, metrics.getTotalSales()))
            );
        }
        
        // 에러율이 5% 이상인 경우
        if (metrics.getErrorRate() > 0.05) {
            emailService.sendAlert(
                "높은 에러율 감지",
                String.format("현재 시스템 에러율이 %.2f%%입니다.", metrics.getErrorRate() * 100)
            );
        }
    }
}

@Service
public class NotificationSchedulingService {
    
    private final NotificationRepository notificationRepository;
    private final EmailService emailService;
    private final SmsService smsService;
    private final PushNotificationService pushService;
    
    @Scheduled(fixedRate = 60000) // 1분마다
    public void processPendingNotifications() {
        List<PendingNotification> pendingNotifications = 
            notificationRepository.findPendingNotifications(LocalDateTime.now());
        
        for (PendingNotification notification : pendingNotifications) {
            try {
                processNotification(notification);
                notification.setStatus(NotificationStatus.SENT);
                notification.setSentAt(LocalDateTime.now());
            } catch (Exception e) {
                log.error("Failed to send notification: {}", notification.getId(), e);
                notification.setStatus(NotificationStatus.FAILED);
                notification.setErrorMessage(e.getMessage());
                notification.setRetryCount(notification.getRetryCount() + 1);
                
                // 재시도 로직
                if (notification.getRetryCount() < 3) {
                    notification.setScheduledAt(LocalDateTime.now().plusMinutes(5));
                    notification.setStatus(NotificationStatus.PENDING);
                }
            } finally {
                notificationRepository.save(notification);
            }
        }
    }
    
    @Scheduled(cron = "0 0 9 * * ?") // 매일 오전 9시
    @SchedulerLock(name = "dailyReminders", lockAtMostFor = "PT1H")
    public void sendDailyReminders() {
        log.info("Sending daily reminders");
        
        try {
            // 1. 장바구니 이탈 고객 리마인더
            sendAbandonedCartReminders();
            
            // 2. 구독 만료 예정 알림
            sendSubscriptionExpiryReminders();
            
            // 3. 미완료 주문 알림
            sendIncompleteOrderReminders();
            
            // 4. 생일 축하 메시지
            sendBirthdayGreetings();
            
            log.info("Daily reminders sent successfully");
            
        } catch (Exception e) {
            log.error("Failed to send daily reminders", e);
            throw e;
        }
    }
    
    private void sendAbandonedCartReminders() {
        LocalDateTime cutoffTime = LocalDateTime.now().minusHours(24);
        
        List<AbandonedCart> abandonedCarts = cartRepository
            .findAbandonedCarts(cutoffTime);
        
        for (AbandonedCart cart : abandonedCarts) {
            try {
                // 개인화된 이메일 생성
                String emailContent = createAbandonedCartEmail(cart);
                
                emailService.sendEmail(
                    cart.getUserEmail(),
                    "잊고 가신 상품이 있어요! 🛒",
                    emailContent
                );
                
                // 푸시 알림도 발송
                if (cart.getUser().isPushEnabled()) {
                    pushService.sendPushNotification(
                        cart.getUser().getDeviceToken(),
                        "장바구니 상품이 기다리고 있어요!",
                        "지금 주문하면 특별 할인 혜택을 받을 수 있습니다."
                    );
                }
                
                // 발송 이력 기록
                recordNotificationSent(cart.getUserId(), "ABANDONED_CART", "EMAIL");
                
            } catch (Exception e) {
                log.error("Failed to send abandoned cart reminder to user: {}", 
                    cart.getUserId(), e);
            }
        }
        
        log.info("Sent {} abandoned cart reminders", abandonedCarts.size());
    }
    
    private void sendSubscriptionExpiryReminders() {
        // 7일 후 만료 예정
        LocalDateTime sevenDaysLater = LocalDateTime.now().plusDays(7);
        List<Subscription> expiringSoon = subscriptionRepository
            .findExpiringBetween(LocalDateTime.now(), sevenDaysLater);
        
        for (Subscription subscription : expiringSoon) {
            try {
                String emailContent = createSubscriptionRenewalEmail(subscription);
                
                emailService.sendEmail(
                    subscription.getUser().getEmail(),
                    "구독 갱신 안내 - 7일 후 만료 예정",
                    emailContent
                );
                
                recordNotificationSent(subscription.getUserId(), "SUBSCRIPTION_EXPIRY", "EMAIL");
                
            } catch (Exception e) {
                log.error("Failed to send subscription expiry reminder to user: {}", 
                    subscription.getUserId(), e);
            }
        }
        
        log.info("Sent {} subscription expiry reminders", expiringSoon.size());
    }
    
    @Scheduled(cron = "0 0 20 * * ?") // 매일 오후 8시
    @SchedulerLock(name = "eveningDigest", lockAtMostFor = "PT30M")
    public void sendEveningDigest() {
        log.info("Sending evening digest notifications");
        
        try {
            LocalDate today = LocalDate.now();
            
            // 오늘의 주요 활동 요약
            List<User> digestSubscribers = userRepository.findDigestSubscribers();
            
            for (User user : digestSubscribers) {
                try {
                    UserDailyDigest digest = createUserDailyDigest(user, today);
                    
                    if (digest.hasContent()) {
                        String emailContent = createDigestEmail(digest);
                        
                        emailService.sendEmail(
                            user.getEmail(),
                            "오늘의 활동 요약 📊",
                            emailContent
                        );
                        
                        recordNotificationSent(user.getId(), "DAILY_DIGEST", "EMAIL");
                    }
                    
                } catch (Exception e) {
                    log.error("Failed to send evening digest to user: {}", user.getId(), e);
                }
            }
            
            log.info("Evening digest sent to {} users", digestSubscribers.size());
            
        } catch (Exception e) {
            log.error("Failed to send evening digest", e);
            throw e;
        }
    }
    
    @Scheduled(cron = "0 0 10 * * SUN") // 매주 일요일 오전 10시
    @SchedulerLock(name = "weeklyNewsletter", lockAtMostFor = "PT2H")
    public void sendWeeklyNewsletter() {
        log.info("Sending weekly newsletter");
        
        try {
            List<User> newsletterSubscribers = userRepository.findNewsletterSubscribers();
            
            // 주간 뉴스레터 콘텐츠 생성
            WeeklyNewsletter newsletter = createWeeklyNewsletter();
            
            // 배치로 발송 (한 번에 100명씩)
            int batchSize = 100;
            for (int i = 0; i < newsletterSubscribers.size(); i += batchSize) {
                List<User> batch = newsletterSubscribers.subList(
                    i, Math.min(i + batchSize, newsletterSubscribers.size())
                );
                
                sendNewsletterBatch(batch, newsletter);
                
                // 배치 간 1초 대기 (이메일 서버 부하 방지)
                Thread.sleep(1000);
            }
            
            log.info("Weekly newsletter sent to {} subscribers", newsletterSubscribers.size());
            
        } catch (Exception e) {
            log.error("Failed to send weekly newsletter", e);
            throw e;
        }
    }
}