Spring 17: 모니터링/Actuator

1. 인프라스트럭처 메트릭

2. 애플리케이션 메트릭

3. 비즈니스 메트릭

Maven

<dependency>
    <groupId>org.springframework.boot</groupId>
    <artifactId>spring-boot-starter-actuator</artifactId>
</dependency>

<!-- Micrometer Prometheus 연동 (선택사항) -->
<dependency>
    <groupId>io.micrometer</groupId>
    <artifactId>micrometer-registry-prometheus</artifactId>
</dependency>

Gradle

implementation 'org.springframework.boot:spring-boot-starter-actuator'

// Micrometer Prometheus 연동 (선택사항)
implementation 'io.micrometer:micrometer-registry-prometheus'

application.yml

# 기본 Actuator 설정
management:
  # 엔드포인트 기본 경로 설정
  endpoints:
    web:
      base-path: /actuator
      exposure:
        # 웹으로 노출할 엔드포인트 지정
        include: health,info,metrics,prometheus
        # 제외할 엔드포인트
        exclude: shutdown
  
  # 개별 엔드포인트 설정
  endpoint:
    health:
      # 상세 정보 표시 설정
      show-details: when-authorized
      show-components: always
    info:
      enabled: true
    metrics:
      enabled: true
    prometheus:
      enabled: true
  
  # 보안 설정
  security:
    enabled: true
  
  # 서버 포트 분리 (선택사항)
  server:
    port: 8081

application.properties

# 엔드포인트 노출 설정
management.endpoints.web.exposure.include=health,info,metrics,prometheus
management.endpoints.web.base-path=/actuator

# Health 엔드포인트 상세 정보 표시
management.endpoint.health.show-details=when-authorized
management.endpoint.health.show-components=always

# Info 엔드포인트 활성화
management.endpoint.info.enabled=true

# 메트릭 엔드포인트 활성화
management.endpoint.metrics.enabled=true

# Prometheus 엔드포인트 활성화
management.endpoint.prometheus.enabled=true

Spring Security 설정

@Configuration
@EnableWebSecurity
public class ActuatorSecurityConfig {

    @Bean
    public SecurityFilterChain actuatorSecurityFilterChain(HttpSecurity http) throws Exception {
        return http
            .requestMatcher(EndpointRequest.toAnyEndpoint())
            .authorizeHttpRequests(auth -> auth
                // Health 엔드포인트는 인증 없이 접근 허용
                .requestMatchers(EndpointRequest.to(HealthEndpoint.class)).permitAll()
                // Info 엔드포인트는 인증 없이 접근 허용
                .requestMatchers(EndpointRequest.to(InfoEndpoint.class)).permitAll()
                // 나머지 엔드포인트는 ADMIN 권한 필요
                .anyRequest().hasRole("ADMIN")
            )
            .httpBasic(Customizer.withDefaults())
            .build();
    }
    
    @Bean
    public UserDetailsService userDetailsService() {
        UserDetails admin = User.builder()
            .username("admin")
            .password("{noop}admin123")
            .roles("ADMIN")
            .build();
        return new InMemoryUserDetailsManager(admin);
    }
}

포트 분리 설정

# 애플리케이션과 관리 포트 분리
server.port=8080
management.server.port=8081

# 관리 포트에 대한 별도 보안 설정
management.security.enabled=true

기본 사용법

@Service
public class OrderService {
    
    private final Counter orderCounter;
    private final Counter errorCounter;
    
    public OrderService(MeterRegistry meterRegistry) {
        this.orderCounter = Counter.builder("orders.created")
            .description("Number of orders created")
            .tag("type", "online")
            .register(meterRegistry);
            
        this.errorCounter = Counter.builder("orders.errors")
            .description("Number of order processing errors")
            .register(meterRegistry);
    }
    
    public void createOrder(Order order) {
        try {
            // 주문 처리 로직
            processOrder(order);
            
            // 성공 시 카운터 증가
            orderCounter.increment();
            
        } catch (Exception e) {
            // 에러 시 에러 카운터 증가
            errorCounter.increment(Tags.of("error.type", e.getClass().getSimpleName()));
            throw e;
        }
    }
    
    // 태그를 동적으로 추가하는 방법
    public void createOrderWithTags(Order order) {
        Counter.builder("orders.created")
            .tag("payment.method", order.getPaymentMethod())
            .tag("customer.type", order.getCustomerType())
            .register(meterRegistry)
            .increment();
    }
}

기본 사용법

@Component
public class SystemMetrics {
    
    private final AtomicInteger activeConnections = new AtomicInteger(0);
    private final Queue<String> messageQueue = new ConcurrentLinkedQueue<>();
    
    public SystemMetrics(MeterRegistry meterRegistry) {
        // AtomicInteger를 직접 게이지로 등록
        Gauge.builder("connections.active")
            .description("Number of active connections")
            .register(meterRegistry, activeConnections, AtomicInteger::get);
            
        // 컬렉션 크기를 게이지로 등록
        Gauge.builder("queue.size")
            .description("Size of message queue")
            .register(meterRegistry, messageQueue, Queue::size);
            
        // 메서드 참조를 이용한 게이지
        Gauge.builder("memory.used")
            .description("Used memory in bytes")
            .register(meterRegistry, this, SystemMetrics::getUsedMemory);
    }
    
    public void addConnection() {
        activeConnections.incrementAndGet();
    }
    
    public void removeConnection() {
        activeConnections.decrementAndGet();
    }
    
    private double getUsedMemory() {
        Runtime runtime = Runtime.getRuntime();
        return runtime.totalMemory() - runtime.freeMemory();
    }
}

MultiGauge 사용법

@Component
public class DatabaseMetrics {
    
    private final MultiGauge connectionPoolGauge;
    
    public DatabaseMetrics(MeterRegistry meterRegistry) {
        this.connectionPoolGauge = MultiGauge.builder("db.connections")
            .description("Database connection pool metrics")
            .register(meterRegistry);
    }
    
    @Scheduled(fixedRate = 30000) // 30초마다 업데이트
    public void updateConnectionMetrics() {
        List<MultiGauge.Row<?>> rows = Arrays.asList(
            MultiGauge.Row.of(Tags.of("pool", "primary", "state", "active"), 
                             getActiveConnections("primary")),
            MultiGauge.Row.of(Tags.of("pool", "primary", "state", "idle"), 
                             getIdleConnections("primary")),
            MultiGauge.Row.of(Tags.of("pool", "secondary", "state", "active"), 
                             getActiveConnections("secondary")),
            MultiGauge.Row.of(Tags.of("pool", "secondary", "state", "idle"), 
                             getIdleConnections("secondary"))
        );
        
        connectionPoolGauge.register(rows, true);
    }
}

기본 사용법

@Service
public class PaymentService {
    
    private final Timer paymentTimer;
    private final MeterRegistry meterRegistry;
    
    public PaymentService(MeterRegistry meterRegistry) {
        this.meterRegistry = meterRegistry;
        this.paymentTimer = Timer.builder("payment.processing.time")
            .description("Payment processing time")
            .register(meterRegistry);
    }
    
    // Timer.Sample을 이용한 측정
    public PaymentResult processPayment(PaymentRequest request) {
        Timer.Sample sample = Timer.start(meterRegistry);
        
        try {
            PaymentResult result = doProcessPayment(request);
            
            // 성공 시 태그 추가
            sample.stop(Timer.builder("payment.processing.time")
                .tag("status", "success")
                .tag("method", request.getPaymentMethod())
                .register(meterRegistry));
                
            return result;
            
        } catch (Exception e) {
            // 실패 시 태그 추가
            sample.stop(Timer.builder("payment.processing.time")
                .tag("status", "error")
                .tag("error.type", e.getClass().getSimpleName())
                .register(meterRegistry));
            throw e;
        }
    }
    
    // recordCallable을 이용한 간편한 측정
    public PaymentResult processPaymentSimple(PaymentRequest request) {
        return paymentTimer.recordCallable(() -> {
            return doProcessPayment(request);
        });
    }
}

@Timed 어노테이션 사용

@Service
public class UserService {
    
    @Timed(name = "user.creation.time", 
           description = "Time taken to create user")
    public User createUser(UserRequest request) {
        // 사용자 생성 로직
        return new User(request);
    }
    
    @Timed(name = "user.search.time",
           description = "Time taken to search users",
           extraTags = {"operation", "search"})
    public List<User> searchUsers(String query) {
        // 사용자 검색 로직
        return userRepository.findByNameContaining(query);
    }
    
    // 조건부 측정
    @Timed(name = "user.update.time",
           description = "Time taken to update user",
           longTask = true) // 장시간 실행되는 작업 측정
    public User updateUser(Long id, UserRequest request) {
        // 사용자 업데이트 로직
        return userRepository.save(user);
    }
}

기본 사용법

@Service
public class FileService {
    
    private final DistributionSummary fileSizeSummary;
    private final DistributionSummary batchSizeSummary;
    
    public FileService(MeterRegistry meterRegistry) {
        this.fileSizeSummary = DistributionSummary.builder("file.size")
            .description("Size of uploaded files")
            .baseUnit("bytes")
            .register(meterRegistry);
            
        this.batchSizeSummary = DistributionSummary.builder("batch.size")
            .description("Number of items in batch")
            .register(meterRegistry);
    }
    
    public void uploadFile(MultipartFile file) {
        // 파일 크기 기록
        fileSizeSummary.record(file.getSize());
        
        // 파일 업로드 처리
        processFile(file);
    }
    
    public void processBatch(List<Item> items) {
        // 배치 크기 기록
        batchSizeSummary.record(items.size());
        
        // 배치 처리
        items.forEach(this::processItem);
    }
}

Maven

<dependencies>
    <!-- Spring Boot Actuator -->
    <dependency>
        <groupId>org.springframework.boot</groupId>
        <artifactId>spring-boot-starter-actuator</artifactId>
    </dependency>
    
    <!-- Micrometer Prometheus Registry -->
    <dependency>
        <groupId>io.micrometer</groupId>
        <artifactId>micrometer-registry-prometheus</artifactId>
    </dependency>
</dependencies>

Gradle

dependencies {
    implementation 'org.springframework.boot:spring-boot-starter-actuator'
    implementation 'io.micrometer:micrometer-registry-prometheus'
}

application.yml

management:
  endpoints:
    web:
      exposure:
        include: health,info,metrics,prometheus
  endpoint:
    prometheus:
      enabled: true
    metrics:
      enabled: true
  metrics:
    export:
      prometheus:
        enabled: true
        # 메트릭 수집 간격 (기본값: 1분)
        step: 1m
        # 히스토그램 버킷 설정
        descriptions: true
    distribution:
      percentiles-histogram:
        http.server.requests: true
      percentiles:
        http.server.requests: 0.5, 0.95, 0.99
      slo:
        http.server.requests: 10ms,50ms,100ms,200ms,500ms
    tags:
      application: ${spring.application.name}
      environment: ${spring.profiles.active:default}

# 애플리케이션 정보
spring:
  application:
    name: monitoring-demo
info:
  app:
    name: ${spring.application.name}
    version: 1.0.0
    description: Spring Boot Monitoring Demo

MeterRegistry 커스터마이징

@Configuration
public class PrometheusConfig {
    
    @Bean
    public MeterRegistryCustomizer<PrometheusMeterRegistry> prometheusCustomizer() {
        return registry -> {
            registry.config()
                // 공통 태그 추가
                .commonTags(
                    "application", "monitoring-demo",
                    "version", "1.0.0",
                    "environment", getEnvironment()
                )
                // 메트릭 이름 변환
                .meterFilter(MeterFilter.renameTag("http.server.requests", "uri", "endpoint"))
                // 특정 메트릭 제외
                .meterFilter(MeterFilter.deny(id -> id.getName().startsWith("jvm.gc.overhead")))
                // 히스토그램 버킷 커스터마이징
                .meterFilter(MeterFilter.maximumExpectedValue("http.server.requests", 
                    Duration.ofSeconds(5)));
        };
    }
    
    @Bean
    public TimedAspect timedAspect(MeterRegistry registry) {
        return new TimedAspect(registry);
    }
    
    private String getEnvironment() {
        return System.getProperty("spring.profiles.active", "default");
    }
}

수동 설정

자동 프로비저닝 설정

# grafana/provisioning/datasources/prometheus.yml
apiVersion: 1

datasources:
  - name: Prometheus
    type: prometheus
    access: proxy
    url: http://prometheus:9090
    isDefault: true
    editable: true
    jsonData:
      httpMethod: POST
      queryTimeout: 60s
      timeInterval: 15s

대시보드 프로바이더 설정

# grafana/provisioning/dashboards/dashboard.yml
apiVersion: 1

providers:
  - name: 'default'
    orgId: 1
    folder: ''
    type: file
    disableDeletion: false
    updateIntervalSeconds: 10
    allowUiUpdates: true
    options:
      path: /etc/grafana/provisioning/dashboards

데이터베이스 헬스 체크

@Component
public class DatabaseHealthIndicator implements HealthIndicator {
    
    private final DataSource dataSource;
    
    public DatabaseHealthIndicator(DataSource dataSource) {
        this.dataSource = dataSource;
    }
    
    @Override
    public Health health() {
        try (Connection connection = dataSource.getConnection()) {
            // 간단한 쿼리 실행으로 DB 연결 확인
            try (PreparedStatement statement = connection.prepareStatement("SELECT 1")) {
                ResultSet resultSet = statement.executeQuery();
                if (resultSet.next()) {
                    return Health.up()
                        .withDetail("database", "Available")
                        .withDetail("connection-pool", getConnectionPoolInfo())
                        .withDetail("query-time", measureQueryTime())
                        .build();
                }
            }
        } catch (SQLException e) {
            return Health.down()
                .withDetail("database", "Unavailable")
                .withDetail("error", e.getMessage())
                .withException(e)
                .build();
        }
        
        return Health.down()
            .withDetail("database", "Unknown state")
            .build();
    }
    
    private Map<String, Object> getConnectionPoolInfo() {
        // HikariCP 정보 조회 예시
        if (dataSource instanceof HikariDataSource) {
            HikariDataSource hikariDS = (HikariDataSource) dataSource;
            HikariPoolMXBean poolBean = hikariDS.getHikariPoolMXBean();
            
            return Map.of(
                "active", poolBean.getActiveConnections(),
                "idle", poolBean.getIdleConnections(),
                "total", poolBean.getTotalConnections(),
                "waiting", poolBean.getThreadsAwaitingConnection()
            );
        }
        return Map.of("type", "unknown");
    }
    
    private String measureQueryTime() {
        long startTime = System.currentTimeMillis();
        try (Connection connection = dataSource.getConnection();
             PreparedStatement statement = connection.prepareStatement("SELECT 1")) {
            statement.executeQuery();
            return (System.currentTimeMillis() - startTime) + "ms";
        } catch (SQLException e) {
            return "error";
        }
    }
}

외부 API 헬스 체크

@Component
public class ExternalApiHealthIndicator implements HealthIndicator {
    
    private final RestTemplate restTemplate;
    private final String apiUrl;
    
    public ExternalApiHealthIndicator(RestTemplate restTemplate, 
                                    @Value("${external.api.url}") String apiUrl) {
        this.restTemplate = restTemplate;
        this.apiUrl = apiUrl;
    }
    
    @Override
    public Health health() {
        try {
            long startTime = System.currentTimeMillis();
            
            ResponseEntity<String> response = restTemplate.exchange(
                apiUrl + "/health",
                HttpMethod.GET,
                null,
                String.class
            );
            
            long responseTime = System.currentTimeMillis() - startTime;
            
            if (response.getStatusCode().is2xxSuccessful()) {
                return Health.up()
                    .withDetail("external-api", "Available")
                    .withDetail("url", apiUrl)
                    .withDetail("status", response.getStatusCode().value())
                    .withDetail("response-time", responseTime + "ms")
                    .build();
            } else {
                return Health.down()
                    .withDetail("external-api", "Unhealthy response")
                    .withDetail("status", response.getStatusCode().value())
                    .withDetail("response-time", responseTime + "ms")
                    .build();
            }
            
        } catch (ResourceAccessException e) {
            return Health.down()
                .withDetail("external-api", "Connection timeout")
                .withDetail("error", e.getMessage())
                .build();
        } catch (Exception e) {
            return Health.down()
                .withDetail("external-api", "Unavailable")
                .withDetail("error", e.getMessage())
                .build();
        }
    }
}

Redis 클러스터 헬스 체크

@Component
public class RedisClusterHealthIndicator implements HealthIndicator {
    
    private final RedisTemplate<String, String> redisTemplate;
    private final List<String> redisNodes;
    
    public RedisClusterHealthIndicator(RedisTemplate<String, String> redisTemplate,
                                     @Value("${redis.nodes}") List<String> redisNodes) {
        this.redisTemplate = redisTemplate;
        this.redisNodes = redisNodes;
    }
    
    @Override
    public Health health() {
        Health.Builder builder = Health.up();
        Map<String, Object> details = new HashMap<>();
        
        try {
            // Redis 연결 테스트
            String testKey = "health-check-" + System.currentTimeMillis();
            String testValue = "test";
            
            // Write 테스트
            redisTemplate.opsForValue().set(testKey, testValue, Duration.ofSeconds(10));
            
            // Read 테스트
            String retrievedValue = redisTemplate.opsForValue().get(testKey);
            
            if (testValue.equals(retrievedValue)) {
                details.put("redis", "Available");
                details.put("operation", "Read/Write successful");
                
                // 클러스터 노드 상태 확인
                details.put("cluster-info", getClusterInfo());
                
            } else {
                builder = Health.down();
                details.put("redis", "Data inconsistency");
            }
            
            // Cleanup
            redisTemplate.delete(testKey);
            
        } catch (Exception e) {
            builder = Health.down();
            details.put("redis", "Unavailable");
            details.put("error", e.getMessage());
        }
        
        return builder.withDetails(details).build();
    }
    
    private Map<String, Object> getClusterInfo() {
        try {
            Properties info = redisTemplate.getConnectionFactory()
                .getConnection()
                .info("replication");
                
            return Map.of(
                "role", info.getProperty("role", "unknown"),
                "connected_slaves", info.getProperty("connected_slaves", "0"),
                "nodes", redisNodes.size()
            );
        } catch (Exception e) {
            return Map.of("error", "Unable to get cluster info");
        }
    }
}

메트릭 수집 최적화

@Configuration
public class MetricsOptimizationConfig {
    
    @Bean
    public MeterRegistryCustomizer<MeterRegistry> metricsOptimization() {
        return registry -> registry.config()
            // 불필요한 메트릭 필터링
            .meterFilter(MeterFilter.deny(id -> 
                id.getName().startsWith("jvm.gc.overhead")))
            
            // 높은 카디널리티 메트릭 제한
            .meterFilter(MeterFilter.maximumExpectedValue(
                "http.server.requests", Duration.ofSeconds(30)))
            
            // 히스토그램 버킷 최적화
            .meterFilter(MeterFilter.replaceTagValues(
                "uri", uri -> uri.length() > 100 ? "long-uri" : uri))
            
            // 샘플링 적용
            .meterFilter(MeterFilter.denyNameStartsWith("debug"));
    }
    
    // 비동기 메트릭 수집
    @Bean
    @ConditionalOnProperty(name = "metrics.async.enabled", havingValue = "true")
    public AsyncMeterRegistry asyncMeterRegistry(MeterRegistry registry) {
        return new AsyncMeterRegistry(registry, Executors.newFixedThreadPool(2));
    }
}

메모리 사용량 최적화

// 메트릭 인스턴스 재사용
@Service
public class OptimizedMetricsService {
    
    private final Map<String, Counter> counterCache = new ConcurrentHashMap<>();
    private final Map<String, Timer> timerCache = new ConcurrentHashMap<>();
    
    public void incrementCounter(String name, Tags tags) {
        String key = name + tags.toString();
        counterCache.computeIfAbsent(key, k -> 
            Counter.builder(name).tags(tags).register(meterRegistry))
            .increment();
    }
    
    public Timer.Sample startTimer(String name, Tags tags) {
        String key = name + tags.toString();
        Timer timer = timerCache.computeIfAbsent(key, k ->
            Timer.builder(name).tags(tags).register(meterRegistry));
        return Timer.start(timer);
    }
    
    // 주기적으로 사용하지 않는 메트릭 정리
    @Scheduled(fixedRate = 300000) // 5분마다
    public void cleanupUnusedMetrics() {
        // 사용하지 않는 메트릭 제거 로직
    }
}

민감한 정보 보호

@Configuration
public class ActuatorSecurityConfig {
    
    @Bean
    public SecurityFilterChain actuatorSecurity(HttpSecurity http) throws Exception {
        return http
            .requestMatcher(EndpointRequest.toAnyEndpoint())
            .authorizeHttpRequests(auth -> auth
                // 공개 엔드포인트
                .requestMatchers(EndpointRequest.to(HealthEndpoint.class))
                    .permitAll()
                .requestMatchers(EndpointRequest.to(InfoEndpoint.class))
                    .permitAll()
                
                // 제한된 엔드포인트
                .requestMatchers(EndpointRequest.to(MetricsEndpoint.class))
                    .hasRole("METRICS_READER")
                .requestMatchers(EndpointRequest.to(PrometheusEndpoint.class))
                    .hasRole("METRICS_READER")
                
                // 관리자 전용 엔드포인트
                .requestMatchers(EndpointRequest.to(
                    EnvironmentEndpoint.class,
                    ConfigPropsEndpoint.class,
                    LoggersEndpoint.class))
                    .hasRole("ADMIN")
                
                .anyRequest().denyAll()
            )
            .httpBasic(Customizer.withDefaults())
            .build();
    }
    
    // 민감한 정보 마스킹
    @Bean
    public EnvironmentEndpointWebExtension environmentEndpoint(
            Environment environment) {
        return new EnvironmentEndpointWebExtension(
            new EnvironmentEndpoint(environment, 
                Arrays.asList("password", "secret", "key", "token")));
    }
}

Runbook 예시

# 높은 응답 시간 알림 대응 가이드

## 1. 즉시 확인사항
- [ ] 현재 트래픽 패턴 확인
- [ ] 에러율 동시 증가 여부 확인
- [ ] 인프라 리소스 사용률 확인
- [ ] 외부 의존성 상태 확인

## 2. 조사 단계
### Grafana 대시보드 확인
- Application Overview Dashboard
- JVM Metrics Dashboard
- Infrastructure Dashboard

### 주요 메트릭 확인
- P95 응답 시간: > 1초
- 에러율: > 5%
- CPU 사용률: > 80%
- 메모리 사용률: > 85%

## 3. 대응 조치
### 즉시 조치
1. 트래픽 제한 (Rate Limiting)
2. 캐시 무효화
3. 스케일 아웃 (Auto Scaling)

### 근본 원인 분석
1. 로그 분석
2. 프로파일링
3. 데이터베이스 쿼리 분석

## 4. 사후 조치
- [ ] 인시던트 리포트 작성
- [ ] 모니터링 개선사항 도출
- [ ] 예방 조치 계획 수립