diff --git a/pom.xml b/pom.xml index 5170d70a..37552818 100644 --- a/pom.xml +++ b/pom.xml @@ -245,6 +245,12 @@ h2 runtime + + + org.elasticsearch.client + elasticsearch-rest-client + 8.10.0 + @@ -286,31 +292,29 @@ 3.3.0 - pl.project13.maven - git-commit-id-plugin - 4.9.10 + io.github.git-commit-id + git-commit-id-maven-plugin + 9.0.2 get-the-git-infos revision + initialize - ${project.basedir}/.git - git - false true - - ${project.build.outputDirectory}/git.properties - - json - - false - false - -dirty - + ${project.build.outputDirectory}/git.properties + + ^git.branch$ + ^git.commit.id.abbrev$ + ^git.build.version$ + ^git.build.time$ + + false + false diff --git a/src/main/java/com/iemr/common/identity/controller/health/HealthController.java b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java new file mode 100644 index 00000000..b7cc3641 --- /dev/null +++ b/src/main/java/com/iemr/common/identity/controller/health/HealthController.java @@ -0,0 +1,90 @@ +/* +* AMRIT – Accessible Medical Records via Integrated Technology +* Integrated EHR (Electronic Health Records) Solution +* +* Copyright (C) "Piramal Swasthya Management and Research Institute" +* +* This file is part of AMRIT. +* +* This program is free software: you can redistribute it and/or modify +* it under the terms of the GNU General Public License as published by +* the Free Software Foundation, either version 3 of the License, or +* (at your option) any later version. +* +* This program is distributed in the hope that it will be useful, +* but WITHOUT ANY WARRANTY; without even the implied warranty of +* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +* GNU General Public License for more details. +* +* You should have received a copy of the GNU General Public License +* along with this program. If not, see https://www.gnu.org/licenses/. +*/ + +package com.iemr.common.identity.controller.health; + +import java.time.Instant; +import java.util.Map; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; +import jakarta.servlet.http.Cookie; +import jakarta.servlet.http.HttpServletRequest; +import com.iemr.common.identity.service.health.HealthService; +import com.iemr.common.identity.utils.JwtAuthenticationUtil; +import io.swagger.v3.oas.annotations.Operation; +import io.swagger.v3.oas.annotations.responses.ApiResponse; +import io.swagger.v3.oas.annotations.responses.ApiResponses; +import io.swagger.v3.oas.annotations.tags.Tag; + + +@RestController +@RequestMapping("/health") +@Tag(name = "Health Check", description = "APIs for checking infrastructure health status") +public class HealthController { + + private static final Logger logger = LoggerFactory.getLogger(HealthController.class); + + private final HealthService healthService; + private final JwtAuthenticationUtil jwtAuthenticationUtil; + + public HealthController(HealthService healthService, JwtAuthenticationUtil jwtAuthenticationUtil) { + this.healthService = healthService; + this.jwtAuthenticationUtil = jwtAuthenticationUtil; + } + @GetMapping + @Operation(summary = "Check infrastructure health", + description = "Returns the health status of MySQL, Redis, Elasticsearch, and other configured services") + @ApiResponses({ + @ApiResponse(responseCode = "200", description = "Services are UP or DEGRADED (operational with warnings)"), + @ApiResponse(responseCode = "503", description = "One or more critical services are DOWN") + }) + public ResponseEntity> checkHealth() { + logger.info("Health check endpoint called"); + + try { + Map healthStatus = healthService.checkHealth(); + String overallStatus = (String) healthStatus.get("status"); + + // Return 503 only if DOWN; 200 for both UP and DEGRADED (DEGRADED = operational with warnings) + HttpStatus httpStatus = "DOWN".equals(overallStatus) ? HttpStatus.SERVICE_UNAVAILABLE : HttpStatus.OK; + + logger.debug("Health check completed with status: {}", overallStatus); + return new ResponseEntity<>(healthStatus, httpStatus); + + } catch (Exception e) { + logger.error("Unexpected error during health check", e); + + // Return sanitized error response + Map errorResponse = Map.of( + "status", "DOWN", + "timestamp", Instant.now().toString() + ); + + return new ResponseEntity<>(errorResponse, HttpStatus.SERVICE_UNAVAILABLE); + } + } +} diff --git a/src/main/java/com/iemr/common/identity/controller/version/VersionController.java b/src/main/java/com/iemr/common/identity/controller/version/VersionController.java index 4435c3db..39eee9f1 100644 --- a/src/main/java/com/iemr/common/identity/controller/version/VersionController.java +++ b/src/main/java/com/iemr/common/identity/controller/version/VersionController.java @@ -21,54 +21,59 @@ */ package com.iemr.common.identity.controller.version; -import java.io.BufferedReader; import java.io.IOException; import java.io.InputStream; -import java.io.InputStreamReader; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.Properties; import org.slf4j.Logger; import org.slf4j.LoggerFactory; + +import org.springframework.http.MediaType; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.RestController; -import com.iemr.common.identity.utils.response.OutputResponse; - import io.swagger.v3.oas.annotations.Operation; @RestController public class VersionController { - private Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private final Logger logger = LoggerFactory.getLogger(this.getClass().getSimpleName()); + private static final String UNKNOWN_VALUE = "unknown"; + @Operation(summary = "Get version information") - @GetMapping(value = "/version",consumes = "application/json", produces = "application/json") - public String versionInformation() { - OutputResponse output = new OutputResponse(); + @GetMapping(value = "/version", produces = MediaType.APPLICATION_JSON_VALUE) + public ResponseEntity> versionInformation() { + Map response = new LinkedHashMap<>(); try { logger.info("version Controller Start"); - output.setResponse(readGitProperties()); - } catch (Exception e) { - output.setError(e); - } - + Properties gitProperties = loadGitProperties(); + response.put("buildTimestamp", gitProperties.getProperty("git.build.time", UNKNOWN_VALUE)); + response.put("version", gitProperties.getProperty("git.build.version", UNKNOWN_VALUE)); + response.put("branch", gitProperties.getProperty("git.branch", UNKNOWN_VALUE)); + response.put("commitHash", gitProperties.getProperty("git.commit.id.abbrev", UNKNOWN_VALUE)); + } catch (Exception e) { + logger.error("Failed to load version information", e); + response.put("buildTimestamp", UNKNOWN_VALUE); + response.put("version", UNKNOWN_VALUE); + response.put("branch", UNKNOWN_VALUE); + response.put("commitHash", UNKNOWN_VALUE); + } logger.info("version Controller End"); - return output.toString(); + return ResponseEntity.ok(response); } - private String readGitProperties() throws Exception { - ClassLoader classLoader = getClass().getClassLoader(); - InputStream inputStream = classLoader.getResourceAsStream("git.properties"); - - return readFromInputStream(inputStream); - } - private String readFromInputStream(InputStream inputStream) - throws IOException { - StringBuilder resultStringBuilder = new StringBuilder(); - try (BufferedReader br = new BufferedReader(new InputStreamReader(inputStream))) { - String line; - while ((line = br.readLine()) != null) { - resultStringBuilder.append(line).append("\n"); - } - } - return resultStringBuilder.toString(); + + private Properties loadGitProperties() throws IOException { + Properties properties = new Properties(); + try (InputStream input = getClass().getClassLoader() + .getResourceAsStream("git.properties")) { + if (input != null) { + properties.load(input); + } + } + return properties; } } diff --git a/src/main/java/com/iemr/common/identity/service/health/HealthService.java b/src/main/java/com/iemr/common/identity/service/health/HealthService.java new file mode 100644 index 00000000..f233d729 --- /dev/null +++ b/src/main/java/com/iemr/common/identity/service/health/HealthService.java @@ -0,0 +1,855 @@ +/* + * AMRIT – Accessible Medical Records via Integrated Technology + * Integrated EHR (Electronic Health Records) Solution + * + * Copyright (C) "Piramal Swasthya Management and Research Institute" + * + * This file is part of AMRIT. + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see https://www.gnu.org/licenses/. + */ + +package com.iemr.common.identity.service.health; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.time.Instant; +import java.util.LinkedHashMap; +import java.util.Map; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicReference; +import java.util.concurrent.locks.ReentrantReadWriteLock; +import java.util.function.Supplier; +import javax.sql.DataSource; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.zaxxer.hikari.HikariDataSource; +import com.zaxxer.hikari.HikariPoolMXBean; +import java.lang.management.ManagementFactory; +import javax.management.MBeanServer; +import javax.management.ObjectName; +import jakarta.annotation.PostConstruct; +import org.apache.http.HttpHost; +import org.apache.http.client.config.RequestConfig; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.RequestOptions; +import org.elasticsearch.client.RestClient; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.data.redis.core.RedisCallback; +import org.springframework.data.redis.core.RedisTemplate; +import org.springframework.stereotype.Service; + +@Service +public class HealthService { + + private static final Logger logger = LoggerFactory.getLogger(HealthService.class); + + // Status values + private static final String STATUS_KEY = "status"; + private static final String STATUS_UP = "UP"; + private static final String STATUS_DOWN = "DOWN"; + private static final String STATUS_DEGRADED = "DEGRADED"; + + // Severity values + private static final String SEVERITY_KEY = "severity"; + private static final String SEVERITY_OK = "OK"; + private static final String SEVERITY_WARNING = "WARNING"; + private static final String SEVERITY_CRITICAL = "CRITICAL"; + + // Response field keys + private static final String ERROR_KEY = "error"; + private static final String MESSAGE_KEY = "message"; + private static final String RESPONSE_TIME_KEY = "responseTimeMs"; + + // Component names + private static final String MYSQL_COMPONENT = "MySQL"; + private static final String REDIS_COMPONENT = "Redis"; + private static final String ELASTICSEARCH_TYPE = "Elasticsearch"; + + // Thresholds + private static final long RESPONSE_TIME_THRESHOLD_MS = 2_000L; + private static final long ADVANCED_CHECKS_THROTTLE_SECONDS = 30L; + private static final long ADVANCED_CHECKS_TIMEOUT_MS = 500L; + + // Diagnostic event codes + private static final String DIAGNOSTIC_LOCK_WAIT = "MYSQL_LOCK_WAIT"; + private static final String DIAGNOSTIC_SLOW_QUERIES = "MYSQL_SLOW_QUERIES"; + private static final String DIAGNOSTIC_POOL_EXHAUSTED = "MYSQL_POOL_EXHAUSTED"; + private static final String DIAGNOSTIC_LOG_TEMPLATE = "Diagnostic: {}"; + + // Elasticsearch constants + private static final long ELASTICSEARCH_FUNCTIONAL_CHECKS_THROTTLE_MS = 60_000L; + private static final int ELASTICSEARCH_CONNECT_TIMEOUT_MS = 2_000; + private static final int ELASTICSEARCH_SOCKET_TIMEOUT_MS = 2_000; + private static final int ELASTICSEARCH_CANARY_TIMEOUT_MS = 500; + private static final String ES_CLUSTER_STATUS_YELLOW = "yellow"; + private static final String ES_CLUSTER_STATUS_RED = "red"; + + private static final boolean ADVANCED_HEALTH_CHECKS_ENABLED = true; + + private final DataSource dataSource; + private final ExecutorService advancedCheckExecutor; + private final RedisTemplate redisTemplate; + private final String elasticsearchHost; + private final int elasticsearchPort; + private final boolean elasticsearchEnabled; + private final boolean elasticsearchIndexingRequired; + private final String elasticsearchTargetIndex; + private static final ObjectMapper objectMapper = new ObjectMapper(); + + private RestClient elasticsearchRestClient; + private boolean elasticsearchClientReady = false; + + private volatile long lastAdvancedCheckTime = 0L; + private volatile AdvancedCheckResult cachedAdvancedCheckResult = null; + private final ReentrantReadWriteLock advancedCheckLock = new ReentrantReadWriteLock(); + private final AtomicBoolean advancedCheckInProgress = new AtomicBoolean(false); + + private final AtomicReference elasticsearchCache = new AtomicReference<>(null); + private volatile long lastElasticsearchFunctionalCheckTime = 0L; + private final AtomicBoolean elasticsearchCheckInProgress = new AtomicBoolean(false); + private final AtomicBoolean elasticsearchFunctionalCheckInProgress = new AtomicBoolean(false); + + + public HealthService( + DataSource dataSource, + @Autowired(required = false) RedisTemplate redisTemplate, + @Value("${elasticsearch.host:localhost}") String elasticsearchHost, + @Value("${elasticsearch.port:9200}") int elasticsearchPort, + @Value("${elasticsearch.enabled:false}") boolean elasticsearchEnabled, + @Value("${elasticsearch.target-index:amrit_data}") String elasticsearchTargetIndex, + @Value("${elasticsearch.indexing-required:false}") boolean elasticsearchIndexingRequired) { + + this.dataSource = dataSource; + this.advancedCheckExecutor = Executors.newSingleThreadExecutor(r -> { + Thread t = new Thread(r, "health-advanced-check"); + t.setDaemon(true); + return t; + }); + this.redisTemplate = redisTemplate; + this.elasticsearchHost = elasticsearchHost; + this.elasticsearchPort = elasticsearchPort; + this.elasticsearchEnabled = elasticsearchEnabled; + this.elasticsearchIndexingRequired = elasticsearchIndexingRequired; + this.elasticsearchTargetIndex = (elasticsearchTargetIndex != null) ? elasticsearchTargetIndex : "amrit_data"; + } + + @PostConstruct + public void init() { + if (elasticsearchEnabled) { + initializeElasticsearchClient(); + } + } + + @jakarta.annotation.PreDestroy + public void cleanup() { + advancedCheckExecutor.shutdownNow(); + if (elasticsearchRestClient != null) { + try { + elasticsearchRestClient.close(); + } catch (IOException e) { + logger.warn("Error closing Elasticsearch client", e); + } + } + } + + private void initializeElasticsearchClient() { + try { + this.elasticsearchRestClient = RestClient.builder( + new HttpHost(elasticsearchHost, elasticsearchPort, "http")) + .setRequestConfigCallback(cb -> cb + .setConnectTimeout(ELASTICSEARCH_CONNECT_TIMEOUT_MS) + .setSocketTimeout(ELASTICSEARCH_SOCKET_TIMEOUT_MS)) + .build(); + this.elasticsearchClientReady = true; + logger.info("Elasticsearch client initialized (connect/socket timeout: {}ms)", + ELASTICSEARCH_CONNECT_TIMEOUT_MS); + } catch (Exception e) { + logger.warn("Failed to initialize Elasticsearch client: {}", e.getMessage()); + this.elasticsearchClientReady = false; + } + } + + + public Map checkHealth() { + Map healthStatus = new LinkedHashMap<>(); + Map components = new LinkedHashMap<>(); + boolean overallHealth = true; + + + Map mysqlStatus = checkMySQLHealth(); + components.put("mysql", mysqlStatus); + if (!isHealthy(mysqlStatus)) { + overallHealth = false; + } + + if (redisTemplate != null) { + Map redisStatus = checkRedisHealth(); + components.put("redis", redisStatus); + if (!isHealthy(redisStatus)) { + overallHealth = false; + } + } + if (elasticsearchEnabled && elasticsearchClientReady) { + Map esStatus = checkElasticsearchHealth(); + components.put("elasticsearch", esStatus); + if (!isHealthy(esStatus)) { + overallHealth = false; + } + } + + healthStatus.put(STATUS_KEY, overallHealth ? STATUS_UP : STATUS_DOWN); + healthStatus.put("timestamp", Instant.now().toString()); + healthStatus.put("components", components); + logger.info("Health check completed – overall: {}", overallHealth ? STATUS_UP : STATUS_DOWN); + return healthStatus; + } + + private Map checkMySQLHealth() { + Map details = new LinkedHashMap<>(); + details.put("type", MYSQL_COMPONENT); + + return performHealthCheck(MYSQL_COMPONENT, details, () -> { + try (Connection connection = dataSource.getConnection()) { + if (!connection.isValid(2)) { + return new HealthCheckResult(false, "Connection validation failed", false); + } + try (PreparedStatement stmt = connection.prepareStatement("SELECT 1 as health_check")) { + stmt.setQueryTimeout(3); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next() && rs.getInt(1) == 1) { + boolean isDegraded = performAdvancedMySQLChecksWithThrottle(); + return new HealthCheckResult(true, null, isDegraded); + } + } + } + return new HealthCheckResult(false, "Unexpected query result", false); + } catch (Exception e) { + throw new IllegalStateException(MYSQL_COMPONENT + " connection failed: " + e.getMessage(), e); + } + }); + } + + private Map checkRedisHealth() { + Map details = new LinkedHashMap<>(); + details.put("type", REDIS_COMPONENT); + + return performHealthCheck(REDIS_COMPONENT, details, () -> { + try { + String pong = redisTemplate.execute((RedisCallback) conn -> conn.ping()); + if ("PONG".equals(pong)) { + return new HealthCheckResult(true, null, false); + } + return new HealthCheckResult(false, "Unexpected ping response", false); + } catch (Exception e) { + throw new IllegalStateException("Redis health check failed", e); + } + }); + } + + private Map checkElasticsearchHealth() { + Map details = new LinkedHashMap<>(); + details.put("type", ELASTICSEARCH_TYPE); + return performHealthCheck(ELASTICSEARCH_TYPE, details, this::getElasticsearchHealthResult); + } + + private HealthCheckResult getElasticsearchHealthResult() { + if (!elasticsearchClientReady || elasticsearchRestClient == null) { + return new HealthCheckResult(false, "Service unavailable", false); + } + + long now = System.currentTimeMillis(); + HealthCheckResult cached = getCachedElasticsearchHealth(now); + if (cached != null) { + return cached; + } + + return performElasticsearchHealthCheckWithCache(now); + } + + private HealthCheckResult getCachedElasticsearchHealth(long now) { + ElasticsearchCacheEntry cached = elasticsearchCache.get(); + if (cached != null && !cached.isExpired(now)) { + logger.debug("Returning cached ES health (age: {}ms, status: {})", + now - cached.timestamp, cached.result.isHealthy ? STATUS_UP : STATUS_DOWN); + return cached.result; + } + return null; + } + + private HealthCheckResult performElasticsearchHealthCheckWithCache(long now) { + // Single-flight: only one thread probes ES; others use stale cache + if (!elasticsearchCheckInProgress.compareAndSet(false, true)) { + ElasticsearchCacheEntry fallback = elasticsearchCache.get(); + if (fallback != null) { + logger.debug("ES check already in progress – using stale cache"); + return fallback.result; + } + // On cold start with concurrent requests, return DEGRADED (not DOWN) until first result + logger.debug("ES check already in progress with no cache – returning DEGRADED"); + return new HealthCheckResult(true, null, true); + } + + try { + HealthCheckResult result = performElasticsearchHealthCheck(); + elasticsearchCache.set(new ElasticsearchCacheEntry(result, now)); + return result; + } catch (Exception e) { + logger.debug("Elasticsearch health check exception: {}", e.getClass().getSimpleName()); + HealthCheckResult errorResult = new HealthCheckResult(false, "Service unavailable", false); + elasticsearchCache.set(new ElasticsearchCacheEntry(errorResult, now)); + return errorResult; + } finally { + elasticsearchCheckInProgress.set(false); + } + } + + private HealthCheckResult performElasticsearchHealthCheck() { + ClusterHealthStatus healthStatus = getClusterHealthStatus(); + if (healthStatus == null) { + // Cluster health unavailable; check if index is reachable to determine degradation vs DOWN + if (indexExists()) { + logger.debug("Cluster health unavailable but index is reachable – returning DEGRADED"); + return new HealthCheckResult(true, null, true); // DEGRADED: index reachable but cluster health offline + } + logger.warn("Cluster health unavailable and index unreachable"); + return new HealthCheckResult(false, "Cluster health unavailable", false); + } + if (ES_CLUSTER_STATUS_RED.equals(healthStatus.status)) { + return new HealthCheckResult(false, "Cluster red", false); + } + + boolean isDegraded = ES_CLUSTER_STATUS_YELLOW.equals(healthStatus.status); + + String functionalCheckError = shouldRunFunctionalChecks() + ? performThrottledFunctionalChecksWithError() + : null; + + if (functionalCheckError != null) { + return new HealthCheckResult(false, functionalCheckError, false); + } + return new HealthCheckResult(true, null, isDegraded); + } + + private ClusterHealthStatus getClusterHealthStatus() { + try { + Request request = new Request("GET", "/_cluster/health"); + applyTimeouts(request, ELASTICSEARCH_CONNECT_TIMEOUT_MS); + var response = elasticsearchRestClient.performRequest(request); + if (response.getStatusLine().getStatusCode() != 200) { + logger.debug("Cluster health returned HTTP {}", response.getStatusLine().getStatusCode()); + return null; + } + String body = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); + JsonNode root = objectMapper.readTree(body); + String status = root.path(STATUS_KEY).asText(); + if (status == null || status.isEmpty()) { + logger.debug("Could not parse cluster status"); + return null; + } + return new ClusterHealthStatus(status); + } catch (java.net.ConnectException | java.net.SocketTimeoutException e) { + logger.debug("Elasticsearch network error: {}", e.getClass().getSimpleName()); + } catch (IOException e) { + logger.debug("Elasticsearch IO error: {}", e.getClass().getSimpleName()); + } catch (Exception e) { + logger.debug("Elasticsearch health check error: {}", e.getClass().getSimpleName()); + } + return null; + } + + private boolean shouldRunFunctionalChecks() { + return (System.currentTimeMillis() - lastElasticsearchFunctionalCheckTime) + >= ELASTICSEARCH_FUNCTIONAL_CHECKS_THROTTLE_MS; + } + + private String performThrottledFunctionalChecksWithError() { + if (!elasticsearchFunctionalCheckInProgress.compareAndSet(false, true)) { + logger.debug("Functional checks already in progress – skipping"); + return null; + } + try { + long now = System.currentTimeMillis(); + + if (!indexExists()) { + logger.warn("Functional check failed: index missing"); + lastElasticsearchFunctionalCheckTime = now; + return "Index missing"; + } + + ReadOnlyCheckResult readOnlyResult = isClusterReadOnly(); + if (readOnlyResult.isReadOnly) { + logger.warn("Functional check failed: cluster is read-only"); + lastElasticsearchFunctionalCheckTime = now; + return "Read-only block"; + } + if (readOnlyResult.isUnableToDetermine) { + logger.warn("Functional check degraded: unable to determine read-only state"); + } + + CanaryWriteResult canaryResult = performCanaryWriteProbe(); + if (!canaryResult.success) { + if (elasticsearchIndexingRequired) { + logger.warn("Functional check failed: canary write unsuccessful – {}", canaryResult.errorCategory); + lastElasticsearchFunctionalCheckTime = now; + return "Canary write failed: " + canaryResult.errorCategory; + } else { + logger.debug("Canary write unsuccessful but indexing not required: {}", canaryResult.errorCategory); + } + } + + lastElasticsearchFunctionalCheckTime = now; + return null; + } finally { + elasticsearchFunctionalCheckInProgress.set(false); + } + } + + private boolean indexExists() { + try { + Request request = new Request("HEAD", "/" + elasticsearchTargetIndex); + applyTimeouts(request, ELASTICSEARCH_CANARY_TIMEOUT_MS); + var response = elasticsearchRestClient.performRequest(request); + return response.getStatusLine().getStatusCode() == 200; + } catch (Exception e) { + logger.debug("Index existence check failed: {}", e.getClass().getSimpleName()); + return false; + } + } + + private ReadOnlyCheckResult isClusterReadOnly() { + try { + Request request = new Request("GET", "/_cluster/settings?include_defaults=true"); + applyTimeouts(request, ELASTICSEARCH_CONNECT_TIMEOUT_MS); + var response = elasticsearchRestClient.performRequest(request); + if (response.getStatusLine().getStatusCode() != 200) { + logger.debug("Cluster settings returned HTTP {}", response.getStatusLine().getStatusCode()); + return new ReadOnlyCheckResult(false, true); + } + String body = new String(response.getEntity().getContent().readAllBytes(), StandardCharsets.UTF_8); + JsonNode root = objectMapper.readTree(body); + boolean hasReadOnlyBlock = hasReadOnlyFlag(root, "read_only"); + boolean hasReadOnlyDeleteBlock = hasReadOnlyFlag(root, "read_only_allow_delete"); + return new ReadOnlyCheckResult(hasReadOnlyBlock || hasReadOnlyDeleteBlock, false); + } catch (java.net.SocketTimeoutException e) { + logger.debug("Read-only check timeout"); + } catch (IOException e) { + logger.debug("Read-only check IO error: {}", e.getClass().getSimpleName()); + } catch (Exception e) { + logger.debug("Read-only check failed: {}", e.getClass().getSimpleName()); + } + return new ReadOnlyCheckResult(false, true); + } + + private boolean hasReadOnlyFlag(JsonNode root, String flag) { + String[] paths = { + "/persistent/cluster/blocks/" + flag, + "/transient/cluster/blocks/" + flag, + "/defaults/cluster/blocks/" + flag + }; + for (String path : paths) { + try { + JsonNode node = root.at(path); + if (node != null && !node.isMissingNode() && + ((node.isBoolean() && node.asBoolean()) || + (node.isTextual() && "true".equalsIgnoreCase(node.asText())))) { + logger.debug("Found read-only flag at {}", path); + return true; + } + } catch (Exception e) { + logger.debug("Error checking JSON pointer {}: {}", path, e.getClass().getSimpleName()); + } + } + return false; + } + + private void applyTimeouts(Request request, int timeoutMs) { + RequestOptions options = RequestOptions.DEFAULT.toBuilder() + .setRequestConfig(RequestConfig.custom() + .setConnectTimeout(timeoutMs) + .setSocketTimeout(timeoutMs) + .build()) + .build(); + request.setOptions(options); + } + + private void performCanaryDelete(String canaryDocId) { + try { + Request deleteRequest = new Request("DELETE", "/" + elasticsearchTargetIndex + "/_doc/" + canaryDocId); + applyTimeouts(deleteRequest, ELASTICSEARCH_CANARY_TIMEOUT_MS); + elasticsearchRestClient.performRequest(deleteRequest); + } catch (Exception e) { + logger.debug("Canary delete warning: {}", e.getClass().getSimpleName()); + } + } + + private CanaryWriteResult performCanaryWriteProbe() { + String canaryDocId = "health-check-canary"; + try { + String canaryBody = "{\"probe\":true,\"timestamp\":\"" + Instant.now() + "\"}"; + + // FIX: Use PUT (not POST) for a document with a specific ID + Request writeRequest = new Request("PUT", "/" + elasticsearchTargetIndex + "/_doc/" + canaryDocId); + applyTimeouts(writeRequest, ELASTICSEARCH_CANARY_TIMEOUT_MS); + writeRequest.setEntity(new org.apache.http.entity.StringEntity(canaryBody, StandardCharsets.UTF_8)); + writeRequest.addParameter("refresh", "true"); + + var writeResponse = elasticsearchRestClient.performRequest(writeRequest); + if (writeResponse.getStatusLine().getStatusCode() > 299) { + logger.debug("Canary write failed with HTTP {}", writeResponse.getStatusLine().getStatusCode()); + return new CanaryWriteResult(false, "Write rejected"); + } + + performCanaryDelete(canaryDocId); + return new CanaryWriteResult(true, null); + } catch (java.net.SocketTimeoutException e) { + logger.debug("Canary probe timeout"); + return new CanaryWriteResult(false, "Timeout"); + } catch (java.net.ConnectException e) { + logger.debug("Canary probe connection refused"); + return new CanaryWriteResult(false, "Connection refused"); + } catch (Exception e) { + logger.debug("Canary probe failed: {}", e.getClass().getSimpleName()); + return new CanaryWriteResult(false, "Write failed"); + } + } + + + private Map performHealthCheck(String componentName, + Map details, + Supplier checker) { + Map status = new LinkedHashMap<>(); + long startTime = System.currentTimeMillis(); + try { + HealthCheckResult result = checker.get(); + long responseTime = System.currentTimeMillis() - startTime; + details.put(RESPONSE_TIME_KEY, responseTime); + + if (result.isHealthy) { + buildHealthyStatus(status, componentName, responseTime, result); + } else { + buildUnhealthyStatus(status, details, componentName, result); + } + status.put("details", details); + return status; + } catch (Exception e) { + long responseTime = System.currentTimeMillis() - startTime; + return buildExceptionStatus(status, details, componentName, e, responseTime); + } + } + + private void buildHealthyStatus(Map status, + String componentName, long responseTime, HealthCheckResult result) { + logger.debug("{} health check: {} ({}ms)", + componentName, result.isDegraded ? STATUS_DEGRADED : STATUS_UP, responseTime); + status.put(STATUS_KEY, result.isDegraded ? STATUS_DEGRADED : STATUS_UP); + status.put(SEVERITY_KEY, determineSeverity(true, responseTime, result.isDegraded)); + if (result.error != null) { + status.put(MESSAGE_KEY, result.error); + } + } + + private void buildUnhealthyStatus(Map status, Map details, + String componentName, HealthCheckResult result) { + String internalError = (result.error != null) ? result.error : "Health check failed"; + logger.warn("{} health check failed: {}", componentName, internalError); + status.put(STATUS_KEY, STATUS_DOWN); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + // Sanitized outward message – no topology leakage + details.put(ERROR_KEY, "Dependency unavailable"); + // For Elasticsearch, sanitize detailed failure reasons; keep real reason in logs only + String exposedCategory = ELASTICSEARCH_TYPE.equals(componentName) + ? "DEPENDENCY_FAILURE" + : internalError; + details.put("errorCategory", exposedCategory); + details.put("errorType", "CheckFailed"); + } + + private Map buildExceptionStatus(Map status, Map details, + String componentName, Exception e, long responseTime) { + logger.error("{} health check threw exception: {}", componentName, e.getMessage(), e); + status.put(STATUS_KEY, STATUS_DOWN); + status.put(SEVERITY_KEY, SEVERITY_CRITICAL); + details.put(RESPONSE_TIME_KEY, responseTime); + // FIX: Sanitize error message – do not expose raw exception detail to consumers + details.put(ERROR_KEY, "Dependency unavailable"); + details.put("errorCategory", "CheckException"); + details.put("errorType", "Exception"); + status.put("details", details); + return status; + } + + private String determineSeverity(boolean isHealthy, long responseTimeMs, boolean isDegraded) { + if (!isHealthy) return SEVERITY_CRITICAL; + if (isDegraded) return SEVERITY_WARNING; + if (responseTimeMs > RESPONSE_TIME_THRESHOLD_MS) return SEVERITY_WARNING; + return SEVERITY_OK; + } + + private boolean isHealthy(Map componentStatus) { + Object s = componentStatus.get(STATUS_KEY); + return STATUS_UP.equals(s) || STATUS_DEGRADED.equals(s); + } + + private boolean performAdvancedMySQLChecksWithThrottle() { + if (!ADVANCED_HEALTH_CHECKS_ENABLED) { + return false; + } + + long currentTime = System.currentTimeMillis(); + + // --- Phase 1: try to serve from cache (read lock) --- + advancedCheckLock.readLock().lock(); + try { + if (cachedAdvancedCheckResult != null && + (currentTime - lastAdvancedCheckTime) < ADVANCED_CHECKS_THROTTLE_SECONDS * 1_000L) { + return cachedAdvancedCheckResult.isDegraded; + } + } finally { + advancedCheckLock.readLock().unlock(); + } + + // --- Phase 2: single-flight guard --- + if (!advancedCheckInProgress.compareAndSet(false, true)) { + // Another thread is refreshing – return stale cache (safe fallback) + advancedCheckLock.readLock().lock(); + try { + return cachedAdvancedCheckResult != null && cachedAdvancedCheckResult.isDegraded; + } finally { + advancedCheckLock.readLock().unlock(); + } + } + + // --- Phase 3: run DB checks outside any lock --- + try { + AdvancedCheckResult result = performAdvancedMySQLChecks(); + + // --- Phase 4: write-lock for atomic cache update --- + advancedCheckLock.writeLock().lock(); + try { + lastAdvancedCheckTime = System.currentTimeMillis(); + cachedAdvancedCheckResult = result; + return result.isDegraded; + } finally { + advancedCheckLock.writeLock().unlock(); + } + } finally { + advancedCheckInProgress.set(false); + } + } + + private AdvancedCheckResult performAdvancedMySQLChecks() { + try (Connection connection = dataSource.getConnection()) { + return executeAdvancedCheckAsync(connection); + } catch (Exception e) { + logger.debug("Failed to get connection for advanced checks: {}", e.getMessage()); + return new AdvancedCheckResult(true); + } + } + + private AdvancedCheckResult executeAdvancedCheckAsync(Connection connection) { + Future future = advancedCheckExecutor.submit( + () -> performAdvancedCheckLogic(connection)); + try { + return future.get(ADVANCED_CHECKS_TIMEOUT_MS, java.util.concurrent.TimeUnit.MILLISECONDS); + } catch (java.util.concurrent.TimeoutException e) { + logger.debug("Advanced checks timed out – marking degraded"); + future.cancel(true); + } catch (ExecutionException e) { + if (e.getCause() instanceof InterruptedException) { + Thread.currentThread().interrupt(); + } + logger.debug("Advanced checks execution failed – marking degraded"); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + logger.debug("Advanced checks interrupted – marking degraded"); + } catch (Exception e) { + logger.debug("Advanced checks encountered exception – marking degraded"); + } + return new AdvancedCheckResult(true); + } + + private AdvancedCheckResult performAdvancedCheckLogic(Connection connection) { + try { + boolean hasIssues = false; + + if (hasLockWaits(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_LOCK_WAIT); + hasIssues = true; + } + if (hasSlowQueries(connection)) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_SLOW_QUERIES); + hasIssues = true; + } + if (hasConnectionPoolExhaustion()) { + logger.warn(DIAGNOSTIC_LOG_TEMPLATE, DIAGNOSTIC_POOL_EXHAUSTED); + hasIssues = true; + } + return new AdvancedCheckResult(hasIssues); + } catch (Exception e) { + logger.debug("Advanced check logic encountered exception"); + return new AdvancedCheckResult(true); + } + } + + private boolean hasLockWaits(Connection connection) { + String sql = + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE (state = 'Waiting for table metadata lock' " + + " OR state = 'Waiting for row lock' " + + " OR state = 'Waiting for lock') " + + "AND user = USER()"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + stmt.setQueryTimeout(2); + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return rs.getInt(1) > 0; + } + } + } catch (Exception e) { + logger.debug("Could not check for lock waits"); + } + return false; + } + + private boolean hasSlowQueries(Connection connection) { + String sql = + "SELECT COUNT(*) FROM INFORMATION_SCHEMA.PROCESSLIST " + + "WHERE command != 'Sleep' AND time > ? AND user = SUBSTRING_INDEX(USER(), '@', 1)"; + try (PreparedStatement stmt = connection.prepareStatement(sql)) { + stmt.setQueryTimeout(2); + stmt.setInt(1, 10); // queries running > 10 seconds + try (ResultSet rs = stmt.executeQuery()) { + if (rs.next()) { + return rs.getInt(1) > 3; // more than 3 slow queries + } + } + } catch (Exception e) { + logger.debug("Could not check for slow queries"); + } + return false; + } + + private boolean hasConnectionPoolExhaustion() { + if (dataSource instanceof HikariDataSource hikariDataSource) { + try { + HikariPoolMXBean poolMXBean = hikariDataSource.getHikariPoolMXBean(); + if (poolMXBean != null) { + int activeConnections = poolMXBean.getActiveConnections(); + int maxPoolSize = hikariDataSource.getMaximumPoolSize(); + int threshold = (int) (maxPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + logger.debug("Could not retrieve HikariCP pool metrics directly"); + } + } + return checkPoolMetricsViaJMX(); + } + + private boolean checkPoolMetricsViaJMX() { + try { + MBeanServer mBeanServer = ManagementFactory.getPlatformMBeanServer(); + ObjectName objectName = new ObjectName("com.zaxxer.hikari:type=Pool (*)"); + var mBeans = mBeanServer.queryMBeans(objectName, null); + for (var mBean : mBeans) { + if (evaluatePoolMetrics(mBeanServer, mBean.getObjectName())) { + return true; + } + } + } catch (Exception e) { + logger.debug("Could not access HikariCP pool metrics via JMX"); + } + logger.debug("Pool exhaustion check disabled: HikariCP metrics unavailable"); + return false; + } + + private boolean evaluatePoolMetrics(MBeanServer mBeanServer, ObjectName objectName) { + try { + Integer activeConnections = (Integer) mBeanServer.getAttribute(objectName, "ActiveConnections"); + Integer maximumPoolSize = (Integer) mBeanServer.getAttribute(objectName, "MaximumPoolSize"); + if (activeConnections != null && maximumPoolSize != null) { + int threshold = (int) (maximumPoolSize * 0.8); + return activeConnections > threshold; + } + } catch (Exception e) { + // Continue to next MBean + } + return false; + } + + + private static class AdvancedCheckResult { + final boolean isDegraded; + AdvancedCheckResult(boolean isDegraded) { this.isDegraded = isDegraded; } + } + + private static class HealthCheckResult { + final boolean isHealthy; + final String error; + final boolean isDegraded; + HealthCheckResult(boolean isHealthy, String error, boolean isDegraded) { + this.isHealthy = isHealthy; + this.error = error; + this.isDegraded = isDegraded; + } + } + + private static class ElasticsearchCacheEntry { + final HealthCheckResult result; + final long timestamp; + ElasticsearchCacheEntry(HealthCheckResult result, long timestamp) { + this.result = result; + this.timestamp = timestamp; + } + /** UP results cache for 30 s; DOWN results for 5 s for faster recovery. */ + boolean isExpired(long now) { + long ttlMs = result.isHealthy ? 30_000L : 5_000L; + return (now - timestamp) >= ttlMs; + } + } + + private static class ClusterHealthStatus { + final String status; + ClusterHealthStatus(String status) { this.status = status; } + } + + private static class ReadOnlyCheckResult { + final boolean isReadOnly; + final boolean isUnableToDetermine; + ReadOnlyCheckResult(boolean isReadOnly, boolean isUnableToDetermine) { + this.isReadOnly = isReadOnly; + this.isUnableToDetermine = isUnableToDetermine; + } + } + + private static class CanaryWriteResult { + final boolean success; + final String errorCategory; + CanaryWriteResult(boolean success, String errorCategory) { + this.success = success; + this.errorCategory = errorCategory; + } + } +} \ No newline at end of file diff --git a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java index 6d5c55f9..c63cf3c2 100644 --- a/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java +++ b/src/main/java/com/iemr/common/identity/utils/JwtUserIdValidationFilter.java @@ -43,7 +43,14 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo return; } String path = request.getRequestURI(); - logger.info("JwtUserIdValidationFilter invoked for path: " + path); + logger.info("JwtUserIdValidationFilter invoked for path: {}", path); + + // Skip JWT validation for public endpoints + if (path.equals("/health") || path.equals("/version")) { + logger.info("Public endpoint accessed: {} - skipping JWT validation", path); + filterChain.doFilter(servletRequest, servletResponse); + return; + } // Log cookies for debugging Cookie[] cookies = request.getCookies(); @@ -76,10 +83,10 @@ public void doFilter(ServletRequest servletRequest, ServletResponse servletRespo } } else { String userAgent = request.getHeader("User-Agent"); - logger.info("User-Agent: " + userAgent); + logger.info("User-Agent: {}", userAgent); if (userAgent != null && isMobileClient(userAgent) && authHeader != null) { try { - logger.info("Common-API incoming userAget : " + userAgent); + logger.info("Common-API incoming userAgent: {}", userAgent); UserAgentContext.setUserAgent(userAgent); filterChain.doFilter(servletRequest, servletResponse); } finally {