implement startup readiness health checks

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/service/LivenessHealthCheck.java

@@ -0,0 +1,28 @@
+package tech.ebp.oqm.core.api.health.service;
+
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import org.eclipse.microprofile.health.Liveness;
+import tech.ebp.oqm.core.api.health.utils.GenericHealthCheck;
+import tech.ebp.oqm.core.api.health.utils.HasLivenessCheck;
+import tech.ebp.oqm.core.api.health.utils.HealthStatus;
+
+@Liveness
+@ApplicationScoped
+public class LivenessHealthCheck extends GenericHealthCheck<HasLivenessCheck> {
+
+    @Inject
+    LivenessHealthCheck(Instance<HasLivenessCheck> providers) {
+        super("Service Health - Liveness", providers);
+    }
+
+    public LivenessHealthCheck() {
+        super("Service Health - Liveness", null);
+    }
+
+    @Override
+    protected HealthStatus getStatus(HasLivenessCheck provider) {
+        return provider.getLivenessStatus();
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/service/ReadinessHealthCheck.java

@@ -0,0 +1,28 @@
+package tech.ebp.oqm.core.api.health.service;
+
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import org.eclipse.microprofile.health.Readiness;
+import tech.ebp.oqm.core.api.health.utils.GenericHealthCheck;
+import tech.ebp.oqm.core.api.health.utils.HasReadinessCheck;
+import tech.ebp.oqm.core.api.health.utils.HealthStatus;
+
+@Readiness
+@ApplicationScoped
+public class ReadinessHealthCheck extends GenericHealthCheck<HasReadinessCheck> {
+
+    @Inject
+    ReadinessHealthCheck(Instance<HasReadinessCheck> providers) {
+        super("Service Health - Readiness", providers);
+    }
+
+    public ReadinessHealthCheck() {
+        super("Service Health - Readiness", null);
+    }
+
+    @Override
+    protected HealthStatus getStatus(HasReadinessCheck provider) {
+        return provider.getReadinessStatus();
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/service/StartupHealthCheck.java

@@ -0,0 +1,28 @@
+package tech.ebp.oqm.core.api.health.service;
+
+import io.quarkus.runtime.Startup;
+import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.inject.Instance;
+import jakarta.inject.Inject;
+import tech.ebp.oqm.core.api.health.utils.GenericHealthCheck;
+import tech.ebp.oqm.core.api.health.utils.HasStartupCheck;
+import tech.ebp.oqm.core.api.health.utils.HealthStatus;
+
+@Startup
+@ApplicationScoped
+public class StartupHealthCheck extends GenericHealthCheck<HasStartupCheck> {
+
+    @Inject
+    StartupHealthCheck(Instance<HasStartupCheck> providers) {
+        super("Service Health - Startup", providers);
+    }
+
+    public StartupHealthCheck() {
+        super("Service Health - Startup", null);
+    }
+
+    @Override
+    protected HealthStatus getStatus(HasStartupCheck provider) {
+        return provider.getStartupStatus();
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/GenericHealthCheck.java

@@ -0,0 +1,34 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+import jakarta.enterprise.inject.Instance;
+import org.eclipse.microprofile.health.HealthCheck;
+import org.eclipse.microprofile.health.HealthCheckResponse;
+import org.eclipse.microprofile.health.HealthCheckResponseBuilder;
+
+public abstract class GenericHealthCheck<T extends HasHealthCheck> implements HealthCheck {
+    private final String healthCheckName;
+    private final Instance<T> providers;
+
+    public GenericHealthCheck(String healthCheckName, Instance<T> providers) {
+        this.healthCheckName = healthCheckName;
+        this.providers = providers;
+    }
+
+    protected abstract HealthStatus getStatus(T provider);
+
+    @Override
+    public HealthCheckResponse call() {
+        HealthCheckResponseBuilder builder = HealthCheckResponse.named(this.healthCheckName);
+        boolean allUp = true;
+
+        for (T provider : this.providers) {
+            HealthStatus status = this.getStatus(provider);
+            boolean up = status.isUp();
+            allUp &= up;
+            builder.withData(status.getName() + ".up", up);
+            builder.withData(status.getName() + ".status", status.getStatusMessage());
+        }
+
+        return (allUp ? builder.up() : builder.down()).build();
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HasHealthCheck.java

@@ -0,0 +1,3 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+public interface HasHealthCheck {}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HasLivenessCheck.java

@@ -0,0 +1,5 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+public interface HasLivenessCheck extends HasHealthCheck {
+    HealthStatus getLivenessStatus();
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HasReadinessCheck.java

@@ -0,0 +1,5 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+public interface HasReadinessCheck extends HasHealthCheck {
+    HealthStatus getReadinessStatus();
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HasStartupCheck.java

@@ -0,0 +1,5 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+public interface HasStartupCheck extends HasHealthCheck {
+    HealthStatus getStartupStatus();
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/health/utils/HealthStatus.java

@@ -0,0 +1,24 @@
+package tech.ebp.oqm.core.api.health.utils;
+
+import lombok.Getter;
+
+@Getter
+public class HealthStatus {
+    private final String name;
+    private volatile boolean up = false;
+    private volatile String statusMessage = "Status not set";
+
+    public HealthStatus(String name) {
+        this.name = name;
+    }
+
+    public void markUp(String message) {
+        up = true;
+        statusMessage = message;
+    }
+
+    public void markDown(String message) {
+        up = false;
+        statusMessage = message;
+    }
+}

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/scheduled/LifecycleBean.java

@@ -8,10 +8,8 @@
 import lombok.extern.slf4j.Slf4j;
 import org.eclipse.microprofile.config.ConfigProvider;
 import org.eclipse.microprofile.config.inject.ConfigProperty;
-import tech.ebp.oqm.core.api.model.object.upgrade.TotalUpgradeResult;
 import tech.ebp.oqm.core.api.service.TempFileService;
 import tech.ebp.oqm.core.api.service.mongo.CustomUnitService;
-import tech.ebp.oqm.core.api.service.schemaVersioning.ObjectSchemaUpgradeService;
 import tech.ebp.oqm.core.api.service.serviceState.db.OqmDatabaseService;
 
 import java.nio.file.Paths;
@@ -38,7 +36,7 @@ public class LifecycleBean {
 	TempFileService tempFileService;
 
 	@Inject
-	OqmDatabaseService dbService;
+    OqmDatabaseService dbService;
 
 	private ZonedDateTime startDateTime;
 

software/core/oqm-core-api/src/main/java/tech/ebp/oqm/core/api/scheduled/MongoDbInit.java

@@ -6,21 +6,23 @@
 import jakarta.enterprise.inject.Instance;
 import jakarta.inject.Inject;
 import jakarta.inject.Singleton;
+import lombok.Getter;
 import lombok.extern.slf4j.Slf4j;
 import tech.ebp.oqm.core.api.model.object.upgrade.TotalUpgradeResult;
 import tech.ebp.oqm.core.api.service.mongo.InventoryItemService;
-import tech.ebp.oqm.core.api.service.mongo.MongoDbAwareService;
 import tech.ebp.oqm.core.api.service.mongo.MongoService;
 import tech.ebp.oqm.core.api.service.schemaVersioning.ObjectSchemaUpgradeService;
 import tech.ebp.oqm.core.api.service.serviceState.InstanceMutexService;
 import tech.ebp.oqm.core.api.service.serviceState.db.DbCacheEntry;
 import tech.ebp.oqm.core.api.service.serviceState.db.OqmDatabaseService;
+import tech.ebp.oqm.core.api.health.utils.HealthStatus;
+import tech.ebp.oqm.core.api.health.utils.HasReadinessCheck;
 
 import java.util.Optional;
 
 @Singleton
 @Slf4j
-public class MongoDbInit {
+public class MongoDbInit implements HasReadinessCheck {
 
     @Inject
     InventoryItemService inventoryItemService;
@@ -46,52 +48,64 @@ public class MongoDbInit {
      * This can be removed once all inventory items have mutexes.
      */
     private void ensureItemMutexesExist() {
-        log.info("Ensuring inventory item mutexes exist.");
-
-        for (DbCacheEntry curDb : this.oqmDatabaseService.getDatabases()) {
-            log.info("Ensuring inventory item mutexes exist for database: {}", curDb.getDbName());
-            this.inventoryItemService.iterator(curDb.getDbId().toHexString()).forEachRemaining((item) -> {
-                this.instanceMutexService.register(
-                    this.instanceMutexService.getMutexIdFor(curDb.getDbId().toHexString(), item)
-                );
-            });
-            log.info("DONE Ensuring inventory item mutexes exist for database: {}", curDb.getDbName());
+        try {
+            log.info("Ensuring inventory item mutexes exist.");
+
+            for (DbCacheEntry curDb : this.oqmDatabaseService.getDatabases()) {
+                log.info("Ensuring inventory item mutexes exist for database: {}", curDb.getDbName());
+                this.inventoryItemService.iterator(curDb.getDbId().toHexString()).forEachRemaining((item) -> {
+                    this.instanceMutexService.register(this.instanceMutexService.getMutexIdFor(curDb.getDbId().toHexString(), item));
+                });
+                log.info("DONE Ensuring inventory item mutexes exist for database: {}", curDb.getDbName());
+            }
+
+            log.info("DONE Ensuring inventory item mutexes exist.");
+        } catch (RuntimeException e) {
+            readinessStatus.markDown("Inventory item mutex initialization failed: " + e.getMessage());
+            throw e;
         }
-
-        log.info("DONE Ensuring inventory item mutexes exist.");
     }
 
 	private void upgradeDbs(){
 		//TODO:: create flag service to check if things initted right. Setup filter to check this flag to reject requests until setup done.
 		//TODO:: integrate into healthcheck. only DOWN if db upgrade failed
-		Optional<TotalUpgradeResult> schemaUpgradeResult = this.objectSchemaUpgradeService.updateSchema();
-		if(schemaUpgradeResult.isEmpty()){
-			log.warn("Did not upgrade schema at start.");
-		} else {
-			log.info("Schema upgrade result: {}", schemaUpgradeResult.get());
-			//TODO:: rescan inv update stats
+		try {
+			Optional<TotalUpgradeResult> schemaUpgradeResult = this.objectSchemaUpgradeService.updateSchema();
+			if(schemaUpgradeResult.isEmpty()){
+				log.warn("Did not upgrade schema at start.");
+			} else {
+				log.info("Schema upgrade result: {}", schemaUpgradeResult.get());
+				//TODO:: rescan inv update stats
+			}
+		} catch (RuntimeException e) {
+			readinessStatus.markDown("Database schema upgrade failed: " + e.getMessage());
+			throw e;
 		}
 	}
 
 	private void initDbs(){
-		log.info("Initializing all databases.");
-		for(MongoService<?, ?, ?> service : this.mongoServices){
-			service.initDb();
+		try {
+			log.info("Initializing all databases.");
+			for(MongoService<?, ?, ?> service : this.mongoServices){
+				service.initDb();
+			}
+			log.info("DONE initializing all databases.");
+		} catch (RuntimeException e) {
+			readinessStatus.markDown("Database initialization failed: " + e.getMessage());
+			throw e;
 		}
-		log.info("DONE initializing all databases.");
 	}
 
 
-    void onStart(
-        @Observes
-        StartupEvent ev
-    ) {
-		log.info("Starting initial db initialization tasks.");
-
-		this.upgradeDbs();
-		this.initDbs();
-		this.ensureItemMutexesExist();
+    @Getter
+    private final HealthStatus readinessStatus = new HealthStatus("Mongo DB Init");
 
-		log.info("FINISHED initial db initialization tasks.");
+    void onStart(@Observes StartupEvent ev) {
+        readinessStatus.markDown("Startup initialization in progress");
+        this.upgradeDbs();
+        this.initDbs();
+        this.ensureItemMutexesExist();
+        readinessStatus.markUp("Initial db initialization tasks finished");
+        log.info("FINISHED initial db initialization tasks.");
     }
 }