Skip to content

Commit

Permalink
fix(#41): merge with dev branch
Browse files Browse the repository at this point in the history
  • Loading branch information
@novoj
novoj committed May 28, 2024
1 parent 6884cab commit d75ccf8
Show file tree
Hide file tree
Showing 5 changed files with 83 additions and 148 deletions.
12 changes: 8 additions & 4 deletions evita_engine/src/main/java/io/evitadb/core/Evita.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -61,7 +61,6 @@
import io.evitadb.core.metric.event.storage.EvitaDBCompositionChangedEvent;
import io.evitadb.core.metric.event.system.EvitaStartedEvent;
import io.evitadb.core.query.algebra.Formula;
import io.evitadb.core.scheduling.BackgroundTask;
import io.evitadb.exception.EvitaInvalidUsageException;
import io.evitadb.exception.UnexpectedIOException;
import io.evitadb.scheduling.RejectingExecutor;
Expand Down Expand Up @@ -626,12 +625,10 @@ public CatalogContract getCatalogInstanceOrThrowException(@Nonnull String catalo
* @param catalogName name of the catalog
*/
private void loadCatalog(@Nonnull String catalogName) {
final Path directory = configuration.storage().storageDirectoryOrDefault().resolve(catalogName);
try {
final long start = System.nanoTime();
final CatalogContract catalog = new Catalog(
catalogName,
directory,
this.cacheSupervisor,
this.configuration.storage(),
this.configuration.transaction(),
Expand All @@ -648,7 +645,14 @@ private void loadCatalog(@Nonnull String catalogName) {
);
} catch (Throwable ex) {
log.error("Catalog {} is corrupted!", catalogName);
this.catalogs.put(catalogName, new CorruptedCatalog(catalogName, directory, ex));
this.catalogs.put(
catalogName,
new CorruptedCatalog(
catalogName,
configuration.storage().storageDirectoryOrDefault().resolve(catalogName),
ex
)
);
}
}

Expand Down
106 changes: 19 additions & 87 deletions ...rc/main/java/io/evitadb/externalApi/observability/metric/ObservabilityProbesDetector.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,6 @@
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.atomic.AtomicReference;
import java.util.stream.Collectors;

/**
* This class is responsible for detecting health problems in the system. It monitors:
Expand All @@ -78,6 +77,7 @@ public class ObservabilityProbesDetector implements ProbesProvider {
.stream()
.filter(gc -> OLD_GENERATION_GC_NAMES.contains(gc.getName()))
.toList();
private ObservabilityManager observabilityManager;
private final AtomicLong lastSeenRejectedTaskCount = new AtomicLong(0L);
private final AtomicLong lastSeenSubmittedTaskCount = new AtomicLong(0L);
private final AtomicLong lastSeenJavaErrorCount = new AtomicLong(0L);
Expand All @@ -87,40 +87,6 @@ public class ObservabilityProbesDetector implements ProbesProvider {
private final AtomicBoolean seenReady = new AtomicBoolean();
private final AtomicReference<ReadinessWithTimestamp> lastReadinessSeen = new AtomicReference<>();

@Nonnull
@Override
public Set<HealthProblem> getHealthProblems(
@Nonnull EvitaContract evitaContract,
@Nonnull ExternalApiServer externalApiServer,
@Nonnull String... apiCodes
) {
final EnumSet<HealthProblem> healthProblems = EnumSet.noneOf(HealthProblem.class);
final ObservabilityManager theObservabilityManager = getObservabilityManager(externalApiServer).orElse(null);

if (evitaContract instanceof Evita evita) {
recordResult(checkInputQueues(evita), healthProblems, theObservabilityManager);
}

if (theObservabilityManager != null) {
recordResult(checkEvitaErrors(theObservabilityManager), healthProblems, theObservabilityManager);
recordResult(checkMemoryShortage(theObservabilityManager), healthProblems, theObservabilityManager);
recordResult(checkJavaErrors(theObservabilityManager), healthProblems, theObservabilityManager);
}

final ReadinessWithTimestamp readinessWithTimestamp = this.lastReadinessSeen.get();
if (readinessWithTimestamp == null ||
(OffsetDateTime.now().minus(HEALTH_CHECK_READINESS_RENEW_INTERVAL).isAfter(readinessWithTimestamp.timestamp()) ||
readinessWithTimestamp.result().state() != ReadinessState.READY)
) {
// enforce renewal of readiness check
getReadiness(evitaContract, externalApiServer, apiCodes);
}

recordResult(checkApiReadiness(), healthProblems, theObservabilityManager);

return healthProblems.isEmpty() ? NO_HEALTH_PROBLEMS : healthProblems;
}

/**
* Records the result of the health problem check.
*
Expand Down Expand Up @@ -149,7 +115,11 @@ private static void recordResult(

@Nonnull
@Override
public Set<HealthProblem> getHealthProblems(@Nonnull EvitaContract evitaContract, @Nonnull ExternalApiServer externalApiServer) {
public Set<HealthProblem> getHealthProblems(
@Nonnull EvitaContract evitaContract,
@Nonnull ExternalApiServer externalApiServer,
@Nonnull String... apiCodes
) {
final EnumSet<HealthProblem> healthProblems = EnumSet.noneOf(HealthProblem.class);
final ObservabilityManager theObservabilityManager = getObservabilityManager(externalApiServer).orElse(null);

Expand All @@ -163,17 +133,18 @@ public Set<HealthProblem> getHealthProblems(@Nonnull EvitaContract evitaContract
recordResult(checkJavaErrors(theObservabilityManager), healthProblems, theObservabilityManager);
}

final ReadinessWithTimestamp readinessWithTimestamp = this.lastReadinessSeen.get();
if (readinessWithTimestamp == null ||
(OffsetDateTime.now().minus(HEALTH_CHECK_READINESS_RENEW_INTERVAL).isAfter(readinessWithTimestamp.timestamp()) ||
readinessWithTimestamp.result().state() != ReadinessState.READY)
) {
// enforce renewal of readiness check
getReadiness(evitaContract, externalApiServer, apiCodes);
}

recordResult(checkApiReadiness(), healthProblems, theObservabilityManager);

if (healthProblems.isEmpty()) {
return NO_HEALTH_PROBLEMS;
} else {
log.error(
"Detected health problems: {}",
healthProblems.stream().map(HealthProblem::name).collect(Collectors.joining(", "))
);
return healthProblems;
}
return healthProblems.isEmpty() ? NO_HEALTH_PROBLEMS : healthProblems;
}

@Nonnull
Expand All @@ -199,19 +170,9 @@ public Readiness getReadiness(@Nonnull EvitaContract evitaContract, @Nonnull Ext
.map(entry -> new ApiState(entry.getKey(), entry.getValue()))
.toArray(ApiState[]::new)
);

final Readiness previousReadiness = this.lastReadinessSeen.get();
if (previousReadiness == null || currentReadiness.state() != previousReadiness.state()) {
if (previousReadiness == null || previousReadiness.state() != ReadinessState.STALLING) {
log.info("System readiness changed to {}", currentReadiness.state());
} else if (currentReadiness.state() == ReadinessState.SHUTDOWN) {
log.warn("System readiness changed to {}", currentReadiness.state());
} else {
log.error("System readiness changed to {}", currentReadiness.state());
}
}

this.lastReadinessSeen.set(currentReadiness);
this.lastReadinessSeen.set(
new ReadinessWithTimestamp(currentReadiness, OffsetDateTime.now())
);
return currentReadiness;
}

Expand Down Expand Up @@ -314,35 +275,6 @@ private HealthProblemCheckResult checkEvitaErrors(@Nonnull ObservabilityManager
return result;
}

@Nonnull
@Override
public Readiness getReadiness(@Nonnull EvitaContract evitaContract, @Nonnull ExternalApiServer externalApiServer, @Nonnull String... apiCodes) {
final Optional<ObservabilityManager> theObservabilityManager = getObservabilityManager(externalApiServer);
// check the end-points availability
//noinspection rawtypes
final Collection<ExternalApiProviderRegistrar> availableExternalApis = ExternalApiServer.gatherExternalApiProviders();
final Map<String, Boolean> readiness = CollectionUtils.createHashMap(availableExternalApis.size());
for (String apiCode : apiCodes) {
final ExternalApiProvider<?> apiProvider = externalApiServer.getExternalApiProviderByCode(apiCode);
readiness.put(apiProvider.getCode(), apiProvider.isReady());
theObservabilityManager.ifPresent(it -> it.recordReadiness(apiProvider.getCode(), apiProvider.isReady()));
}
final boolean ready = readiness.values().stream().allMatch(Boolean::booleanValue);
if (ready) {
this.seenReady.set(true);
}
final Readiness currentReadiness = new Readiness(
ready ? ReadinessState.READY : (this.seenReady.get() ? ReadinessState.STALLING : ReadinessState.STARTING),
readiness.entrySet().stream()
.map(entry -> new ApiState(entry.getKey(), entry.getValue()))
.toArray(ApiState[]::new)
);
this.lastReadinessSeen.set(
new ReadinessWithTimestamp(currentReadiness, OffsetDateTime.now())
);
return currentReadiness;
}

/**
* Returns the observability manager from the external API server.
*
Expand Down
Loading

0 comments on commit d75ccf8

Please sign in to comment.