From d186728943a1553042b493024658441494b286b3 Mon Sep 17 00:00:00 2001 From: Jai Balani Date: Thu, 21 Nov 2024 01:12:36 +0530 Subject: [PATCH] Added initial changes for state build --- .../PartitionStateChangeListener.java | 7 ++++++ .../clustermap/StateTransitionException.java | 5 ++++ .../github/ambry/config/ClusterMapConfig.java | 4 ---- .../com/github/ambry/config/StoreConfig.java | 8 +++---- .../github/ambry/cloud/RecoveryManager.java | 5 ++++ .../ambry/clustermap/HelixParticipant.java | 7 ++++++ .../AmbryStateModelFactoryTest.java | 5 ++++ .../ambry/replication/ReplicationManager.java | 5 ++++ .../com/github/ambry/server/StatsManager.java | 5 ++++ .../com/github/ambry/store/DiskManager.java | 23 ++++++++----------- .../github/ambry/store/StorageManager.java | 9 ++++++-- 11 files changed, 60 insertions(+), 23 deletions(-) diff --git a/ambry-api/src/main/java/com/github/ambry/clustermap/PartitionStateChangeListener.java b/ambry-api/src/main/java/com/github/ambry/clustermap/PartitionStateChangeListener.java index 63c6b69b21..05a56494c2 100644 --- a/ambry-api/src/main/java/com/github/ambry/clustermap/PartitionStateChangeListener.java +++ b/ambry-api/src/main/java/com/github/ambry/clustermap/PartitionStateChangeListener.java @@ -74,10 +74,17 @@ default void onPartitionBecomeDroppedFromError(String partitionName) { default void onPartitionBecomeOfflineFromError(String partitionName) { } + /** + * Action to take when state building is triggered for filecopy + * @param partitionName of the partition. + */ + void buildStateForFileCopy(String partitionName); + /** * Action to take when reset method is called on certain partition. * @param partitionName of the partition. */ default void onReset(String partitionName) { } + } diff --git a/ambry-api/src/main/java/com/github/ambry/clustermap/StateTransitionException.java b/ambry-api/src/main/java/com/github/ambry/clustermap/StateTransitionException.java index 6f89632e40..7b358a3f19 100644 --- a/ambry-api/src/main/java/com/github/ambry/clustermap/StateTransitionException.java +++ b/ambry-api/src/main/java/com/github/ambry/clustermap/StateTransitionException.java @@ -45,6 +45,11 @@ public enum TransitionErrorCode { * If replica is not present in Helix and not found on current node. */ ReplicaNotFound, + + /** + * If disk manager not found during state build. + */ + DiskManagerNotFoundForFileCopyStateBuild, /** * If failure occurs during replica operation (i.e. replica addition/removal in StoreManager, ReplicationManager). */ diff --git a/ambry-api/src/main/java/com/github/ambry/config/ClusterMapConfig.java b/ambry-api/src/main/java/com/github/ambry/config/ClusterMapConfig.java index 3fea12c012..252e5a3d70 100644 --- a/ambry-api/src/main/java/com/github/ambry/config/ClusterMapConfig.java +++ b/ambry-api/src/main/java/com/github/ambry/config/ClusterMapConfig.java @@ -373,9 +373,6 @@ public class ClusterMapConfig { public final boolean clusterMapIgnoreDownwardStateTransition; public static final String ENABLE_FILE_COPY_FOR_BOOTSTRAP = "clustermap.enable.file.copy.for.bootstrap"; - @Config(ENABLE_FILE_COPY_FOR_BOOTSTRAP) - @Default("false") - public final boolean enableFileCopyForBootstrap; public ClusterMapConfig(VerifiableProperties verifiableProperties) { clusterMapFixedTimeoutDatanodeErrorThreshold = @@ -460,6 +457,5 @@ public ClusterMapConfig(VerifiableProperties verifiableProperties) { verifiableProperties.getLongInRange("clustermap.default.replica.capacity.in.bytes", 384L * 1024 * 1024 * 1024, 0, Long.MAX_VALUE); clusterMapIgnoreDownwardStateTransition = verifiableProperties.getBoolean(IGNORE_DOWNWARD_STATE_TRANSITION, false); - enableFileCopyForBootstrap = verifiableProperties.getBoolean(ENABLE_FILE_COPY_FOR_BOOTSTRAP, false); } } diff --git a/ambry-api/src/main/java/com/github/ambry/config/StoreConfig.java b/ambry-api/src/main/java/com/github/ambry/config/StoreConfig.java index 0bb94d9130..1dcd5d0647 100644 --- a/ambry-api/src/main/java/com/github/ambry/config/StoreConfig.java +++ b/ambry-api/src/main/java/com/github/ambry/config/StoreConfig.java @@ -675,10 +675,10 @@ public class StoreConfig { public final static String storeBlockStaleBlobStoreToStartName = "store.block.stale.blob.store.to.start"; - @Config(ENABLE_FILE_COPY_FOR_BOOTSTRAP) + @Config(STORE_ENABLE_FILE_COPY_FOR_BOOTSTRAP) @Default("false") - public final boolean enableFileCopyForBootstrap; - public static final String ENABLE_FILE_COPY_FOR_BOOTSTRAP = "clustermap.enable.file.copy.for.bootstrap"; + public final boolean storeEnableFileCopyForBootstrap; + public static final String STORE_ENABLE_FILE_COPY_FOR_BOOTSTRAP = "store.enable.file.copy.for.bootstrap"; /** @@ -878,6 +878,6 @@ public StoreConfig(VerifiableProperties verifiableProperties) { storeStaleTimeInDays = verifiableProperties.getIntInRange(storeStaleTimeInDaysName, 7, 0, Integer.MAX_VALUE); storeBlockStaleBlobStoreToStart = verifiableProperties.getBoolean(storeBlockStaleBlobStoreToStartName, false); storeReshuffleDisksOnReorder = verifiableProperties.getBoolean(storeReshuffleDisksOnReorderName, false); - enableFileCopyForBootstrap = verifiableProperties.getBoolean(ENABLE_FILE_COPY_FOR_BOOTSTRAP, false); + storeEnableFileCopyForBootstrap = verifiableProperties.getBoolean(STORE_ENABLE_FILE_COPY_FOR_BOOTSTRAP, false); } } diff --git a/ambry-cloud/src/main/java/com/github/ambry/cloud/RecoveryManager.java b/ambry-cloud/src/main/java/com/github/ambry/cloud/RecoveryManager.java index 1ccfe41c6c..a8b8b4054c 100644 --- a/ambry-cloud/src/main/java/com/github/ambry/cloud/RecoveryManager.java +++ b/ambry-cloud/src/main/java/com/github/ambry/cloud/RecoveryManager.java @@ -493,6 +493,11 @@ public void onPartitionBecomeDroppedFromOffline(String partitionName) { partitionName); } + @Override + public void buildStateForFileCopy(String partitionName) { + // no op + } + /** * If only config specified list of partitions are being replicated from cloud, then check that the partition * belongs to the specified list. diff --git a/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixParticipant.java b/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixParticipant.java index 297d8fb4ec..ae711a3dec 100644 --- a/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixParticipant.java +++ b/ambry-clustermap/src/main/java/com/github/ambry/clustermap/HelixParticipant.java @@ -65,6 +65,8 @@ */ public class HelixParticipant implements ClusterParticipant, PartitionStateChangeListener { public static final String DISK_KEY = "DISK"; + + public static final boolean ENABLE_FILE_COPY = false; final HelixParticipantMetrics participantMetrics; private final HelixClusterManager clusterManager; private final String clusterName; @@ -1074,6 +1076,11 @@ public void onPartitionBecomeOfflineFromError(String partitionName) { localPartitionAndState.put(partitionName, ReplicaState.OFFLINE); } + @Override + public void buildStateForFileCopy(String partitionName) { + // no op + } + @Override public void onReset(String partitionName) { localPartitionAndState.put(partitionName, ReplicaState.OFFLINE); diff --git a/ambry-clustermap/src/test/java/com/github/ambry/clustermap/AmbryStateModelFactoryTest.java b/ambry-clustermap/src/test/java/com/github/ambry/clustermap/AmbryStateModelFactoryTest.java index 21552b9127..264d048469 100644 --- a/ambry-clustermap/src/test/java/com/github/ambry/clustermap/AmbryStateModelFactoryTest.java +++ b/ambry-clustermap/src/test/java/com/github/ambry/clustermap/AmbryStateModelFactoryTest.java @@ -111,6 +111,11 @@ public void onPartitionBecomeOfflineFromInactive(String partitionName) { public void onPartitionBecomeDroppedFromOffline(String partitionName) { // no op } + + @Override + public void buildStateForFileCopy(String partitionName) { + // no op + } }, mock(HelixClusterManager.class)); StateModel stateModel; switch (config.clustermapStateModelDefinition) { diff --git a/ambry-replication/src/main/java/com/github/ambry/replication/ReplicationManager.java b/ambry-replication/src/main/java/com/github/ambry/replication/ReplicationManager.java index 03d086b2e2..dd792efe8d 100644 --- a/ambry-replication/src/main/java/com/github/ambry/replication/ReplicationManager.java +++ b/ambry-replication/src/main/java/com/github/ambry/replication/ReplicationManager.java @@ -412,5 +412,10 @@ public void onPartitionBecomeDroppedFromOffline(String partitionName) { // the transition removeReplica(replica); } + + @Override + public void buildStateForFileCopy(String partitionName) { + // no op + } } } diff --git a/ambry-server/src/main/java/com/github/ambry/server/StatsManager.java b/ambry-server/src/main/java/com/github/ambry/server/StatsManager.java index 78c43329a2..1c1773bbf5 100644 --- a/ambry-server/src/main/java/com/github/ambry/server/StatsManager.java +++ b/ambry-server/src/main/java/com/github/ambry/server/StatsManager.java @@ -532,5 +532,10 @@ public void onPartitionBecomeDroppedFromOffline(String partitionName) { // remove replica from in-mem data structure. If replica doesn't exist, log info but don't fail the transition removeReplica(replica); } + + @Override + public void buildStateForFileCopy(String partitionName) { + // no op + } } } diff --git a/ambry-store/src/main/java/com/github/ambry/store/DiskManager.java b/ambry-store/src/main/java/com/github/ambry/store/DiskManager.java index 24686c8088..a7ac7a0a92 100644 --- a/ambry-store/src/main/java/com/github/ambry/store/DiskManager.java +++ b/ambry-store/src/main/java/com/github/ambry/store/DiskManager.java @@ -419,6 +419,13 @@ boolean controlCompactionForBlobStore(PartitionId id, boolean enabled) { return succeed; } + /** + * Add store to compaction manager + */ + void addBlobStoreToCompactionManager(BlobStore store){ + compactionManager.addBlobStore(store); + } + /** * Add a new BlobStore with given {@link ReplicaId}. * @param replica the {@link ReplicaId} of the {@link Store} which would be added. @@ -447,22 +454,12 @@ boolean addBlobStore(ReplicaId replica) { BlobStore store = new BlobStore(replica, storeConfig, scheduler, longLivedTaskScheduler, this, diskIOScheduler, diskSpaceAllocator, storeMainMetrics, storeUnderCompactionMetrics, keyFactory, recovery, hardDelete, replicaStatusDelegates, time, accountService, null, indexPersistScheduler); - // For Filecopy, we do not init the BlobStore since new log/index isn't required, the files will be directly - // copied from remote node. - if (!storeConfig.enableFileCopyForBootstrap) { - store.start(); - } - + store.start(); // collect store segment requirements and add into DiskSpaceAllocator List storeRequirements = Collections.singletonList(store.getDiskSpaceRequirements()); diskSpaceAllocator.addRequiredSegments(diskSpaceAllocator.getOverallRequirements(storeRequirements), false); - - // We don't need to add the store into compaction manager for filecopy approach - if (!storeConfig.enableFileCopyForBootstrap) { - // add store into CompactionManager - compactionManager.addBlobStore(store); - } - + // add store into CompactionManager + compactionManager.addBlobStore(store); // add new created store into in-memory data structures. stores.put(replica.getPartitionId(), store); partitionToReplicaMap.put(replica.getPartitionId(), replica); diff --git a/ambry-store/src/main/java/com/github/ambry/store/StorageManager.java b/ambry-store/src/main/java/com/github/ambry/store/StorageManager.java index 767a5d6a9c..7a27804ed0 100644 --- a/ambry-store/src/main/java/com/github/ambry/store/StorageManager.java +++ b/ambry-store/src/main/java/com/github/ambry/store/StorageManager.java @@ -763,8 +763,6 @@ public void onPartitionBecomeBootstrapFromOffline(String partitionName) { } } while (!replicaAdded); - - if (primaryClusterParticipant != null) { // update InstanceConfig in Helix try { @@ -1012,6 +1010,13 @@ public void onPartitionBecomeDroppedFromOffline(String partitionName) { logger.info("Partition {} is successfully dropped on current node", partitionName); } + + + @Override + public void buildStateForFileCopy(String partitionName) { + // no op + } + /** * Return true if the blob store should resume the decommission * @param store The {@link BlobStore}