Skip to content

Commit

Permalink
PR comments WIP
Browse files Browse the repository at this point in the history
  • Loading branch information
@DevenAhluwalia
DevenAhluwalia committed Dec 20, 2024
1 parent 1e6cefa commit 0b9cb66
Show file tree
Hide file tree
Showing 3 changed files with 259 additions and 20 deletions.
14 changes: 14 additions & 0 deletions ambry-api/src/main/java/com/github/ambry/store/ReplicationProtocolTransitionType.java
View file
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
/**
* Copyright 2024 LinkedIn Corp. All rights reserved.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
*/

package com.github.ambry.store;

/**
Expand Down
23 changes: 10 additions & 13 deletions ambry-store/src/main/java/com/github/ambry/store/BootstrapController.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,10 +40,9 @@
*/
public class BootstrapController {
private boolean isRunning = false;
private final String BOOTSTRAP_IN_PROGRESS_FILE_NAME;
private final String FILECOPY_IN_PROGRESS_FILE_NAME;
private final Pattern allLogSegmentFilesPattern = Pattern.compile(".*_log.*");
private final Pattern allLogSegmentFilesPattern = Pattern.compile(".*_log");
private final ServerConfig serverConfig;
private final StoreConfig storeConfig;
private final StoreManager storeManager;
private final PartitionStateChangeListener storageManagerListener;
private final PartitionStateChangeListener fileCopyManagerListener;
Expand All @@ -54,13 +53,11 @@ public class BootstrapController {
public BootstrapController(
@Nonnull StoreManager storeManager, @Nonnull StoreConfig storeConfig,
@Nonnull ServerConfig serverConfig, @Nonnull ClusterParticipant primaryClusterParticipant) {
this.serverConfig = serverConfig;
this.storeManager = storeManager;
this.storeConfig = storeConfig;
this.serverConfig = serverConfig;
this.primaryClusterParticipant = primaryClusterParticipant;

this.BOOTSTRAP_IN_PROGRESS_FILE_NAME = storeConfig.storeBootstrapInProgressFile;
this.FILECOPY_IN_PROGRESS_FILE_NAME = storeConfig.storeFileCopyInProgressFileName;

primaryClusterParticipant.registerPartitionStateChangeListener(
StateModelListenerType.BootstrapControllerListener, new BootstrapControllerImpl());
logger.info("Bootstrap Controller's state change listener registered!");
Expand Down Expand Up @@ -99,8 +96,8 @@ class BootstrapControllerImpl implements PartitionStateChangeListener {
*/
@Override
public void onPartitionBecomeBootstrapFromOffline(@Nonnull String partitionName) {
logger.info("Bootstrap Controller's state change listener invoked for partition `{}`, state change `{}`",
partitionName, "Offline -> Bootstrap");
logger.info("Bootstrap Controller's state change listener invoked for partition `{}`, "
+ "state change `Offline -> Bootstrap`", partitionName);

ReplicaId replica = storeManager.getReplica(partitionName);
PartitionStateChangeListener listenerToInvoke = null;
Expand All @@ -125,7 +122,7 @@ public void onPartitionBecomeBootstrapFromOffline(@Nonnull String partitionName)
}
} else {
if (isFileCopyFeatureEnabled()) {
if (isFileExists(replica.getPartitionId(), BOOTSTRAP_IN_PROGRESS_FILE_NAME)) {
if (isFileExists(replica.getPartitionId(), storeConfig.storeBootstrapInProgressFile)) {
// R.Incomplete -> FC
// Last attempt with blob based bootstrap protocol had failed for this partition.
// FileCopy bootstrap protocol is enabled but we will still continue with blob based bootstrap protocol.
Expand All @@ -135,7 +132,7 @@ public void onPartitionBecomeBootstrapFromOffline(@Nonnull String partitionName)
ReplicationProtocolTransitionType.BLOB_BASED_HYDRATION_INCOMPLETE_TO_FILE_BASED_HYDRATION);
logStateChange("R.Incomplete -> FC", partitionName);
} else if (isAnyLogSegmentExists(replica.getPartitionId())) {
if (isFileExists(replica.getPartitionId(), FILECOPY_IN_PROGRESS_FILE_NAME)) {
if (isFileExists(replica.getPartitionId(), storeConfig.storeFileCopyInProgressFileName)) {
// FC.Incomplete -> FC
// Last attempt with FileCopy bootstrap protocol had failed for this partition.
// We will resume the boostrap with FileCopy bootstrap protocol.
Expand All @@ -158,7 +155,7 @@ public void onPartitionBecomeBootstrapFromOffline(@Nonnull String partitionName)
}
}
} else {
if (isFileExists(replica.getPartitionId(), BOOTSTRAP_IN_PROGRESS_FILE_NAME)) {
if (isFileExists(replica.getPartitionId(), storeConfig.storeBootstrapInProgressFile)) {
// R.Incomplete -> R
// Last attempt with blob based bootstrap protocol had failed for this partition.
// FileCopy bootstrap protocol is disabled and we will continue with blob based bootstrap protocol.
Expand All @@ -168,7 +165,7 @@ public void onPartitionBecomeBootstrapFromOffline(@Nonnull String partitionName)
ReplicationProtocolTransitionType.BLOB_BASED_HYDRATION_INCOMPLETE_TO_BLOB_BASED_HYDRATION);
logStateChange("R.Incomplete -> R", partitionName);
} else if (isAnyLogSegmentExists(replica.getPartitionId())) {
if (isFileExists(replica.getPartitionId(), FILECOPY_IN_PROGRESS_FILE_NAME)) {
if (isFileExists(replica.getPartitionId(), storeConfig.storeFileCopyInProgressFileName)) {
// FC.Incomplete -> R
// Last attempt with FileCopy bootstrap protocol had failed for this partition.
// First we delete FileCopy data and then continue with blob based bootstrap protocol.
Expand Down
242 changes: 235 additions & 7 deletions ambry-store/src/test/java/com/github/ambry/store/BootstrapControllerTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -69,9 +69,20 @@ public static void initialize() {
.thenReturn(mock(ReplicaSyncUpManager.class));
}

/**
* Test for {@link BootstrapController.BootstrapControllerImpl#onPartitionBecomeBootstrapFromOffline(String)}
* when :-
* 1. The server replication mode is {@link ServerReplicationMode#FILE_BASED}
* 2. Replica is null
* Bootstrap Controller is expected to instantiate fileCopyManagerListener
* ReplicationProtocolTransitionType is {@link ReplicationProtocolTransitionType#NEW_PARTITION_TO_FILE_BASED_HYDRATION}
*/
@Test
public void test__ReplicationProtocolTransitionType__NEW_PARTITION_TO_FILE_BASED_HYDRATION() {
public void testNewPartitionToFileBasedHydration() {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

final BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.FILE_BASED);

Expand All @@ -92,9 +103,20 @@ public void test__ReplicationProtocolTransitionType__NEW_PARTITION_TO_FILE_BASED
ReplicationProtocolTransitionType.NEW_PARTITION_TO_FILE_BASED_HYDRATION);
}

/**
* Test for {@link BootstrapController.BootstrapControllerImpl#onPartitionBecomeBootstrapFromOffline(String)}
* when :-
* 1. The server replication mode is {@link ServerReplicationMode#BLOB_BASED}
* 2. Replica is null
* Bootstrap Controller is expected to instantiate storageManagerListener
* ReplicationProtocolTransitionType is {@link ReplicationProtocolTransitionType#NEW_PARTITION_TO_BLOB_BASED_HYDRATION}
*/
@Test
public void test__ReplicationProtocolTransitionType__NEW_PARTITION_TO_BLOB_BASED_HYDRATION() {
public void testNewPartitionToBlobBasedHydration() {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

final BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.BLOB_BASED);

Expand All @@ -115,9 +137,21 @@ public void test__ReplicationProtocolTransitionType__NEW_PARTITION_TO_BLOB_BASED
ReplicationProtocolTransitionType.NEW_PARTITION_TO_BLOB_BASED_HYDRATION);
}

/**
* Test for {@link BootstrapController.BootstrapControllerImpl#onPartitionBecomeBootstrapFromOffline(String)}
* when :-
* 1. The server replication mode is {@link ServerReplicationMode#FILE_BASED}
* 2. Replica is not null
* 3. Bootstrap_in_progress file exists
* Bootstrap Controller is expected to instantiate fileCopyManagerListener
* ReplicationProtocolTransitionType is {@link ReplicationProtocolTransitionType#BLOB_BASED_HYDRATION_INCOMPLETE_TO_FILE_BASED_HYDRATION}
*/
@Test
public void test__ReplicationProtocolTransitionType__BLOB_BASED_HYDRATION_COMPLETE_TO_FILE_BASED_HYDRATION() {
public void testBlobBasedHydrationInCompleteToFileBasedHydration() {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

final BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.FILE_BASED);

Expand Down Expand Up @@ -153,14 +187,16 @@ public void test__ReplicationProtocolTransitionType__BLOB_BASED_HYDRATION_COMPLE
* 4. Filecopy_in_progress file exists
* 5. Atleast one LogSegment File exists for the partition
* Bootstrap Controller is expected to instantiate storageManagerListener
* ReplicationProtocolTransitionType is either of
* ReplicationProtocolTransitionType is either of the following types :-
* a. {@link ReplicationProtocolTransitionType#FILE_BASED_HYDRATION_COMPLETE_TO_BLOB_BASED_HYDRATION} state
* b. {@link ReplicationProtocolTransitionType#BLOB_BASED_HYDRATION_COMPLETE_TO_BLOB_BASED_HYDRATION} state
* @throws IOException
*/
@Test
public void test__ReplicationProtocolTransitionType__COMPLETED_BOOTSTRAP_TO_BLOB_BASED_HYDRATION() {
public void testCompletedHydrationToBlobBasedHydration() {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

final BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.BLOB_BASED);

Expand All @@ -170,7 +206,7 @@ public void test__ReplicationProtocolTransitionType__COMPLETED_BOOTSTRAP_TO_BLOB
.thenReturn(partitionId);
when(storeManager.getReplica(partitionName1))
.thenReturn(testReplica);
when(storeManager.isFileExists(partitionId, storeConfig.storeBootstrapInProgressFile))
when(bootstrapControllerImpl.isFileExists(partitionId, storeConfig.storeBootstrapInProgressFile))
.thenReturn(false);
when(bootstrapControllerImpl.isFileExists(partitionId, storeConfig.storeFileCopyInProgressFileName))
.thenReturn(false);
Expand All @@ -193,6 +229,198 @@ public void test__ReplicationProtocolTransitionType__COMPLETED_BOOTSTRAP_TO_BLOB
ReplicationProtocolTransitionType.BLOB_BASED_HYDRATION_COMPLETE_TO_BLOB_BASED_HYDRATION);
}

/**
* Test for {@link BootstrapController.BootstrapControllerImpl#onPartitionBecomeBootstrapFromOffline(String)}
* when :-
* 1. The server replication mode is {@link ServerReplicationMode#FILE_BASED}
* 2. Replica is not null
* 3. Bootstrap_in_progress file does not exist
* 4. Filecopy_in_progress file exists
* 5. Atleast one LogSegment File exists for the partition
* Bootstrap Controller is expected to instantiate fileCopyManagerListener
* ReplicationProtocolTransitionType is {@link ReplicationProtocolTransitionType#FILE_BASED_HYDRATION_INCOMPLETE_TO_FILE_BASED_HYDRATION}
*/
@Test
public void testFileBasedHydrationInCompleteToFileBasedHydration() {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

final BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.FILE_BASED);

TestReplica testReplica = mock(TestReplica.class);
PartitionId partitionId = mock(PartitionId.class);
when(testReplica.getPartitionId())
.thenReturn(partitionId);
when(storeManager.getReplica(partitionName1))
.thenReturn(testReplica);
when(bootstrapControllerImpl.isFileExists(partitionId, storeConfig.storeBootstrapInProgressFile))
.thenReturn(false);
when(bootstrapControllerImpl.isFileExists(partitionId, storeConfig.storeFileCopyInProgressFileName))
.thenReturn(true);
when(bootstrapControllerImpl.isAnyLogSegmentExists(partitionId))
.thenReturn(true);

// Act
bootstrapControllerImpl.onPartitionBecomeBootstrapFromOffline(partitionName1);

// Assert
// TODO: Fix the assertions

// verify(fileCopyManagerListener, times(1))
// .onPartitionBecomeBootstrapFromOffline(partitionName1);
// verify(storageManagerListener, never())
// .onPartitionBecomeBootstrapFromOffline(partitionName1);
//
// assert bootstrapControllerImpl.replicationProtocolTransitionType.size() == 1;
// assert bootstrapControllerImpl.replicationProtocolTransitionType.contains(
// ReplicationProtocolTransitionType.FILE_BASED_HYDRATION_INCOMPLETE_TO_FILE_BASED_HYDRATION);
}

/**
* Test for {@link BootstrapController.BootstrapControllerImpl#onPartitionBecomeBootstrapFromOffline(String)}
* when :-
* 1. The server replication mode is {@link ServerReplicationMode#BLOB_BASED}
* 2. Replica is not null
* 3. Bootstrap_in_progress file exists
* Bootstrap Controller is expected to instantiate storageManagerListener
* ReplicationProtocolTransitionType is {@link ReplicationProtocolTransitionType#BLOB_BASED_HYDRATION_INCOMPLETE_TO_BLOB_BASED_HYDRATION}
*/
@Test
public void testBlobBasedHydrationInCompleteToBlobBasedHydration() {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

final BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.BLOB_BASED);

TestReplica testReplica = mock(TestReplica.class);
PartitionId partitionId = mock(PartitionId.class);
when(testReplica.getPartitionId())
.thenReturn(partitionId);
when(storeManager.getReplica(partitionName1))
.thenReturn(testReplica);
when(storeManager.isFileExists(partitionId, storeConfig.storeBootstrapInProgressFile))
.thenReturn(true);

// Act
bootstrapControllerImpl.onPartitionBecomeBootstrapFromOffline(partitionName1);

// Assert
verify(fileCopyManagerListener, never())
.onPartitionBecomeBootstrapFromOffline(partitionName1);
verify(storageManagerListener, times(1))
.onPartitionBecomeBootstrapFromOffline(partitionName1);

assert bootstrapControllerImpl.replicationProtocolTransitionType.size() == 1;
assert bootstrapControllerImpl.replicationProtocolTransitionType.contains(
ReplicationProtocolTransitionType.BLOB_BASED_HYDRATION_INCOMPLETE_TO_BLOB_BASED_HYDRATION);
}

/**
* Test for {@link BootstrapController.BootstrapControllerImpl#onPartitionBecomeBootstrapFromOffline(String)}
* when :-
* 1. The server replication mode is {@link ServerReplicationMode#BLOB_BASED}
* 2. Replica is not null
* 3. Bootstrap_in_progress file doesn't exist
* 4. Filecopy_in_progress file exists
* 5. Atleast one LogSegment File exists for the partition
* Bootstrap Controller is expected to instantiate storageManagerListener
* ReplicationProtocolTransitionType is {@link ReplicationProtocolTransitionType#FILE_BASED_HYDRATION_INCOMPLETE_TO_BLOB_BASED_HYDRATION}
*/
@Test
public void testFileBasedHydrationInCompleteToBlobBasedHydration() throws IOException, StoreException {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.BLOB_BASED);

TestReplica testReplica = mock(TestReplica.class);
PartitionId partitionId = mock(PartitionId.class);
when(testReplica.getPartitionId())
.thenReturn(partitionId);
when(storeManager.getReplica(partitionName1))
.thenReturn(testReplica);

when(bootstrapControllerImpl.isFileExists(partitionId, storeConfig.storeFileCopyInProgressFileName))
.thenReturn(true);
when(storeManager.isFileExists(partitionId, storeConfig.storeBootstrapInProgressFile))
.thenReturn(false);

when(bootstrapControllerImpl.isAnyLogSegmentExists(partitionId))
.thenReturn(true);

// Act
bootstrapControllerImpl.onPartitionBecomeBootstrapFromOffline(partitionName1);

// Assert
// TODO: Fix the assertions

// verify(fileCopyManagerListener, never())
// .onPartitionBecomeBootstrapFromOffline(partitionName1);
// verify(storageManagerListener, times(1))
// .onPartitionBecomeBootstrapFromOffline(partitionName1);
//
// assert bootstrapControllerImpl.replicationProtocolTransitionType.size() == 1;
// assert bootstrapControllerImpl.replicationProtocolTransitionType.contains(
// ReplicationProtocolTransitionType.FILE_BASED_HYDRATION_INCOMPLETE_TO_BLOB_BASED_HYDRATION);
}

/**
* Test for {@link BootstrapController.BootstrapControllerImpl#onPartitionBecomeBootstrapFromOffline(String)}
* when :-
* 1. The server replication mode is {@link ServerReplicationMode#FILE_BASED}
* 2. Replica is not null
* 3. Bootstrap_in_progress file doesn't exist
* 4. Filecopy_in_progress file doesn't exist
* 5. Atleast one LogSegment File exists for the partition
* Bootstrap Controller is expected to instantiate storageManagerListener
* ReplicationProtocolTransitionType is either of the following types :-
* a. {@link ReplicationProtocolTransitionType#BLOB_BASED_HYDRATION_COMPLETE_TO_FILE_BASED_HYDRATION} state
* b. {@link ReplicationProtocolTransitionType#FILE_BASED_HYDRATION_COMPLETE_TO_FILE_BASED_HYDRATION} state
*/
@Test
public void testCompletedHydrationToFileBasedHydration() {
// Arrange
clearInvocations(fileCopyManagerListener);
clearInvocations(storageManagerListener);

final BootstrapController.BootstrapControllerImpl bootstrapControllerImpl =
getBootstrapControllerImpl(ServerReplicationMode.FILE_BASED);

TestReplica testReplica = mock(TestReplica.class);
PartitionId partitionId = mock(PartitionId.class);
when(testReplica.getPartitionId())
.thenReturn(partitionId);
when(storeManager.getReplica(partitionName1))
.thenReturn(testReplica);
when(bootstrapControllerImpl.isFileExists(partitionId, storeConfig.storeBootstrapInProgressFile))
.thenReturn(false);
when(bootstrapControllerImpl.isFileExists(partitionId, storeConfig.storeFileCopyInProgressFileName))
.thenReturn(false);
when(bootstrapControllerImpl.isAnyLogSegmentExists(partitionId))
.thenReturn(true);

// Act
bootstrapControllerImpl.onPartitionBecomeBootstrapFromOffline(partitionName1);

// Assert
verify(fileCopyManagerListener, never())
.onPartitionBecomeBootstrapFromOffline(partitionName1);
verify(storageManagerListener, times(1))
.onPartitionBecomeBootstrapFromOffline(partitionName1);

assert bootstrapControllerImpl.replicationProtocolTransitionType.size() == 2;
assert bootstrapControllerImpl.replicationProtocolTransitionType.contains(
ReplicationProtocolTransitionType.BLOB_BASED_HYDRATION_COMPLETE_TO_FILE_BASED_HYDRATION);
assert bootstrapControllerImpl.replicationProtocolTransitionType.contains(
ReplicationProtocolTransitionType.FILE_BASED_HYDRATION_COMPLETE_TO_FILE_BASED_HYDRATION);
}

private static BootstrapController.BootstrapControllerImpl getBootstrapControllerImpl(
ServerReplicationMode serverReplicationMode) {

Expand Down

0 comments on commit 0b9cb66

Please sign in to comment.