Skip to content

Commit

Permalink
Adding state build on top of copied files using Filestore
Browse files Browse the repository at this point in the history
  • Loading branch information
Jai Balani committed Jan 30, 2025
1 parent 40ab7d0 commit 70326f3
Show file tree
Hide file tree
Showing 2 changed files with 10 additions and 1 deletion.
Original file line number Diff line number Diff line change
Expand Up @@ -595,6 +595,7 @@ private void testFileStoreUtils() throws StoreException, IOException {
}

private void testChunkAggregateWithStateBuildForFileCopy() throws IOException {
String partitionName = "0";
String chunkPath = "/tmp/0/test_chunk"; // The path to the chunk file
String logFilePath = "/tmp/0/0_log"; // The path to the log file where chunks are written
String outputFilePath = "/tmp/0/output_log_copy"; // New file where the log data will be copied
Expand Down Expand Up @@ -654,6 +655,13 @@ private void testChunkAggregateWithStateBuildForFileCopy() throws IOException {
}

// TODO: Run state build on the aggregated output file to see if the state is built correctly post filecopy
// delete the source file
Files.delete(Paths.get(logFilePath));
// rename the output file to act as the new base log file
Files.move(Paths.get(outputFilePath), Paths.get(logFilePath));

storageManager.buildStateForFileCopy(storageManager.getReplica(partitionName));

}

/**
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -191,7 +191,8 @@ void start(boolean shouldRemoveUnexpectedDirs) throws InterruptedException {
ConcurrentHashMap<PartitionId, Exception> startExceptions = new ConcurrentHashMap<>();
List<Thread> startupThreads = new ArrayList<>();
for (final Map.Entry<PartitionId, BlobStore> partitionAndStore : stores.entrySet()) {
if(partitionAndStore.getKey().getId() == 146) {

if(Arrays.asList(146L, 803L).contains(partitionAndStore.getKey().getId())) {
logger.info("Skip the store {} because it is running file copy", partitionAndStore.getKey());
continue;
}
Expand Down

0 comments on commit 70326f3

Please sign in to comment.