From 5a5dfb7496d68f461e6926789b6f4bd44da561fc Mon Sep 17 00:00:00 2001 From: Elvis Souza Date: Wed, 4 Sep 2024 21:55:23 -0300 Subject: [PATCH] CommandLine module upgrades (#560) * command lines module upgrades * new tests * missing dep * missing dep * release notes * [Gradle Release Plugin] - new version commit: '3.27.0-snapshot'. --- RELEASE-NOTES.md | 3 + gradle.properties | 2 +- .../mageddo/commons/exec/CommandLines.java | 106 ++++++++-------- .../commons/exec/DelegateOutputStream.java | 36 ++++++ .../ExecutionValidationFailedException.java | 10 +- .../commons/exec/NopResultHandler.java | 16 +++ .../com/mageddo/commons/exec/PipedStream.java | 42 +++++++ .../exec/ProcessAccessibleDaemonExecutor.java | 20 +++ .../commons/exec/ProcessesWatchDog.java | 49 ++++++++ .../com/mageddo/commons/exec/Request.java | 117 ++++++++++++++++++ .../java/com/mageddo/commons/exec/Result.java | 93 ++++++++++++++ .../com/mageddo/concurrent/ThreadsV2.java | 7 ++ src/main/java/com/mageddo/io/LogPrinter.java | 35 ++++++ .../commons/exec/CommandLinesTest.java | 56 +++++++++ .../exec/DelegateOutputStreamTest.java | 26 ++++ .../mageddo/commons/exec/PipedStreamTest.java | 27 ++++ 16 files changed, 589 insertions(+), 56 deletions(-) create mode 100644 src/main/java/com/mageddo/commons/exec/DelegateOutputStream.java create mode 100644 src/main/java/com/mageddo/commons/exec/NopResultHandler.java create mode 100644 src/main/java/com/mageddo/commons/exec/PipedStream.java create mode 100644 src/main/java/com/mageddo/commons/exec/ProcessAccessibleDaemonExecutor.java create mode 100644 src/main/java/com/mageddo/commons/exec/ProcessesWatchDog.java create mode 100644 src/main/java/com/mageddo/commons/exec/Request.java create mode 100644 src/main/java/com/mageddo/commons/exec/Result.java create mode 100644 src/main/java/com/mageddo/concurrent/ThreadsV2.java create mode 100644 src/main/java/com/mageddo/io/LogPrinter.java create mode 100644 src/test/java/com/mageddo/commons/exec/CommandLinesTest.java create mode 100644 src/test/java/com/mageddo/commons/exec/DelegateOutputStreamTest.java create mode 100644 src/test/java/com/mageddo/commons/exec/PipedStreamTest.java diff --git a/RELEASE-NOTES.md b/RELEASE-NOTES.md index 63b69e54d..c34c723b2 100644 --- a/RELEASE-NOTES.md +++ b/RELEASE-NOTES.md @@ -1,3 +1,6 @@ +## 3.27.0 +* Commandline module upgrade to be able to execute new usecases when creating new native int tests. #533 + ## 3.26.0 * Option to set config file path from the ENV, see the docs. diff --git a/gradle.properties b/gradle.properties index f6bed534f..25497eff6 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1 +1 @@ -version=3.26.0-snapshot +version=3.27.0-snapshot diff --git a/src/main/java/com/mageddo/commons/exec/CommandLines.java b/src/main/java/com/mageddo/commons/exec/CommandLines.java index 452d8622e..a372a483f 100644 --- a/src/main/java/com/mageddo/commons/exec/CommandLines.java +++ b/src/main/java/com/mageddo/commons/exec/CommandLines.java @@ -1,25 +1,21 @@ package com.mageddo.commons.exec; -import lombok.Builder; -import lombok.Getter; -import lombok.NonNull; -import lombok.ToString; +import lombok.extern.slf4j.Slf4j; import org.apache.commons.exec.CommandLine; -import org.apache.commons.exec.DaemonExecutor; import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteResultHandler; import org.apache.commons.exec.ExecuteWatchdog; -import org.apache.commons.exec.Executor; -import org.apache.commons.exec.PumpStreamHandler; -import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Duration; +@Slf4j public class CommandLines { public static Result exec(String commandLine, Object... args) { return exec(CommandLine.parse(String.format(commandLine, args)), - ExecuteWatchdog.INFINITE_TIMEOUT + ExecuteWatchdog.INFINITE_TIMEOUT ); } @@ -32,56 +28,62 @@ public static Result exec(CommandLine commandLine) { } public static Result exec(CommandLine commandLine, long timeout) { - final var out = new ByteArrayOutputStream(); - final var executor = new DaemonExecutor(); - final var streamHandler = new PumpStreamHandler(out); - executor.setStreamHandler(streamHandler); - int exitCode; + return exec( + Request.builder() + .commandLine(commandLine) + .timeout(Duration.ofMillis(timeout)) + .build() + ); + } + + private static void registerProcessWatch(ProcessAccessibleDaemonExecutor executor) { + ProcessesWatchDog.instance() + .watch(executor::getProcess) + ; + } + + public static Result exec(CommandLine commandLine, ExecuteResultHandler handler) { + return exec(Request + .builder() + .commandLine(commandLine) + .handler(handler) + .build() + ); + } + + public static Result exec(Request request) { + final var executor = createExecutor(); + executor.setStreamHandler(request.getStreamHandler()); + Integer exitCode = null; try { - executor.setWatchdog(new ExecuteWatchdog(timeout)); - exitCode = executor.execute(commandLine); + executor.setWatchdog(new ExecuteWatchdog(request.getTimeoutInMillis())); + if (request.getHandler() != null) { + executor.execute(request.getCommandLine(), request.getEnv(), request.getHandler()); + registerProcessWatch(executor); + } else { + exitCode = executor.execute(request.getCommandLine(), request.getEnv()); + } } catch (ExecuteException e) { - exitCode = e.getExitValue(); + if (request.getHandler() != null) { + request.getHandler().onProcessFailed(e); + } else { + exitCode = e.getExitValue(); + } } catch (IOException e) { throw new UncheckedIOException(e); } return Result - .builder() - .executor(executor) - .out(out) - .exitCode(exitCode) - .build(); + .builder() + .executor(executor) + .processSupplier(executor::getProcess) + .out(request.getBestOut()) + .exitCode(exitCode) + .request(request) + .build(); } - @Getter - @Builder - @ToString(of = {"exitCode"}) - public static class Result { - - @NonNull - private Executor executor; - - @NonNull - private ByteArrayOutputStream out; - - private int exitCode; - - public String getOutAsString() { - return this.out.toString(); - } - - public Result checkExecution() { - if (this.executor.isFailure(this.getExitCode())) { - throw new ExecutionValidationFailedException(this); - } - return this; - } - - public String toString(boolean printOut) { - return String.format( - "code=%d, out=%s", - this.exitCode, printOut ? this.getOutAsString() : null - ); - } + private static ProcessAccessibleDaemonExecutor createExecutor() { + return new ProcessAccessibleDaemonExecutor(); } + } diff --git a/src/main/java/com/mageddo/commons/exec/DelegateOutputStream.java b/src/main/java/com/mageddo/commons/exec/DelegateOutputStream.java new file mode 100644 index 000000000..0fddc4ffe --- /dev/null +++ b/src/main/java/com/mageddo/commons/exec/DelegateOutputStream.java @@ -0,0 +1,36 @@ +package com.mageddo.commons.exec; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.List; +import java.util.stream.Stream; + +public class DelegateOutputStream extends OutputStream { + + private final List delegateOuts; + + public DelegateOutputStream(OutputStream... delegateOuts) { + this.delegateOuts = Stream.of(delegateOuts).toList(); + } + + public DelegateOutputStream(List delegateOuts) { + this.delegateOuts = delegateOuts; + } + + @Override + public void write(int b) throws IOException { + for (final var delegateOut : this.delegateOuts) { + delegateOut.write(b); + } + } + + @Override + public void close() throws IOException { + for (final var out : this.delegateOuts) { + try { + out.close(); + } catch (IOException e) { + } + } + } +} diff --git a/src/main/java/com/mageddo/commons/exec/ExecutionValidationFailedException.java b/src/main/java/com/mageddo/commons/exec/ExecutionValidationFailedException.java index 618d8ae0e..482ded0ac 100644 --- a/src/main/java/com/mageddo/commons/exec/ExecutionValidationFailedException.java +++ b/src/main/java/com/mageddo/commons/exec/ExecutionValidationFailedException.java @@ -1,14 +1,18 @@ package com.mageddo.commons.exec; public class ExecutionValidationFailedException extends RuntimeException { - private final CommandLines.Result result; + private final Result result; - public ExecutionValidationFailedException(CommandLines.Result result) { + public ExecutionValidationFailedException(Result result) { super(String.format("error, code=%d, error=%s", result.getExitCode(), result.getOutAsString())); this.result = result; } - public CommandLines.Result result() { + public Result result() { return this.result; } + + public int getExitCode() { + return this.result.getExitCode(); + } } diff --git a/src/main/java/com/mageddo/commons/exec/NopResultHandler.java b/src/main/java/com/mageddo/commons/exec/NopResultHandler.java new file mode 100644 index 000000000..382d65b74 --- /dev/null +++ b/src/main/java/com/mageddo/commons/exec/NopResultHandler.java @@ -0,0 +1,16 @@ +package com.mageddo.commons.exec; + +import org.apache.commons.exec.ExecuteException; +import org.apache.commons.exec.ExecuteResultHandler; + +public class NopResultHandler implements ExecuteResultHandler { + @Override + public void onProcessComplete(int exitValue) { + + } + + @Override + public void onProcessFailed(ExecuteException e) { + + } +} diff --git a/src/main/java/com/mageddo/commons/exec/PipedStream.java b/src/main/java/com/mageddo/commons/exec/PipedStream.java new file mode 100644 index 000000000..78164fd80 --- /dev/null +++ b/src/main/java/com/mageddo/commons/exec/PipedStream.java @@ -0,0 +1,42 @@ +package com.mageddo.commons.exec; + +import lombok.Getter; + +import java.io.IOException; +import java.io.OutputStream; +import java.io.PipedInputStream; +import java.io.PipedOutputStream; +import java.io.UncheckedIOException; + +public class PipedStream extends OutputStream { + + @Getter + private final PipedInputStream pipedIn; + + private final DelegateOutputStream delegateOut; + private final OutputStream originalOut; + + public PipedStream(final OutputStream out) { + try { + this.pipedIn = new PipedInputStream(); + this.originalOut = out; + final var pout = new PipedOutputStream(this.pipedIn); + this.delegateOut = new DelegateOutputStream(out, pout); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + } + + @Override + public void write(int b) throws IOException { + this.delegateOut.write(b); + } + + public void close() throws IOException { + this.delegateOut.close(); + } + + OutputStream getOriginalOut() { + return originalOut; + } +} diff --git a/src/main/java/com/mageddo/commons/exec/ProcessAccessibleDaemonExecutor.java b/src/main/java/com/mageddo/commons/exec/ProcessAccessibleDaemonExecutor.java new file mode 100644 index 000000000..16cf233bd --- /dev/null +++ b/src/main/java/com/mageddo/commons/exec/ProcessAccessibleDaemonExecutor.java @@ -0,0 +1,20 @@ +package com.mageddo.commons.exec; + +import lombok.Getter; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.DaemonExecutor; + +import java.io.File; +import java.io.IOException; +import java.util.Map; + +@Getter +class ProcessAccessibleDaemonExecutor extends DaemonExecutor { + + private Process process = null; + + @Override + protected Process launch(CommandLine command, Map env, File dir) throws IOException { + return this.process = super.launch(command, env, dir); + } +} diff --git a/src/main/java/com/mageddo/commons/exec/ProcessesWatchDog.java b/src/main/java/com/mageddo/commons/exec/ProcessesWatchDog.java new file mode 100644 index 000000000..9fbb57388 --- /dev/null +++ b/src/main/java/com/mageddo/commons/exec/ProcessesWatchDog.java @@ -0,0 +1,49 @@ +package com.mageddo.commons.exec; + +import com.mageddo.commons.lang.Singletons; +import lombok.extern.slf4j.Slf4j; + +import java.util.ArrayList; +import java.util.List; +import java.util.Objects; +import java.util.function.Supplier; + +@Slf4j +public class ProcessesWatchDog { + + private List> processes = new ArrayList<>(); + + public static ProcessesWatchDog instance() { + return Singletons.createOrGet(ProcessesWatchDog.class, ProcessesWatchDog::new); + } + + public void watch(Supplier sup) { + this.processes.add(sup); + } + + public void watch(Process process) { + this.processes.add(() -> process); + } + + public void killAllProcesses() { + final var validProcesses = this.findValidProcesses(); + + log.debug("status=killing all processes, processes={}, valid={}", this.processes.size(), validProcesses.size()); + + validProcesses.forEach(process -> { + try { + process.destroy(); + log.trace("status=killed, pid={}", process.pid()); + } catch (Exception e) { + log.warn("status=unable to destroy, processId={}, msg={}", process.pid(), e.getMessage(), e); + } + }); + } + + private List findValidProcesses() { + return this.processes.stream() + .map(Supplier::get) + .filter(Objects::nonNull) + .toList(); + } +} diff --git a/src/main/java/com/mageddo/commons/exec/Request.java b/src/main/java/com/mageddo/commons/exec/Request.java new file mode 100644 index 000000000..987e2fca9 --- /dev/null +++ b/src/main/java/com/mageddo/commons/exec/Request.java @@ -0,0 +1,117 @@ +package com.mageddo.commons.exec; + +import com.mageddo.io.LogPrinter; +import lombok.Builder; +import lombok.NonNull; +import lombok.Value; +import lombok.experimental.NonFinal; +import org.apache.commons.exec.CommandLine; +import org.apache.commons.exec.ExecuteResultHandler; +import org.apache.commons.exec.ExecuteStreamHandler; +import org.apache.commons.exec.ExecuteWatchdog; +import org.apache.commons.exec.PumpStreamHandler; + +import java.io.ByteArrayOutputStream; +import java.io.InputStream; +import java.io.OutputStream; +import java.time.Duration; +import java.util.Map; + +@Value +@Builder(toBuilder = true, builderClassName = "RequestBuilder", buildMethodName = "build0") +public class Request { + + @NonNull + private final CommandLine commandLine; + + private final Duration timeout; + + private final ExecuteResultHandler handler; + private Map env; + + @NonFinal + private boolean watchingOutput; + + @Builder.Default + private final Streams streams = Streams.builder() + .outAndErr(new ByteArrayOutputStream()) + .build(); + + public ExecuteStreamHandler getStreamHandler() { + return this.streams.toStreamHandler(); + } + + public Request printOutToLogsInBackground() { + if (this.watchingOutput) { + throw new IllegalStateException("Already watching output"); + } + this.watchingOutput = true; + LogPrinter.printInBackground(this.streams.outAndErr.getPipedIn()); + return this; + } + + public OutputStream getBestOut() { + return this.streams.getBestOriginalOutput(); + } + + public long getTimeoutInMillis() { + if (this.timeout == null) { + return ExecuteWatchdog.INFINITE_TIMEOUT; + } + return this.timeout.toMillis(); + } + + public static class RequestBuilder { + + private boolean printLogsInBackground = false; + + public Request build() { + final var request = this.build0(); + if (this.printLogsInBackground) { + request.printOutToLogsInBackground(); + } + return request; + } + + public RequestBuilder printLogsInBackground() { + this.printLogsInBackground = true; + return this; + } + } + + + @Value + @Builder(toBuilder = true, builderClassName = "StreamsBuilder") + public static class Streams { + + private final PipedStream outAndErr; + private final OutputStream out; + private final OutputStream err; + private final InputStream input; + + public PipedStream getBestOut() { + if (this.outAndErr != null) { + return this.outAndErr; + } + throw new UnsupportedOperationException(); + } + + public OutputStream getBestOriginalOutput() { + return this.getBestOut().getOriginalOut(); + } + + public static class StreamsBuilder { + public Streams.StreamsBuilder outAndErr(OutputStream outAndErr) { + this.outAndErr = new PipedStream(outAndErr); + return this; + } + } + + public ExecuteStreamHandler toStreamHandler() { + if (this.outAndErr != null) { + return new PumpStreamHandler(this.outAndErr); + } + return new PumpStreamHandler(this.out, this.err, this.input); + } + } +} diff --git a/src/main/java/com/mageddo/commons/exec/Result.java b/src/main/java/com/mageddo/commons/exec/Result.java new file mode 100644 index 000000000..62ea79063 --- /dev/null +++ b/src/main/java/com/mageddo/commons/exec/Result.java @@ -0,0 +1,93 @@ +package com.mageddo.commons.exec; + +import com.mageddo.wait.Wait; +import lombok.Builder; +import lombok.Getter; +import lombok.NonNull; +import lombok.SneakyThrows; +import lombok.ToString; +import org.apache.commons.exec.Executor; +import org.apache.commons.lang3.Validate; + +import java.io.ByteArrayOutputStream; +import java.io.OutputStream; +import java.util.function.Supplier; + +@Getter +@Builder +@ToString(of = {"exitCode"}) +public class Result { + + @NonNull + private Request request; + + @NonNull + private Executor executor; + + @NonNull + private OutputStream out; + + @NonNull + private Supplier processSupplier; + + private Integer exitCode; + + public Result printOutToLogsInBackground() { + this.request.printOutToLogsInBackground(); + return this; + } + + public String getOutAsString() { + Validate.isTrue(this.out instanceof ByteArrayOutputStream, "Only ByteArrayOutputStream is supported"); + return this.out.toString(); + } + + public Result checkExecution() { + if (this.executor.isFailure(this.getExitCode())) { + throw new ExecutionValidationFailedException(this); + } + return this; + } + + public String toString(boolean printOut) { + return String.format( + "code=%d, out=%s", + this.exitCode, printOut ? this.getOutAsString() : null + ); + } + + @SneakyThrows + public Process getProcess() { + return this.processSupplier.get(); + } + + public Long getProcessId() { + final var process = this.getProcess(); + if (process == null) { + return null; + } + return process.pid(); + } + + public void waitProcessToFinish() { + new Wait<>() + .infinityTimeout() + .ignoreException(IllegalArgumentException.class) + .until(() -> { + Validate.isTrue(this.isProcessFinished(), "Process not finished yet"); + return true; + }); + } + + private boolean isProcessFinished() { + return getProcess() != null && !getProcess().isAlive(); + } + + public Integer getProcessExitCodeWhenAvailable() { + try { + return getProcess().exitValue(); + } catch (IllegalThreadStateException e) { + return null; + } + } +} diff --git a/src/main/java/com/mageddo/concurrent/ThreadsV2.java b/src/main/java/com/mageddo/concurrent/ThreadsV2.java new file mode 100644 index 000000000..4c59f6b7f --- /dev/null +++ b/src/main/java/com/mageddo/concurrent/ThreadsV2.java @@ -0,0 +1,7 @@ +package com.mageddo.concurrent; + +public class ThreadsV2 { + public static boolean isInterrupted() { + return Thread.currentThread().isInterrupted(); + } +} diff --git a/src/main/java/com/mageddo/io/LogPrinter.java b/src/main/java/com/mageddo/io/LogPrinter.java new file mode 100644 index 000000000..3ded49a69 --- /dev/null +++ b/src/main/java/com/mageddo/io/LogPrinter.java @@ -0,0 +1,35 @@ +package com.mageddo.io; + +import com.mageddo.concurrent.ThreadsV2; +import lombok.extern.slf4j.Slf4j; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStream; +import java.io.InputStreamReader; + +@Slf4j +public class LogPrinter { + + public static void printInBackground(InputStream in) { + final var task = (Runnable) () -> { + final var bf = new BufferedReader(new InputStreamReader(in)); + while (!ThreadsV2.isInterrupted()) { + try { + final var line = bf.readLine(); + if (line == null) { + log.debug("status=outputEnded"); + break; + } + log.debug(">>> {}", line); + } catch (IOException e) { + + } + } + }; + Thread + .ofVirtual() + .start(task); + } + +} diff --git a/src/test/java/com/mageddo/commons/exec/CommandLinesTest.java b/src/test/java/com/mageddo/commons/exec/CommandLinesTest.java new file mode 100644 index 000000000..bebd08efa --- /dev/null +++ b/src/test/java/com/mageddo/commons/exec/CommandLinesTest.java @@ -0,0 +1,56 @@ +package com.mageddo.commons.exec; + +import org.apache.commons.exec.CommandLine; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; + +class CommandLinesTest { + + + @Test + void mustValidateWhenExitsWithErrorCode(){ + + final var result = CommandLines.exec( + new CommandLine("sh") + .addArgument("-c") + .addArgument("exit 3", false) + ); + + final var ex = assertThrows(ExecutionValidationFailedException.class, result::checkExecution); + + assertEquals(3, ex.getExitCode()); + + } + + @Test + void mustExecuteCommand(){ + + final var result = CommandLines.exec("echo %s", "hey"); + + assertEquals(0, result.getExitCode()); + assertEquals("hey\n", result.getOutAsString()); + } + + @Test + void mustExecuteAndPrintOutputConcurrently() { + + final var result = CommandLines.exec( + new CommandLine("sh") + .addArgument("-c") + .addArgument("echo hi && sleep 0.2 && echo hi2", false), + new NopResultHandler() + ); + + result.printOutToLogsInBackground(); + + result.waitProcessToFinish(); + + final var expectedOut = """ + hi + hi2 + """; + assertEquals(expectedOut, result.getOutAsString()); + } +} diff --git a/src/test/java/com/mageddo/commons/exec/DelegateOutputStreamTest.java b/src/test/java/com/mageddo/commons/exec/DelegateOutputStreamTest.java new file mode 100644 index 000000000..86b07f4ca --- /dev/null +++ b/src/test/java/com/mageddo/commons/exec/DelegateOutputStreamTest.java @@ -0,0 +1,26 @@ +package com.mageddo.commons.exec; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +class DelegateOutputStreamTest { + @Test + void mustWriteToTheTwoOuts() throws IOException { + + final var out1 = new ByteArrayOutputStream(); + final var out2 = new ByteArrayOutputStream(); + final var arr = new byte[]{1, 2, 3}; + + // act + final var delegateOut = new DelegateOutputStream(out1, out2); + delegateOut.write(arr); + + // assert + assertArrayEquals(arr, out1.toByteArray()); + assertArrayEquals(arr, out2.toByteArray()); + } +} diff --git a/src/test/java/com/mageddo/commons/exec/PipedStreamTest.java b/src/test/java/com/mageddo/commons/exec/PipedStreamTest.java new file mode 100644 index 000000000..b93f1121a --- /dev/null +++ b/src/test/java/com/mageddo/commons/exec/PipedStreamTest.java @@ -0,0 +1,27 @@ +package com.mageddo.commons.exec; + +import org.junit.jupiter.api.Test; + +import java.io.ByteArrayOutputStream; +import java.io.IOException; + +import static org.junit.jupiter.api.Assertions.assertArrayEquals; + +class PipedStreamTest { + + @Test + void mustWriteToOutAndBeAbleToReadWhatIsBeingWritten() throws IOException { + // arrange + final var bytes = new byte[]{1, 2, 3}; + + // act + final var stream = new PipedStream(new ByteArrayOutputStream()); + stream.write(bytes); + stream.close(); + + // assert + final var bout = (ByteArrayOutputStream) stream.getOriginalOut(); + assertArrayEquals(bytes, bout.toByteArray()); + assertArrayEquals(bytes, stream.getPipedIn().readAllBytes()); + } +}