diff --git a/CHANGELOG.md b/CHANGELOG.md index 7390c0ed..62fc932b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [1.59.0](https://github.com/cloudogu/ces-build-lib/releases/tag/1.59.0) - 2022-11-28 +### Added +- Function `collectAndArchiveLogs` to collect dogu and resource information to help debugging k3s Jenkins buils. #89 +- Function `applyDoguResource(String doguName, String doguNamespace, String doguVersion)` to apply a custom dogu + resource into the cluster. This effectively installs a dogu if it is available. #89 + ## [1.58.0](https://github.com/cloudogu/ces-build-lib/releases/tag/1.58.0) - 2022-11-07 ### Changed - Push k8s yaml content via file reference #87 diff --git a/README.md b/README.md index 707d1248..04bcec6d 100644 --- a/README.md +++ b/README.md @@ -1050,7 +1050,7 @@ if (response.status == '201' && response.content-type == 'application/json') { # K3d -`K3d` provides functions to set up and administer a lokal k3s cluster in Docker +`K3d` provides functions to set up and administer a lokal k3s cluster in Docker. Example: @@ -1079,7 +1079,16 @@ try { k3d.waitForDeploymentRollout("my-dogu-name", 300, 5) } - + + stage('install a dependent dogu by applying a dogu resource') { + k3d.applyDoguResource("my-dependency", "nyNamespace", "10.0.0-1") + k3d.waitForDeploymentRollout("my-dependency", 300, 5) + } + +} catch (Exception ignored) { + // in case of a failed build collect dogus, resources and pod logs and archive them as log file on the build. + k3d.collectAndArchiveLogs() + throw e } finally { stage('Remove k3d cluster') { k3d.deleteK3d() diff --git a/pom.xml b/pom.xml index 76ca9bf6..659bf116 100644 --- a/pom.xml +++ b/pom.xml @@ -7,7 +7,7 @@ com.cloudogu.ces ces-build-lib ces-build-lib - 1.58.0 + 1.59.0 UTF-8 @@ -18,13 +18,13 @@ com.cloudbees groovy-cps - 1.21 + 1.31 org.codehaus.groovy groovy-all - 2.4.11 + 2.4.21 diff --git a/src/com/cloudogu/ces/cesbuildlib/K3d.groovy b/src/com/cloudogu/ces/cesbuildlib/K3d.groovy index d6da72b6..e316c222 100644 --- a/src/com/cloudogu/ces/cesbuildlib/K3d.groovy +++ b/src/com/cloudogu/ces/cesbuildlib/K3d.groovy @@ -11,6 +11,7 @@ class K3d { * The version of k3d to be installed */ private static String K3D_VERSION = "4.4.7" + private static String K3D_LOG_FILENAME = "k8sLogs" private String clusterName private script @@ -225,6 +226,33 @@ class K3d { patchDoguDeployment(dogu, image) } + /** + * Applies the specified dogu resource into the k8s cluster. This should be used for dogus which are not build or + * locally installed in the build process. An example for the usage would be to install a dogu dependency before + * starting integration tests. + * + * @param doguName Name of the dogu, e.g., "nginx-ingress" + * @param doguNamespace Namespace of the dogu, e.g., "official" + * @param doguVersion Version of the dogu, e.g., "13.9.9-1" + */ + void applyDoguResource(String doguName, String doguNamespace, String doguVersion) { + def filename = "target/make/k8s/${doguName}.yaml" + def doguContentYaml = """ +apiVersion: k8s.cloudogu.com/v1 +kind: Dogu +metadata: + name: ${doguName} + labels: + dogu: ${doguName} +spec: + name: ${doguNamespace}/${doguName} + version: ${doguVersion} +""" + + script.writeFile(file: filename.toString(), text: doguContentYaml.toString()) + kubectl("apply -f ${filename}") + } + private void applyDevDoguDescriptor(Docker docker, String dogu, String imageUrl, String port) { String imageDev String doguJsonDevFile = "${this.workspace}/target/dogu.json" @@ -459,5 +487,82 @@ data: "registryConfigEncrypted": {${config.registryConfigEncrypted}} }""" } + + + /** + * Collects all necessary resources and log information used to identify problems with our kubernetes cluster. + * + * The collected information are archived as zip files at the build. + */ + void collectAndArchiveLogs() { + script.dir(K3D_LOG_FILENAME) { + script.deleteDir() + } + script.sh("rm -rf ${K3D_LOG_FILENAME}.zip".toString()) + + collectResourcesSummaries() + collectDoguDescriptions() + collectPodLogs() + + String fileNameString = "${K3D_LOG_FILENAME}.zip".toString() + script.zip(zipFile: fileNameString, archive: "false", dir: "${K3D_LOG_FILENAME}".toString()) + script.archiveArtifacts(artifacts: fileNameString, allowEmptyArchive: "true") + } + + /** + * Collects all information about resources and their quantity and saves them as .yaml files. + */ + void collectResourcesSummaries() { + def relevantResources = [ + "persistentvolumeclaim", + "statefulset", + "replicaset", + "deployment", + "service", + "secret", + "pod", + ] + + for (def resource : relevantResources) { + def resourceYaml = kubectl("get ${resource} --show-kind --ignore-not-found -l app=ces -o yaml || true", true) + script.dir("${K3D_LOG_FILENAME}") { + script.writeFile(file: "${resource}.yaml".toString(), text: resourceYaml) + } + } + } + + /** + * Collects all descriptions of dogus resources and saves them as .yaml files into the k8s logs directory. + */ + void collectDoguDescriptions() { + def allDoguNames = kubectl("get dogu --ignore-not-found -o name || true", true) + def doguNames = allDoguNames.split("\n") + for (def doguName : doguNames) { + def doguFileName = doguName.split("/")[1] + def doguDescribe = kubectl("describe ${doguName} || true", true) + script.dir("${K3D_LOG_FILENAME}") { + script.dir('dogus') { + script.writeFile(file: "${doguFileName}.txt".toString(), text: doguDescribe) + } + } + } + } + + /** + * Collects all pod logs and saves them into the k8s logs directory. + */ + void collectPodLogs() { + def allPodNames = kubectl("get pods -o name || true", true) + def podNames = allPodNames.split("\n") + for (def podName : podNames) { + def podFileName = podName.split("/")[1] + def podLogs = kubectl("logs ${podName} || true", true) + script.dir("${K3D_LOG_FILENAME}") { + script.dir('pods') { + script.writeFile(file: "${podFileName}.log".toString(), text: podLogs) + } + } + } + } } diff --git a/test/com/cloudogu/ces/cesbuildlib/K3dTest.groovy b/test/com/cloudogu/ces/cesbuildlib/K3dTest.groovy index 55090a1c..9339a481 100644 --- a/test/com/cloudogu/ces/cesbuildlib/K3dTest.groovy +++ b/test/com/cloudogu/ces/cesbuildlib/K3dTest.groovy @@ -265,7 +265,6 @@ class K3dTest extends GroovyTestCase { K3d sut = new K3d(scriptMock, workspaceDir, k3dWorkspaceDir, "path") String prefixedRegistryName = "k3d-${sut.getRegistryName()}" String port = "5000" - String imageUrl = "myIP:1234/test/myimage:0.1.2" scriptMock.expectedShRetValueForScript.put('whoami'.toString(), "itsme") scriptMock.expectedShRetValueForScript.put('cat /etc/passwd | grep itsme'.toString(), "test:x:900:1001::/home/test:/bin/sh") @@ -324,4 +323,103 @@ spec: assertThat(scriptMock.allActualArgs[20].trim()).startsWith("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl patch deployment 'test' -p '{\"spec\":{\"template\":{\"spec\":{\"containers\":[{\"name\":\"test\",\"image\":\"myIP:1234/test/myimage:0.1.2\"}]}}}}'") assertThat(scriptMock.allActualArgs.size()).isEqualTo(21) } + + + void testK3d_collectAndArchiveLogs() { + // given + def workspaceDir = "leWorkspace" + def k3dWorkspaceDir = "leK3dWorkSpace" + def scriptMock = new ScriptMock() + K3d sut = new K3d(scriptMock, workspaceDir, k3dWorkspaceDir, "path") + + def relevantResources = ["persistentvolumeclaim","statefulset","replicaset","deployment","service","secret","pod"] + for(def resource : relevantResources) { + scriptMock.expectedShRetValueForScript.put("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get ${resource} --show-kind --ignore-not-found -l app=ces -o yaml || true".toString(), "value for ${resource}") + } + + scriptMock.expectedShRetValueForScript.put("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get dogu --ignore-not-found -o name || true".toString(), "k8s.cloudogu.com/testdogu") + scriptMock.expectedShRetValueForScript.put("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl describe k8s.cloudogu.com/testdogu || true".toString(), "this is the description of a dogu") + + scriptMock.expectedShRetValueForScript.put("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get pods -o name || true".toString(), "pod/testpod-1234\npod/testpod2-1234") + scriptMock.expectedShRetValueForScript.put("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl logs pod/testpod-1234 || true".toString(), "this is the log from testpod") + scriptMock.expectedShRetValueForScript.put("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl logs pod/testpod2-1234 || true".toString(), "this is the log from testpod2") + + // when + sut.collectAndArchiveLogs() + + // then + int i = 0 + assertThat(scriptMock.allActualArgs[i++].trim()).contains("called deleteDir()") + assertThat(scriptMock.allActualArgs[i++].trim()).contains("rm -rf k8sLogs.zip") + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get persistentvolumeclaim --show-kind --ignore-not-found -l app=ces -o yaml || true") + assertThat(scriptMock.writeFileParams[0]).isEqualTo(["file": "persistentvolumeclaim.yaml", "text": "value for persistentvolumeclaim"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get statefulset --show-kind --ignore-not-found -l app=ces -o yaml || true") + assertThat(scriptMock.writeFileParams[1]).isEqualTo(["file": "statefulset.yaml", "text": "value for statefulset"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get replicaset --show-kind --ignore-not-found -l app=ces -o yaml || true") + assertThat(scriptMock.writeFileParams[2]).isEqualTo(["file": "replicaset.yaml", "text": "value for replicaset"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get deployment --show-kind --ignore-not-found -l app=ces -o yaml || true") + assertThat(scriptMock.writeFileParams[3]).isEqualTo(["file": "deployment.yaml", "text": "value for deployment"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get service --show-kind --ignore-not-found -l app=ces -o yaml || true") + assertThat(scriptMock.writeFileParams[4]).isEqualTo(["file": "service.yaml", "text": "value for service"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get secret --show-kind --ignore-not-found -l app=ces -o yaml || true") + assertThat(scriptMock.writeFileParams[5]).isEqualTo(["file": "secret.yaml", "text": "value for secret"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get pod --show-kind --ignore-not-found -l app=ces -o yaml || true") + assertThat(scriptMock.writeFileParams[6]).isEqualTo(["file": "pod.yaml", "text": "value for pod"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get dogu --ignore-not-found -o name || true") + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl describe k8s.cloudogu.com/testdogu || true") + assertThat(scriptMock.writeFileParams[7]).isEqualTo(["file": "testdogu.txt", "text": "this is the description of a dogu"]) + + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl get pods -o name || true") + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl logs pod/testpod-1234 || true") + assertThat(scriptMock.allActualArgs[i++].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl logs pod/testpod2-1234 || true") + assertThat(scriptMock.writeFileParams[8]).isEqualTo(["file": "testpod-1234.log", "text": "this is the log from testpod"]) + assertThat(scriptMock.writeFileParams[9]).isEqualTo(["file": "testpod2-1234.log", "text": "this is the log from testpod2"]) + + assertThat(scriptMock.zipParams.size()).isEqualTo(1) + assertThat(scriptMock.zipParams[0]).isEqualTo(["archive":"false", "dir":"k8sLogs", "zipFile":"k8sLogs.zip"]) + assertThat(scriptMock.archivedArtifacts.size()).isEqualTo(1) + assertThat(scriptMock.archivedArtifacts[0]).isEqualTo(["allowEmptyArchive":"true", "artifacts":"k8sLogs.zip"]) + + assertThat(scriptMock.allActualArgs.size()).isEqualTo(i) + assertThat(scriptMock.writeFileParams.size()).isEqualTo(10) + } + + void testK3d_applyDoguResource() { + // given + def workspaceDir = "leWorkspace" + def k3dWorkspaceDir = "leK3dWorkSpace" + def scriptMock = new ScriptMock() + K3d sut = new K3d(scriptMock, workspaceDir, k3dWorkspaceDir, "path") + + def filename = "target/make/k8s/testName.yaml" + def doguContentYaml = """ +apiVersion: k8s.cloudogu.com/v1 +kind: Dogu +metadata: + name: testName + labels: + dogu: testName +spec: + name: nyNamespace/testName + version: 14.1.1-1 +""" + + // when + sut.applyDoguResource("testName", "nyNamespace", "14.1.1-1") + + // then + assertThat(scriptMock.writeFileParams[0]).isEqualTo(["file": filename, "text": doguContentYaml]) + assertThat(scriptMock.writeFileParams.size()).isEqualTo(1) + + assertThat(scriptMock.allActualArgs[0].trim()).contains("sudo KUBECONFIG=leK3dWorkSpace/.k3d/.kube/config kubectl apply -f target/make/k8s/testName.yaml") + assertThat(scriptMock.allActualArgs.size()).isEqualTo(1) + } } diff --git a/test/com/cloudogu/ces/cesbuildlib/ScriptMock.groovy b/test/com/cloudogu/ces/cesbuildlib/ScriptMock.groovy index 7ef1be96..08c54d47 100644 --- a/test/com/cloudogu/ces/cesbuildlib/ScriptMock.groovy +++ b/test/com/cloudogu/ces/cesbuildlib/ScriptMock.groovy @@ -30,6 +30,8 @@ class ScriptMock { List actualShMapArgs = new LinkedList<>() List> writeFileParams = new LinkedList<>() + List> zipParams = new LinkedList<>() + List> archivedArtifacts = new LinkedList<>() Map actualFileArgs Map actualStringArgs Map files = new HashMap() @@ -60,6 +62,10 @@ class ScriptMock { actualJUnitFlags = map } + void deleteDir() { + allActualArgs.add("called deleteDir()") + } + String sh(Map args) { actualShMapArgs.add(args.script.toString()) allActualArgs.add(args.script.toString()) @@ -150,6 +156,14 @@ class ScriptMock { writeFileParams.add(params) } + void zip(Map params) { + zipParams.add(params) + } + + void archiveArtifacts(Map params) { + archivedArtifacts.add(params) + } + String readFile(String file) { return files.get(file) }