OpTestKernelTest: Kernel build test with bisection

OpTestConfiguration.py

@@ -460,6 +460,8 @@ def get_parser():
                             help="Don't exit if we find unknown command line arguments")
     misc_group.add_argument("--secvar-payload-url",
                             help="Specify a URL for the secvar test data payload")
+    misc_group.add_argument("--bisect-flag",
+                            help="Specify if bisection is to be done or not")
 
     return parser
 

common/OpTestUtil.py

@@ -2353,6 +2353,122 @@ def prepare_source(self,spec_file, host, dest_path, package, build_option=None,
         except OpTestError:
             return ""
 
+    def err_message(self, er):
+        """
+        To get entire error msg
+        """
+        error_message = [str(item) for item in er]
+        combined_error = '\n'.join(error_message)
+        combined_error = re.sub(r'\x1b\[[0-9;]*[mK]|,', '', combined_error)
+        parts = combined_error.split('\x1b[')
+        cleaned_error = ''.join(char for char in combined_error if ord(char) < 128)
+        pattern = r'in file.*?compilation terminated'
+        match = re.search(pattern, cleaned_error, re.IGNORECASE | re.DOTALL)
+        if match:
+            relevant_part = match.group(0)
+            return relevant_part
+        return " "
+
+    def get_email(self, commit_id):
+        """
+        To get email of Author from the identified bad commit
+        """
+        connection = self.conf.host()
+        try :
+            connection.host_run_command("git config --global color.ui true")
+            result = connection.host_run_command("git show --format=%ce {} | sed -r 's/\x1B\[[0-9:]*[JKsu]//g'".format(commit_id))
+            return result[0]
+        except subprocess.CalledProcessError as e:
+            log.info(e)
+            return None
+
+    def build_bisector(self, linux_path, good_commit, repo):
+        connection = self.conf.host()
+        connection.host_run_command(" if [ '$(pwd)' != {} ]; then cd {} || exit 1 ; fi ".format(linux_path,linux_path))
+        shallow = connection.host_run_command("git rev-parse --is-shallow-repository")
+        if shallow[-1] in [True,'true']:
+            connection.host_run_command("git fetch --unshallow",timeout=3000)
+        makefile_path = os.path.join(self.conf.basedir, "test_binaries/make.sh")
+        connection.copy_test_file_to_host(makefile_path, dstdir=linux_path)
+        connection.host_run_command("git bisect start")
+        folder_type=re.split(r'[\/\\.]',str(repo))[-2]
+        if folder_type == 'linux-next':
+            connection.host_run_command("git fetch --tags")
+            good_tag=connection.host_run_command("git tag -l 'v[0-9]*' | sort -V | tail -n 1")
+            connection.host_run_command("git bisect good {} ".format(good_tag))
+        else:
+            connection.host_run_command("git bisect good {} ".format(good_commit))
+        connection.host_run_command(" git bisect bad ")
+        connection.host_run_command("chmod +x ./make.sh ")
+        commit =  connection.host_run_command(" git bisect run ./make.sh")
+        badCommit = [word for word in commit if word.endswith("is the first bad commit")]
+        badCommit= badCommit[0].split()[0]
+        email = self.get_email(badCommit)
+        connection.host_run_command("git bisect log")
+        connection.host_run_command("git bisect reset")
+        return email, badCommit
+
+    def get_commit_message(self, linux_path, commit_sha):
+        connection = self.conf.host()
+        try:
+            connection.host_run_command(" if [ '$(pwd)' != {} ]; then cd {} || exit 1 ; fi ".format(linux_path,linux_path))
+            commit_message = connection.host_run_command("git log -n 1 --pretty=format:%s {} | sed -r 's/\x1B\[[0-9:]*[JKsu]//g'".format(commit_sha))
+        except subprocess.CalledProcessError as e:
+            log.info(e)
+            commit_message = None
+        return commit_message[0].strip()
+
+    def format_email(self, linux_path , repo):
+        connection = self.conf.host()
+        machine_type = connection.host_run_command("uname -m")
+        gcc_version = connection.host_run_command("gcc --version")[0]
+        kernel_version = connection.host_run_command("uname -r")
+        try:
+            with open("output.json", "r") as file:
+                data = json.load(file)
+                error_message = data.get("error", "")
+                err_long = data.get("err_msg","")
+                commit = str(data.get("commit", ""))[:7]
+        except FileNotFoundError:
+            log.error("Error: output.json not found.")
+            error_message = ""
+            commit = ""
+        fix_description = self.get_commit_message(linux_path,commit)
+        if "netdev/net" in repo:
+            linux = "netdev/net"
+        elif "netdev/net-next" in repo:
+            linux = "netdev/net-next"
+        elif "mkp/scsi" in repo :
+            linux = "scsi/scsi-queue"
+        elif "torvalds/linux" in repo:
+            linux = "mainline/master"
+        elif "next/linux-next" in repo:
+            linux = "linux-next/master"
+        else:
+            linux = "linux"
+        subject = "[{}][bisected {}] [{}] build fail with error: {}".format(linux,commit, machine_type, error_message)
+        body = """
+        Greetings,
+
+        Today's {} kernel fails to build on {} machine.
+
+        Kernel build fail at error: {}
+        {}
+
+        Kernel Version: {}
+        Machine Type: {}
+        gcc: {}
+        Bisected Commit: {}
+
+        kernel builds fine when the bad commit ({})  is reverted
+        {} - {}
+        --
+        Regards
+        Linux CI
+        """.format(linux, machine_type, error_message,err_long,kernel_version, machine_type, gcc_version, commit,commit, commit, fix_description)
+        with open("email.json","w") as email:
+             json.dump({"subject":subject,"body":body},email)
+
 
 class Server(object):
     '''

op-test

@@ -110,6 +110,7 @@ from testcases import GcovSetup
 from testcases import Lcov
 from testcases import BisectKernel
 from testcases import OpTestHtxBootme
+from testcases import OpTestKernelTest
 import OpTestConfiguration
 import sys
 import time

test_binaries/make.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+# OpenPOWER Automated Test Project
+#
+# Contributors Listed Below - COPYRIGHT 2024
+# [+] International Business Machines Corp.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+nproc=`nproc`
+yes "" | make olddefconfig /dev/null 2>&1
+make -j$nproc -S vmlinux > /dev/null 2>&1

testcases/OpTestKernelTest.py

@@ -0,0 +1,205 @@
+#!/usr/bin/env python3
+# OpenPOWER Automated Test Project
+#
+# Contributors Listed Below - COPYRIGHT 2024
+# [+] International Business Machines Corp.
+#
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+# implied. See the License for the specific language governing
+# permissions and limitations under the License.
+#
+# Author : Tejas Manhas <Tejas.Manhas@ibm.com>
+# Co-Author : Abdul Haleem <abdhalee@linux.vnet.ibm.com>
+
+import json
+import OpTestConfiguration
+import OpTestLogger
+import os
+import unittest
+from urllib.parse import urlparse
+import re
+import subprocess
+import sys
+import time
+
+from common.OpTestSystem import OpSystemState
+from common.OpTestSOL import OpSOLMonitorThread
+from common.Exceptions import CommandFailed
+from common.OpTestUtil import OpTestUtil
+
+log = OpTestLogger.optest_logger_glob.get_logger(__name__)
+
+
+class KernelTest(unittest.TestCase):
+
+    def setUp(self):
+        """
+        Set up the test environment.
+        Initializes test parameters and checks required configurations.
+        """
+        self.conf = OpTestConfiguration.conf
+        self.cv_HOST = self.conf.host()
+        self.cv_SYSTEM = self.conf.system()
+        self.con = self.cv_SYSTEM.cv_HOST.get_ssh_connection()
+        self.host_cmd_timeout = self.conf.args.host_cmd_timeout
+        self.repo = self.conf.args.git_repo
+        self.repo_reference = self.conf.args.git_repo_reference
+        self.branch = self.conf.args.git_branch
+        self.home = self.conf.args.git_home
+        self.config_path = self.conf.args.git_repoconfigpath
+        self.config = self.conf.args.git_repoconfig
+        self.good_commit = self.conf.args.good_commit
+        self.bad_commit = self.conf.args.bad_commit
+        self.bisect_script = self.conf.args.bisect_script
+        self.bisect_category = self.conf.args.bisect_category
+        self.append_kernel_cmdline = self.conf.args.append_kernel_cmdline
+        self.linux_path = os.path.join(self.home, "linux")
+        self.bisect_flag = self.conf.args.bisect_flag
+        self.util = OpTestUtil(OpTestConfiguration.conf)
+        self.host_distro_name = self.util.distro_name()
+        self.console_thread = OpSOLMonitorThread(1, "console")
+        # in case bisection see if we need powercycle not for build, but for boot
+        self.cv_SYSTEM.goto_state(OpSystemState.OFF)
+        self.cv_SYSTEM.goto_state(OpSystemState.OS)
+        self.console_thread.start()
+        if not self.repo:
+            self.fail("Provide git repo of kernel to install")
+        if not (self.conf.args.host_ip and self.conf.args.host_user and self.conf.args.host_password):
+            self.fail(
+                "Provide host ip user details refer, --host-{ip,user,password}")
+
+    def wait_for(self, func, timeout, first=0.0, step=1.0, text=None, args=None, kwargs=None):
+        args = args or []
+        kwargs = kwargs or {}
+
+        start_time = time.monotonic()
+        end_time = start_time + timeout
+
+        time.sleep(first)
+
+        while time.monotonic() < end_time:
+            if text:
+                log.debug("%s (%.9f secs)", text, (time.monotonic() - start_time))
+
+            output = func(*args, **kwargs)
+            if output:
+                return output
+
+            time.sleep(step)
+
+        return None
+
+    def build_kernel(self):
+        """
+        Build and install the Linux kernel.
+        """
+        self.config_path = self.conf.args.git_repoconfigpath
+
+        def is_url(path):
+            '''
+            param path: path to download
+            return: boolean True if given path is url False Otherwise
+            '''
+            valid_schemes = ['http', 'https', 'git', 'ftp']
+            if urlparse(path).scheme in valid_schemes:
+                return True
+            return False
+
+        if self.config_path:
+            if is_url(self.config_path):
+                self.con.run_command("wget %s -O linux/.config" % self.config_path)
+            else:
+                self.cv_HOST.copy_test_file_to_host(self.config_path, sourcedir="", dstdir=os.path.join(linux_path, ".config"))
+        self.con.run_command("cd linux && make olddefconfig")
+        # the below part of the code is needed for only first run and will be decided bisect flag false
+        ker_ver = self.con.run_command("make kernelrelease")[-1]
+        sha = self.con.run_command("git rev-parse HEAD")
+        tcommit = self.con.run_command("export 'TERM=xterm-256color';git show -s --format=%ci")
+        tcommit = re.sub(r"\x1b\[[0-9;]*[mGKHF]", "", tcommit[1])
+        log.info("Upstream kernel version: %s", ker_ver)
+        log.info("Upstream kernel commit-id: %s", sha[-1])
+        log.info("Upstream kernel commit-time: %s", tcommit)
+        log.debug("Compile the upstream kernel")
+        try:
+            cpu= self.con.run_command("lscpu | grep '^CPU(s):' | awk '{print $2}'")
+            err=self.con.run_command("make -j {} -s vmlinux".format(cpu[-1]), timeout=self.host_cmd_timeout)
+            log.info("Kernel build successful")
+            return 0,err
+        except CommandFailed as e:
+            log.error("Kernel build failed: {}".format(e))
+            return  4,e
+
+    def Store_loc(self, er) :
+            """
+            To get location of file in which error is introduced
+            """
+            pattern = r"([\w\d_]+\/(?:(?:[\w\d_]+\/)*[\w\d_]+\b))"
+            matches = [match.group(1) for match in re.finditer(pattern,er)]
+            return matches
+
+
+class KernelBuild(KernelTest):
+    """
+    Does the build for any Linux repo and in case of build failure, calls build bisector 
+    from OpTestUtils to give first bad commit and related information along with email template. 
+    """
+
+    def setUp(self):
+        """
+        Does setup for KernelBUild from parent KernelTest
+        """
+        super(KernelBuild,self).setUp()
+
+    def runTest(self):
+        """
+        Clones git repo and builds to check for failure and do bisection
+        """
+        self.con.run_command("if [ -d {} ]; then rm -rf {}; fi".format(self.home,self.home))
+        self.con.run_command("if [ ! -d {} ]; then mkdir -p {}; fi".format(self.home,self.home))
+        self.con.run_command("cd {}".format(self.home))
+        if not self.branch:
+            self.branch='master' 
+        self.con.run_command("git clone --depth 1 -b {} {} linux".format( self.branch, self.repo),timeout=3000)
+        self.con.run_command("cd linux")
+        commit = self.con.run_command(" git log -1 --format=%H  | sed -r 's/\x1B\[[0-9:]*[JKsu]//g'")
+        self.con.run_command("cd ..")
+        error = self.build_kernel()
+        exit_code = error[0]
+        errVal = str(error[1])
+        log.info("printing the exit code '{}'".format(exit_code))
+        entry=[]
+        err_msg=[]
+        if exit_code != 0:
+            entry = self.Store_loc(errVal)[-1]
+            err_msg= self.util.err_message(error)
+            badCommit = commit[-1]
+            if self.bisect_flag == '1':
+                log.info("STARTING BUILD_BISECTION")
+                res = self.util.build_bisector(self.linux_path, self.good_commit, self.repo)
+                log.info("BUILD_BISECTION ENDED")
+                emaili=res[0]
+                commiti=res[1]
+                log.info("revert commit check is manual for now")
+            else :  
+                emaili=""
+                commiti=commit[-1]
+        else :  
+             emaili=""
+             commiti=commit[-1]
+        with open('output.json','w') as f:
+            json.dump({"exit_code":exit_code,"email":emaili,"commit": commiti,"error":entry,"err_msg":err_msg,"flag":self.bisect_flag},f)
+        if exit_code != 0:
+            self.util.format_email(self.linux_path, self.repo)
+
+    def tearDown(self):
+        self.console_thread.console_terminate()
+        self.con.close()