diff --git a/kernel_install_dep.sh b/kernel_install_dep.sh index 372cd81..58cdb39 100755 --- a/kernel_install_dep.sh +++ b/kernel_install_dep.sh @@ -37,6 +37,7 @@ install_kselftest_deps_8() { libreswan \ libubsan \ llvm \ + mock \ ncurses-devel \ net-tools \ netsniff-ng \ @@ -46,6 +47,7 @@ install_kselftest_deps_8() { openssl-devel \ perf \ popt-devel \ + python3-GitPython \ python3-pip \ rsync \ socat \ @@ -110,6 +112,7 @@ install_kselftest_deps_9() { libreswan \ libubsan \ llvm \ + mock \ ncurses-devel \ net-tools \ netsniff-ng \ @@ -120,6 +123,7 @@ install_kselftest_deps_9() { packetdrill \ perf \ popt-devel \ + python3-GitPython \ python3-pip \ rsync \ socat \ @@ -177,6 +181,7 @@ install_kselftest_deps_10() { libreswan \ libubsan \ llvm \ + mock \ ncurses-devel \ net-tools \ nftables \ @@ -186,6 +191,7 @@ install_kselftest_deps_10() { packetdrill \ perf \ popt-devel \ + python3-GitPython \ python3-pip \ rsync \ socat \ @@ -219,3 +225,5 @@ case "$ROCKY_SUPPORT_PRODUCT" in install_kselftest_deps_8 ;; esac + +sudo usermod -a -G mock $USER diff --git a/kt/commands/vm/command.py b/kt/commands/vm/command.py index ba966fd..d3f1cea 100644 --- a/kt/commands/vm/command.py +++ b/kt/commands/vm/command.py @@ -57,6 +57,15 @@ help="Lists existings vms", ) @click.option("--test", is_flag=True, help="Build the kernel and run kselftests") +@click.option("--content_release", is_flag=True, help="Perform an lts content release") @click.argument("kernel_workspace", required=False, shell_complete=ShellCompletion.show_kernel_workspaces) -def vm(kernel_workspace, console, destroy, override, list_all, test): - main(name=kernel_workspace, console=console, destroy=destroy, override=override, list_all=list_all, test=test) +def vm(kernel_workspace, console, destroy, override, list_all, test, content_release): + main( + name=kernel_workspace, + console=console, + destroy=destroy, + override=override, + list_all=list_all, + test=test, + content_release=content_release, + ) diff --git a/kt/commands/vm/impl.py b/kt/commands/vm/impl.py index e48038f..3e2795d 100644 --- a/kt/commands/vm/impl.py +++ b/kt/commands/vm/impl.py @@ -12,6 +12,7 @@ from kt.ktlib.config import Config from kt.ktlib.kernel_workspace import KernelWorkspace +from kt.ktlib.kernels import KernelsInfo from kt.ktlib.ssh import SshCommand from kt.ktlib.util import Constants from kt.ktlib.virt import VirtHelper, VmCommand @@ -259,11 +260,106 @@ def test(self, config): self.kselftests(config=config) + def content_release(self, config): + logging.debug("Running mkdistgitdiff") + mkdistgitdiff = str(config.base_path / Path("kernel-tools") / Path("mkdistgitdiff.py")) + output_file = self.kernel_workspace.folder.absolute() / Path("mkdistgitdiff.log") + + # mkdistgitdiff needs these to be set + ssh_cmd = f'git config --global user.name "{config.user}"' + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + ssh_cmd = f"git config --global user.email {config.email}" + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = ( + f"cd {self.kernel_workspace.dist_worktree.folder.absolute()} && {mkdistgitdiff} " + f"--srcgit {self.kernel_workspace.src_worktree.folder.absolute()} " + f"--srcgit-branch {self.kernel_workspace.src_worktree.local_branch} " + f"--distgit . " + f"--distgit-branch {self.kernel_workspace.dist_worktree.local_branch} " + f"--last-tag " + f"--bump" + ) + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = ( + f"git -C {self.kernel_workspace.dist_worktree.folder.absolute()} " + f"checkout {{${{USER}}}}_{self.kernel_workspace.src_worktree.local_branch}" + ) + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = ( + f"cd {self.kernel_workspace.dist_worktree.folder.absolute()} && " + f"curl -O https://raw.githubusercontent.com/rocky-linux/rocky-tools/main/getsrc/getsrc.sh" + ) + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = f"cd {self.kernel_workspace.dist_worktree.folder.absolute()} && chmod +x ./getsrc.sh && ./getsrc.sh" + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = f"cd {self.kernel_workspace.folder} && mkdir -p build_files" + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + mock_configs_path = str(config.base_path / Path("mock-configs")) + + # Get the mock_config from kernels.yaml + kernels_info = KernelsInfo.from_yaml(config=config) + kernel_name = Vm._extract_kernel_name(self.kernel_workspace.folder.name) + mock_config = kernels_info.kernels[kernel_name].mock_config + + ssh_cmd = ( + f"cd {self.kernel_workspace.folder} && " + f"cp {mock_configs_path}/{mock_config} ${{USER}}_{mock_config} && " + f"sed -i 's/DEPOT_USER/{config.depot_user}/g' ${{USER}}_{mock_config} && " + f"sed -i 's/DEPOT_TOKEN/{config.depot_token}/g' ${{USER}}_{mock_config} " + ) + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = "sudo usermod -a -G mock ${USER}" + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = ( + f"cd {self.kernel_workspace.dist_worktree.folder.absolute()} && " + f"mock -v -r ../${{USER}}_{mock_config} --resultdir={self.kernel_workspace.folder.absolute()}/build_files --buildsrpm " + f"--spec=SPECS/kernel.spec --sources=SOURCES" + ) + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = ( + f"cd {self.kernel_workspace.dist_worktree.folder.absolute()} && " + f"mock -v -r ../${{USER}}_{mock_config} --resultdir={self.kernel_workspace.folder.absolute()}/build_files " + f"{self.kernel_workspace.folder.absolute()}/build_files/*.src.rpm" + ) + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = f"cd {self.kernel_workspace.folder} && rm ${{USER}}_{mock_config}" + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + + ssh_cmd = ( + f"cd {self.kernel_workspace.folder} && " + f"sudo dnf install $(find build_files -maxdepth 1 -name '*.rpm' -not -name '*.src.rpm' -not -name 'kernel-rt*.rpm' -not -name 'kernel-debug-*.rpm') -y" + ) + output_file = self.kernel_workspace.folder.absolute() / Path("install.log") + SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd]) + + self.reboot() + + ssh_cmd = f"sudo /usr/libexec/kselftests/run_kselftest.sh | tee {self.kernel_workspace.folder.absolute()}/selftest-$(uname -r).log" + SshCommand.run(domain=self.domain, command=[ssh_cmd]) + def console(self): VmCommand.console(vm_name=self.name) -def main(name: str, console: bool, destroy: bool, override: bool, list_all: bool, test: bool = False): +def main( + name: str, + console: bool, + destroy: bool, + override: bool, + list_all: bool, + test: bool = False, + content_release: bool = False, +): if list_all: VmCommand.list_all() return @@ -289,5 +385,11 @@ def main(name: str, console: bool, destroy: bool, override: bool, list_all: bool time.sleep(Constants.VM_DEPS_INSTALL_WAIT_SECONDS) vm_instance.test(config=config) + if content_release: + # Wait for the dependencies to be installed + logging.info("Waiting for the deps to be installed") + # time.sleep(Constants.VM_DEPS_INSTALL_WAIT_SECONDS) + vm_instance.content_release(config=config) + if console: vm_instance.console() diff --git a/kt/data/kernels.yaml b/kt/data/kernels.yaml index 32d82b4..e8363fd 100644 --- a/kt/data/kernels.yaml +++ b/kt/data/kernels.yaml @@ -8,39 +8,46 @@ kernels: src_tree_branch: ciqcbr7_9 dist_git_root: dist-git-tree-cbr dist_git_branch: cbr79-7 + mock_config: centos-cbr79-x86_64.cfg lts-8.6: src_tree_root: kernel-src-tree src_tree_branch: ciqlts8_6 dist_git_root: dist-git-tree-lts dist_git_branch: lts86-8 + mock_config: rocky-lts86-depot-x86_64.cfg lts-9.2: src_tree_root: kernel-src-tree src_tree_branch: ciqlts9_2 dist_git_root: dist-git-tree-lts dist_git_branch: lts92-9 + mock_config: rocky-lts92-depot-x86_64.cfg lts-9.4: src_tree_root: kernel-src-tree src_tree_branch: ciqlts9_4 dist_git_root: dist-git-tree-lts dist_git_branch: lts94-9 + mock_config: rocky-lts94-depot-x86_64.cfg fipslegacy-8.6: src_tree_root: kernel-src-tree src_tree_branch: fips-legacy-8-compliant/4.18.0-425.13.1 dist_git_root: dist-git-tree-fips dist_git_branch: fips-compliant8 + mock_config: rocky-fips-legacy-8-x86_64.cfg fips-9.2: src_tree_root: kernel-src-tree src_tree_branch: fips-9-compliant/5.14.0-284.30.1 dist_git_root: dist-git-tree-fips dist_git_branch: el92-fips-compliant-9 + mock_config: rocky-fips92-depot-x86_64.cfg lts-9.6: src_tree_root: kernel-src-tree src_tree_branch: ciqlts9_6 dist_git_root: dist-git-tree-lts dist_git_branch: lts96-9 + mock_config: rocky-lts96-depot-x86_64.cfg diff --git a/kt/ktlib/config.py b/kt/ktlib/config.py index 6ff5963..9745e72 100644 --- a/kt/ktlib/config.py +++ b/kt/ktlib/config.py @@ -23,6 +23,12 @@ class Config: stored ssh_key: Path to the ssh key (public) shared between host and each vms + user: User's full name (for git config) + email: User's email (for git config) + + depot_user: Depot username + depot_token: Depot token + All paths should be absolute, to avoid issues later. """ @@ -33,20 +39,44 @@ class Config: ssh_key: Path + user: str + email: str + + depot_user: str + depot_token: str + DEFAULT: ClassVar = { "base_path": "~/ciq", "kernels_dir": "~/ciq/kernels", "images_source_dir": "~/ciq/default_test_images", "images_dir": "~/ciq/tmp/virt-images", "ssh_key": "~/.ssh/id_ed25519_generic.pub", + "user": "CIQ Developer", + "email": "dev@ciq.com", + "depot_user": "user", + "depot_token": "token", } @classmethod def from_str_dict(cls, data: dict[str, str]): - # Transform the str values to Path - new_data = {k: Path(v).expanduser() for k, v in data.items()} - - if not all(v.is_absolute() for v in new_data.values()): + # Path fields that need to be converted + path_fields = {"base_path", "kernels_dir", "images_source_dir", "images_dir", "ssh_key"} + + # Merge user data with defaults to support partial configs + merged_data = dict(cls.DEFAULT) + merged_data.update(data) + + # Transform only the path values to Path, keep strings as is + new_data = {} + for k, v in merged_data.items(): + if k in path_fields: + new_data[k] = Path(v).expanduser() + else: + new_data[k] = v + + # Check that all path fields are absolute + path_values = {k: v for k, v in new_data.items() if k in path_fields} + if not all(v.is_absolute() for v in path_values.values()): raise ValueError("all paths should be absolute; check your config") return cls(**new_data) diff --git a/kt/ktlib/kernels.py b/kt/ktlib/kernels.py index a881d44..402a4cf 100644 --- a/kt/ktlib/kernels.py +++ b/kt/ktlib/kernels.py @@ -21,6 +21,7 @@ class KernelInfo: src_tree_branch: the corresponding branch in the source tree dist_git_root: rocky staging rpm repo dist_git_branch: corresponding branch in the rocky staging rpm repo + mock_config: mock config used to build this kernel The src_tree_root and dist_git_root contain absolute paths to the local clone of these repos and their corresponding remote url. @@ -34,6 +35,8 @@ class KernelInfo: dist_git_root: RepoInfo dist_git_branch: str + mock_config: str = "" + @dataclass class KernelsInfo: diff --git a/tests/kt/ktlib/test_config.py b/tests/kt/ktlib/test_config.py index 419a6d4..4aea03e 100644 --- a/tests/kt/ktlib/test_config.py +++ b/tests/kt/ktlib/test_config.py @@ -47,8 +47,9 @@ def test_config_load_from_json_data_None(): def test_config_load_from_json_data_empty(): json_data = "{}" - with pytest.raises(TypeError, match="missing 5 required positional arguments:"): - config = Config.from_json(json_data) # noqa F841 + # Empty config should use defaults + config = Config.from_json(json_data) + assert config.base_path == DEFAULT_CONFIG["base_path"] def test_config_load_from_json_proper_base_path():