diff --git a/Makefile b/Makefile index 33d6d5a..ee63065 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,8 @@ .PHONY: hardly test-image BASE_IMAGE ?= quay.io/packit/packit-worker +# true|false +PULL_BASE_IMAGE ?= true HARDLY_IMAGE ?= quay.io/packit/hardly:dev TEST_IMAGE ?= hardly-tests TEST_TARGET ?= ./tests/ @@ -14,16 +16,29 @@ COV_REPORT ?= term-missing COLOR ?= yes SOURCE_BRANCH ?= $(shell git branch --show-current) +# The 'hardly' image is built FROM quay.io/packit/packit-worker +# which is pulled/updated before the build. +# If you want to build from your local worker image, +# i.e. don't want to pull the base image, +# set PULL_BASE_IMAGE=false hardly: files/recipe-hardly.yaml files/install-deps.yaml - $(CONTAINER_ENGINE) pull $(BASE_IMAGE) - $(CONTAINER_ENGINE) build --rm -t $(HARDLY_IMAGE) -f files/Containerfile --build-arg SOURCE_BRANCH=$(SOURCE_BRANCH) . + $(CONTAINER_ENGINE) build --rm \ + --pull=$(PULL_BASE_IMAGE) \ + -t $(HARDLY_IMAGE) \ + -f files/Containerfile \ + --build-arg SOURCE_BRANCH=$(SOURCE_BRANCH) \ + . check: find . -name "*.pyc" -exec rm {} \; PYTHONPATH=$(CURDIR) PYTHONDONTWRITEBYTECODE=1 python3 -m pytest --color=$(COLOR) --verbose --showlocals --cov=hardly --cov-report=$(COV_REPORT) $(TEST_TARGET) test-image: files/recipe-tests.yaml - $(CONTAINER_ENGINE) build --rm -t $(TEST_IMAGE) -f files/Containerfile.tests --build-arg SOURCE_BRANCH=$(SOURCE_BRANCH) . + $(CONTAINER_ENGINE) build --rm \ + -t $(TEST_IMAGE) \ + -f files/Containerfile.tests \ + --build-arg SOURCE_BRANCH=$(SOURCE_BRANCH) \ + . check-in-container: @# don't use -ti here in CI, TTY is not allocated in zuul diff --git a/files/install-deps.yaml b/files/install-deps.yaml index 73480bb..91109ba 100644 --- a/files/install-deps.yaml +++ b/files/install-deps.yaml @@ -9,7 +9,7 @@ tasks: # - import_tasks: tasks/process-source-branch.yaml - name: Install all RPM/python packages needed to run hardly - dnf: + ansible.builtin.dnf: name: - centpkg - tig # for debugging, can be removed later diff --git a/files/recipe-hardly.yaml b/files/recipe-hardly.yaml index d9b8b28..00f8f6e 100644 --- a/files/recipe-hardly.yaml +++ b/files/recipe-hardly.yaml @@ -6,9 +6,9 @@ hosts: all tasks: - name: Rename run_worker.sh - command: mv /usr/bin/run_worker.sh /usr/bin/run_worker_.sh + ansible.builtin.command: mv /usr/bin/run_worker.sh /usr/bin/run_worker_.sh - name: Install our run_worker.sh - copy: + ansible.builtin.copy: src: run_worker.sh dest: /usr/bin/ mode: 0777 diff --git a/files/recipe-tests.yaml b/files/recipe-tests.yaml index 69d239e..d88a910 100644 --- a/files/recipe-tests.yaml +++ b/files/recipe-tests.yaml @@ -6,7 +6,7 @@ hosts: all tasks: - name: Install test RPM dependencies - dnf: + ansible.builtin.dnf: name: - python3-flexmock - python3-pytest diff --git a/files/tasks/common.yaml b/files/tasks/common.yaml index 92ca681..5b4a1d9 100644 --- a/files/tasks/common.yaml +++ b/files/tasks/common.yaml @@ -4,7 +4,7 @@ # Tasks common to all images - name: make packit home dir - file: + ansible.builtin.file: state: directory path: "{{ home_path }}" mode: 0776 diff --git a/files/tasks/install-ogr-deps.yaml b/files/tasks/install-ogr-deps.yaml deleted file mode 100644 index 5a5d244..0000000 --- a/files/tasks/install-ogr-deps.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# Copyright Contributors to the Packit project. -# SPDX-License-Identifier: MIT - -# Install dependencies from setup.cfg as rpm packages for ogr -- name: Download script setupcfg2rpm.py - get_url: - url: https://raw.githubusercontent.com/packit/deployment/main/scripts/setupcfg2rpm.py - dest: ./setupcfg2rpm.py - mode: "0744" - register: fetch_setupcfg2rpm - until: fetch_setupcfg2rpm is not failed - retries: 6 - -- name: Download ogr setup.cfg - get_url: - # ogr has only main branch - url: https://raw.githubusercontent.com/packit/ogr/main/setup.cfg - dest: ./ogr_setup.cfg - register: fetch_ogr_setup - until: fetch_ogr_setup is not failed - retries: 6 - -- name: Install ogr dependencies provided by setupcfg2rpm - shell: dnf install $(./setupcfg2rpm.py ogr_setup.cfg) -y - args: - warn: no - become: true diff --git a/files/tasks/install-packit-deps.yaml b/files/tasks/install-packit-deps.yaml deleted file mode 100644 index 1321d5f..0000000 --- a/files/tasks/install-packit-deps.yaml +++ /dev/null @@ -1,42 +0,0 @@ -# Copyright Contributors to the Packit project. -# SPDX-License-Identifier: MIT - -# Install dependencies for packit -- name: Download script setupcfg2rpm.py - get_url: - url: https://raw.githubusercontent.com/packit/deployment/main/scripts/setupcfg2rpm.py - dest: ./setupcfg2rpm.py - mode: "0744" - register: fetch_setupcfg2rpm - until: fetch_setupcfg2rpm is not failed - retries: 6 - -- name: Download packit spec file - get_url: - url: https://raw.githubusercontent.com/packit/packit/{{ source_branch }}/packit.spec - dest: ./packit.spec - register: fetch_packit_spec - until: fetch_packit_spec is not failed - retries: 6 - -- name: Install packit RPM build dependencies from packit.spec - shell: dnf builddep packit.spec -y - args: - warn: no -- name: Install packit RPM runtime dependencies - shell: dnf install $(rpmspec -q --requires packit.spec | grep -v 'packit') -y - args: - warn: no - become: true - -- name: Download packit setup.cfg - get_url: - url: https://raw.githubusercontent.com/packit/packit/{{ source_branch }}/setup.cfg - dest: ./packit_setup.cfg - register: fetch_packit_setup - until: fetch_packit_setup is not failed - retries: 6 - -- name: Install dependencies provided by setupcfg2rpm - shell: dnf install $(./setupcfg2rpm.py packit_setup.cfg) - become: true diff --git a/files/tasks/process-source-branch.yaml b/files/tasks/process-source-branch.yaml deleted file mode 100644 index e355065..0000000 --- a/files/tasks/process-source-branch.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Copyright Contributors to the Packit project. -# SPDX-License-Identifier: MIT - -# Check and if necessary adjust source_branch variable - -- name: Fail on empty source_branch variable - fail: - msg: Variable source_branch, which is set from env variable SOURCE_BRANCH is empty. - when: source_branch == '' -# Docker Hub CI image builds are performed from non contributor's branches (eg. packit:fix_hook), -# but we need it to be main or stable -- name: Change source branch to main if is not main/stable - set_fact: - source_branch: "main" - when: - - source_branch != 'stable' - - source_branch != 'main' -- name: Check source_branch value - debug: - msg: source_branch is set to {{ source_branch }} diff --git a/files/zuul-tests.yaml b/files/zuul-tests.yaml index c9dcc1a..f40d6ab 100644 --- a/files/zuul-tests.yaml +++ b/files/zuul-tests.yaml @@ -7,24 +7,24 @@ hosts: all tasks: - name: Install podman - dnf: + ansible.builtin.dnf: name: - podman state: present become: true # Fix the SELinux context for podman - name: Create ~/.local/share/ - file: + ansible.builtin.file: path: ~/.local/share/ state: directory - recurse: yes + recurse: true setype: data_home_t - name: Build test image - command: "make test-image" + ansible.builtin.command: "make test-image" args: chdir: "{{ zuul.project.src_dir }}" - name: Run tests within a container - command: "make check-in-container" + ansible.builtin.command: "make check-in-container" args: chdir: "{{ zuul.project.src_dir }}" environment: diff --git a/hardly/handlers/distgit.py b/hardly/handlers/distgit.py index 0559ee2..0a9609f 100644 --- a/hardly/handlers/distgit.py +++ b/hardly/handlers/distgit.py @@ -8,12 +8,14 @@ from typing import Optional from hardly.handlers.abstract import TaskName +from ogr.abstract import PullRequest from packit.api import PackitAPI from packit.config.job_config import JobConfig from packit.config.package_config import PackageConfig from packit.local_project import LocalProject from packit_service.models import PullRequestModel, SourceGitPRDistGitPRModel from packit_service.worker.events import MergeRequestGitlabEvent, PipelineGitlabEvent +from packit_service.worker.events.enums import GitlabEventAction from packit_service.worker.events.pagure import PullRequestFlagPagureEvent from packit_service.worker.handlers import JobHandler from packit_service.worker.handlers.abstract import ( @@ -41,6 +43,7 @@ def __init__( job_config=job_config, event=event, ) + self.action = event["action"] self.mr_identifier = event["identifier"] self.mr_title = event["title"] self.mr_description = event["description"] @@ -51,6 +54,60 @@ def __init__( ) self.target_repo_branch = event["target_repo_branch"] + # lazy + self._source_git_pr_model = None + self._dist_git_pr_model = None + self._dist_git_pr = None + self._packit = None + + @property + def source_git_pr_model(self) -> PullRequestModel: + if not self._source_git_pr_model: + self._source_git_pr_model = PullRequestModel.get_or_create( + pr_id=self.mr_identifier, + namespace=self.project.namespace, + repo_name=self.project.repo, + project_url=self.project.get_web_url(), + ) + return self._source_git_pr_model + + @property + def dist_git_pr_model(self) -> Optional[PullRequestModel]: + if not self._dist_git_pr_model: + if sg_dg := SourceGitPRDistGitPRModel.get_by_source_git_id( + self.source_git_pr_model.id + ): + self._dist_git_pr_model = sg_dg.dist_git_pull_request + return self._dist_git_pr_model + + @property + def dist_git_pr(self) -> Optional[PullRequest]: + if not self._dist_git_pr and self.dist_git_pr_model: + dist_git_project = self.service_config.get_project( + url=self.dist_git_pr_model.project.project_url + ) + self._dist_git_pr = dist_git_project.get_pr(self.dist_git_pr_model.pr_id) + return self._dist_git_pr + + @property + def packit(self) -> PackitAPI: + if not self._packit: + source_project = self.service_config.get_project( + url=self.source_project_url + ) + local_project = LocalProject( + git_project=source_project, + ref=self.data.commit_sha, + working_dir=self.service_config.command_handler_work_dir, + ) + + self._packit = PackitAPI( + config=self.service_config, + package_config=self.package_config, + upstream_local_project=local_project, + ) + return self._packit + def run(self) -> TaskResults: """ If user creates a merge-request on the source-git repository, @@ -61,26 +118,44 @@ def run(self) -> TaskResults: "Not creating a dist-git MR from " f"{self.target_repo}:{self.target_repo_branch}" ) - return TaskResults(success=True, details={}) + return TaskResults(success=True) if not self.package_config: logger.debug("No package config found.") - return TaskResults(success=True, details={}) + return TaskResults(success=True) - logger.debug(f"About to create a dist-git MR from source-git MR {self.mr_url}") + if ( + self.target_repo_branch + not in self.packit.dg.local_project.git_project.get_branches() + ): + msg = ( + "Can't create a dist-git pull/merge request out of this contribution " + f"because matching {self.target_repo_branch} branch does not exist " + f"in dist-git {self.target_repo} repo." + ) + self.project.get_pr(int(self.mr_identifier)).comment(msg) + logger.info(msg) + return TaskResults(success=True) - source_project = self.service_config.get_project(url=self.source_project_url) - self.local_project = LocalProject( - git_project=source_project, - ref=self.data.commit_sha, - working_dir=self.service_config.command_handler_work_dir, - ) + if self.dist_git_pr_model: + logger.info( + f"{self.source_git_pr_model} already has corresponding {self.dist_git_pr_model}" + ) + if self.dist_git_pr: + if self.action == GitlabEventAction.closed.value: + msg = f"[Source-git MR]({self.mr_url}) has been closed." + self.dist_git_pr.close() + elif self.action == GitlabEventAction.reopen.value: + msg = f"[Source-git MR]({self.mr_url}) has been reopened." + # https://github.com/packit/ogr/pull/714 + # self.dist_git_pr.reopen() + else: + logger.error(f"Unknown action {self.action}") + return TaskResults(success=False) + logger.info(msg) + self.dist_git_pr.comment(msg) + return TaskResults(success=True) - self.api = PackitAPI( - config=self.service_config, - package_config=self.package_config, - upstream_local_project=self.local_project, - ) dg_mr_info = f"""###### Info for package maintainer This MR has been automatically created from [this source-git MR]({self.mr_url}).""" @@ -89,34 +164,18 @@ def run(self) -> TaskResults: Please review the contribution and once you are comfortable with the content, you should trigger a CI pipeline run via `Pipelines → Run pipeline`.""" - if ( - self.target_repo_branch - in self.api.dg.local_project.git_project.get_branches() + logger.info(f"About to create a dist-git MR from source-git MR {self.mr_url}") + if dg_mr := self.packit.sync_release( + dist_git_branch=self.target_repo_branch, + version=self.packit.up.get_specfile_version(), + add_new_sources=False, + title=self.mr_title, + description=f"{self.mr_description}\n\n---\n{dg_mr_info}", + sync_default_files=False, + # we rely on this in PipelineHandler below + local_pr_branch_suffix=f"src-{self.mr_identifier}", + mark_commit_origin=True, ): - dist_git_branch = self.target_repo_branch - else: - msg = f""" -Downstream {self.target_repo}:{self.target_repo_branch} branch does not exist. -""" - self.project.get_pr(int(self.mr_identifier)).comment(msg) - logger.debug(msg) - dist_git_branch = None - - dg_mr = None - if dist_git_branch: - dg_mr = self.api.sync_release( - dist_git_branch=dist_git_branch, - version=self.api.up.get_specfile_version(), - add_new_sources=False, - title=self.mr_title, - description=f"{self.mr_description}\n\n---\n{dg_mr_info}", - sync_default_files=False, - # we rely on this in PipelineHandler below - local_pr_branch_suffix=f"src-{self.mr_identifier}", - mark_commit_origin=True, - ) - - if dg_mr: comment = f"""[Dist-git MR #{dg_mr.id}]({dg_mr.url}) has been created for sake of triggering the downstream checks. It ensures that your contribution is valid and can be incorporated in @@ -181,12 +240,12 @@ def run(self) -> TaskResults: """ if not (dist_git_pr_model := self.dist_git_pr_model()): logger.debug("No dist-git PR model.") - return TaskResults(success=True, details={}) + return TaskResults(success=True) if not ( sg_dg := SourceGitPRDistGitPRModel.get_by_dist_git_id(dist_git_pr_model.id) ): logger.debug(f"Source-git PR for {dist_git_pr_model} not found.") - return TaskResults(success=True, details={}) + return TaskResults(success=True) source_git_pr_model = sg_dg.source_git_pull_request source_git_project = self.service_config.get_project( @@ -214,7 +273,7 @@ def run(self) -> TaskResults: check_name=self.status_check_name, url=self.status_url, ) - return TaskResults(success=True, details={}) + return TaskResults(success=True) @reacts_to(event=PipelineGitlabEvent) diff --git a/tests/integration/test_dist_git_mr.py b/tests/integration/test_dist_git_mr.py index aa8d2e3..1119499 100644 --- a/tests/integration/test_dist_git_mr.py +++ b/tests/integration/test_dist_git_mr.py @@ -10,6 +10,7 @@ from packit.upstream import Upstream from packit_service.config import ServiceConfig from packit_service.constants import SANDCASTLE_WORK_DIR +from packit_service.models import PullRequestModel, SourceGitPRDistGitPRModel from packit_service.service.db_triggers import AddPullRequestDbTrigger from packit_service.utils import dump_package_config from packit_service.worker.monitoring import Pushgateway @@ -97,6 +98,14 @@ def test_dist_git_mr( flexmock(GitlabProject).should_receive("get_file_content").and_return( source_git_yaml ) + + flexmock(PullRequestModel).should_receive("get_or_create").and_return( + flexmock(id=1) + ) + flexmock(SourceGitPRDistGitPRModel).should_receive( + "get_by_source_git_id" + ).and_return(None) + lp = flexmock( LocalProject, refresh_the_arguments=lambda: None, checkout_ref=lambda ref: None )