Skip to content

WIP: storing update data after update #607

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
renatav wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

WIP: storing update data after update #607

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 11 additions & 1 deletion taf/auth_repo.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
get_role_metadata_path,
get_target_path,
)
from taf.constants import INFO_JSON_PATH
from taf.constants import INFO_JSON_PATH, LOG_DIR_NAME, TARGETS_DIRECTORY_NAME
from taf.yubikey.yubikey_manager import PinManager


Expand Down Expand Up @@ -140,6 +140,16 @@ def certs_dir(self):
certs_dir.mkdir(parents=True, exist_ok=True)
return str(certs_dir)

@property
def log_dir(self) -> Path:
"""
Location inside the targets directory where dependencies update data is stored
"""
log_dir_path = self.path / TARGETS_DIRECTORY_NAME / LOG_DIR_NAME
if not log_dir_path.is_dir():
log_dir_path.mkdir(parents=True)
return log_dir_path

@property
def dependencies(self) -> Dict:
return self._dependencies
Expand Down
1 change: 1 addition & 0 deletions taf/constants.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

TARGETS_DIRECTORY_NAME = "targets"
METADATA_DIRECTORY_NAME = "metadata"
LOG_DIR_NAME = "log"


DEFAULT_RSA_SIGNATURE_SCHEME = "rsa-pkcs1v15-sha256"
Expand Down
4 changes: 4 additions & 0 deletions taf/tests/test_updater/update_utils.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -257,6 +257,7 @@ def update_and_check_commit_shas(
no_upstream=False,
skip_check_last_validated=False,
num_of_commits_to_remove=None,
sign=False,
):
client_repos = load_target_repositories(origin_auth_repo, clients_dir)
client_repos = {
Expand All @@ -282,6 +283,7 @@ def update_and_check_commit_shas(
bare=bare,
force=force,
no_upstream=no_upstream,
sign=sign,
)

if operation == OperationType.CLONE:
Expand Down Expand Up @@ -510,6 +512,7 @@ def update_and_validate_repositories(
client_dir,
invalid_target_names=None,
excluded_target_globs=None,
sign=False,
):
if invalid_target_names is None:
invalid_target_names = []
Expand All @@ -532,6 +535,7 @@ def update_and_validate_repositories(
origin_root_repo,
client_dir,
excluded_target_globs=excluded_target_globs,
sign=sign,
)
except UpdateFailedError as e:
if any(
Expand Down
8 changes: 5 additions & 3 deletions taf/tools/repo/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -245,15 +245,16 @@ def update_repo_command():
@click.option("--no-deps", is_flag=True, default=False, help="Optionally disables updating of dependencies.")
@click.option("--upstream/--no-upstream", default=False, help="Skips comparison with remote repositories upstream")
@click.option("-v", "--verbosity", count=True, help="Displays varied levels of logging information based on verbosity level")
def update(path, library_dir, expected_repo_type, scripts_root_dir, profile, format_output, exclude_target, strict, no_deps, force, upstream, verbosity):
@click.option("--sign", is_flag=True, default=False, help="Write update date to target files and sign. Applicable when updating a repository that has dependencies")
def update(path, library_dir, expected_repo_type, scripts_root_dir, profile, format_output, exclude_target, strict, no_deps, force, upstream, verbosity, sign):
settings.VERBOSITY = verbosity
initialize_logger_handlers()

if profile:
start_profiling()

config = UpdateConfig(
operation=OperationType.UPDATE,
config = UpdConfig(
operation=OpeaterationType.UPDATE,
path=path,
library_dir=library_dir,
expected_repo_type=UpdateType(expected_repo_type),
Expand All @@ -263,6 +264,7 @@ def update(path, library_dir, expected_repo_type, scripts_root_dir, profile, for
force=force,
no_upstream=not upstream,
no_deps=no_deps,
sign=sign,
)

_call_updater(config, format_output)
Expand Down
37 changes: 37 additions & 0 deletions taf/updater/updater.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
loads data from a most recent commit.
"""
import copy
import datetime
import json
from logging import ERROR

from typing import Dict, Tuple, Any
Expand Down Expand Up @@ -245,6 +247,12 @@ class UpdateConfig:
"docs": "Flag to skip comparison with remote repositories upstream. Optional."
},
)
sign: bool = field(
default=False,
metadata={
"docs": "Flag to write update date to target files and sign. Applicable when updating a repository that has dependencies."
}
)

def __attrs_post_init__(self):
if self.operation == OperationType.CLONE:
Expand Down Expand Up @@ -469,6 +477,10 @@ def _process_repo_update(
child_auth_repos = repositoriesdb.get_deduplicated_auth_repositories(
auth_repo, commits
).values()

current_time = datetime.datetime.now()
formatted_time = current_time.strftime('%Y-%m-%d %H:%M:%S')

outputs, errors = _update_dependencies(update_config, child_auth_repos)
if len(errors):
errors = "\n".join(errors)
Expand All @@ -495,6 +507,31 @@ def _process_repo_update(
_process_repo_update(
child_config, output, visited, repos_update_data, transient_data
)
if update_config.sign:
dependency_name = output.auth_repo_name
dependency_log_path = auth_repo.log_dir / f"{dependency_name}.json"
dependency_target_dir = dependency_log_path.parent
if not dependency_target_dir.is_dir():
dependency_target_dir.mkdir()
target_content = {
"branch": output.users_auth_repo.default_branch,
"commit": output.commits_data.after_pull.value,
"update_time": formatted_time,
}
repos_data = []
for repo_name, repo_data in output.targets_data.items():
commits_data = {}
repo_content = {
"name": repo_name,
"commits": commits_data,
}
for branch, branch_data in repo_data["commits"].items():
if branch_data["after_pull"]:
commits_data[branch] = branch_data["after_pull"][0]["commit"]

repos_data.append(repo_content)
target_content["targets"] = repos_data
dependency_log_path.write_text(json.dumps(target_content, indent=4))

# do not call the handlers if only validating the repositories
# if a handler fails and we are in the development mode, revert the update
Expand Down
Loading