From e14b2de9a43be6d27778615606c12ceddf7eb66b Mon Sep 17 00:00:00 2001 From: JoshKowi Date: Thu, 12 Dec 2024 11:29:42 +0100 Subject: [PATCH 1/6] Implement Reporting Prototype Make Module Reports available Create Language Reports for each module --- pywikitools/resourcesbot/bot.py | 24 ++++++---- .../modules/consistency_checks.py | 41 ++++++++++++++++ .../resourcesbot/modules/export_html.py | 47 +++++++++++++++++++ .../resourcesbot/modules/export_pdf.py | 31 ++++++++++++ .../resourcesbot/modules/export_repository.py | 40 ++++++++++++++-- .../resourcesbot/modules/write_lists.py | 47 ++++++++++++++++--- .../resourcesbot/modules/write_report.py | 42 +++++++++++++++-- .../modules/write_sidebar_messages.py | 36 +++++++++++++- pywikitools/resourcesbot/reporting.py | 46 ++++++++++++++++++ pywikitools/test/test_resourcesbot.py | 7 ++- 10 files changed, 336 insertions(+), 25 deletions(-) create mode 100644 pywikitools/resourcesbot/reporting.py diff --git a/pywikitools/resourcesbot/bot.py b/pywikitools/resourcesbot/bot.py index b67f20c..37b9efe 100755 --- a/pywikitools/resourcesbot/bot.py +++ b/pywikitools/resourcesbot/bot.py @@ -22,6 +22,7 @@ ) from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor from pywikitools.resourcesbot.modules.write_summary import WriteSummary +import pywikitools.resourcesbot.reporting as reporting AVAILABLE_MODULES: Final[List[str]] = [ "consistency_checks", @@ -206,19 +207,24 @@ def run(self): self.logger.info(f"Modules specified for execution: {self.modules}") + module_reports = {} + for selected_module in self.modules: module = load_module(selected_module)( self.fortraininglib, self._config, self.site ) + module_reports[type(module).__name__] = [] + for lang in self._result: - module.run( - self._result[lang], - self._result["en"], - ChangeLog(), - ChangeLog(), - force_rewrite=(self._rewrite == "all") - or (self._rewrite == module.abbreviation()), - ) + module_reports[type(module).__name__].append( + module.run( + self._result[lang], + self._result["en"], + ChangeLog(), + ChangeLog(), + force_rewrite=(self._rewrite == "all") + or (self._rewrite == module.abbreviation()), + )) # Now run all GlobalPostProcessors if not self._limit_to_lang: @@ -229,6 +235,8 @@ def run(self): force_rewrite=(self._rewrite == "all") or (self._rewrite == "summary"), ) + reporting.print_summaries(module_reports) + def get_english_version(self, page_source: str) -> Tuple[str, int]: """ Extract the version of an English worksheet diff --git a/pywikitools/resourcesbot/modules/consistency_checks.py b/pywikitools/resourcesbot/modules/consistency_checks.py index 9d298a7..a88510c 100644 --- a/pywikitools/resourcesbot/modules/consistency_checks.py +++ b/pywikitools/resourcesbot/modules/consistency_checks.py @@ -13,6 +13,7 @@ from pywikitools.lang.translated_page import TranslationUnit from pywikitools.resourcesbot.data_structures import LanguageInfo, WorksheetInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor +from pywikitools.resourcesbot.reporting import LanguageReport class ConsistencyCheck(LanguagePostProcessor): @@ -263,6 +264,9 @@ def run( self.logger.info( f"Consistency checks for {language_info.english_name}: {checks_passed}/5 passed" ) + lang_report = ConsistencyLanguageReport(language_info.language_code) + lang_report.checks_passed = checks_passed + return lang_report """ @@ -284,3 +288,40 @@ def run( be the same -> needs to be checked manually """ + + +class ConsistencyLanguageReport(LanguageReport): + """ + A specialized report for export_pdf, + containing information about saved pdfs + """ + + def __init__(self, language_code: str): + super().__init__(language_code) + + self.checks_passed = 0 + + @classmethod + def get_module_name(cls) -> str: + return "export_pdf" + + def consistent(self): + if self.checks_passed == 5: + return True + else: + return False + + def get_summary(self) -> str: + return (f"Ran Consistency checks for: {self.language}: {self.checks_passed}/5 checks passed.") + + @classmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + + total_checks_passed = sum(report.checks_passed for report in lang_reports) + consistent_reports = [report for report in lang_reports if report.consistent()] + + return (f"Ran Consistency checks for {len(lang_reports)} languages.\n" + f"Consistent languages: {len(consistent_reports)}/{len(lang_reports)}, " + f"Overall: {total_checks_passed}/{len(lang_reports) * 5} checks passed.") diff --git a/pywikitools/resourcesbot/modules/export_html.py b/pywikitools/resourcesbot/modules/export_html.py index 6cd1707..750dc37 100644 --- a/pywikitools/resourcesbot/modules/export_html.py +++ b/pywikitools/resourcesbot/modules/export_html.py @@ -17,6 +17,7 @@ WorksheetInfo, ) from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor +from pywikitools.resourcesbot.reporting import LanguageReport class CustomBeautifyHTML(BeautifyHTML): @@ -221,6 +222,8 @@ def run( f"ExportHTML {lang_code}: " f"Downloaded {html_counter} HTML files, {file_counter} images" ) + lang_report = HtmlLanguageReport(lang_code, html_counter, file_counter) + return lang_report class StructureEncoder(json.JSONEncoder): @@ -251,3 +254,47 @@ def default(self, o): worksheet_json["pdf"] = pdf_info.url[pos + 1:] return worksheet_json return super().default(o) + + +class HtmlLanguageReport(LanguageReport): + """ + A specialized report for export_html, + containing information about saved htmls and images + """ + + def __init__(self, language_code: str, html_counter: int, image_counter: int): + super().__init__(language_code) + + self.html_counter = html_counter + self.image_counter = image_counter + + def get_html_number(self) -> int: + """ + Returns the number of HTML elements processed. + """ + return self.html_counter + + def get_image_number(self) -> int: + """ + Returns the number of images downloaded. + """ + return self.image_counter + + def get_summary(self) -> str: + return (f"Ran ExportHTML for {self.language}: " + f"Processed {self.html_counter} htmls, downloaded {self.image_counter} images.") + + @classmethod + def get_module_name(cls) -> str: + return "export_html" + + @classmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + + total_htmls = sum(report.get_html_number() for report in lang_reports) + total_images = sum(report.get_image_number() for report in lang_reports) + + return (f"Ran export_html for {len(lang_reports)} languages, " + f"processed {total_htmls} htmls, downloaded {total_images} images.") diff --git a/pywikitools/resourcesbot/modules/export_pdf.py b/pywikitools/resourcesbot/modules/export_pdf.py index 25a052e..4db5096 100644 --- a/pywikitools/resourcesbot/modules/export_pdf.py +++ b/pywikitools/resourcesbot/modules/export_pdf.py @@ -11,6 +11,7 @@ from pywikitools.resourcesbot.changes import ChangeLog from pywikitools.resourcesbot.data_structures import FileInfo, LanguageInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor +from pywikitools.resourcesbot.reporting import LanguageReport class ExportPDF(LanguagePostProcessor): @@ -117,3 +118,33 @@ def run( self.logger.info(f"Successfully downloaded and saved {file_path}") self.logger.info(f"ExportPDF {lang_code}: Downloaded {file_counter} PDF files") + lang_report = PdfLanguageReport(lang_info.language_code, file_counter) + return lang_report + + +class PdfLanguageReport(LanguageReport): + """ + A specialized report for export_pdf, + containing information about saved pdfs + """ + + def __init__(self, language_code: str, pdf_counter: int): + super().__init__(language_code) + + self.pdf_counter = pdf_counter + + @classmethod + def get_module_name(cls) -> str: + return "export_pdf" + + def get_summary(self) -> str: + return (f"Ran ExportHTML for {self.language}: Downloaded {self.pdf_counter} pdfs.") + + @classmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + + total_pdfs = sum(report.pdf_counter for report in lang_reports) + + return (f"Ran export_pdf for {len(lang_reports)} languages: Downloaded {total_pdfs} pdfs.") diff --git a/pywikitools/resourcesbot/modules/export_repository.py b/pywikitools/resourcesbot/modules/export_repository.py index 30809db..d6e6635 100644 --- a/pywikitools/resourcesbot/modules/export_repository.py +++ b/pywikitools/resourcesbot/modules/export_repository.py @@ -10,6 +10,7 @@ from pywikitools.fortraininglib import ForTrainingLib from pywikitools.resourcesbot.data_structures import LanguageInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor +from pywikitools.resourcesbot.reporting import LanguageReport class ExportRepository(LanguagePostProcessor): @@ -61,18 +62,19 @@ def run( Currently, we're ignoring the change parameter and just check for changes in the git repository """ + lang_report = ExportRepoLanguageReport(language_info.language_code) # Make sure we have a valid repository if self._base_folder == "": - return + return lang_report folder: str = os.path.join(self._base_folder, language_info.language_code) try: repo = Repo(folder) except GitError: self.logger.warning(f"No valid repository found in {folder}, skipping.") - return + return lang_report if "origin" not in repo.remotes: self.logger.warning(f"Git remote origin missing in {folder}, skipping.") - return + return lang_report # Staging all changes untracked: int = len(repo.untracked_files) @@ -105,7 +107,39 @@ def run( repo.index.commit(f"{commit_message}", author=self._author) result = repo.remotes.origin.push() self.logger.info(f"Pushed to remote, result: {result[0].summary}") + lang_report.pushed = True else: self.logger.info( f"ExportRepository {language_info.language_code}: No changes." ) + + return lang_report + + +class ExportRepoLanguageReport(LanguageReport): + """ + A specialized report for export_repository. + """ + + def __init__(self, language_code: str): + super().__init__(language_code) + self.pushed = False + + @classmethod + def get_module_name(cls) -> str: + return "export_repository" + + def get_summary(self) -> str: + if self.pushed: + return (f"Pushed htmls of {self.language} to remote repository.") + else: + return "" + + @classmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + + exported_languages = [report for report in lang_reports if report.pushed] + + return (f"Pushed html_exports for {len(exported_languages)}/{len(lang_reports)} languages.") diff --git a/pywikitools/resourcesbot/modules/write_lists.py b/pywikitools/resourcesbot/modules/write_lists.py index caf02fe..8584b1f 100644 --- a/pywikitools/resourcesbot/modules/write_lists.py +++ b/pywikitools/resourcesbot/modules/write_lists.py @@ -9,6 +9,7 @@ from pywikitools.resourcesbot.changes import ChangeLog, ChangeType from pywikitools.resourcesbot.data_structures import FileInfo, LanguageInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor +from pywikitools.resourcesbot.reporting import LanguageReport class WriteList(LanguagePostProcessor): @@ -191,9 +192,10 @@ def run( _english_changes, *, force_rewrite: bool = False - ) -> None: + ) -> LanguageReport: + lang_report = WriteListLanguageReport(language_info.language_code) if not force_rewrite and not self.needs_rewrite(language_info, changes): - return + return lang_report # Saving this to the language information page, e.g. https://www.4training.net/German language = language_info.english_name @@ -202,12 +204,12 @@ def run( f"English language name of {language_info.language_code} missing! " f"Skipping WriteList" ) - return + return lang_report self.logger.debug(f"Writing list of available resources in {language}...") page = pywikibot.Page(self._site, language) if not page.exists(): self.logger.warning(f"Language information page {language} doesn't exist!") - return + return lang_report if page.isRedirectPage(): self.logger.info( f"Language information page {language} is a redirect. Following the " @@ -218,7 +220,7 @@ def run( self.logger.warning( f"Redirect target for language {language} doesn't exist!" ) - return + return lang_report language = page.title() list_start, list_end = self._find_resources_list(page.text, language) @@ -228,7 +230,7 @@ def run( f"Doing nothing." ) self.logger.info(page.text) - return + return lang_report self.logger.debug( f"Found existing list of available training resources " f"@{list_start}-{list_end}. Replacing..." @@ -241,7 +243,7 @@ def run( # Save page and mark it for translation if necessary if page.text.strip() == new_page_content.strip(): - return + return lang_report page.text = new_page_content page.save( "Updated list of available training resources" @@ -254,8 +256,39 @@ def run( f"Updated language information page {language} and marked it " f"for translation." ) + lang_report.updated_language_info = True else: self.logger.info( f"Updated language information page {language}. Couldn't mark it " f"for translation." ) + return lang_report + + +class WriteListLanguageReport(LanguageReport): + """ + A specialized report for write_list. + """ + + def __init__(self, language_code: str): + super().__init__(language_code) + self.updated_language_info = False + + @classmethod + def get_module_name(cls) -> str: + return "write_list" + + def get_summary(self) -> str: + if self.updated_language_info: + return (f"Updated language information page for {self.language}.") + else: + return "" + + @classmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + + exported_languages = [report for report in lang_reports if report.updated_language_info] + + return (f"Updated language information page for {len(exported_languages)}/{len(lang_reports)} languages.") diff --git a/pywikitools/resourcesbot/modules/write_report.py b/pywikitools/resourcesbot/modules/write_report.py index 6dac4cf..88142ee 100644 --- a/pywikitools/resourcesbot/modules/write_report.py +++ b/pywikitools/resourcesbot/modules/write_report.py @@ -10,6 +10,7 @@ from pywikitools.resourcesbot.changes import ChangeLog from pywikitools.resourcesbot.data_structures import LanguageInfo, WorksheetInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor +from pywikitools.resourcesbot.reporting import LanguageReport class Color(Enum): @@ -57,6 +58,7 @@ def __init__( site: our pywikibot object to be able to write to the mediawiki system """ super().__init__(fortraininglib, config, site) + self.lang_report = WriteReportLanguageReport("") self.logger: Final[logging.Logger] = logging.getLogger( "pywikitools.resourcesbot.modules.write_report" ) @@ -69,16 +71,17 @@ def run( english_changes: ChangeLog, *, force_rewrite: bool = False - ): + ) -> LanguageReport: """Entry function We run everything, and don't look at whether we have changes because we need to look at all CorrectBot reports and according to them, may need to rewrite the report even if changes and english_changes are empty """ + self.lang_report = WriteReportLanguageReport(language_info.language_code) # We don't need a report for English as it is the source language if language_info.language_code == "en": - return + return self.lang_report # Don't write reports for language variants (except Brazilian Portuguese) # TODO: this should go somewhere else @@ -86,8 +89,9 @@ def run( "-" in language_info.language_code and language_info.language_code != "pt-br" ): - return + return self.lang_report self.save_language_report(language_info, english_info) + return self.lang_report def create_correctbot_mediawiki(self, worksheet: str, language_code: str) -> str: """Check Correctbot report status for one worksheet @@ -164,7 +168,7 @@ def save_language_report( f"English name of language {language_info.language_code} empty! " f"Skipping WriteReport" ) - return + return self.lang_report page_url = f"4training:{language_info.english_name}" page = pywikibot.Page(self._site, page_url) report = self.create_mediawiki(language_info, english_info) @@ -183,6 +187,7 @@ def save_language_report( self.logger.info( f"Updated language report for {language_info.english_name}" ) + self.lang_report.updated_language_report = True def create_mediawiki( self, language_info: LanguageInfo, english_info: LanguageInfo @@ -408,3 +413,32 @@ def create_worksheet_line( line_color = Color.GREEN content = f'|- style="background-color:{line_color}"\n' + content return content + + +class WriteReportLanguageReport(LanguageReport): + """ + A specialized report for write_report. + """ + + def __init__(self, language_code: str): + super().__init__(language_code) + self.updated_language_report = False + + @classmethod + def get_module_name(cls) -> str: + return "write_report" + + def get_summary(self) -> str: + if self.updated_language_report: + return (f"Updated language report for {self.language}.") + else: + return "" + + @classmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + + updated_languages = [report for report in lang_reports if report.updated_language_report] + + return (f"Updated language report for {len(updated_languages)}/{len(lang_reports)} languages.") diff --git a/pywikitools/resourcesbot/modules/write_sidebar_messages.py b/pywikitools/resourcesbot/modules/write_sidebar_messages.py index 4451cd5..bcd83b1 100644 --- a/pywikitools/resourcesbot/modules/write_sidebar_messages.py +++ b/pywikitools/resourcesbot/modules/write_sidebar_messages.py @@ -8,6 +8,7 @@ from pywikitools.resourcesbot.changes import ChangeLog, ChangeType from pywikitools.resourcesbot.data_structures import LanguageInfo, WorksheetInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor +from pywikitools.resourcesbot.reporting import LanguageReport class WriteSidebarMessages(LanguagePostProcessor): @@ -44,6 +45,7 @@ def __init__( site: pywikibot.site.APISite ): super().__init__(fortraininglib, config, site) + self.lang_report = WriteSidebarReport("") self.logger: Final[logging.Logger] = logging.getLogger( "pywikitools.resourcesbot.modules.write_sidebar_messages" ) @@ -66,6 +68,7 @@ def save_worksheet_title(self, worksheet: WorksheetInfo): self.logger.info(f"Updating system message {title}") page.text = worksheet.title page.save("Updated translated worksheet title") + self.lang_report.updated_worksheet_counter += 1 @staticmethod def has_relevant_change(worksheet: str, changes: ChangeLog) -> bool: @@ -91,10 +94,41 @@ def run( _english_changes, *, force_rewrite: bool = False - ) -> None: + ) -> LanguageReport: """Our entry function""" + self.lang_report = WriteSidebarReport(language_info.language_code) for worksheet in language_info.worksheets.values(): if worksheet.title == "": continue if force_rewrite or self.has_relevant_change(worksheet.page, changes): self.save_worksheet_title(worksheet) + return self.lang_report + + +class WriteSidebarReport(LanguageReport): + """ + A specialized report for export_pdf, + containing information about saved pdfs + """ + + def __init__(self, language_code: str): + super().__init__(language_code) + self.updated_worksheet_counter = 0 + + @classmethod + def get_module_name(cls) -> str: + return "write_sidebar_messages" + + def get_summary(self) -> str: + return (f"Ran write_sidebar_messages for {self.language}: " + f"Updated {self.updated_worksheet_counter} worksheet titles.") + + @classmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + + total_updated_titles = sum(report.updated_worksheet_counter for report in lang_reports) + + return (f"Ran write_sidebar_messages for {len(lang_reports)} languages: " + f"Updated {total_updated_titles} worksheet titles.") diff --git a/pywikitools/resourcesbot/reporting.py b/pywikitools/resourcesbot/reporting.py new file mode 100644 index 0000000..59d3d63 --- /dev/null +++ b/pywikitools/resourcesbot/reporting.py @@ -0,0 +1,46 @@ +""" +Contains the data-structures and main methods for the reporting. +Through reporting, the ressources-bot keeps track of what it did. +This information can then be made available to people who need to know what is going on. +""" + +from abc import ABC, abstractmethod +from typing import Dict, List + + +class LanguageReport(ABC): + """ + Basic data structure for the report of one module running once for one language. + """ + + def __init__(self, language_code: str): + self.language = language_code + self.summary_text = "" + + def was_anything_to_report(self) -> bool: + return self.summary_text != "" + + @abstractmethod + def get_summary(self) -> str: + return self.summary_text + + @classmethod + @abstractmethod + def get_module_name(cls) -> str: + return "" + + @classmethod + @abstractmethod + def get_module_summary(cls, lang_reports: list) -> str: + if len(lang_reports) == 0: + return "" + else: + return f"Ran module {cls.get_module_name()} for {len(lang_reports)} languages" + + +def print_summaries(module_reports: Dict[str, List[LanguageReport]]): + for key, value in module_reports.items(): + if len(value) == 0: + print(f"Module {key}: Empty Report") + else: + print(f"Module {key}'s report: {type(value[0]).get_module_summary(value)}") diff --git a/pywikitools/test/test_resourcesbot.py b/pywikitools/test/test_resourcesbot.py index bef12d6..235612a 100644 --- a/pywikitools/test/test_resourcesbot.py +++ b/pywikitools/test/test_resourcesbot.py @@ -164,8 +164,10 @@ def json_test_loader(site, page: str): @patch("pywikitools.resourcesbot.modules.export_html.ExportHTML.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) + @patch("pywikitools.resourcesbot.reporting.print_summaries", autospec=True) def test_run_with_cache( self, + mock_reporting_summary, mock_consistency_check, mock_export_pdf, mock_export_html, @@ -175,13 +177,12 @@ def test_run_with_cache( mock_write_report, mock_write_summary, mock_pywikibot_page, - mock_pywikibot_site, + mock_pywikibot_site ): mock_pywikibot_page.side_effect = self.json_test_loader mock_pywikibot_site.return_value.logged_in.return_value = True bot = ResourcesBot(config=self.config, read_from_cache=True) bot.run() - # run() function of each LanguagePostProcessor should get called 2x (for English and Russian) self.assertEqual(mock_consistency_check.call_count, 2) self.assertEqual(mock_export_pdf.call_count, 2) @@ -211,8 +212,10 @@ def test_run_with_cache( @patch("pywikitools.resourcesbot.modules.export_html.ExportHTML.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) + @patch("pywikitools.resourcesbot.reporting.print_summaries", autospec=True) def test_rewrite_options( self, + mock_reporting_summary, mock_consistency_check, mock_export_pdf, mock_export_html, From ee7258a2bd01e8a2351102fdcf9e92690f062c10 Mon Sep 17 00:00:00 2001 From: JoshKowi Date: Thu, 12 Dec 2024 18:06:08 +0100 Subject: [PATCH 2/6] Publish Report on Mediawiki Prototype --- pywikitools/resourcesbot/bot.py | 1 + .../modules/consistency_checks.py | 2 +- .../resourcesbot/modules/write_report.py | 1 + pywikitools/resourcesbot/reporting.py | 48 ++++++++++++++++++- pywikitools/test/test_resourcesbot.py | 6 ++- 5 files changed, 54 insertions(+), 4 deletions(-) diff --git a/pywikitools/resourcesbot/bot.py b/pywikitools/resourcesbot/bot.py index 37b9efe..1c42acd 100755 --- a/pywikitools/resourcesbot/bot.py +++ b/pywikitools/resourcesbot/bot.py @@ -236,6 +236,7 @@ def run(self): ) reporting.print_summaries(module_reports) + reporting.save_report(self.site, module_reports) def get_english_version(self, page_source: str) -> Tuple[str, int]: """ diff --git a/pywikitools/resourcesbot/modules/consistency_checks.py b/pywikitools/resourcesbot/modules/consistency_checks.py index a88510c..d0ecc75 100644 --- a/pywikitools/resourcesbot/modules/consistency_checks.py +++ b/pywikitools/resourcesbot/modules/consistency_checks.py @@ -322,6 +322,6 @@ def get_module_summary(cls, lang_reports: list) -> str: total_checks_passed = sum(report.checks_passed for report in lang_reports) consistent_reports = [report for report in lang_reports if report.consistent()] - return (f"Ran Consistency checks for {len(lang_reports)} languages.\n" + return (f"Ran Consistency checks for {len(lang_reports)} languages. " f"Consistent languages: {len(consistent_reports)}/{len(lang_reports)}, " f"Overall: {total_checks_passed}/{len(lang_reports) * 5} checks passed.") diff --git a/pywikitools/resourcesbot/modules/write_report.py b/pywikitools/resourcesbot/modules/write_report.py index 88142ee..8cef723 100644 --- a/pywikitools/resourcesbot/modules/write_report.py +++ b/pywikitools/resourcesbot/modules/write_report.py @@ -178,6 +178,7 @@ def save_language_report( ) page.text = report page.save("Created language report") + self.lang_report.updated_language_report = True else: if page.text.strip() != report.strip(): page.text = report diff --git a/pywikitools/resourcesbot/reporting.py b/pywikitools/resourcesbot/reporting.py index 59d3d63..7e68a04 100644 --- a/pywikitools/resourcesbot/reporting.py +++ b/pywikitools/resourcesbot/reporting.py @@ -6,6 +6,9 @@ from abc import ABC, abstractmethod from typing import Dict, List +from datetime import datetime + +import pywikibot class LanguageReport(ABC): @@ -39,8 +42,49 @@ def get_module_summary(cls, lang_reports: list) -> str: def print_summaries(module_reports: Dict[str, List[LanguageReport]]): + print(make_summaries(module_reports)) + + +def make_summaries(module_reports: Dict[str, List[LanguageReport]]): + summary = "" + for key, value in module_reports.items(): + if len(value) == 0: + summary = f"Module {key}: Empty Report" + else: + summary += f"Module {key}'s report: {type(value[0]).get_module_summary(value)}\n" + return summary + + +def generate_mediawiki_overview(module_reports: Dict[str, List[LanguageReport]]): + mediawiki = "===Overview===\n\n" for key, value in module_reports.items(): if len(value) == 0: - print(f"Module {key}: Empty Report") + mediawiki = f"*Module '''{key}''': Empty Report\n\n" else: - print(f"Module {key}'s report: {type(value[0]).get_module_summary(value)}") + mediawiki += f"*Module '''{key}''''s report: {type(value[0]).get_module_summary(value)}\n\n" + return mediawiki + + +def generate_mediawiki(module_reports: Dict[str, List[LanguageReport]]): + timestamp_str = datetime.now().strftime("%Y-%m-%d--%H:%M") + head = f"==Resourcesbot-run at {timestamp_str}==\n\n" + overview = generate_mediawiki_overview(module_reports) + details = "===Detailed Reports per Module===\n\n" + for key, value in module_reports.items(): + if len(value) == 0: + pass + else: + details += f"===={key}====\n\n" + for lang_report in value: + details += f"*{lang_report.get_summary()}\n\n" + return head + overview + details + + +def save_report(site, module_reports: Dict[str, List[LanguageReport]]): + page_url = "4training:Resourcesbot.report" + page = pywikibot.Page(site, page_url) + if page.exists(): + page.text = generate_mediawiki(module_reports) + page.text + else: + page.text = generate_mediawiki(module_reports) + page.save("Created summary for Resourcesbot run") diff --git a/pywikitools/test/test_resourcesbot.py b/pywikitools/test/test_resourcesbot.py index 235612a..27ce7e9 100644 --- a/pywikitools/test/test_resourcesbot.py +++ b/pywikitools/test/test_resourcesbot.py @@ -165,9 +165,11 @@ def json_test_loader(site, page: str): @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) @patch("pywikitools.resourcesbot.reporting.print_summaries", autospec=True) + @patch("pywikitools.resourcesbot.reporting.save_report", autospec=True) def test_run_with_cache( self, - mock_reporting_summary, + mock_save_report, + mock_print_summary, mock_consistency_check, mock_export_pdf, mock_export_html, @@ -213,8 +215,10 @@ def test_run_with_cache( @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) @patch("pywikitools.resourcesbot.reporting.print_summaries", autospec=True) + @patch("pywikitools.resourcesbot.reporting.save_report", autospec=True) def test_rewrite_options( self, + mock_save_report, mock_reporting_summary, mock_consistency_check, mock_export_pdf, From ea7f139e7ae5d6932d6b2e5c31461cc3d1bdf041 Mon Sep 17 00:00:00 2001 From: JoshKowi Date: Tue, 17 Dec 2024 11:36:59 +0100 Subject: [PATCH 3/6] Reporting: refactor from top-level functions to class. --- pywikitools/resourcesbot/bot.py | 11 +-- pywikitools/resourcesbot/reporting.py | 99 ++++++++++++++------------- pywikitools/test/test_resourcesbot.py | 12 ++-- 3 files changed, 63 insertions(+), 59 deletions(-) diff --git a/pywikitools/resourcesbot/bot.py b/pywikitools/resourcesbot/bot.py index 1c42acd..57acfd4 100755 --- a/pywikitools/resourcesbot/bot.py +++ b/pywikitools/resourcesbot/bot.py @@ -207,16 +207,17 @@ def run(self): self.logger.info(f"Modules specified for execution: {self.modules}") - module_reports = {} + module_reports = reporting.Reporting() for selected_module in self.modules: module = load_module(selected_module)( self.fortraininglib, self._config, self.site ) - module_reports[type(module).__name__] = [] + module_reports.add_module(type(module).__name__) for lang in self._result: - module_reports[type(module).__name__].append( + module_reports.add_language_report( + type(module).__name__, module.run( self._result[lang], self._result["en"], @@ -235,8 +236,8 @@ def run(self): force_rewrite=(self._rewrite == "all") or (self._rewrite == "summary"), ) - reporting.print_summaries(module_reports) - reporting.save_report(self.site, module_reports) + module_reports.print_summaries() + module_reports.save_report(self.site) def get_english_version(self, page_source: str) -> Tuple[str, int]: """ diff --git a/pywikitools/resourcesbot/reporting.py b/pywikitools/resourcesbot/reporting.py index 7e68a04..2dfa853 100644 --- a/pywikitools/resourcesbot/reporting.py +++ b/pywikitools/resourcesbot/reporting.py @@ -41,50 +41,57 @@ def get_module_summary(cls, lang_reports: list) -> str: return f"Ran module {cls.get_module_name()} for {len(lang_reports)} languages" -def print_summaries(module_reports: Dict[str, List[LanguageReport]]): - print(make_summaries(module_reports)) - - -def make_summaries(module_reports: Dict[str, List[LanguageReport]]): - summary = "" - for key, value in module_reports.items(): - if len(value) == 0: - summary = f"Module {key}: Empty Report" - else: - summary += f"Module {key}'s report: {type(value[0]).get_module_summary(value)}\n" - return summary - - -def generate_mediawiki_overview(module_reports: Dict[str, List[LanguageReport]]): - mediawiki = "===Overview===\n\n" - for key, value in module_reports.items(): - if len(value) == 0: - mediawiki = f"*Module '''{key}''': Empty Report\n\n" - else: - mediawiki += f"*Module '''{key}''''s report: {type(value[0]).get_module_summary(value)}\n\n" - return mediawiki - - -def generate_mediawiki(module_reports: Dict[str, List[LanguageReport]]): - timestamp_str = datetime.now().strftime("%Y-%m-%d--%H:%M") - head = f"==Resourcesbot-run at {timestamp_str}==\n\n" - overview = generate_mediawiki_overview(module_reports) - details = "===Detailed Reports per Module===\n\n" - for key, value in module_reports.items(): - if len(value) == 0: - pass +class Reporting: + + def __init__(self): + self.module_reports: Dict[str, List[LanguageReport]] = dict() + + def add_module(self, module_name: str): + self.module_reports[module_name] = list() + + def add_language_report(self, module_name: str, language_report: LanguageReport): + self.module_reports.setdefault(module_name, []).append(language_report) + + def print_summaries(self): + print(self.make_summaries()) + + def make_summaries(self): + summary = "" + for key, value in self.module_reports.items(): + if len(value) == 0: + summary = f"Module {key}: Empty Report" + else: + summary += f"Module {key}'s report: {type(value[0]).get_module_summary(value)}\n" + return summary + + def generate_mediawiki_overview(self): + mediawiki = "===Overview===\n\n" + for key, value in self.module_reports.items(): + if len(value) == 0: + mediawiki = f"*Module '''{key}''': Empty Report\n\n" + else: + mediawiki += f"*Module '''{key}''''s report: {type(value[0]).get_module_summary(value)}\n\n" + return mediawiki + + def generate_mediawiki(self): + timestamp_str = datetime.now().strftime("%Y-%m-%d--%H:%M") + head = f"==Resourcesbot-run at {timestamp_str}==\n\n" + overview = self.generate_mediawiki_overview() + details = "===Detailed Reports per Module===\n\n" + for key, value in self.module_reports.items(): + if len(value) == 0: + pass + else: + details += f"===={key}====\n\n" + for lang_report in value: + details += f"*{lang_report.get_summary()}\n\n" + return head + overview + details + + def save_report(self, site): + page_url = "4training:Resourcesbot.report" + page = pywikibot.Page(site, page_url) + if page.exists(): + page.text = self.generate_mediawiki() + page.text else: - details += f"===={key}====\n\n" - for lang_report in value: - details += f"*{lang_report.get_summary()}\n\n" - return head + overview + details - - -def save_report(site, module_reports: Dict[str, List[LanguageReport]]): - page_url = "4training:Resourcesbot.report" - page = pywikibot.Page(site, page_url) - if page.exists(): - page.text = generate_mediawiki(module_reports) + page.text - else: - page.text = generate_mediawiki(module_reports) - page.save("Created summary for Resourcesbot run") + page.text = self.generate_mediawiki() + page.save("Created summary for Resourcesbot run") diff --git a/pywikitools/test/test_resourcesbot.py b/pywikitools/test/test_resourcesbot.py index 27ce7e9..771acd5 100644 --- a/pywikitools/test/test_resourcesbot.py +++ b/pywikitools/test/test_resourcesbot.py @@ -164,12 +164,10 @@ def json_test_loader(site, page: str): @patch("pywikitools.resourcesbot.modules.export_html.ExportHTML.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) - @patch("pywikitools.resourcesbot.reporting.print_summaries", autospec=True) - @patch("pywikitools.resourcesbot.reporting.save_report", autospec=True) + @patch("pywikitools.resourcesbot.reporting.Reporting", autospec=True) def test_run_with_cache( self, - mock_save_report, - mock_print_summary, + mock_report, mock_consistency_check, mock_export_pdf, mock_export_html, @@ -214,12 +212,10 @@ def test_run_with_cache( @patch("pywikitools.resourcesbot.modules.export_html.ExportHTML.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) - @patch("pywikitools.resourcesbot.reporting.print_summaries", autospec=True) - @patch("pywikitools.resourcesbot.reporting.save_report", autospec=True) + @patch("pywikitools.resourcesbot.reporting.Reporting", autospec=True) def test_rewrite_options( self, - mock_save_report, - mock_reporting_summary, + mock_report, mock_consistency_check, mock_export_pdf, mock_export_html, From a32084b3e152e7fd3c39a151bce5925827aedf1b Mon Sep 17 00:00:00 2001 From: JoshKowi Date: Tue, 17 Dec 2024 12:31:48 +0100 Subject: [PATCH 4/6] Reporting: Show detailed reports by language, not by modules on the mediawiki. Omit empty reports. --- .../modules/consistency_checks.py | 3 +-- .../resourcesbot/modules/export_html.py | 5 +++- .../resourcesbot/modules/export_pdf.py | 10 ++++--- .../resourcesbot/modules/write_lists.py | 2 +- .../modules/write_sidebar_messages.py | 2 ++ pywikitools/resourcesbot/reporting.py | 26 +++++++++++++++---- 6 files changed, 35 insertions(+), 13 deletions(-) diff --git a/pywikitools/resourcesbot/modules/consistency_checks.py b/pywikitools/resourcesbot/modules/consistency_checks.py index d0ecc75..abe3b51 100644 --- a/pywikitools/resourcesbot/modules/consistency_checks.py +++ b/pywikitools/resourcesbot/modules/consistency_checks.py @@ -298,7 +298,6 @@ class ConsistencyLanguageReport(LanguageReport): def __init__(self, language_code: str): super().__init__(language_code) - self.checks_passed = 0 @classmethod @@ -312,7 +311,7 @@ def consistent(self): return False def get_summary(self) -> str: - return (f"Ran Consistency checks for: {self.language}: {self.checks_passed}/5 checks passed.") + return (f"Ran Consistency checks for {self.language}: {self.checks_passed}/5 checks passed.") @classmethod def get_module_summary(cls, lang_reports: list) -> str: diff --git a/pywikitools/resourcesbot/modules/export_html.py b/pywikitools/resourcesbot/modules/export_html.py index 750dc37..a27b438 100644 --- a/pywikitools/resourcesbot/modules/export_html.py +++ b/pywikitools/resourcesbot/modules/export_html.py @@ -281,6 +281,8 @@ def get_image_number(self) -> int: return self.image_counter def get_summary(self) -> str: + if self.html_counter + self.image_counter == 0: + return "" return (f"Ran ExportHTML for {self.language}: " f"Processed {self.html_counter} htmls, downloaded {self.image_counter} images.") @@ -295,6 +297,7 @@ def get_module_summary(cls, lang_reports: list) -> str: total_htmls = sum(report.get_html_number() for report in lang_reports) total_images = sum(report.get_image_number() for report in lang_reports) - + if total_htmls + total_images == 0: + return "" return (f"Ran export_html for {len(lang_reports)} languages, " f"processed {total_htmls} htmls, downloaded {total_images} images.") diff --git a/pywikitools/resourcesbot/modules/export_pdf.py b/pywikitools/resourcesbot/modules/export_pdf.py index 4db5096..2f88e81 100644 --- a/pywikitools/resourcesbot/modules/export_pdf.py +++ b/pywikitools/resourcesbot/modules/export_pdf.py @@ -138,13 +138,15 @@ def get_module_name(cls) -> str: return "export_pdf" def get_summary(self) -> str: - return (f"Ran ExportHTML for {self.language}: Downloaded {self.pdf_counter} pdfs.") + if self.pdf_counter == 0: + return "" + return f"Ran ExportPDF for {self.language}: Downloaded {self.pdf_counter} pdfs." @classmethod def get_module_summary(cls, lang_reports: list) -> str: if len(lang_reports) == 0: return "" - total_pdfs = sum(report.pdf_counter for report in lang_reports) - - return (f"Ran export_pdf for {len(lang_reports)} languages: Downloaded {total_pdfs} pdfs.") + if total_pdfs == 0: + return "" + return f"Ran export_pdf for {len(lang_reports)} languages: Downloaded {total_pdfs} pdfs." diff --git a/pywikitools/resourcesbot/modules/write_lists.py b/pywikitools/resourcesbot/modules/write_lists.py index 8584b1f..7d58154 100644 --- a/pywikitools/resourcesbot/modules/write_lists.py +++ b/pywikitools/resourcesbot/modules/write_lists.py @@ -280,7 +280,7 @@ def get_module_name(cls) -> str: def get_summary(self) -> str: if self.updated_language_info: - return (f"Updated language information page for {self.language}.") + return f"Updated language information page for {self.language}." else: return "" diff --git a/pywikitools/resourcesbot/modules/write_sidebar_messages.py b/pywikitools/resourcesbot/modules/write_sidebar_messages.py index bcd83b1..c3287f7 100644 --- a/pywikitools/resourcesbot/modules/write_sidebar_messages.py +++ b/pywikitools/resourcesbot/modules/write_sidebar_messages.py @@ -120,6 +120,8 @@ def get_module_name(cls) -> str: return "write_sidebar_messages" def get_summary(self) -> str: + if self.updated_worksheet_counter == 0: + return "" return (f"Ran write_sidebar_messages for {self.language}: " f"Updated {self.updated_worksheet_counter} worksheet titles.") diff --git a/pywikitools/resourcesbot/reporting.py b/pywikitools/resourcesbot/reporting.py index 2dfa853..89ea9b5 100644 --- a/pywikitools/resourcesbot/reporting.py +++ b/pywikitools/resourcesbot/reporting.py @@ -21,7 +21,7 @@ def __init__(self, language_code: str): self.summary_text = "" def was_anything_to_report(self) -> bool: - return self.summary_text != "" + return self.get_summary() != "" @abstractmethod def get_summary(self) -> str: @@ -64,6 +64,16 @@ def make_summaries(self): summary += f"Module {key}'s report: {type(value[0]).get_module_summary(value)}\n" return summary + def get_reports_by_languages(self): + reports_by_language: Dict[str, List[LanguageReport]] = dict() + for key, value in self.module_reports.items(): + if len(value) == 0: + pass + else: + for lang_report in value: + reports_by_language.setdefault(lang_report.language, []).append(lang_report) + return reports_by_language + def generate_mediawiki_overview(self): mediawiki = "===Overview===\n\n" for key, value in self.module_reports.items(): @@ -77,14 +87,20 @@ def generate_mediawiki(self): timestamp_str = datetime.now().strftime("%Y-%m-%d--%H:%M") head = f"==Resourcesbot-run at {timestamp_str}==\n\n" overview = self.generate_mediawiki_overview() - details = "===Detailed Reports per Module===\n\n" - for key, value in self.module_reports.items(): + details = "===Detailed Reports per Language===\n\n" + for key, value in self.get_reports_by_languages().items(): if len(value) == 0: pass else: - details += f"===={key}====\n\n" + anything_to_report = False + lang_summary = "" for lang_report in value: - details += f"*{lang_report.get_summary()}\n\n" + if lang_report.was_anything_to_report(): + anything_to_report = True + lang_summary += f"*{lang_report.get_summary()}\n\n" + if anything_to_report: + details += f"===={key}====\n\n" + details += lang_summary return head + overview + details def save_report(self, site): From cdc1343ff410f5635262a7162606de2b8d00ed92 Mon Sep 17 00:00:00 2001 From: JoshKowi Date: Wed, 18 Dec 2024 12:43:26 +0100 Subject: [PATCH 5/6] Reporting: Refactor names Module write_report -> write_progress class WriteReport -> WriteTranslationProgress class WriteSummary -> WriteTranslationSummary class LanguageReport -> Report class Reporting -> ReportSummary --- pywikitools/resourcesbot/bot.py | 8 ++++---- .../modules/consistency_checks.py | 6 +++--- .../resourcesbot/modules/export_html.py | 6 +++--- .../resourcesbot/modules/export_pdf.py | 6 +++--- .../resourcesbot/modules/export_repository.py | 6 +++--- .../resourcesbot/modules/write_lists.py | 8 ++++---- .../{write_report.py => write_progress.py} | 14 ++++++------- .../modules/write_sidebar_messages.py | 6 +++--- .../resourcesbot/modules/write_summary.py | 2 +- pywikitools/resourcesbot/reporting.py | 10 +++++----- pywikitools/test/test_resourcesbot.py | 20 +++++++++---------- ...write_report.py => test_write_progress.py} | 16 +++++++-------- pywikitools/test/test_write_summary.py | 8 ++++---- 13 files changed, 58 insertions(+), 58 deletions(-) rename pywikitools/resourcesbot/modules/{write_report.py => write_progress.py} (98%) rename pywikitools/test/{test_write_report.py => test_write_progress.py} (88%) diff --git a/pywikitools/resourcesbot/bot.py b/pywikitools/resourcesbot/bot.py index 57acfd4..83bd83a 100755 --- a/pywikitools/resourcesbot/bot.py +++ b/pywikitools/resourcesbot/bot.py @@ -21,7 +21,7 @@ json_decode, ) from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.modules.write_summary import WriteSummary +from pywikitools.resourcesbot.modules.write_summary import WriteProgressSummary import pywikitools.resourcesbot.reporting as reporting AVAILABLE_MODULES: Final[List[str]] = [ @@ -30,7 +30,7 @@ "export_pdf", "export_repository", "write_lists", - "write_report", + "write_progress", "write_sidebar_messages", ] @@ -207,7 +207,7 @@ def run(self): self.logger.info(f"Modules specified for execution: {self.modules}") - module_reports = reporting.Reporting() + module_reports = reporting.ReportSummary() for selected_module in self.modules: module = load_module(selected_module)( @@ -229,7 +229,7 @@ def run(self): # Now run all GlobalPostProcessors if not self._limit_to_lang: - write_summary = WriteSummary(self.site) + write_summary = WriteProgressSummary(self.site) write_summary.run( self._result, self._changelog, diff --git a/pywikitools/resourcesbot/modules/consistency_checks.py b/pywikitools/resourcesbot/modules/consistency_checks.py index abe3b51..f9741d2 100644 --- a/pywikitools/resourcesbot/modules/consistency_checks.py +++ b/pywikitools/resourcesbot/modules/consistency_checks.py @@ -13,7 +13,7 @@ from pywikitools.lang.translated_page import TranslationUnit from pywikitools.resourcesbot.data_structures import LanguageInfo, WorksheetInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.reporting import LanguageReport +from pywikitools.resourcesbot.reporting import Report class ConsistencyCheck(LanguagePostProcessor): @@ -264,7 +264,7 @@ def run( self.logger.info( f"Consistency checks for {language_info.english_name}: {checks_passed}/5 passed" ) - lang_report = ConsistencyLanguageReport(language_info.language_code) + lang_report = ConsistencyReport(language_info.language_code) lang_report.checks_passed = checks_passed return lang_report @@ -290,7 +290,7 @@ def run( """ -class ConsistencyLanguageReport(LanguageReport): +class ConsistencyReport(Report): """ A specialized report for export_pdf, containing information about saved pdfs diff --git a/pywikitools/resourcesbot/modules/export_html.py b/pywikitools/resourcesbot/modules/export_html.py index a27b438..c39261d 100644 --- a/pywikitools/resourcesbot/modules/export_html.py +++ b/pywikitools/resourcesbot/modules/export_html.py @@ -17,7 +17,7 @@ WorksheetInfo, ) from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.reporting import LanguageReport +from pywikitools.resourcesbot.reporting import Report class CustomBeautifyHTML(BeautifyHTML): @@ -222,7 +222,7 @@ def run( f"ExportHTML {lang_code}: " f"Downloaded {html_counter} HTML files, {file_counter} images" ) - lang_report = HtmlLanguageReport(lang_code, html_counter, file_counter) + lang_report = HtmlReport(lang_code, html_counter, file_counter) return lang_report @@ -256,7 +256,7 @@ def default(self, o): return super().default(o) -class HtmlLanguageReport(LanguageReport): +class HtmlReport(Report): """ A specialized report for export_html, containing information about saved htmls and images diff --git a/pywikitools/resourcesbot/modules/export_pdf.py b/pywikitools/resourcesbot/modules/export_pdf.py index 2f88e81..bb98426 100644 --- a/pywikitools/resourcesbot/modules/export_pdf.py +++ b/pywikitools/resourcesbot/modules/export_pdf.py @@ -11,7 +11,7 @@ from pywikitools.resourcesbot.changes import ChangeLog from pywikitools.resourcesbot.data_structures import FileInfo, LanguageInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.reporting import LanguageReport +from pywikitools.resourcesbot.reporting import Report class ExportPDF(LanguagePostProcessor): @@ -118,11 +118,11 @@ def run( self.logger.info(f"Successfully downloaded and saved {file_path}") self.logger.info(f"ExportPDF {lang_code}: Downloaded {file_counter} PDF files") - lang_report = PdfLanguageReport(lang_info.language_code, file_counter) + lang_report = PdfReport(lang_info.language_code, file_counter) return lang_report -class PdfLanguageReport(LanguageReport): +class PdfReport(Report): """ A specialized report for export_pdf, containing information about saved pdfs diff --git a/pywikitools/resourcesbot/modules/export_repository.py b/pywikitools/resourcesbot/modules/export_repository.py index d6e6635..110ccd2 100644 --- a/pywikitools/resourcesbot/modules/export_repository.py +++ b/pywikitools/resourcesbot/modules/export_repository.py @@ -10,7 +10,7 @@ from pywikitools.fortraininglib import ForTrainingLib from pywikitools.resourcesbot.data_structures import LanguageInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.reporting import LanguageReport +from pywikitools.resourcesbot.reporting import Report class ExportRepository(LanguagePostProcessor): @@ -62,7 +62,7 @@ def run( Currently, we're ignoring the change parameter and just check for changes in the git repository """ - lang_report = ExportRepoLanguageReport(language_info.language_code) + lang_report = ExportRepoReport(language_info.language_code) # Make sure we have a valid repository if self._base_folder == "": return lang_report @@ -116,7 +116,7 @@ def run( return lang_report -class ExportRepoLanguageReport(LanguageReport): +class ExportRepoReport(Report): """ A specialized report for export_repository. """ diff --git a/pywikitools/resourcesbot/modules/write_lists.py b/pywikitools/resourcesbot/modules/write_lists.py index 7d58154..fcbd23d 100644 --- a/pywikitools/resourcesbot/modules/write_lists.py +++ b/pywikitools/resourcesbot/modules/write_lists.py @@ -9,7 +9,7 @@ from pywikitools.resourcesbot.changes import ChangeLog, ChangeType from pywikitools.resourcesbot.data_structures import FileInfo, LanguageInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.reporting import LanguageReport +from pywikitools.resourcesbot.reporting import Report class WriteList(LanguagePostProcessor): @@ -192,8 +192,8 @@ def run( _english_changes, *, force_rewrite: bool = False - ) -> LanguageReport: - lang_report = WriteListLanguageReport(language_info.language_code) + ) -> Report: + lang_report = WriteListReport(language_info.language_code) if not force_rewrite and not self.needs_rewrite(language_info, changes): return lang_report @@ -265,7 +265,7 @@ def run( return lang_report -class WriteListLanguageReport(LanguageReport): +class WriteListReport(Report): """ A specialized report for write_list. """ diff --git a/pywikitools/resourcesbot/modules/write_report.py b/pywikitools/resourcesbot/modules/write_progress.py similarity index 98% rename from pywikitools/resourcesbot/modules/write_report.py rename to pywikitools/resourcesbot/modules/write_progress.py index 8cef723..2767046 100644 --- a/pywikitools/resourcesbot/modules/write_report.py +++ b/pywikitools/resourcesbot/modules/write_progress.py @@ -10,7 +10,7 @@ from pywikitools.resourcesbot.changes import ChangeLog from pywikitools.resourcesbot.data_structures import LanguageInfo, WorksheetInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.reporting import LanguageReport +from pywikitools.resourcesbot.reporting import Report class Color(Enum): @@ -23,7 +23,7 @@ def __str__(self) -> str: return self.value -class WriteReport(LanguagePostProcessor): +class WriteTranslationProgress(LanguagePostProcessor): """ Write/update status reports for all languages (for translators and translation coordinators). @@ -41,7 +41,7 @@ def help_summary(cls) -> str: @classmethod def abbreviation(cls) -> str: - return "report" + return "progress" @classmethod def can_be_rewritten(cls) -> bool: @@ -58,7 +58,7 @@ def __init__( site: our pywikibot object to be able to write to the mediawiki system """ super().__init__(fortraininglib, config, site) - self.lang_report = WriteReportLanguageReport("") + self.lang_report = WriteReportReport("") self.logger: Final[logging.Logger] = logging.getLogger( "pywikitools.resourcesbot.modules.write_report" ) @@ -71,14 +71,14 @@ def run( english_changes: ChangeLog, *, force_rewrite: bool = False - ) -> LanguageReport: + ) -> Report: """Entry function We run everything, and don't look at whether we have changes because we need to look at all CorrectBot reports and according to them, may need to rewrite the report even if changes and english_changes are empty """ - self.lang_report = WriteReportLanguageReport(language_info.language_code) + self.lang_report = WriteReportReport(language_info.language_code) # We don't need a report for English as it is the source language if language_info.language_code == "en": return self.lang_report @@ -416,7 +416,7 @@ def create_worksheet_line( return content -class WriteReportLanguageReport(LanguageReport): +class WriteReportReport(Report): """ A specialized report for write_report. """ diff --git a/pywikitools/resourcesbot/modules/write_sidebar_messages.py b/pywikitools/resourcesbot/modules/write_sidebar_messages.py index c3287f7..a6724fa 100644 --- a/pywikitools/resourcesbot/modules/write_sidebar_messages.py +++ b/pywikitools/resourcesbot/modules/write_sidebar_messages.py @@ -8,7 +8,7 @@ from pywikitools.resourcesbot.changes import ChangeLog, ChangeType from pywikitools.resourcesbot.data_structures import LanguageInfo, WorksheetInfo from pywikitools.resourcesbot.modules.post_processing import LanguagePostProcessor -from pywikitools.resourcesbot.reporting import LanguageReport +from pywikitools.resourcesbot.reporting import Report class WriteSidebarMessages(LanguagePostProcessor): @@ -94,7 +94,7 @@ def run( _english_changes, *, force_rewrite: bool = False - ) -> LanguageReport: + ) -> Report: """Our entry function""" self.lang_report = WriteSidebarReport(language_info.language_code) for worksheet in language_info.worksheets.values(): @@ -105,7 +105,7 @@ def run( return self.lang_report -class WriteSidebarReport(LanguageReport): +class WriteSidebarReport(Report): """ A specialized report for export_pdf, containing information about saved pdfs diff --git a/pywikitools/resourcesbot/modules/write_summary.py b/pywikitools/resourcesbot/modules/write_summary.py index 4be0779..f68cbd0 100644 --- a/pywikitools/resourcesbot/modules/write_summary.py +++ b/pywikitools/resourcesbot/modules/write_summary.py @@ -7,7 +7,7 @@ from pywikitools.resourcesbot.modules.post_processing import GlobalPostProcessor -class WriteSummary(GlobalPostProcessor): +class WriteProgressSummary(GlobalPostProcessor): """ Write/update global status report with an overview over the translation progress in all languages: diff --git a/pywikitools/resourcesbot/reporting.py b/pywikitools/resourcesbot/reporting.py index 89ea9b5..65e6b44 100644 --- a/pywikitools/resourcesbot/reporting.py +++ b/pywikitools/resourcesbot/reporting.py @@ -11,7 +11,7 @@ import pywikibot -class LanguageReport(ABC): +class Report(ABC): """ Basic data structure for the report of one module running once for one language. """ @@ -41,15 +41,15 @@ def get_module_summary(cls, lang_reports: list) -> str: return f"Ran module {cls.get_module_name()} for {len(lang_reports)} languages" -class Reporting: +class ReportSummary: def __init__(self): - self.module_reports: Dict[str, List[LanguageReport]] = dict() + self.module_reports: Dict[str, List[Report]] = dict() def add_module(self, module_name: str): self.module_reports[module_name] = list() - def add_language_report(self, module_name: str, language_report: LanguageReport): + def add_language_report(self, module_name: str, language_report: Report): self.module_reports.setdefault(module_name, []).append(language_report) def print_summaries(self): @@ -65,7 +65,7 @@ def make_summaries(self): return summary def get_reports_by_languages(self): - reports_by_language: Dict[str, List[LanguageReport]] = dict() + reports_by_language: Dict[str, List[Report]] = dict() for key, value in self.module_reports.items(): if len(value) == 0: pass diff --git a/pywikitools/test/test_resourcesbot.py b/pywikitools/test/test_resourcesbot.py index 771acd5..609238d 100644 --- a/pywikitools/test/test_resourcesbot.py +++ b/pywikitools/test/test_resourcesbot.py @@ -156,15 +156,15 @@ def json_test_loader(site, page: str): @patch("pywikibot.Site", autospec=True) @patch("pywikibot.Page", autospec=True) - @patch("pywikitools.resourcesbot.modules.write_summary.WriteSummary.run", autospec=True) - @patch("pywikitools.resourcesbot.modules.write_report.WriteReport.run", autospec=True) + @patch("pywikitools.resourcesbot.modules.write_summary.WriteProgressSummary.run", autospec=True) + @patch("pywikitools.resourcesbot.modules.write_progress.WriteTranslationProgress.run", autospec=True) @patch("pywikitools.resourcesbot.modules.write_lists.WriteList.run", autospec=True) @patch("pywikitools.resourcesbot.modules.write_sidebar_messages.WriteSidebarMessages.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_repository.ExportRepository.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_html.ExportHTML.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) - @patch("pywikitools.resourcesbot.reporting.Reporting", autospec=True) + @patch("pywikitools.resourcesbot.reporting.ReportSummary", autospec=True) def test_run_with_cache( self, mock_report, @@ -174,7 +174,7 @@ def test_run_with_cache( mock_export_repository, mock_write_sidebar_messages, mock_write_list, - mock_write_report, + mock_write_progress, mock_write_summary, mock_pywikibot_page, mock_pywikibot_site @@ -190,7 +190,7 @@ def test_run_with_cache( self.assertEqual(mock_export_repository.call_count, 2) self.assertEqual(mock_write_sidebar_messages.call_count, 2) self.assertEqual(mock_write_list.call_count, 2) - self.assertEqual(mock_write_report.call_count, 2) + self.assertEqual(mock_write_progress.call_count, 2) mock_write_summary.assert_called_once() self.assertIn("en", bot._result) @@ -204,15 +204,15 @@ def test_run_with_cache( @patch("pywikibot.Site", autospec=True) @patch("pywikibot.Page", autospec=True) - @patch("pywikitools.resourcesbot.modules.write_summary.WriteSummary.run", autospec=True) - @patch("pywikitools.resourcesbot.modules.write_report.WriteReport.run", autospec=True) + @patch("pywikitools.resourcesbot.modules.write_summary.WriteProgressSummary.run", autospec=True) + @patch("pywikitools.resourcesbot.modules.write_progress.WriteTranslationProgress.run", autospec=True) @patch("pywikitools.resourcesbot.modules.write_lists.WriteList.run", autospec=True) @patch("pywikitools.resourcesbot.modules.write_sidebar_messages.WriteSidebarMessages.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_repository.ExportRepository.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_html.ExportHTML.run", autospec=True) @patch("pywikitools.resourcesbot.modules.export_pdf.ExportPDF.run", autospec=True) @patch("pywikitools.resourcesbot.modules.consistency_checks.ConsistencyCheck.run", autospec=True) - @patch("pywikitools.resourcesbot.reporting.Reporting", autospec=True) + @patch("pywikitools.resourcesbot.reporting.ReportSummary", autospec=True) def test_rewrite_options( self, mock_report, @@ -222,7 +222,7 @@ def test_rewrite_options( mock_export_repository, mock_write_sidebar_messages, mock_write_list, - mock_write_report, + mock_write_progress, mock_write_summary, mock_pywikibot_page, mock_pywikibot_site, @@ -234,7 +234,7 @@ def test_rewrite_options( rewrite_check: Dict[str, Mock] = { "summary": mock_write_summary, "list": mock_write_list, - "report": mock_write_report, + "progress": mock_write_progress, "html": mock_export_html, "sidebar": mock_write_sidebar_messages, } diff --git a/pywikitools/test/test_write_report.py b/pywikitools/test/test_write_progress.py similarity index 88% rename from pywikitools/test/test_write_report.py rename to pywikitools/test/test_write_progress.py index cbb3249..79d538e 100644 --- a/pywikitools/test/test_write_report.py +++ b/pywikitools/test/test_write_progress.py @@ -8,10 +8,10 @@ from pywikitools.fortraininglib import ForTrainingLib from pywikitools.resourcesbot.changes import ChangeLog from pywikitools.resourcesbot.data_structures import LanguageInfo, json_decode -from pywikitools.resourcesbot.modules.write_report import WriteReport +from pywikitools.resourcesbot.modules.write_progress import WriteTranslationProgress -class TestWriteReport(unittest.TestCase): +class TestWriteProgress(unittest.TestCase): def setUp(self): self.config = ConfigParser() with open(join(dirname(abspath(__file__)), "data", "ru.json"), "r") as f: @@ -61,7 +61,7 @@ def mock_pywikibot_pages(site, page: str): def test_created_mediawiki(self, mock_page): # Compare mediawiki output with the content in # data/ru_worksheet_overview.mediawiki - write_report = WriteReport(self.fortraininglib, self.config, None) + write_report = WriteTranslationProgress(self.fortraininglib, self.config, None) mock_page.side_effect = self.mock_pywikibot_pages with open( @@ -86,11 +86,11 @@ def test_created_mediawiki(self, mock_page): ) @patch( - "pywikitools.resourcesbot.modules.write_report.WriteReport" ".create_mediawiki" + "pywikitools.resourcesbot.modules.write_progress.WriteTranslationProgress" ".create_mediawiki" ) # don't go into create_mediawiki() @patch("pywikibot.Page") def test_save_language_report(self, mock_page, mock_create_mediawiki): - write_report = WriteReport(self.fortraininglib, self.config, None) + write_report = WriteTranslationProgress(self.fortraininglib, self.config, None) # When there is no proper language name, save_language_report() # should directly exit with self.assertLogs( @@ -114,11 +114,11 @@ def test_save_language_report(self, mock_page, mock_create_mediawiki): mock_page.return_value.save.assert_called_with("Updated language report") @patch( - "pywikitools.resourcesbot.modules.write_report.WriteReport" + "pywikitools.resourcesbot.modules.write_progress.WriteTranslationProgress" ".save_language_report" ) def test_run(self, mock_save): - write_report = WriteReport(self.fortraininglib, self.config, None) + write_report = WriteTranslationProgress(self.fortraininglib, self.config, None) # save_language_report() shouldn't get called when we have a language variant write_report.run( LanguageInfo("de-test", "Deutsch (Test)"), @@ -130,7 +130,7 @@ def test_run(self, mock_save): # save_language_report() should be called once (for Russian) and force_rewrite # should be ignored - write_report = WriteReport(self.fortraininglib, self.config, None) + write_report = WriteTranslationProgress(self.fortraininglib, self.config, None) write_report.run( self.language_info, self.english_info, ChangeLog(), ChangeLog() ) diff --git a/pywikitools/test/test_write_summary.py b/pywikitools/test/test_write_summary.py index 9730a0d..56f3900 100644 --- a/pywikitools/test/test_write_summary.py +++ b/pywikitools/test/test_write_summary.py @@ -6,7 +6,7 @@ from pywikitools.resourcesbot.changes import ChangeLog, ChangeType from pywikitools.resourcesbot.data_structures import LanguageInfo, json_decode -from pywikitools.resourcesbot.modules.write_summary import WriteSummary +from pywikitools.resourcesbot.modules.write_summary import WriteProgressSummary class TestWriteSummary(unittest.TestCase): @@ -19,7 +19,7 @@ def setUp(self): self.language_data[language_code] = json.load(f, object_hook=json_decode) self.empty_change_log[language_code] = ChangeLog() - self.write_summary = WriteSummary(None) + self.write_summary = WriteProgressSummary(None) def test_created_mediawiki(self): # Compare mediawiki output with the content in data/summary.mediawiki @@ -49,7 +49,7 @@ def test_save_summary(self, mock_page): self.write_summary.save_summary(self.language_data) mock_page.return_value.save.assert_called_with("Updated summary report") - @patch("pywikitools.resourcesbot.modules.write_summary.WriteSummary" + @patch("pywikitools.resourcesbot.modules.write_summary.WriteProgressSummary" ".save_summary") def test_run(self, mock_save): # save_summary() shouldn't get called when there are no changes @@ -64,7 +64,7 @@ def test_run(self, mock_save): mock_save.assert_called_once() # save_summary() should be called when we have force_rewrite (even if there are no changes) - write_summary = WriteSummary(None) + write_summary = WriteProgressSummary(None) write_summary.run(self.language_data, self.empty_change_log, force_rewrite=True) self.assertEqual(mock_save.call_count, 2) From ca02ff173f9f62070d935d712d7aab78246d1dc3 Mon Sep 17 00:00:00 2001 From: JoshKowi Date: Wed, 18 Dec 2024 13:14:57 +0100 Subject: [PATCH 6/6] Reporting: Automatically create new pages. Add category --- pywikitools/resourcesbot/reporting.py | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pywikitools/resourcesbot/reporting.py b/pywikitools/resourcesbot/reporting.py index 65e6b44..3a3f3db 100644 --- a/pywikitools/resourcesbot/reporting.py +++ b/pywikitools/resourcesbot/reporting.py @@ -104,10 +104,18 @@ def generate_mediawiki(self): return head + overview + details def save_report(self, site): - page_url = "4training:Resourcesbot.report" + timestamp_str = datetime.now().strftime("%Y-%m-%d") + run_content = self.generate_mediawiki() + page_url = f"4training:Resourcesbot.report.{timestamp_str}" page = pywikibot.Page(site, page_url) if page.exists(): - page.text = self.generate_mediawiki() + page.text + page.text = run_content + page.text else: - page.text = self.generate_mediawiki() + page.text = run_content + "\n\n[[Category:ResourcesBot-Run]]" page.save("Created summary for Resourcesbot run") + + main_page_url = "4training:Resourcesbot.report" + main_page = pywikibot.Page(site, main_page_url) + main_page.text = ("This page only shows the most recent run.\n\n" + + run_content + "\n\n[[Category:ResourcesBot-Run]]") + main_page.save("Created summary for Resourcesbot run")