From 27ee92c14d64b4ccb6ee3b71b51166b978dbb075 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 <57287228+TobiasBeck777@users.noreply.github.com> Date: Sat, 18 Oct 2025 11:19:52 +0200 Subject: [PATCH 01/24] Create transfer.py --- pywikitools/transfer.py | 87 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 pywikitools/transfer.py diff --git a/pywikitools/transfer.py b/pywikitools/transfer.py new file mode 100644 index 0000000..ba78960 --- /dev/null +++ b/pywikitools/transfer.py @@ -0,0 +1,87 @@ +import logging +from os.path import abspath, dirname, join +import requests +import pywikibot +import argparse +from pywikitools.family import Family +from pywikitools.fortraininglib import ForTrainingLib +from configparser import ConfigParser + +TIMEOUT: int = 30 # Timeout after 30s (prevent indefinite hanging when there is network issues) + +class TransferTool: + def __init__(self, config: ConfigParser): + + + + if not config.has_option('transfer', 'source_site') or \ + not config.has_option('transfer', 'source_username') or \ + not config.has_option('transfer', 'source_password') or \ + not config.has_option('transfer', 'destination_site') or \ + not config.has_option('transfer', 'destination_username') or \ + not config.has_option('transfer', 'destination_password'): + raise RuntimeError("Missing settings for transfer in config.ini") + + self.logger: logging.Logger = logging.getLogger('pywikitools.transfer') + + self.source_site = config.get('transfer', 'source_site') + self.destination_site = config.get('transfer', 'destination_site') + + family = Family() + + self.source_wiki_site = pywikibot.Site(code=self.source_site, fam=family, + user=config.get('transfer', 'source_username')) + if not self.source_wiki_site.logged_in(): + self.source_wiki_site.login() + if not self.source_wiki_site.logged_in(): + raise RuntimeError("Login with pywikibot failed to source site failed.") + # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) + self.source_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) + + self.destination_wiki_site = pywikibot.Site(code=self.destination_site, fam=family, + user=config.get('transfer', 'destination_username')) + if not self.destination_wiki_site.logged_in(): + self.destination_wiki_site.login() + if not self.destination_wiki_site.logged_in(): + raise RuntimeError("Login with pywikibot failed to destination site failed.") + # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) + self.destination_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) + + self.source_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.source_site, ''), + family.scriptpath(self.source_site)) + self.destination_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.destination_site, ''), + family.scriptpath(self.destination_site)) + + + def transfer(self, page_name, language_code, force=False): + source_translation_page = self.source_fortraininglib.get_translation_units(page_name, "en") + + if not force and self.destination_fortraininglib.get_translated_title(page_name, language_code) is not None: + self.logger.warning("Destination already exists. If you want to force overwrite, use the -f flag.") + return + + for translation_unit in source_translation_page: + self.upload(f"{translation_unit.identifier}/{language_code}", + translation_unit.get_translation()) + + + def upload(self, identifier: str, translated_text: str): + """Transfer a workshoot from one mediawiki system to another one""" + destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") + destination_mediawiki_page.text = translated_text + destination_mediawiki_page.save(summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}' completed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Transfer a worksheet for a certain language from one site to another site.") + parser.add_argument("worksheet_name", help="Name of the worksheet to transfer.") + parser.add_argument("language_code", help="Target language code for transfer.") + parser.add_argument("-f", "--force", action="store_true", help="Force overwrite if source exists.") + args = parser.parse_args() + + config = ConfigParser() + config.read(join(dirname(abspath(__file__)), "config.ini")) + + transfer_tool = TransferTool(config) + transfer_tool.transfer(args.worksheet_name, args.language_code, args.force) + From 2ce786e08ca396ef2a5e0aa8593c9c2eddc6413b Mon Sep 17 00:00:00 2001 From: TobiasBeck777 <57287228+TobiasBeck777@users.noreply.github.com> Date: Sat, 18 Oct 2025 11:22:25 +0200 Subject: [PATCH 02/24] Create transfer.py (right place) --- transfer.py | 87 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 87 insertions(+) create mode 100644 transfer.py diff --git a/transfer.py b/transfer.py new file mode 100644 index 0000000..ba78960 --- /dev/null +++ b/transfer.py @@ -0,0 +1,87 @@ +import logging +from os.path import abspath, dirname, join +import requests +import pywikibot +import argparse +from pywikitools.family import Family +from pywikitools.fortraininglib import ForTrainingLib +from configparser import ConfigParser + +TIMEOUT: int = 30 # Timeout after 30s (prevent indefinite hanging when there is network issues) + +class TransferTool: + def __init__(self, config: ConfigParser): + + + + if not config.has_option('transfer', 'source_site') or \ + not config.has_option('transfer', 'source_username') or \ + not config.has_option('transfer', 'source_password') or \ + not config.has_option('transfer', 'destination_site') or \ + not config.has_option('transfer', 'destination_username') or \ + not config.has_option('transfer', 'destination_password'): + raise RuntimeError("Missing settings for transfer in config.ini") + + self.logger: logging.Logger = logging.getLogger('pywikitools.transfer') + + self.source_site = config.get('transfer', 'source_site') + self.destination_site = config.get('transfer', 'destination_site') + + family = Family() + + self.source_wiki_site = pywikibot.Site(code=self.source_site, fam=family, + user=config.get('transfer', 'source_username')) + if not self.source_wiki_site.logged_in(): + self.source_wiki_site.login() + if not self.source_wiki_site.logged_in(): + raise RuntimeError("Login with pywikibot failed to source site failed.") + # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) + self.source_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) + + self.destination_wiki_site = pywikibot.Site(code=self.destination_site, fam=family, + user=config.get('transfer', 'destination_username')) + if not self.destination_wiki_site.logged_in(): + self.destination_wiki_site.login() + if not self.destination_wiki_site.logged_in(): + raise RuntimeError("Login with pywikibot failed to destination site failed.") + # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) + self.destination_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) + + self.source_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.source_site, ''), + family.scriptpath(self.source_site)) + self.destination_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.destination_site, ''), + family.scriptpath(self.destination_site)) + + + def transfer(self, page_name, language_code, force=False): + source_translation_page = self.source_fortraininglib.get_translation_units(page_name, "en") + + if not force and self.destination_fortraininglib.get_translated_title(page_name, language_code) is not None: + self.logger.warning("Destination already exists. If you want to force overwrite, use the -f flag.") + return + + for translation_unit in source_translation_page: + self.upload(f"{translation_unit.identifier}/{language_code}", + translation_unit.get_translation()) + + + def upload(self, identifier: str, translated_text: str): + """Transfer a workshoot from one mediawiki system to another one""" + destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") + destination_mediawiki_page.text = translated_text + destination_mediawiki_page.save(summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}' completed") + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Transfer a worksheet for a certain language from one site to another site.") + parser.add_argument("worksheet_name", help="Name of the worksheet to transfer.") + parser.add_argument("language_code", help="Target language code for transfer.") + parser.add_argument("-f", "--force", action="store_true", help="Force overwrite if source exists.") + args = parser.parse_args() + + config = ConfigParser() + config.read(join(dirname(abspath(__file__)), "config.ini")) + + transfer_tool = TransferTool(config) + transfer_tool.transfer(args.worksheet_name, args.language_code, args.force) + From 5f0b999e7e04cb62e1b3458518a599b5b6387579 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 <57287228+TobiasBeck777@users.noreply.github.com> Date: Sat, 18 Oct 2025 11:23:16 +0200 Subject: [PATCH 03/24] Create test_transfer.py --- pywikitools/test/test_transfer.py | 35 +++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 pywikitools/test/test_transfer.py diff --git a/pywikitools/test/test_transfer.py b/pywikitools/test/test_transfer.py new file mode 100644 index 0000000..7b92c12 --- /dev/null +++ b/pywikitools/test/test_transfer.py @@ -0,0 +1,35 @@ +from configparser import ConfigParser +import unittest +from unittest.mock import patch, Mock +import sys +sys.path.append('../../') # Is there a better way to do it? +from transfer import TransferTool + + +class TestTransferTool(unittest.TestCase): + @patch('pywikibot.Site', autospec=True) + def setUp(self, mock_pywikibot_site): + mock_pywikibot_site.return_value.logged_in.return_value = True + config = ConfigParser() + config.read_dict({"transfer": {"source_username": "User1", + "source_password": "Pw1", + "source_site": "local", + "destination_username": "User2", + "destination_password": "Pw2", + "destination_site": "test" + }}) + self.transfer_tool = TransferTool(config) + + @patch('pywikibot.Page') + def test_upload(self, mock_page): + self.transfer_tool.upload("Test_Page/1/fr", "Test transfer") + mock_page.return_value.save.assert_called_once_with(summary="Transfer of 'Test_Page/1/fr' from 'local' to 'test' completed") + + @patch('pywikibot.Page') + def test_transfer(self, mock_page): + self.transfer_tool.transfer("", "fr", ) + mock_page.return_value.save.assert_called_once_with(summary="Transfer of 'Test_Page/1/fr' from 'local' to 'test' completed") + + +if __name__ == "__main__": + unittest.main() From 2897c1061ef4e77cee44a0d0fb0a6924376c06f0 Mon Sep 17 00:00:00 2001 From: Jonatan Giuseppe Schnyder Date: Sat, 18 Oct 2025 11:25:31 +0200 Subject: [PATCH 04/24] removed wrong file transfer.py --- pywikitools/transfer.py | 87 ----------------------------------------- 1 file changed, 87 deletions(-) delete mode 100644 pywikitools/transfer.py diff --git a/pywikitools/transfer.py b/pywikitools/transfer.py deleted file mode 100644 index ba78960..0000000 --- a/pywikitools/transfer.py +++ /dev/null @@ -1,87 +0,0 @@ -import logging -from os.path import abspath, dirname, join -import requests -import pywikibot -import argparse -from pywikitools.family import Family -from pywikitools.fortraininglib import ForTrainingLib -from configparser import ConfigParser - -TIMEOUT: int = 30 # Timeout after 30s (prevent indefinite hanging when there is network issues) - -class TransferTool: - def __init__(self, config: ConfigParser): - - - - if not config.has_option('transfer', 'source_site') or \ - not config.has_option('transfer', 'source_username') or \ - not config.has_option('transfer', 'source_password') or \ - not config.has_option('transfer', 'destination_site') or \ - not config.has_option('transfer', 'destination_username') or \ - not config.has_option('transfer', 'destination_password'): - raise RuntimeError("Missing settings for transfer in config.ini") - - self.logger: logging.Logger = logging.getLogger('pywikitools.transfer') - - self.source_site = config.get('transfer', 'source_site') - self.destination_site = config.get('transfer', 'destination_site') - - family = Family() - - self.source_wiki_site = pywikibot.Site(code=self.source_site, fam=family, - user=config.get('transfer', 'source_username')) - if not self.source_wiki_site.logged_in(): - self.source_wiki_site.login() - if not self.source_wiki_site.logged_in(): - raise RuntimeError("Login with pywikibot failed to source site failed.") - # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) - self.source_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) - - self.destination_wiki_site = pywikibot.Site(code=self.destination_site, fam=family, - user=config.get('transfer', 'destination_username')) - if not self.destination_wiki_site.logged_in(): - self.destination_wiki_site.login() - if not self.destination_wiki_site.logged_in(): - raise RuntimeError("Login with pywikibot failed to destination site failed.") - # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) - self.destination_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) - - self.source_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.source_site, ''), - family.scriptpath(self.source_site)) - self.destination_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.destination_site, ''), - family.scriptpath(self.destination_site)) - - - def transfer(self, page_name, language_code, force=False): - source_translation_page = self.source_fortraininglib.get_translation_units(page_name, "en") - - if not force and self.destination_fortraininglib.get_translated_title(page_name, language_code) is not None: - self.logger.warning("Destination already exists. If you want to force overwrite, use the -f flag.") - return - - for translation_unit in source_translation_page: - self.upload(f"{translation_unit.identifier}/{language_code}", - translation_unit.get_translation()) - - - def upload(self, identifier: str, translated_text: str): - """Transfer a workshoot from one mediawiki system to another one""" - destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") - destination_mediawiki_page.text = translated_text - destination_mediawiki_page.save(summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}' completed") - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Transfer a worksheet for a certain language from one site to another site.") - parser.add_argument("worksheet_name", help="Name of the worksheet to transfer.") - parser.add_argument("language_code", help="Target language code for transfer.") - parser.add_argument("-f", "--force", action="store_true", help="Force overwrite if source exists.") - args = parser.parse_args() - - config = ConfigParser() - config.read(join(dirname(abspath(__file__)), "config.ini")) - - transfer_tool = TransferTool(config) - transfer_tool.transfer(args.worksheet_name, args.language_code, args.force) - From aee2b40cc812e22f9a7a7069073fe90f0de84a4f Mon Sep 17 00:00:00 2001 From: TobiasBeck777 <57287228+TobiasBeck777@users.noreply.github.com> Date: Sat, 18 Oct 2025 12:18:32 +0200 Subject: [PATCH 05/24] Update transfer.py remove password --- transfer.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/transfer.py b/transfer.py index ba78960..f501a19 100644 --- a/transfer.py +++ b/transfer.py @@ -12,14 +12,10 @@ class TransferTool: def __init__(self, config: ConfigParser): - - if not config.has_option('transfer', 'source_site') or \ not config.has_option('transfer', 'source_username') or \ - not config.has_option('transfer', 'source_password') or \ not config.has_option('transfer', 'destination_site') or \ - not config.has_option('transfer', 'destination_username') or \ - not config.has_option('transfer', 'destination_password'): + not config.has_option('transfer', 'destination_username'): raise RuntimeError("Missing settings for transfer in config.ini") self.logger: logging.Logger = logging.getLogger('pywikitools.transfer') From b865bb81e4fa289868ee0187b2f43f91e30fc17d Mon Sep 17 00:00:00 2001 From: TobiasBeck777 <57287228+TobiasBeck777@users.noreply.github.com> Date: Sat, 18 Oct 2025 12:19:23 +0200 Subject: [PATCH 06/24] Update test_transfer.py remove pw for test --- pywikitools/test/test_transfer.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pywikitools/test/test_transfer.py b/pywikitools/test/test_transfer.py index 7b92c12..118e334 100644 --- a/pywikitools/test/test_transfer.py +++ b/pywikitools/test/test_transfer.py @@ -12,10 +12,8 @@ def setUp(self, mock_pywikibot_site): mock_pywikibot_site.return_value.logged_in.return_value = True config = ConfigParser() config.read_dict({"transfer": {"source_username": "User1", - "source_password": "Pw1", "source_site": "local", "destination_username": "User2", - "destination_password": "Pw2", "destination_site": "test" }}) self.transfer_tool = TransferTool(config) From e5ca00651f2ea3da6525f3932d2396978d5a4a07 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 12:28:18 +0200 Subject: [PATCH 07/24] test --- transfer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transfer.py b/transfer.py index f501a19..e9361a4 100644 --- a/transfer.py +++ b/transfer.py @@ -11,7 +11,6 @@ class TransferTool: def __init__(self, config: ConfigParser): - if not config.has_option('transfer', 'source_site') or \ not config.has_option('transfer', 'source_username') or \ not config.has_option('transfer', 'destination_site') or \ From ace241f46fa9d38818ccf57e9c266bfaf5d54b96 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 13:59:02 +0200 Subject: [PATCH 08/24] test of transfer --- pywikitools/test/test_transfer.py | 10 +++++----- transfer.py | 11 +++-------- 2 files changed, 8 insertions(+), 13 deletions(-) diff --git a/pywikitools/test/test_transfer.py b/pywikitools/test/test_transfer.py index 118e334..a0a23c7 100644 --- a/pywikitools/test/test_transfer.py +++ b/pywikitools/test/test_transfer.py @@ -12,21 +12,21 @@ def setUp(self, mock_pywikibot_site): mock_pywikibot_site.return_value.logged_in.return_value = True config = ConfigParser() config.read_dict({"transfer": {"source_username": "User1", - "source_site": "local", + "source_site": "test", "destination_username": "User2", - "destination_site": "test" + "destination_site": "local" }}) self.transfer_tool = TransferTool(config) @patch('pywikibot.Page') def test_upload(self, mock_page): self.transfer_tool.upload("Test_Page/1/fr", "Test transfer") - mock_page.return_value.save.assert_called_once_with(summary="Transfer of 'Test_Page/1/fr' from 'local' to 'test' completed") + mock_page.return_value.save.assert_called_once_with(summary="Transfer of 'Test_Page/1/fr' from 'test' to 'local'") @patch('pywikibot.Page') def test_transfer(self, mock_page): - self.transfer_tool.transfer("", "fr", ) - mock_page.return_value.save.assert_called_once_with(summary="Transfer of 'Test_Page/1/fr' from 'local' to 'test' completed") + self.transfer_tool.transfer("A_Daily_Prayer", "fr") + mock_page.return_value.save.assert_called_with(summary="Transfer of 'A_Daily_Prayer/21/fr' from 'test' to 'local'") if __name__ == "__main__": diff --git a/transfer.py b/transfer.py index e9361a4..c29891f 100644 --- a/transfer.py +++ b/transfer.py @@ -48,13 +48,9 @@ def __init__(self, config: ConfigParser): family.scriptpath(self.destination_site)) - def transfer(self, page_name, language_code, force=False): + def transfer(self, page_name, language_code): source_translation_page = self.source_fortraininglib.get_translation_units(page_name, "en") - if not force and self.destination_fortraininglib.get_translated_title(page_name, language_code) is not None: - self.logger.warning("Destination already exists. If you want to force overwrite, use the -f flag.") - return - for translation_unit in source_translation_page: self.upload(f"{translation_unit.identifier}/{language_code}", translation_unit.get_translation()) @@ -64,19 +60,18 @@ def upload(self, identifier: str, translated_text: str): """Transfer a workshoot from one mediawiki system to another one""" destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") destination_mediawiki_page.text = translated_text - destination_mediawiki_page.save(summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}' completed") + destination_mediawiki_page.save(summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}'") if __name__ == "__main__": parser = argparse.ArgumentParser(description="Transfer a worksheet for a certain language from one site to another site.") parser.add_argument("worksheet_name", help="Name of the worksheet to transfer.") parser.add_argument("language_code", help="Target language code for transfer.") - parser.add_argument("-f", "--force", action="store_true", help="Force overwrite if source exists.") args = parser.parse_args() config = ConfigParser() config.read(join(dirname(abspath(__file__)), "config.ini")) transfer_tool = TransferTool(config) - transfer_tool.transfer(args.worksheet_name, args.language_code, args.force) + transfer_tool.transfer(args.worksheet_name, args.language_code) From d6805704d783997ca8ecc6ae94fafe480bc01b89 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 14:21:49 +0200 Subject: [PATCH 09/24] adjusted config.example.ini --- config.example.ini | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/config.example.ini b/config.example.ini index 9c23386..777ab80 100644 --- a/config.example.ini +++ b/config.example.ini @@ -80,3 +80,13 @@ username = AutoTranslateBot # DeepL configuration deeplendpoint = https://api-free.deepl.com/v2/translate deeplapikey = MySecretDeepLAPIKey + + +[transfer] +# The mediawiki environment from where we copy it, for example 'test' +source_site = local +source_username = CorrectBot +# The mediawiki environment where we store the content above, for example 'local' +destination_site = local +destination_username = CorrectBot + From 6854047266ad9e415ac45c86998286ace7ba34bf Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 14:25:26 +0200 Subject: [PATCH 10/24] adjusted config.example.ini --- config.example.ini | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config.example.ini b/config.example.ini index 777ab80..298185e 100644 --- a/config.example.ini +++ b/config.example.ini @@ -89,4 +89,5 @@ source_username = CorrectBot # The mediawiki environment where we store the content above, for example 'local' destination_site = local destination_username = CorrectBot - +# Example call for transfer: +# python3 transfer.py "A_Daily_Prayer" "fr" From 7d1407b772ba770103b28b614631d59366d42d58 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 14:36:51 +0200 Subject: [PATCH 11/24] formatting --- pywikitools/test/test_transfer.py | 17 +++++++++-------- transfer.py | 26 ++++++++++++-------------- 2 files changed, 21 insertions(+), 22 deletions(-) diff --git a/pywikitools/test/test_transfer.py b/pywikitools/test/test_transfer.py index a0a23c7..aa8bb47 100644 --- a/pywikitools/test/test_transfer.py +++ b/pywikitools/test/test_transfer.py @@ -1,9 +1,9 @@ from configparser import ConfigParser import unittest -from unittest.mock import patch, Mock +from unittest.mock import patch import sys -sys.path.append('../../') # Is there a better way to do it? from transfer import TransferTool +sys.path.append('../../') # Is there a better way to do it? class TestTransferTool(unittest.TestCase): @@ -11,22 +11,23 @@ class TestTransferTool(unittest.TestCase): def setUp(self, mock_pywikibot_site): mock_pywikibot_site.return_value.logged_in.return_value = True config = ConfigParser() - config.read_dict({"transfer": {"source_username": "User1", + config.read_dict({"transfer": {"source_username": "User1", "source_site": "test", - "destination_username": "User2", - "destination_site": "local" - }}) + "destination_username": "User2", + "destination_site": "local"}}) self.transfer_tool = TransferTool(config) @patch('pywikibot.Page') def test_upload(self, mock_page): self.transfer_tool.upload("Test_Page/1/fr", "Test transfer") - mock_page.return_value.save.assert_called_once_with(summary="Transfer of 'Test_Page/1/fr' from 'test' to 'local'") + mock_page.return_value.save.assert_called_once_with( + summary="Transfer of 'Test_Page/1/fr' from 'test' to 'local'") @patch('pywikibot.Page') def test_transfer(self, mock_page): self.transfer_tool.transfer("A_Daily_Prayer", "fr") - mock_page.return_value.save.assert_called_with(summary="Transfer of 'A_Daily_Prayer/21/fr' from 'test' to 'local'") + mock_page.return_value.save.assert_called_with( + summary="Transfer of 'A_Daily_Prayer/21/fr' from 'test' to 'local'") if __name__ == "__main__": diff --git a/transfer.py b/transfer.py index c29891f..a57a52d 100644 --- a/transfer.py +++ b/transfer.py @@ -1,6 +1,5 @@ import logging from os.path import abspath, dirname, join -import requests import pywikibot import argparse from pywikitools.family import Family @@ -9,6 +8,7 @@ TIMEOUT: int = 30 # Timeout after 30s (prevent indefinite hanging when there is network issues) + class TransferTool: def __init__(self, config: ConfigParser): if not config.has_option('transfer', 'source_site') or \ @@ -24,8 +24,8 @@ def __init__(self, config: ConfigParser): family = Family() - self.source_wiki_site = pywikibot.Site(code=self.source_site, fam=family, - user=config.get('transfer', 'source_username')) + self.source_wiki_site = pywikibot.Site(code=self.source_site, fam=family, + user=config.get('transfer', 'source_username')) if not self.source_wiki_site.logged_in(): self.source_wiki_site.login() if not self.source_wiki_site.logged_in(): @@ -33,8 +33,8 @@ def __init__(self, config: ConfigParser): # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) self.source_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) - self.destination_wiki_site = pywikibot.Site(code=self.destination_site, fam=family, - user=config.get('transfer', 'destination_username')) + self.destination_wiki_site = pywikibot.Site(code=self.destination_site, fam=family, + user=config.get('transfer', 'destination_username')) if not self.destination_wiki_site.logged_in(): self.destination_wiki_site.login() if not self.destination_wiki_site.logged_in(): @@ -43,28 +43,27 @@ def __init__(self, config: ConfigParser): self.destination_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) self.source_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.source_site, ''), - family.scriptpath(self.source_site)) + family.scriptpath(self.source_site)) self.destination_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.destination_site, ''), - family.scriptpath(self.destination_site)) - + family.scriptpath(self.destination_site)) def transfer(self, page_name, language_code): source_translation_page = self.source_fortraininglib.get_translation_units(page_name, "en") for translation_unit in source_translation_page: - self.upload(f"{translation_unit.identifier}/{language_code}", - translation_unit.get_translation()) - + self.upload(f"{translation_unit.identifier}/{language_code}", translation_unit.get_translation()) def upload(self, identifier: str, translated_text: str): """Transfer a workshoot from one mediawiki system to another one""" destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") destination_mediawiki_page.text = translated_text - destination_mediawiki_page.save(summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}'") + destination_mediawiki_page.save( + summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}'") if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Transfer a worksheet for a certain language from one site to another site.") + parser = argparse.ArgumentParser( + description="Transfer a worksheet for a certain language from one site to another site.") parser.add_argument("worksheet_name", help="Name of the worksheet to transfer.") parser.add_argument("language_code", help="Target language code for transfer.") args = parser.parse_args() @@ -74,4 +73,3 @@ def upload(self, identifier: str, translated_text: str): transfer_tool = TransferTool(config) transfer_tool.transfer(args.worksheet_name, args.language_code) - From b9a36f02818ed8e5093d90a853901cd650c6d570 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 14:53:16 +0200 Subject: [PATCH 12/24] fix bug where source is always English --- transfer.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/transfer.py b/transfer.py index a57a52d..d53c62a 100644 --- a/transfer.py +++ b/transfer.py @@ -48,7 +48,7 @@ def __init__(self, config: ConfigParser): family.scriptpath(self.destination_site)) def transfer(self, page_name, language_code): - source_translation_page = self.source_fortraininglib.get_translation_units(page_name, "en") + source_translation_page = self.source_fortraininglib.get_translation_units(page_name, language_code) for translation_unit in source_translation_page: self.upload(f"{translation_unit.identifier}/{language_code}", translation_unit.get_translation()) From 44ac9556b9fc25529fef37bcaae8979e0641b094 Mon Sep 17 00:00:00 2001 From: Jonatan Giuseppe Schnyder Date: Sat, 18 Oct 2025 14:53:38 +0200 Subject: [PATCH 13/24] upkeeping cause module changed functionname (.setDelays() -> .set_delays() --- pywikitools/correctbot/bot.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pywikitools/correctbot/bot.py b/pywikitools/correctbot/bot.py index 61a995b..962168a 100644 --- a/pywikitools/correctbot/bot.py +++ b/pywikitools/correctbot/bot.py @@ -25,7 +25,7 @@ def __init__(self, config: ConfigParser, simulate: bool = False): family = Family() self.site = pywikibot.Site(code=code, fam=family, user=self._config.get('correctbot', 'username')) # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) - self.site.throttle.setDelays(delay=0, writedelay=0, absolute=True) + self.site.throttle.set_delays(delay=0, writedelay=0, absolute=True) self.fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(code, ''), family.scriptpath(code)) From 273bc8f2af6223f662ac23ec67a7c9dcdbfd7e11 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 15:20:36 +0200 Subject: [PATCH 14/24] set_delays new method name (old one was deprecated) --- transfer.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/transfer.py b/transfer.py index d53c62a..16398d0 100644 --- a/transfer.py +++ b/transfer.py @@ -31,7 +31,7 @@ def __init__(self, config: ConfigParser): if not self.source_wiki_site.logged_in(): raise RuntimeError("Login with pywikibot failed to source site failed.") # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) - self.source_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) + self.source_wiki_site.throttle.set_delays(delay=0, writedelay=0, absolute=True) self.destination_wiki_site = pywikibot.Site(code=self.destination_site, fam=family, user=config.get('transfer', 'destination_username')) @@ -40,7 +40,7 @@ def __init__(self, config: ConfigParser): if not self.destination_wiki_site.logged_in(): raise RuntimeError("Login with pywikibot failed to destination site failed.") # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) - self.destination_wiki_site.throttle.setDelays(delay=0, writedelay=0, absolute=True) + self.destination_wiki_site.throttle.set_delays(delay=0, writedelay=0, absolute=True) self.source_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.source_site, ''), family.scriptpath(self.source_site)) From f8c4011c371291768e08c3d682697fa71381d23a Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 16:35:47 +0200 Subject: [PATCH 15/24] summary --- pywikitools/lang/translated_page.py | 6 +++++ transfer.py | 42 ++++++++++++++++++++++++++--- 2 files changed, 44 insertions(+), 4 deletions(-) diff --git a/pywikitools/lang/translated_page.py b/pywikitools/lang/translated_page.py index 50c212d..a337126 100644 --- a/pywikitools/lang/translated_page.py +++ b/pywikitools/lang/translated_page.py @@ -419,6 +419,12 @@ def add_translation_unit(self, unit: TranslationUnit): """Append a translation unit. Infos are not invalidated""" self.units.append(unit) + def get_iteration_unit(self, identifier: str): + for slice in self.units: + if (slice.identifier == identifier): + return slice + return None + def __iter__(self): """Make this class iterable in a simple way (not suitable for concurrency!)""" self._iterate_pos = 0 diff --git a/transfer.py b/transfer.py index 16398d0..f8bdeda 100644 --- a/transfer.py +++ b/transfer.py @@ -1,10 +1,13 @@ import logging from os.path import abspath, dirname, join +from typing import Optional import pywikibot import argparse from pywikitools.family import Family from pywikitools.fortraininglib import ForTrainingLib from configparser import ConfigParser +from pywikitools.lang.translated_page import TranslatedPage +from pywikitools.lang.translated_page import TranslationUnit TIMEOUT: int = 30 # Timeout after 30s (prevent indefinite hanging when there is network issues) @@ -48,10 +51,41 @@ def __init__(self, config: ConfigParser): family.scriptpath(self.destination_site)) def transfer(self, page_name, language_code): - source_translation_page = self.source_fortraininglib.get_translation_units(page_name, language_code) - - for translation_unit in source_translation_page: - self.upload(f"{translation_unit.identifier}/{language_code}", translation_unit.get_translation()) + source_translation_page: Optional[TranslatedPage] = self.source_fortraininglib.get_translation_units( + page_name, language_code) + + unchanged: int = 0 + modified: int = 0 + created: int = 0 + + if source_translation_page is None: + raise RuntimeError("Could not get translation units from source site") + + destination_translation_page: Optional[TranslatedPage] = self.destination_fortraininglib.get_translation_units( + page_name, language_code) + + if destination_translation_page is None: + raise RuntimeError("Could not get translation units from destination site") + + for source_translation_unit in source_translation_page: + source_translation = source_translation_unit.get_translation() + destination_translation_unit: Optional[TranslationUnit] = destination_translation_page.get_iteration_unit( + source_translation_unit.identifier) + if destination_translation_unit is None: + created += 1 + elif destination_translation_unit.get_translation() == source_translation: + unchanged += 1 + else: + modified += 1 + + self.upload(f"{source_translation_unit.identifier}/{language_code}", + source_translation) + + numTotal = unchanged + modified + created + print(f"Transfer of {numTotal} elements completed.") + print(f"unchanged: {unchanged}") + print(f"modified: {modified}") + print(f"created: {created}") def upload(self, identifier: str, translated_text: str): """Transfer a workshoot from one mediawiki system to another one""" From 8830ae0642150f0abc89bc199565d7402a43dc82 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 17:03:09 +0200 Subject: [PATCH 16/24] remove username for source because it is not required --- config.example.ini | 2 +- pywikitools/test/test_translated_page.py | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/config.example.ini b/config.example.ini index 298185e..9b56777 100644 --- a/config.example.ini +++ b/config.example.ini @@ -85,9 +85,9 @@ deeplapikey = MySecretDeepLAPIKey [transfer] # The mediawiki environment from where we copy it, for example 'test' source_site = local -source_username = CorrectBot # The mediawiki environment where we store the content above, for example 'local' destination_site = local +# The username for the destination site destination_username = CorrectBot # Example call for transfer: # python3 transfer.py "A_Daily_Prayer" "fr" diff --git a/pywikitools/test/test_translated_page.py b/pywikitools/test/test_translated_page.py index 46b07fe..d1b8f64 100644 --- a/pywikitools/test/test_translated_page.py +++ b/pywikitools/test/test_translated_page.py @@ -270,6 +270,17 @@ def test_returning_empty_strings(self): self.assertEqual(translated_page.get_worksheet_info().progress.total, 3) self.assertEqual(translated_page.get_worksheet_info().progress.translated, 3) + def test_get_iteration_unit(self): + unit1 = TranslationUnit("Test/1", "de", TEST_UNIT_WITH_LISTS, TEST_UNIT_WITH_LISTS_DE) + unit2 = TranslationUnit("Test/2", "de", TEST_UNIT_WITH_DEFINITION, TEST_UNIT_WITH_DEFINITION_DE_ERROR) + page = TranslatedPage("Test", "de", [unit1, unit2]) + retrieved1 = page.get_iteration_unit(unit1.identifier) + retrieved2 = page.get_iteration_unit(unit2.identifier) + notExisting = page.get_iteration_unit("not existing") + self.assertEqual(unit1, retrieved1) + self.assertEqual(unit2, retrieved2) + self.assertIsNone(notExisting) + def test_with_real_data(self): # TODO this test is closely tied to content on 4training.net that might change in the future fortraininglib = ForTrainingLib("https://test.4training.net") From deaeef9aa394164534894681fce82b8881c7a7d5 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 17:09:27 +0200 Subject: [PATCH 17/24] rename method to get translation unit --- pywikitools/lang/translated_page.py | 3 ++- pywikitools/test/test_translated_page.py | 6 +++--- transfer.py | 12 +----------- 3 files changed, 6 insertions(+), 15 deletions(-) diff --git a/pywikitools/lang/translated_page.py b/pywikitools/lang/translated_page.py index a337126..41238c6 100644 --- a/pywikitools/lang/translated_page.py +++ b/pywikitools/lang/translated_page.py @@ -419,7 +419,8 @@ def add_translation_unit(self, unit: TranslationUnit): """Append a translation unit. Infos are not invalidated""" self.units.append(unit) - def get_iteration_unit(self, identifier: str): + def get_translation_unit(self, identifier: str): + """Return translation unit by the given identifier""" for slice in self.units: if (slice.identifier == identifier): return slice diff --git a/pywikitools/test/test_translated_page.py b/pywikitools/test/test_translated_page.py index d1b8f64..60fac79 100644 --- a/pywikitools/test/test_translated_page.py +++ b/pywikitools/test/test_translated_page.py @@ -274,9 +274,9 @@ def test_get_iteration_unit(self): unit1 = TranslationUnit("Test/1", "de", TEST_UNIT_WITH_LISTS, TEST_UNIT_WITH_LISTS_DE) unit2 = TranslationUnit("Test/2", "de", TEST_UNIT_WITH_DEFINITION, TEST_UNIT_WITH_DEFINITION_DE_ERROR) page = TranslatedPage("Test", "de", [unit1, unit2]) - retrieved1 = page.get_iteration_unit(unit1.identifier) - retrieved2 = page.get_iteration_unit(unit2.identifier) - notExisting = page.get_iteration_unit("not existing") + retrieved1 = page.get_translation_unit(unit1.identifier) + retrieved2 = page.get_translation_unit(unit2.identifier) + notExisting = page.get_translation_unit("not existing") self.assertEqual(unit1, retrieved1) self.assertEqual(unit2, retrieved2) self.assertIsNone(notExisting) diff --git a/transfer.py b/transfer.py index f8bdeda..e2b5c9a 100644 --- a/transfer.py +++ b/transfer.py @@ -15,7 +15,6 @@ class TransferTool: def __init__(self, config: ConfigParser): if not config.has_option('transfer', 'source_site') or \ - not config.has_option('transfer', 'source_username') or \ not config.has_option('transfer', 'destination_site') or \ not config.has_option('transfer', 'destination_username'): raise RuntimeError("Missing settings for transfer in config.ini") @@ -27,15 +26,6 @@ def __init__(self, config: ConfigParser): family = Family() - self.source_wiki_site = pywikibot.Site(code=self.source_site, fam=family, - user=config.get('transfer', 'source_username')) - if not self.source_wiki_site.logged_in(): - self.source_wiki_site.login() - if not self.source_wiki_site.logged_in(): - raise RuntimeError("Login with pywikibot failed to source site failed.") - # Set throttle to 0 to speed up write operations (otherwise pywikibot would wait up to 10s after each write) - self.source_wiki_site.throttle.set_delays(delay=0, writedelay=0, absolute=True) - self.destination_wiki_site = pywikibot.Site(code=self.destination_site, fam=family, user=config.get('transfer', 'destination_username')) if not self.destination_wiki_site.logged_in(): @@ -69,7 +59,7 @@ def transfer(self, page_name, language_code): for source_translation_unit in source_translation_page: source_translation = source_translation_unit.get_translation() - destination_translation_unit: Optional[TranslationUnit] = destination_translation_page.get_iteration_unit( + destination_translation_unit: Optional[TranslationUnit] = destination_translation_page.get_translation_unit( source_translation_unit.identifier) if destination_translation_unit is None: created += 1 From de68fc5ffe4e08b1660cf6638ad308f84cec3e73 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 17:23:31 +0200 Subject: [PATCH 18/24] removed translation unit because we see the difference from the text property of media wiki bot --- pywikitools/lang/translated_page.py | 7 ----- pywikitools/test/test_translated_page.py | 11 -------- transfer.py | 34 +++++++++++------------- 3 files changed, 16 insertions(+), 36 deletions(-) diff --git a/pywikitools/lang/translated_page.py b/pywikitools/lang/translated_page.py index 41238c6..50c212d 100644 --- a/pywikitools/lang/translated_page.py +++ b/pywikitools/lang/translated_page.py @@ -419,13 +419,6 @@ def add_translation_unit(self, unit: TranslationUnit): """Append a translation unit. Infos are not invalidated""" self.units.append(unit) - def get_translation_unit(self, identifier: str): - """Return translation unit by the given identifier""" - for slice in self.units: - if (slice.identifier == identifier): - return slice - return None - def __iter__(self): """Make this class iterable in a simple way (not suitable for concurrency!)""" self._iterate_pos = 0 diff --git a/pywikitools/test/test_translated_page.py b/pywikitools/test/test_translated_page.py index 60fac79..46b07fe 100644 --- a/pywikitools/test/test_translated_page.py +++ b/pywikitools/test/test_translated_page.py @@ -270,17 +270,6 @@ def test_returning_empty_strings(self): self.assertEqual(translated_page.get_worksheet_info().progress.total, 3) self.assertEqual(translated_page.get_worksheet_info().progress.translated, 3) - def test_get_iteration_unit(self): - unit1 = TranslationUnit("Test/1", "de", TEST_UNIT_WITH_LISTS, TEST_UNIT_WITH_LISTS_DE) - unit2 = TranslationUnit("Test/2", "de", TEST_UNIT_WITH_DEFINITION, TEST_UNIT_WITH_DEFINITION_DE_ERROR) - page = TranslatedPage("Test", "de", [unit1, unit2]) - retrieved1 = page.get_translation_unit(unit1.identifier) - retrieved2 = page.get_translation_unit(unit2.identifier) - notExisting = page.get_translation_unit("not existing") - self.assertEqual(unit1, retrieved1) - self.assertEqual(unit2, retrieved2) - self.assertIsNone(notExisting) - def test_with_real_data(self): # TODO this test is closely tied to content on 4training.net that might change in the future fortraininglib = ForTrainingLib("https://test.4training.net") diff --git a/transfer.py b/transfer.py index e2b5c9a..5305331 100644 --- a/transfer.py +++ b/transfer.py @@ -7,7 +7,6 @@ from pywikitools.fortraininglib import ForTrainingLib from configparser import ConfigParser from pywikitools.lang.translated_page import TranslatedPage -from pywikitools.lang.translated_page import TranslationUnit TIMEOUT: int = 30 # Timeout after 30s (prevent indefinite hanging when there is network issues) @@ -21,6 +20,10 @@ def __init__(self, config: ConfigParser): self.logger: logging.Logger = logging.getLogger('pywikitools.transfer') + self.unchanged: int = 0 + self.modified: int = 0 + self.created: int = 0 + self.source_site = config.get('transfer', 'source_site') self.destination_site = config.get('transfer', 'destination_site') @@ -44,10 +47,6 @@ def transfer(self, page_name, language_code): source_translation_page: Optional[TranslatedPage] = self.source_fortraininglib.get_translation_units( page_name, language_code) - unchanged: int = 0 - modified: int = 0 - created: int = 0 - if source_translation_page is None: raise RuntimeError("Could not get translation units from source site") @@ -59,27 +58,26 @@ def transfer(self, page_name, language_code): for source_translation_unit in source_translation_page: source_translation = source_translation_unit.get_translation() - destination_translation_unit: Optional[TranslationUnit] = destination_translation_page.get_translation_unit( - source_translation_unit.identifier) - if destination_translation_unit is None: - created += 1 - elif destination_translation_unit.get_translation() == source_translation: - unchanged += 1 - else: - modified += 1 - self.upload(f"{source_translation_unit.identifier}/{language_code}", source_translation) - numTotal = unchanged + modified + created + numTotal = self.unchanged + self.modified + self.created print(f"Transfer of {numTotal} elements completed.") - print(f"unchanged: {unchanged}") - print(f"modified: {modified}") - print(f"created: {created}") + print(f"unchanged: {self.unchanged}") + print(f"modified: {self.modified}") + print(f"created: {self.created}") def upload(self, identifier: str, translated_text: str): """Transfer a workshoot from one mediawiki system to another one""" destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") + + if destination_mediawiki_page.text is None or "": + self.created += 1 + elif destination_mediawiki_page.text == translated_text: + self.unchanged += 1 + else: + self.modified += 1 + destination_mediawiki_page.text = translated_text destination_mediawiki_page.save( summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}'") From 42c291afef474315a21fde2fc74d4520573ae046 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 17:32:29 +0200 Subject: [PATCH 19/24] typos --- config.example.ini | 4 ++-- transfer.py | 14 +++----------- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/config.example.ini b/config.example.ini index 9b56777..32eaa30 100644 --- a/config.example.ini +++ b/config.example.ini @@ -84,10 +84,10 @@ deeplapikey = MySecretDeepLAPIKey [transfer] # The mediawiki environment from where we copy it, for example 'test' -source_site = local +source_site = test # The mediawiki environment where we store the content above, for example 'local' destination_site = local # The username for the destination site destination_username = CorrectBot # Example call for transfer: -# python3 transfer.py "A_Daily_Prayer" "fr" +# python3 transfer.py "A_Daily_Prayer" "fr" diff --git a/transfer.py b/transfer.py index 5305331..ab37da7 100644 --- a/transfer.py +++ b/transfer.py @@ -50,25 +50,17 @@ def transfer(self, page_name, language_code): if source_translation_page is None: raise RuntimeError("Could not get translation units from source site") - destination_translation_page: Optional[TranslatedPage] = self.destination_fortraininglib.get_translation_units( - page_name, language_code) - - if destination_translation_page is None: - raise RuntimeError("Could not get translation units from destination site") - for source_translation_unit in source_translation_page: source_translation = source_translation_unit.get_translation() self.upload(f"{source_translation_unit.identifier}/{language_code}", source_translation) numTotal = self.unchanged + self.modified + self.created - print(f"Transfer of {numTotal} elements completed.") - print(f"unchanged: {self.unchanged}") - print(f"modified: {self.modified}") - print(f"created: {self.created}") + print(f"Transfer of {numTotal} elements for '{page_name}/{language_code}' from '{self.source_site}' to '{self.destination_site}' completed.") + print(f"unchanged: {self.unchanged} | modified: {self.modified} | created: {self.created}") def upload(self, identifier: str, translated_text: str): - """Transfer a workshoot from one mediawiki system to another one""" + """Transfer a worksheet from one mediawiki system to another one""" destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") if destination_mediawiki_page.text is None or "": From 1b915b08fddfdbafcdd0906872b5315e7abadbe2 Mon Sep 17 00:00:00 2001 From: Jonatan Giuseppe Schnyder Date: Sat, 18 Oct 2025 17:52:42 +0200 Subject: [PATCH 20/24] Custom summary when uploading (with --message ) --- transfer.py | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/transfer.py b/transfer.py index ab37da7..7948757 100644 --- a/transfer.py +++ b/transfer.py @@ -43,7 +43,7 @@ def __init__(self, config: ConfigParser): self.destination_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.destination_site, ''), family.scriptpath(self.destination_site)) - def transfer(self, page_name, language_code): + def transfer(self, page_name, language_code, message): source_translation_page: Optional[TranslatedPage] = self.source_fortraininglib.get_translation_units( page_name, language_code) @@ -53,13 +53,13 @@ def transfer(self, page_name, language_code): for source_translation_unit in source_translation_page: source_translation = source_translation_unit.get_translation() self.upload(f"{source_translation_unit.identifier}/{language_code}", - source_translation) + source_translation, message) numTotal = self.unchanged + self.modified + self.created print(f"Transfer of {numTotal} elements for '{page_name}/{language_code}' from '{self.source_site}' to '{self.destination_site}' completed.") print(f"unchanged: {self.unchanged} | modified: {self.modified} | created: {self.created}") - def upload(self, identifier: str, translated_text: str): + def upload(self, identifier: str, translated_text: str, message): """Transfer a worksheet from one mediawiki system to another one""" destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") @@ -71,8 +71,11 @@ def upload(self, identifier: str, translated_text: str): self.modified += 1 destination_mediawiki_page.text = translated_text - destination_mediawiki_page.save( - summary=f"Transfer of '{identifier}' from '{self.source_site}' to '{self.destination_site}'") + if message is None: + destination_mediawiki_page.save() + else: + destination_mediawiki_page.save( + summary=message) if __name__ == "__main__": @@ -80,10 +83,12 @@ def upload(self, identifier: str, translated_text: str): description="Transfer a worksheet for a certain language from one site to another site.") parser.add_argument("worksheet_name", help="Name of the worksheet to transfer.") parser.add_argument("language_code", help="Target language code for transfer.") + parser.add_argument("--message", nargs=1, help="This is a test") args = parser.parse_args() config = ConfigParser() config.read(join(dirname(abspath(__file__)), "config.ini")) transfer_tool = TransferTool(config) - transfer_tool.transfer(args.worksheet_name, args.language_code) + print(args) + transfer_tool.transfer(args.worksheet_name, args.language_code, args.message) From f5559effab9c550baabd8bdff3469e022d624e3d Mon Sep 17 00:00:00 2001 From: Jonatan Giuseppe Schnyder Date: Sat, 18 Oct 2025 17:55:55 +0200 Subject: [PATCH 21/24] remove unnecesairy print statement --- transfer.py | 1 - 1 file changed, 1 deletion(-) diff --git a/transfer.py b/transfer.py index 7948757..4b6ff47 100644 --- a/transfer.py +++ b/transfer.py @@ -90,5 +90,4 @@ def upload(self, identifier: str, translated_text: str, message): config.read(join(dirname(abspath(__file__)), "config.ini")) transfer_tool = TransferTool(config) - print(args) transfer_tool.transfer(args.worksheet_name, args.language_code, args.message) From d3d1eb4a6583c623bed618e126ee1ee9dd63c186 Mon Sep 17 00:00:00 2001 From: Jonatan Giuseppe Schnyder Date: Sat, 18 Oct 2025 18:30:14 +0200 Subject: [PATCH 22/24] Testcompatibility for Custom summary (--message ) --- pywikitools/test/test_transfer.py | 11 ++++++++--- transfer.py | 4 ++-- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/pywikitools/test/test_transfer.py b/pywikitools/test/test_transfer.py index aa8bb47..270c06f 100644 --- a/pywikitools/test/test_transfer.py +++ b/pywikitools/test/test_transfer.py @@ -20,14 +20,19 @@ def setUp(self, mock_pywikibot_site): @patch('pywikibot.Page') def test_upload(self, mock_page): self.transfer_tool.upload("Test_Page/1/fr", "Test transfer") + mock_page.return_value.save.assert_called_once() + + @patch('pywikibot.Page') + def test_upload_with_message(self, mock_page): + self.transfer_tool.upload("Test_Page/1/fr", "Test transfer", "Test") mock_page.return_value.save.assert_called_once_with( - summary="Transfer of 'Test_Page/1/fr' from 'test' to 'local'") + summary="Test") + @patch('pywikibot.Page') def test_transfer(self, mock_page): self.transfer_tool.transfer("A_Daily_Prayer", "fr") - mock_page.return_value.save.assert_called_with( - summary="Transfer of 'A_Daily_Prayer/21/fr' from 'test' to 'local'") + mock_page.return_value.save.assert_called() if __name__ == "__main__": diff --git a/transfer.py b/transfer.py index 4b6ff47..112bd22 100644 --- a/transfer.py +++ b/transfer.py @@ -43,7 +43,7 @@ def __init__(self, config: ConfigParser): self.destination_fortraininglib: ForTrainingLib = ForTrainingLib(family.base_url(self.destination_site, ''), family.scriptpath(self.destination_site)) - def transfer(self, page_name, language_code, message): + def transfer(self, page_name, language_code, message=None): source_translation_page: Optional[TranslatedPage] = self.source_fortraininglib.get_translation_units( page_name, language_code) @@ -59,7 +59,7 @@ def transfer(self, page_name, language_code, message): print(f"Transfer of {numTotal} elements for '{page_name}/{language_code}' from '{self.source_site}' to '{self.destination_site}' completed.") print(f"unchanged: {self.unchanged} | modified: {self.modified} | created: {self.created}") - def upload(self, identifier: str, translated_text: str, message): + def upload(self, identifier: str, translated_text: str, message=None): """Transfer a worksheet from one mediawiki system to another one""" destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") From 7d8f3fe7941d7fe19969a7f4dfca8cffb7420378 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 18:35:13 +0200 Subject: [PATCH 23/24] tests for statistics --- pywikitools/test/test_transfer.py | 21 ++++++++++++++++++--- transfer.py | 10 +++++++--- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/pywikitools/test/test_transfer.py b/pywikitools/test/test_transfer.py index 270c06f..b4ee31a 100644 --- a/pywikitools/test/test_transfer.py +++ b/pywikitools/test/test_transfer.py @@ -11,8 +11,7 @@ class TestTransferTool(unittest.TestCase): def setUp(self, mock_pywikibot_site): mock_pywikibot_site.return_value.logged_in.return_value = True config = ConfigParser() - config.read_dict({"transfer": {"source_username": "User1", - "source_site": "test", + config.read_dict({"transfer": {"source_site": "test", "destination_username": "User2", "destination_site": "local"}}) self.transfer_tool = TransferTool(config) @@ -27,13 +26,29 @@ def test_upload_with_message(self, mock_page): self.transfer_tool.upload("Test_Page/1/fr", "Test transfer", "Test") mock_page.return_value.save.assert_called_once_with( summary="Test") - @patch('pywikibot.Page') def test_transfer(self, mock_page): self.transfer_tool.transfer("A_Daily_Prayer", "fr") mock_page.return_value.save.assert_called() + @patch('pywikibot.Page') + def test_upload_created(self, mock_page): + mock_page.return_value.exists.return_value = False + self.transfer_tool.upload("Test_Page/2/it", "Test transfer", "v1") + self.assertEqual(self.transfer_tool.created, 1) + self.assertEqual(self.transfer_tool.modified, 0) + self.assertEqual(self.transfer_tool.unchanged, 0) + + @patch('pywikibot.Page') + def test_upload_modified(self, mock_page): + mock_page.return_value.exists.return_value = True + mock_page.return_value.text.return_value = "Old entry" + self.transfer_tool.upload("Test_Page/2/it", "New entry", "v4") + self.assertEqual(self.transfer_tool.created, 0) + self.assertEqual(self.transfer_tool.modified, 1) + self.assertEqual(self.transfer_tool.unchanged, 0) + if __name__ == "__main__": unittest.main() diff --git a/transfer.py b/transfer.py index 112bd22..b89fef5 100644 --- a/transfer.py +++ b/transfer.py @@ -49,6 +49,10 @@ def transfer(self, page_name, language_code, message=None): if source_translation_page is None: raise RuntimeError("Could not get translation units from source site") + + self.unchanged: int = 0 + self.modified: int = 0 + self.created: int = 0 for source_translation_unit in source_translation_page: source_translation = source_translation_unit.get_translation() @@ -63,7 +67,7 @@ def upload(self, identifier: str, translated_text: str, message=None): """Transfer a worksheet from one mediawiki system to another one""" destination_mediawiki_page = pywikibot.Page(self.destination_wiki_site, f"Translations:{identifier}") - if destination_mediawiki_page.text is None or "": + if not destination_mediawiki_page.exists(): self.created += 1 elif destination_mediawiki_page.text == translated_text: self.unchanged += 1 @@ -71,11 +75,11 @@ def upload(self, identifier: str, translated_text: str, message=None): self.modified += 1 destination_mediawiki_page.text = translated_text + if message is None: destination_mediawiki_page.save() else: - destination_mediawiki_page.save( - summary=message) + destination_mediawiki_page.save(summary=message) if __name__ == "__main__": From d9928411e3fe22e0beae4a7cc00603eed3833978 Mon Sep 17 00:00:00 2001 From: TobiasBeck777 Date: Sat, 18 Oct 2025 18:40:01 +0200 Subject: [PATCH 24/24] fix test + linting --- transfer.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/transfer.py b/transfer.py index b89fef5..85d83d0 100644 --- a/transfer.py +++ b/transfer.py @@ -49,7 +49,7 @@ def transfer(self, page_name, language_code, message=None): if source_translation_page is None: raise RuntimeError("Could not get translation units from source site") - + self.unchanged: int = 0 self.modified: int = 0 self.created: int = 0 @@ -60,7 +60,8 @@ def transfer(self, page_name, language_code, message=None): source_translation, message) numTotal = self.unchanged + self.modified + self.created - print(f"Transfer of {numTotal} elements for '{page_name}/{language_code}' from '{self.source_site}' to '{self.destination_site}' completed.") + print(f"Transfer of {numTotal} elements for '{page_name}/{language_code}' " + + "from '{self.source_site}' to '{self.destination_site}' completed.") print(f"unchanged: {self.unchanged} | modified: {self.modified} | created: {self.created}") def upload(self, identifier: str, translated_text: str, message=None):