From 8abd8dfd23e212047bffa6e4bc81913cd572b30c Mon Sep 17 00:00:00 2001 From: Frisk Date: Fri, 16 Apr 2021 20:35:01 +0200 Subject: [PATCH 1/7] #195 Add config mitigation for new settings --- src/exceptions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/exceptions.py b/src/exceptions.py index 0f5b6aa..620f134 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -2,4 +2,4 @@ class MWError(Exception): pass class ArticleCommentError(Exception): - pass \ No newline at end of file + pass From 065d67c90c13e0449faae9ad30c48cdc2d0aed74 Mon Sep 17 00:00:00 2001 From: Frisk Date: Fri, 16 Apr 2021 20:37:09 +0200 Subject: [PATCH 2/7] #195 Add config mitigation for new settings --- src/migrations/1.13.1.1.py | 23 +++++++++++++++++++++++ src/migrations/__init__.py | 0 src/migrations/utils.py | 18 ++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 src/migrations/1.13.1.1.py create mode 100644 src/migrations/__init__.py create mode 100644 src/migrations/utils.py diff --git a/src/migrations/1.13.1.1.py b/src/migrations/1.13.1.1.py new file mode 100644 index 0000000..83d8600 --- /dev/null +++ b/src/migrations/1.13.1.1.py @@ -0,0 +1,23 @@ +from src.configloader import settings +import logging + +from src.migrations.utils import return_example_file + +logger = logging.getLogger("rcgcdw.migrations.1.13.1.1") +base_file = return_example_file() +new_settings = settings.copy() + + +def run(): + if "event_appearance" not in settings: + try: + settings["event_appearance"] = {} + struct = settings['appearance']['embed'] + for key, value in struct.items(): + settings["event_appearance"][key] = value + settings["event_appearance"][key]["emoji"] = base_file["event_appearance"] + except KeyError: + logger.error("Failed to migrate appearance embed.") + else: # Don't do migrations + return + diff --git a/src/migrations/__init__.py b/src/migrations/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/migrations/utils.py b/src/migrations/utils.py new file mode 100644 index 0000000..55adfb5 --- /dev/null +++ b/src/migrations/utils.py @@ -0,0 +1,18 @@ +import requests +import logging +import json + +discussion_logger = logging.getLogger("rcgcdw.migrations.utils") + + +def return_example_file() -> dict: + try: + with open('settings.json.example', 'r') as example_file: + return json.loads(example_file.read()) + except FileNotFoundError: + try: + f = requests.get("https://gitlab.com/piotrex43/RcGcDw/-/raw/master/settings.json.example") + except: + raise + return json.loads(f.text) + From cab2d864811ce7b07627638c150ba3ebe8afe18d Mon Sep 17 00:00:00 2001 From: Frisk Date: Sat, 17 Apr 2021 16:13:24 +0200 Subject: [PATCH 3/7] Added migration for config --- src/configloader.py | 51 +++++++++++++++++++++++--------------- src/migrations/1.13.1.1.py | 23 ----------------- src/migrations/11311.py | 46 ++++++++++++++++++++++++++++++++++ src/migrations/__init__.py | 1 + src/migrations/utils.py | 4 ++- src/rcgcdw.py | 2 +- 6 files changed, 82 insertions(+), 45 deletions(-) delete mode 100644 src/migrations/1.13.1.1.py create mode 100644 src/migrations/11311.py diff --git a/src/configloader.py b/src/configloader.py index 50fa9c6..5bd7467 100644 --- a/src/configloader.py +++ b/src/configloader.py @@ -1,22 +1,33 @@ -import json, sys, logging +import json +import logging +import sys + +global settings + + +def load_settings(): + global settings + try: # load settings + with open("settings.json", encoding="utf8") as sfile: + settings = json.load(sfile) + if settings["limitrefetch"] < settings["limit"] and settings["limitrefetch"] != -1: + settings["limitrefetch"] = settings["limit"] + if "user-agent" in settings["header"]: + settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.13.1") # set the version in the useragent + except FileNotFoundError: + logging.critical("No config file could be found. Please make sure settings.json is in the directory.") + sys.exit(1) + # Set the cooldown to 15 seconds if it's a wiki farm like Fandom or Gamepedia and the cooldown is even lower than that. + # Look, it's unreasonable to have even higher refresh rate than that, seriously. Setting it even lower can cause issues + # for all users of the script for high usage of farm's servers. So please, do not remove this code unless you absolutely + # know what you are doing <3 + if any(("fandom.com" in settings["wiki_url"], "gamepedia.com" in settings["wiki_url"])): + if settings["cooldown"] < 15: + settings["cooldown"] = 15 + if settings["fandom_discussions"]["cooldown"] < 15: + settings["fandom_discussions"]["cooldown"] = 15 + + +load_settings() -try: # load settings - with open("settings.json", encoding="utf8") as sfile: - settings = json.load(sfile) - if settings["limitrefetch"] < settings["limit"] and settings["limitrefetch"] != -1: - settings["limitrefetch"] = settings["limit"] - if "user-agent" in settings["header"]: - settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.13.1") # set the version in the useragent -except FileNotFoundError: - logging.critical("No config file could be found. Please make sure settings.json is in the directory.") - sys.exit(1) -# Set the cooldown to 15 seconds if it's a wiki farm like Fandom or Gamepedia and the cooldown is even lower than that. -# Look, it's unreasonable to have even higher refresh rate than that, seriously. Setting it even lower can cause issues -# for all users of the script for high usage of farm's servers. So please, do not remove this code unless you absolutely -# know what you are doing <3 -if any(("fandom.com" in settings["wiki_url"], "gamepedia.com" in settings["wiki_url"])): - if settings["cooldown"] < 15: - settings["cooldown"] = 15 -if settings["fandom_discussions"]["cooldown"] < 15: - settings["fandom_discussions"]["cooldown"] = 15 diff --git a/src/migrations/1.13.1.1.py b/src/migrations/1.13.1.1.py deleted file mode 100644 index 83d8600..0000000 --- a/src/migrations/1.13.1.1.py +++ /dev/null @@ -1,23 +0,0 @@ -from src.configloader import settings -import logging - -from src.migrations.utils import return_example_file - -logger = logging.getLogger("rcgcdw.migrations.1.13.1.1") -base_file = return_example_file() -new_settings = settings.copy() - - -def run(): - if "event_appearance" not in settings: - try: - settings["event_appearance"] = {} - struct = settings['appearance']['embed'] - for key, value in struct.items(): - settings["event_appearance"][key] = value - settings["event_appearance"][key]["emoji"] = base_file["event_appearance"] - except KeyError: - logger.error("Failed to migrate appearance embed.") - else: # Don't do migrations - return - diff --git a/src/migrations/11311.py b/src/migrations/11311.py new file mode 100644 index 0000000..ec966b2 --- /dev/null +++ b/src/migrations/11311.py @@ -0,0 +1,46 @@ +from src.configloader import settings, load_settings +import logging +import shutil +import time +import json +import sys + +from src.migrations.utils import return_example_file + +logger = logging.getLogger("rcgcdw.migrations.1.13.1.1") +base_file = return_example_file() +new_settings = settings.copy() + +def run(): + global base_file + if "event_appearance" not in settings: + logger.info("Running migration 1.13.1.1") + if "event_appearance" not in base_file: # if local base file is outdated, download from repo + base_file = return_example_file(force=True) + try: + struct = settings['appearance']['embed'] + new_settings["event_appearance"] = {} + keys = [] + for key, value in struct.items(): + if key not in ("show_edit_changes", "show_footer", "embed_images"): + new_settings["event_appearance"][key] = value + try: + new_settings["event_appearance"][key]["emoji"] = base_file["event_appearance"][key]["emoji"] + except KeyError: + new_settings["event_appearance"][key]["emoji"] = "" + keys.append(key) + for item in keys: + del new_settings['appearance']['embed'][item] + except KeyError: + logger.exception("Failed to migrate appearance embed.") + sys.exit(1) + shutil.copy("settings.json", "settings.json.{}.bak".format(int(time.time()))) + with open("settings.json", "w") as new_write: + new_write.write(json.dumps(new_settings, indent=4)) + load_settings() + logger.info("Migration 1.13.1.1 has been successful.") + else: + logger.debug("Ignoring migration 1.13.1.1") + + +run() diff --git a/src/migrations/__init__.py b/src/migrations/__init__.py index e69de29..8e786d0 100644 --- a/src/migrations/__init__.py +++ b/src/migrations/__init__.py @@ -0,0 +1 @@ +__all__ = ["11311"] diff --git a/src/migrations/utils.py b/src/migrations/utils.py index 55adfb5..fc448f1 100644 --- a/src/migrations/utils.py +++ b/src/migrations/utils.py @@ -5,8 +5,10 @@ import json discussion_logger = logging.getLogger("rcgcdw.migrations.utils") -def return_example_file() -> dict: +def return_example_file(force=False) -> dict: try: + if force: + raise FileNotFoundError with open('settings.json.example', 'r') as example_file: return json.loads(example_file.read()) except FileNotFoundError: diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 1970e18..2bd33e5 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -46,7 +46,7 @@ TESTING = True if "--test" in sys.argv else False # debug mode, pipeline testin logging.config.dictConfig(settings["logging"]) logger = logging.getLogger("rcgcdw") logger.debug("Current settings: {settings}".format(settings=settings)) - +from src.migrations import * # migrations after logging storage = datafile # Remove previous data holding file if exists and limitfetch allows From fcde6265b5031d03bbee41ae25d68ffe8e5548fe Mon Sep 17 00:00:00 2001 From: Frisk Date: Sat, 17 Apr 2021 16:35:54 +0200 Subject: [PATCH 4/7] Fixed #194 --- src/discussion_formatters.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/discussion_formatters.py b/src/discussion_formatters.py index 8cfdb23..6b8b624 100644 --- a/src/discussion_formatters.py +++ b/src/discussion_formatters.py @@ -150,7 +150,7 @@ def embed_formatter(post_type, post, article_paths): if post["_embedded"]["thread"][0]["tags"]: tag_displayname = [] for tag in post["_embedded"]["thread"][0]["tags"]: - tag_displayname.append("[{title}]({url})".format(title=tag["articleTitle"], url=create_article_path(tag["articleTitle"]))) + tag_displayname.append("[{title}]({url})".format(title=tag["articleTitle"], url=create_article_path(quote_plus(tag["articleTitle"].replace(" ", "_"), "/:?=&")))) if len(", ".join(tag_displayname)) > 1000: embed.add_field(_("Tags"), _("{} tags").format(len(post["_embedded"]["thread"][0]["tags"]))) else: From 1ad8528f79ddff291ee4eb1b26d6b285cd586193 Mon Sep 17 00:00:00 2001 From: Frisk <702385-piotrex43@users.noreply.gitlab.com> Date: Tue, 20 Apr 2021 13:11:43 +0000 Subject: [PATCH 5/7] Bump version because I apparently messed up --- src/configloader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/configloader.py b/src/configloader.py index 5bd7467..9ca412a 100644 --- a/src/configloader.py +++ b/src/configloader.py @@ -13,7 +13,7 @@ def load_settings(): if settings["limitrefetch"] < settings["limit"] and settings["limitrefetch"] != -1: settings["limitrefetch"] = settings["limit"] if "user-agent" in settings["header"]: - settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.13.1") # set the version in the useragent + settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.13.1.2") # set the version in the useragent except FileNotFoundError: logging.critical("No config file could be found. Please make sure settings.json is in the directory.") sys.exit(1) From 65400f3fb5b5220aa444808d278a84626394f963 Mon Sep 17 00:00:00 2001 From: Frisk Date: Tue, 27 Apr 2021 23:51:59 +0200 Subject: [PATCH 6/7] Fixes to #200 and #198 --- src/misc.py | 6 +++--- src/rc.py | 24 +++++++++++++++++------- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/src/misc.py b/src/misc.py index f241806..ee9ab47 100644 --- a/src/misc.py +++ b/src/misc.py @@ -54,7 +54,7 @@ class DataFile: def generate_datafile(): """Generate a data.json file from a template.""" try: - with open("data.json", 'w') as data: + with open("data.json", 'w', encoding="utf-8") as data: data.write(json.dumps(data_template, indent=4)) except PermissionError: misc_logger.critical("Could not create a data file (no permissions). No way to store last edit.") @@ -65,7 +65,7 @@ class DataFile: :rtype: dict """ try: - with open("data.json") as data: + with open("data.json", encoding="utf-8") as data: return json.loads(data.read()) except FileNotFoundError: self.generate_datafile() @@ -77,7 +77,7 @@ class DataFile: if self.changed is False: # don't cause unnecessary write operations return try: - with open("data.json", "w") as data_file: + with open("data.json", "w", encoding="utf-8") as data_file: data_file.write(json.dumps(self.data, indent=4)) self.changed = False except PermissionError: diff --git a/src/rc.py b/src/rc.py index 2accac9..2b6b5a4 100644 --- a/src/rc.py +++ b/src/rc.py @@ -63,6 +63,7 @@ class Recent_Changes_Class(object): self.session = session self.logged_in = False self.initial_run_complete = False + self.memory_id = None # Used only when limitrefetch is set to -1 to avoid reading from storage @staticmethod def handle_mw_errors(request): @@ -114,9 +115,12 @@ class Recent_Changes_Class(object): messagequeue.resend_msgs() last_check = self.fetch_changes(amount=amount) if last_check is not None: - storage["rcid"] = last_check[0] if last_check[0] else storage["rcid"] - storage["abuse_log_id"] = last_check[1] if last_check[1] else storage["abuse_log_id"] - storage.save_datafile() + if settings["limitrefetch"] != -1: + storage["rcid"] = last_check[0] if last_check[0] else storage["rcid"] + storage["abuse_log_id"] = last_check[1] if last_check[1] else storage["abuse_log_id"] + storage.save_datafile() + else: + self.memory_id = last_check self.initial_run_complete = True def fetch_recentchanges_request(self, amount): @@ -155,8 +159,11 @@ class Recent_Changes_Class(object): categorize_events = {} new_events = 0 changes.reverse() - highest_id = recent_id = storage["rcid"] - dry_run = True if recent_id is None else False + if settings["limitrefetch"] == -1 and self.memory_id is not None: + highest_id = recent_id = self.memory_id[0] + else: + highest_id = recent_id = storage["rcid"] + dry_run = True if recent_id is None or (self.memory_id is None and settings["limitrefetch"] == -1) else False for change in changes: if not dry_run and not (change["rcid"] <= recent_id): new_events += 1 @@ -217,8 +224,11 @@ class Recent_Changes_Class(object): if not abuse_log: return None abuse_log.reverse() - recent_id = storage["abuse_log_id"] - dryrun = True if recent_id is None else False + if self.memory_id is not None and settings["limitrefetch"] == -1: + recent_id = self.memory_id[1] + else: + recent_id = storage["abuse_log_id"] + dryrun = True if recent_id is None or (self.initial_run_complete is False and settings["limitrefetch"] == -1) else False for entry in abuse_log: if dryrun: continue From 01f2fb682a7ae7ea3f32b30141795557383151f1 Mon Sep 17 00:00:00 2001 From: MarkusRost <2701034-MarkusRost@users.noreply.gitlab.com> Date: Thu, 29 Apr 2021 06:12:27 +0000 Subject: [PATCH 7/7] Encode ALL files to utf8 --- scripts/configbuilder.py | 8 ++++---- src/migrations/11311.py | 2 +- src/migrations/utils.py | 2 +- src/rcgcdw.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/scripts/configbuilder.py b/scripts/configbuilder.py index 9c02d73..8677506 100644 --- a/scripts/configbuilder.py +++ b/scripts/configbuilder.py @@ -196,7 +196,7 @@ def yes_no(answer): print("Welcome in RcGcDw config builder! This script is still work in progress so beware! You can accept the default value if provided in the question by using Enter key without providing any other input.\nWARNING! Your current settings.json will be overwritten if you continue!") try: # load settings - with open("../settings.json.example") as sfile: + with open("../settings.json.example", encoding="utf-8") as sfile: settings = json.load(sfile) except FileNotFoundError: if yes_no(default_or_custom(input("Template config (settings.json.example) could not be found. Download the most recent stable one from master branch? (https://gitlab.com/piotrex43/RcGcDw/raw/master/settings.json.example)? (Y/n)"), "y")): @@ -468,7 +468,7 @@ class AdvancedSettings: try: BasicSettings() shutil.copy("settings.json", "settings.json.bak") - with open("settings.json", "w") as settings_file: + with open("settings.json", "w", encoding="utf-8") as settings_file: settings_file.write(json.dumps(settings, indent=4)) if "--advanced" in sys.argv: print("Basic part of the config has been completed. Starting the advanced part...") @@ -478,5 +478,5 @@ except KeyboardInterrupt: if not yes_no(default_or_custom(input("\nSave the config before exiting? (y/N)"),"n")): sys.exit(0) else: - with open("settings.json", "w") as settings_file: - settings_file.write(json.dumps(settings, indent=4)) \ No newline at end of file + with open("settings.json", "w", encoding="utf-8") as settings_file: + settings_file.write(json.dumps(settings, indent=4)) diff --git a/src/migrations/11311.py b/src/migrations/11311.py index ec966b2..9ead1da 100644 --- a/src/migrations/11311.py +++ b/src/migrations/11311.py @@ -35,7 +35,7 @@ def run(): logger.exception("Failed to migrate appearance embed.") sys.exit(1) shutil.copy("settings.json", "settings.json.{}.bak".format(int(time.time()))) - with open("settings.json", "w") as new_write: + with open("settings.json", "w", encoding="utf-8") as new_write: new_write.write(json.dumps(new_settings, indent=4)) load_settings() logger.info("Migration 1.13.1.1 has been successful.") diff --git a/src/migrations/utils.py b/src/migrations/utils.py index fc448f1..5e66bc8 100644 --- a/src/migrations/utils.py +++ b/src/migrations/utils.py @@ -9,7 +9,7 @@ def return_example_file(force=False) -> dict: try: if force: raise FileNotFoundError - with open('settings.json.example', 'r') as example_file: + with open('settings.json.example', 'r', encoding="utf-8") as example_file: return json.loads(example_file.read()) except FileNotFoundError: try: diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 2bd33e5..a9b0d41 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -52,7 +52,7 @@ storage = datafile # Remove previous data holding file if exists and limitfetch allows if settings["limitrefetch"] != -1 and os.path.exists("lastchange.txt") is True: - with open("lastchange.txt", 'r') as sfile: + with open("lastchange.txt", 'r', encoding="utf-8") as sfile: logger.info("Converting old lastchange.txt file into new data storage data.json...") storage["rcid"] = int(sfile.read().strip()) datafile.save_datafile()