diff --git a/extensions/base/discussions.py b/extensions/base/discussions.py index 474a008..0bb1bd5 100644 --- a/extensions/base/discussions.py +++ b/extensions/base/discussions.py @@ -103,7 +103,7 @@ class DiscussionsFromHellParser: for mark in marks: if mark["type"] == "mention": prefix += "[" - suffix = "]({wiki}f/u/{userid}){suffix}".format(wiki=self.ctx.settings["fandom_discussions"]["wiki_url"], + suffix = "]({wiki}f/u/{userid}){suffix}".format(wiki=self.ctx.settings.WIKI_SCRIPT_PATH, userid=mark["attrs"]["userId"], suffix=suffix) elif mark["type"] == "strong": prefix += "**" diff --git a/locale/de/LC_MESSAGES/misc.mo b/locale/de/LC_MESSAGES/misc.mo index b3af6e6..5f31ff6 100644 Binary files a/locale/de/LC_MESSAGES/misc.mo and b/locale/de/LC_MESSAGES/misc.mo differ diff --git a/locale/de/LC_MESSAGES/misc.po b/locale/de/LC_MESSAGES/misc.po index 5a7dd58..fb16ff1 100644 --- a/locale/de/LC_MESSAGES/misc.po +++ b/locale/de/LC_MESSAGES/misc.po @@ -2,7 +2,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-06-16 23:56+0200\n" +"POT-Creation-Date: 2024-02-24 17:51+0100\n" "PO-Revision-Date: 2020-08-03 13:44+0000\n" "Last-Translator: MarkusRost <>\n" "Language-Team: German \n" -"Language-Team: Hindi " -"\n" +"Language-Team: Hindi \n" "Language: hi\n" "MIME-Version: 1.0\n" "Content-Type: text/plain; charset=UTF-8\n" @@ -19,70 +19,69 @@ msgstr "" "Plural-Forms: nplurals=2; plural=n > 1;\n" "X-Generator: Weblate 4.12.1\n" -#: src/misc.py:45 +#: src/misc.py:115 msgid "Location" msgstr "स्थान" -#: src/misc.py:45 +#: src/misc.py:115 msgid "About me" msgstr "मेरे बारे में" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Google link" msgstr "Google कड़ी" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Facebook link" msgstr "Facebook कड़ी" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Twitter link" msgstr "Twitter कड़ी" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Reddit link" msgstr "Reddit कड़ी" -#: src/misc.py:45 +#: src/misc.py:118 msgid "Twitch link" msgstr "Twitch कड़ी" -#: src/misc.py:45 +#: src/misc.py:118 msgid "PSN link" msgstr "PSN कड़ी" -#: src/misc.py:45 +#: src/misc.py:119 msgid "VK link" msgstr "VK कड़ी" -#: src/misc.py:45 +#: src/misc.py:119 msgid "XBL link" msgstr "XBL कड़ी" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Steam link" msgstr "Steam कड़ी" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Discord handle" msgstr "डिस्कॉर्ड खाता" -#: src/misc.py:45 +#: src/misc.py:121 msgid "Battle.net handle" msgstr "Battle.net खाता" -#: src/misc.py:140 -msgid "" -"\n" -"__And more__" -msgstr "" -"\n" -"और कई सारे" - -#: src/misc.py:330 +#: src/misc.py:127 msgid "Unknown" msgstr "अज्ञात" -#: src/misc.py:332 +#: src/misc.py:129 msgid "unknown" msgstr "अनजान" + +#~ msgid "" +#~ "\n" +#~ "__And more__" +#~ msgstr "" +#~ "\n" +#~ "और कई सारे" diff --git a/locale/pl/LC_MESSAGES/misc.mo b/locale/pl/LC_MESSAGES/misc.mo index 4155b44..b3cf25b 100644 Binary files a/locale/pl/LC_MESSAGES/misc.mo and b/locale/pl/LC_MESSAGES/misc.mo differ diff --git a/locale/pl/LC_MESSAGES/misc.po b/locale/pl/LC_MESSAGES/misc.po index 467465c..74dcb1b 100644 --- a/locale/pl/LC_MESSAGES/misc.po +++ b/locale/pl/LC_MESSAGES/misc.po @@ -6,7 +6,7 @@ msgid "" msgstr "" "Project-Id-Version: RcGcDw\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-06-16 23:56+0200\n" +"POT-Creation-Date: 2024-02-24 17:51+0100\n" "PO-Revision-Date: 2020-03-17 20:57+0100\n" "Last-Translator: Frisk <>\n" "Language-Team: \n" @@ -19,70 +19,69 @@ msgstr "" "Plural-Forms: nplurals=3; plural=(n==1 ? 0 : n%10>=2 && n%10<=4 && (n%100<10 " "|| n%100>=20) ? 1 : 2);\n" -#: src/misc.py:45 +#: src/misc.py:115 msgid "Location" msgstr "Lokacja" -#: src/misc.py:45 +#: src/misc.py:115 msgid "About me" msgstr "O mnie" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Google link" msgstr "link Google" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Facebook link" msgstr "link Facebook" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Twitter link" msgstr "link Twitter" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Reddit link" msgstr "link Reddit" -#: src/misc.py:45 +#: src/misc.py:118 msgid "Twitch link" msgstr "link Twitch" -#: src/misc.py:45 +#: src/misc.py:118 msgid "PSN link" msgstr "link PSN" -#: src/misc.py:45 +#: src/misc.py:119 msgid "VK link" msgstr "link VK" -#: src/misc.py:45 +#: src/misc.py:119 msgid "XBL link" msgstr "link XBL" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Steam link" msgstr "link Steam" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Discord handle" msgstr "konto Discord" -#: src/misc.py:45 +#: src/misc.py:121 msgid "Battle.net handle" msgstr "konto Battle.net" -#: src/misc.py:140 -msgid "" -"\n" -"__And more__" -msgstr "" -"\n" -"__Oraz więcej__" - -#: src/misc.py:330 +#: src/misc.py:127 msgid "Unknown" msgstr "Nieznana" -#: src/misc.py:332 +#: src/misc.py:129 msgid "unknown" msgstr "nieznana sekcja" + +#~ msgid "" +#~ "\n" +#~ "__And more__" +#~ msgstr "" +#~ "\n" +#~ "__Oraz więcej__" diff --git a/locale/pt-br/LC_MESSAGES/misc.mo b/locale/pt-br/LC_MESSAGES/misc.mo index 5030289..7d84af6 100644 Binary files a/locale/pt-br/LC_MESSAGES/misc.mo and b/locale/pt-br/LC_MESSAGES/misc.mo differ diff --git a/locale/pt-br/LC_MESSAGES/misc.po b/locale/pt-br/LC_MESSAGES/misc.po index cd36524..488853d 100644 --- a/locale/pt-br/LC_MESSAGES/misc.po +++ b/locale/pt-br/LC_MESSAGES/misc.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-06-16 23:56+0200\n" +"POT-Creation-Date: 2024-02-24 17:51+0100\n" "PO-Revision-Date: 2020-11-27 13:07+0000\n" "Last-Translator: dr03ramos \n" "Language-Team: Portuguese (Brazil) 1;\n" "X-Generator: Weblate 4.2.1\n" -#: src/misc.py:45 +#: src/misc.py:115 msgid "Location" msgstr "Localização" -#: src/misc.py:45 +#: src/misc.py:115 msgid "About me" msgstr "Sobre mim" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Google link" msgstr "Link do Google" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Facebook link" msgstr "Link do Facebook" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Twitter link" msgstr "Link do Twitter" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Reddit link" msgstr "Link do Reddit" -#: src/misc.py:45 +#: src/misc.py:118 msgid "Twitch link" msgstr "Link da Twitch" -#: src/misc.py:45 +#: src/misc.py:118 msgid "PSN link" msgstr "Link do PSN" -#: src/misc.py:45 +#: src/misc.py:119 msgid "VK link" msgstr "Link do VK" -#: src/misc.py:45 +#: src/misc.py:119 msgid "XBL link" msgstr "Link do XBL" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Steam link" msgstr "Link da Steam" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Discord handle" msgstr "Link do Discord" -#: src/misc.py:45 +#: src/misc.py:121 msgid "Battle.net handle" msgstr "Link do Battle.net" -#: src/misc.py:140 -msgid "" -"\n" -"__And more__" -msgstr "" -"\n" -"__E mais__" - -#: src/misc.py:330 +#: src/misc.py:127 msgid "Unknown" msgstr "Desconhecido" -#: src/misc.py:332 +#: src/misc.py:129 msgid "unknown" msgstr "desconhecido" + +#~ msgid "" +#~ "\n" +#~ "__And more__" +#~ msgstr "" +#~ "\n" +#~ "__E mais__" diff --git a/locale/ru/LC_MESSAGES/misc.mo b/locale/ru/LC_MESSAGES/misc.mo index 88a0928..f042137 100644 Binary files a/locale/ru/LC_MESSAGES/misc.mo and b/locale/ru/LC_MESSAGES/misc.mo differ diff --git a/locale/ru/LC_MESSAGES/misc.po b/locale/ru/LC_MESSAGES/misc.po index 15ad40c..29d0167 100644 --- a/locale/ru/LC_MESSAGES/misc.po +++ b/locale/ru/LC_MESSAGES/misc.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: \n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-06-16 23:56+0200\n" +"POT-Creation-Date: 2024-02-24 17:51+0100\n" "PO-Revision-Date: 2020-03-18 13:59+0100\n" "Last-Translator: BabylonAS\n" "Language-Team: \n" @@ -19,70 +19,69 @@ msgstr "" "n%10<=4 && (n%100<10 || n%100>=20) ? 1 : 2);\n" "X-Generator: Poedit 2.3\n" -#: src/misc.py:45 +#: src/misc.py:115 msgid "Location" msgstr "Местоположение" -#: src/misc.py:45 +#: src/misc.py:115 msgid "About me" msgstr "О себе" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Google link" msgstr "Ссылка на учётную запись Google" -#: src/misc.py:45 +#: src/misc.py:116 msgid "Facebook link" msgstr "Ссылка на учётную запись Facebook" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Twitter link" msgstr "Ссылка на учётную запись Twitter" -#: src/misc.py:45 +#: src/misc.py:117 msgid "Reddit link" msgstr "Ссылка на учётную запись Reddit" -#: src/misc.py:45 +#: src/misc.py:118 msgid "Twitch link" msgstr "Ссылка на учётную запись Twitch" -#: src/misc.py:45 +#: src/misc.py:118 msgid "PSN link" msgstr "Ссылка на учётную запись PSN" -#: src/misc.py:45 +#: src/misc.py:119 msgid "VK link" msgstr "Ссылка на учётную запись ВКонтакте" -#: src/misc.py:45 +#: src/misc.py:119 msgid "XBL link" msgstr "Ссылка на учётную запись XBL" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Steam link" msgstr "Ссылка на учётную запись Steam" -#: src/misc.py:45 +#: src/misc.py:120 msgid "Discord handle" msgstr "Ссылка на учётную запись Discord" -#: src/misc.py:45 +#: src/misc.py:121 msgid "Battle.net handle" msgstr "Ссылка на учётную запись Battle.net" -#: src/misc.py:140 -msgid "" -"\n" -"__And more__" -msgstr "" -"\n" -"__И ещё__" - -#: src/misc.py:330 +#: src/misc.py:127 msgid "Unknown" msgstr "Неизвестно" -#: src/misc.py:332 +#: src/misc.py:129 msgid "unknown" msgstr "неизвестно" + +#~ msgid "" +#~ "\n" +#~ "__And more__" +#~ msgstr "" +#~ "\n" +#~ "__И ещё__" diff --git a/locale/zh-hans/LC_MESSAGES/misc.mo b/locale/zh-hans/LC_MESSAGES/misc.mo index 9f7e8e0..eebc845 100644 Binary files a/locale/zh-hans/LC_MESSAGES/misc.mo and b/locale/zh-hans/LC_MESSAGES/misc.mo differ diff --git a/locale/zh-hans/LC_MESSAGES/misc.po b/locale/zh-hans/LC_MESSAGES/misc.po index 9df172c..bf87fac 100644 --- a/locale/zh-hans/LC_MESSAGES/misc.po +++ b/locale/zh-hans/LC_MESSAGES/misc.po @@ -7,7 +7,7 @@ msgid "" msgstr "" "Project-Id-Version: RcGcDw\n" "Report-Msgid-Bugs-To: \n" -"POT-Creation-Date: 2022-06-16 23:56+0200\n" +"POT-Creation-Date: 2024-02-24 17:51+0100\n" "PO-Revision-Date: 2020-12-06 14:17+0000\n" "Last-Translator: lakejason0 \n" "Language-Team: Chinese (Simplified) \n" "Language-Team: Chinese (Traditional) str: return (f"") def __repr__(self): @@ -101,6 +102,33 @@ class Domain: wiki.statistics.update(Log(type=LogType.SCAN_REASON, title=str(reason))) self.wikis.move_to_end(wiki.script_url) + def failure_rate_investigation(self) -> Optional[set]: + """Function is supposed to determine if a notification should be sent regarding a wiki/domain not working properly + + Cases considered worthy of notification: + An entire farm (20% of wikis when 15+ wikis from domain) is responding with errors for the past 10 minutes + A single wiki returning connection errors either for full queue_length or for an hour + """ + if len(self) > 15: + affected = set() + for wiki_url, wiki_obj in self.wikis.items(): + failures = 0 + logs_last_10 = wiki_obj.statistics.filter_by_time(10 * 60) + for log in logs_last_10: + if log.type == LogType.CONNECTION_ERROR: + failures += 1 + if len(logs_last_10)/2 <= failures: + affected.add(wiki_url) + if len(affected) > len(self)/5: + return affected + else: + affected = set() + for wiki_url, wiki_obj in self.wikis.items(): + if all([x for x in wiki_obj.statistics.filter_by_time(60*60) if x.type in (LogType.CONNECTION_ERROR, LogType.SCAN_REASON, LogType.HTTP_ERROR)]): + affected.add(wiki_url) + if affected: + return affected + async def irc_scheduler(self): try: while True: @@ -123,12 +151,14 @@ class Domain: except Exception as e: if command_line_args.debug: logger.exception("IRC scheduler task for domain {} failed!".format(self.name)) - else: - self.failures += 1 + else: # production + if not (time.time()-172800 > self.last_failure_report): # If we haven't reported for more than 2 days or at all + return traceback.print_exc() - await self.send_exception_to_monitoring(e) - if self.failures > 5: - raise asyncio.exceptions.CancelledError + wikis = self.failure_rate_investigation() + if wikis: + await self.send_exception_to_monitoring(e, wikis) + self.last_failure_report = time.time() async def regular_scheduler(self): try: @@ -139,11 +169,13 @@ class Domain: if command_line_args.debug: logger.exception("Regular scheduler task for domain {} failed!".format(self.name)) else: - self.failures += 1 - await self.send_exception_to_monitoring(e) + if not (time.time()-172800 > self.last_failure_report): # If we haven't reported for more than 2 days or at all + return traceback.print_exc() - if self.failures > 5: - raise asyncio.exceptions.CancelledError + wikis = self.failure_rate_investigation() + if wikis: + await self.send_exception_to_monitoring(e, wikis) + self.last_failure_report = time.time() @cache def calculate_sleep_time(self, queue_length: int): @@ -170,11 +202,11 @@ class Domain: await wiki.session.close() raise - async def send_exception_to_monitoring(self, ex: Exception): + async def send_exception_to_monitoring(self, ex: Exception, wikis: set): discord_message = DiscordMessage("embed", "generic", [""]) discord_message["title"] = "Domain scheduler exception for {} (recovered)".format(self.name) - discord_message["content"] = str(ex)[0:1995] - discord_message.add_field("Failure count", str(self.failures)) + discord_message["content"] = "Affected wikis: {}".format(", ".join(wikis)) + "\n" + str(ex) + # discord_message.add_field("Failure count", str(self.failures)) discord_message.finish_embed_message() header = settings["header"] header['Content-Type'] = 'application/json' diff --git a/src/irc_feed.py b/src/irc_feed.py index 3d6db2b..b50505c 100644 --- a/src/irc_feed.py +++ b/src/irc_feed.py @@ -18,7 +18,13 @@ if TYPE_CHECKING: class AioIRCCat(irc.client_aio.AioSimpleIRCClient): def connect(self, *args, **kwargs): logger.debug("Connecting with {}...".format(args)) - super().connect(*args, **kwargs) + while True: + try: + super().connect(*args, **kwargs) + break + except ConnectionRefusedError as error: # hopefully picked up by self.activity_tester + logger.exception("Could not connect with IRC!") + pass self.connection_details = (args, kwargs) def __init__(self, targets: dict[str, str], domain_object: Domain, rc_callback: Optional[Callable], discussion_callback: Optional[Callable]): diff --git a/src/statistics.py b/src/statistics.py index b218168..01b2029 100644 --- a/src/statistics.py +++ b/src/statistics.py @@ -3,7 +3,7 @@ from datetime import datetime import aiohttp.web_request from src.config import settings -from typing import Union, Optional +from typing import Union, Optional, List from enum import Enum @@ -65,12 +65,12 @@ class Statistics: for log in args: self.logs.append(log) - def filter_by_time(self, time_ago: int, logs: list = None): # cannot have self.logs in here as this is evaluated once + def filter_by_time(self, time_ago: int, logs: list = None) -> List[Log]: # cannot have self.logs in here as this is evaluated once """Returns logs with time between time_ago seconds ago and now""" time_limit = int(time.time()) - time_ago return [x for x in (self.logs if logs is None else logs) if x.time > time_limit] - def filter_by_type(self, log_type: LogType, logs: list = None): + def filter_by_type(self, log_type: LogType, logs: list = None) -> List[Log]: """Returns logs with same type as in log_type""" return [x for x in (self.logs if logs is None else logs) if x.type == log_type] diff --git a/src/wiki.py b/src/wiki.py index 7d7f5fd..0ea079b 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -342,7 +342,7 @@ class Wiki: # If WikiServerError comes up 2 times in recent 2 minutes, this will reraise the exception, otherwise waits 2 seconds and retries self.statistics.update(Log(type=LogType.CONNECTION_ERROR, title=str(e.exception))) if self.statistics.recent_connection_errors() > 9: - raise + raise e await asyncio.sleep(2.0) continue if not self.mw_messages or self.recache_requested: @@ -360,7 +360,10 @@ class Wiki: else: self.statistics.update(last_action=0) dbmanager.add(("UPDATE rcgcdb SET rcid = 0 WHERE wiki = $1 AND ( rcid != -1 OR rcid IS NULL )", (self.script_url))) - return # TODO Add a log entry? + self.statistics.update(Log(type=LogType.VALUE_UPDATE, + title="Commited first fetch and updated rcid value from {} to {}".format( + self.rc_id, self.statistics.last_action))) + return categorize_events = {} new_events = 0 self.statistics.last_checked_rc = int(time.time())