diff --git a/src/bot.py b/src/bot.py index ba4f710..fed0965 100644 --- a/src/bot.py +++ b/src/bot.py @@ -64,26 +64,31 @@ async def wiki_scanner(): local_wiki = all_wikis[db_wiki[3]] # set a reference to a wiki object from memory if local_wiki.mw_messages is None: extended = True - logger.debug("test") - try: - wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3]) - await local_wiki.check_status(db_wiki[3], wiki_response.status) - except (WikiServerError, WikiError): - logger.exception("Exeption when fetching the wiki") - continue # ignore this wiki if it throws errors - try: - recent_changes_resp = await wiki_response.json(encoding="UTF-8") - if "error" in recent_changes_resp or "errors" in recent_changes_resp: - # TODO Remove on some errors (example "code": "readapidenied") - raise WikiError - recent_changes = recent_changes_resp['query']['recentchanges'] - recent_changes.reverse() - except asyncio.exceptions.TimeoutError: - logger.debug("Timeout on fetching {}.".format(db_wiki[3])) - continue - except: - logger.exception("On loading json of response.") - continue + async with aiohttp.ClientSession(headers=settings["header"], + timeout=aiohttp.ClientTimeout(2.0)) as session: + try: + wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3], session) + await local_wiki.check_status(db_wiki[3], wiki_response.status) + except (WikiServerError, WikiError): + logger.exception("Exeption when fetching the wiki") + continue # ignore this wiki if it throws errors + try: + recent_changes_resp = await wiki_response.json() + if "error" in recent_changes_resp or "errors" in recent_changes_resp: + error = recent_changes_resp.get("error", recent_changes_resp["errors"]) + if error["code"] == "readapidenied": + await local_wiki.fail_add(db_wiki[3], 410) + continue + raise WikiError + recent_changes = recent_changes_resp['query']['recentchanges'] + recent_changes.reverse() + except aiohttp.ContentTypeError: + logger.exception("Wiki seems to be resulting in non-json content.") + await local_wiki.fail_add(db_wiki[3], 410) + continue + except: + logger.exception("On loading json of response.") + continue if extended: await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs) if db_wiki[6] is None: # new wiki, just get the last rc to not spam the channel @@ -108,7 +113,7 @@ async def wiki_scanner(): DBHandler.update_db() await asyncio.sleep(delay=calc_delay) except asyncio.CancelledError: - return + raise async def message_sender(): @@ -121,6 +126,7 @@ def shutdown(loop, signal=None): loop.stop() logger.info("Script has shut down due to signal {}.".format(signal)) for task in asyncio.all_tasks(loop): + logger.debug("Killing task {}".format(task.get_name())) task.cancel() sys.exit(0) @@ -134,17 +140,20 @@ async def main_loop(): loop = asyncio.get_event_loop() try: signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT) + for s in signals: + loop.add_signal_handler( + s, lambda s=s: shutdown(loop, signal=s)) except AttributeError: logger.info("Running on Windows huh? This complicates things") signals = (signal.SIGBREAK, signal.SIGTERM, signal.SIGINT) - for s in signals: - loop.add_signal_handler( - s, lambda s=s: shutdown(loop, signal=s)) loop.set_exception_handler(global_exception_handler) - task1 = asyncio.create_task(wiki_scanner()) - task2 = asyncio.create_task(message_sender()) - await task1 - await task2 + try: + task1 = asyncio.create_task(wiki_scanner()) + task2 = asyncio.create_task(message_sender()) + await task1 + await task2 + except KeyboardInterrupt: + shutdown(loop) asyncio.run(main_loop()) diff --git a/src/discord.py b/src/discord.py index 147a66c..dbd9d64 100644 --- a/src/discord.py +++ b/src/discord.py @@ -21,7 +21,7 @@ async def wiki_removal(wiki_id, status): """Our own translation string to make it compatible with async""" return langs[observer[4]].gettext(string) reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"), - 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")} + 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible"), 410: _("wiki becoming inaccessible")} reason = reasons.get(status, _("unknown error")) await send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[observer[2]], content=_("The webhook for {} has been removed due to {}.".format(wiki_id, reason)), wiki=None)) header = settings["header"] diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 09e1331..ecde89f 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -104,10 +104,13 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c english_length_num = re.sub(r"(\D+)", "", change["logparams"]["duration"]) try: english_length = english_length.rstrip("s").strip() - block_time = "{num} {translated_length}".format(num=english_length_num, + try: + block_time = "{num} {translated_length}".format(num=english_length_num, translated_length=ngettext(english_length, english_length + "s", int(english_length_num))) + except ValueError: + logger.exception("Couldn't properly resolve block expiry.") except AttributeError: logger.error("Could not strip s from the block event, seems like the regex didn't work?") return diff --git a/src/msgqueue.py b/src/msgqueue.py index 496424c..b892e73 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -40,9 +40,9 @@ class MessageQueue: logger.debug( "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), str(item))) - if await send_to_discord_webhook(item, self.session) < 2: + if await send_to_discord_webhook(item) < 2: logger.debug("Sending message succeeded") - await asyncio.sleep(1.5) + await asyncio.sleep(1.9) else: logger.debug("Sending message failed") break diff --git a/src/wiki.py b/src/wiki.py index f222d3e..6e297af 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -7,6 +7,7 @@ from src.formatters.rc import embed_formatter, compact_formatter from src.misc import parse_link from src.i18n import langs import src.discord +import asyncio from src.config import settings from bs4 import BeautifulSoup @@ -22,7 +23,7 @@ class Wiki: session: aiohttp.ClientSession = None - async def fetch_wiki(self, extended, script_path) -> aiohttp.ClientResponse: + async def fetch_wiki(self, extended, script_path, session) -> aiohttp.ClientResponse: url_path = script_path + "api.php" amount = 20 if extended: @@ -30,18 +31,17 @@ class Wiki: "meta": "allmessages|siteinfo", "utf8": 1, "tglimit": "max", "tgprop": "displayname", "rcprop": "title|redirect|timestamp|ids|loginfo|parsedcomment|sizes|flags|tags|user", - "rclimit": amount, "rctype": "edit|new|log|external", + "rclimit": amount, "rcshow": "!bot", "rctype": "edit|new|log|external", "ammessages": "recentchanges-page-added-to-category|recentchanges-page-removed-from-category|recentchanges-page-added-to-category-bundled|recentchanges-page-removed-from-category-bundled", "amenableparser": 1, "amincludelocal": 1, "siprop": "namespaces|general"} else: params = {"action": "query", "format": "json", "uselang": "content", "list": "tags|recentchanges", "meta": "siteinfo", "utf8": 1, - "tglimit": "max", "tgprop": "displayname", + "tglimit": "max", "rcshow": "!bot", "tgprop": "displayname", "rcprop": "title|redirect|timestamp|ids|loginfo|parsedcomment|sizes|flags|tags|user", "rclimit": amount, "rctype": "edit|new|log|external", "siprop": "namespaces|general"} try: - async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(6.0)) as session: - response = await session.get(url_path, params=params) + response = await session.get(url_path, params=params) except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError): logger.exception("A connection error occurred while requesting {}".format(url_path)) raise WikiServerError @@ -49,7 +49,7 @@ class Wiki: async def safe_request(self, url): try: - async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0)) as session: + async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(2.0)) as session: request = await session.get(url, timeout=5, allow_redirects=False) request.raise_for_status() except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError): @@ -58,15 +58,19 @@ class Wiki: else: return request + async def fail_add(self, wiki_url, status): + logger.debug("Increasing fail_times to {}".format(self.fail_times+3)) + self.fail_times += 3 + if self.fail_times > 9: + await self.remove(wiki_url, status) + async def check_status(self, wiki_url, status): if 199 < status < 300: - self.fail_times = 0 + self.fail_times -= 1 pass elif 400 < status < 500: # ignore 400 error since this might be our fault - self.fail_times += 1 + await self.fail_add(wiki_url, status) logger.warning("Wiki {} responded with HTTP code {}, increased fail_times to {}, skipping...".format(wiki_url, status, self.fail_times)) - if self.fail_times > 3: - await self.remove(wiki_url, status) raise WikiError elif 499 < status < 600: logger.warning("Wiki {} responded with HTTP code {}, skipping...".format(wiki_url, status, self.fail_times)) @@ -75,8 +79,8 @@ class Wiki: async def remove(self, wiki_id, reason): await src.discord.wiki_removal(wiki_id, reason) await src.discord.wiki_removal_monitor(wiki_id, reason) - db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", (wiki_id,)) - logger.warning("{} rows affected by DELETE FROM rcgcdw WHERE wiki = {}".format(db_cursor.rowcount, wiki_id)) + db_cursor.execute('DELETE FROM rcgcdw WHERE wiki = "?"', (wiki_id,)) + logger.warning('{} rows affected by DELETE FROM rcgcdw WHERE wiki = "{}"'.format(db_cursor.rowcount, wiki_id)) db_connection.commit() async def pull_comment(self, comment_id, WIKI_API_PATH): @@ -143,7 +147,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict): if not "missing" in message: # ignore missing strings msgs.append((message["name"], re.sub(r'\[\[.*?\]\]', '', message["*"]))) else: - logging.warning("Could not fetch the MW message translation for: {}".format(message["name"])) + logger.warning("Could not fetch the MW message translation for: {}".format(message["name"])) msgs = tuple(msgs) for key, set in mw_msgs.items(): if msgs == set: @@ -163,7 +167,6 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ lang = langs[target[0][0]] ngettext = lang.ngettext # recent_changes = RecentChangesClass() # TODO Look into replacing RecentChangesClass with local_wiki - logger.debug(change) appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter if ("actionhidden" in change or "suppressed" in change): # if event is hidden using suppression await appearance_mode("suppressed", change, "", changed_categories, local_wiki, target, _, ngettext, paths)