From dd54fa32072588ceb31feb1b0b1d790061ac88f6 Mon Sep 17 00:00:00 2001 From: Frisk Date: Thu, 7 Nov 2024 10:49:08 +0100 Subject: [PATCH] Added some functionality for handling wiki 404/410 errors --- src/domain.py | 9 ++++++++- src/exceptions.py | 3 ++- src/wiki.py | 4 +++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/domain.py b/src/domain.py index a2fb0c7..9216001 100644 --- a/src/domain.py +++ b/src/domain.py @@ -11,6 +11,7 @@ import sys import aiohttp +from src.exceptions import WikiNotFoundError from src.misc import LimitedList from src.discord.message import DiscordMessage from src.config import settings @@ -124,7 +125,13 @@ class Domain: logger.debug(f"Added new wiki {wiki.script_url} to domain {self.name}") async def run_wiki_scan(self, wiki: src.wiki.Wiki, reason: Optional[str] = None): - await wiki.scan() + try: + await wiki.scan() + except WikiNotFoundError as e: + self.wikis.move_to_end(wiki.script_url) + logs_for_wiki = wiki.statistics.filter_by_time(60*60) + if all([x.type == LogType.HTTP_ERROR for x in logs_for_wiki]) and len(logs_for_wiki) > 10: + await wiki.remove_wiki_from_db("This recent changes webhook has been removed for `wiki returning code {}`!".format(e.code), send_reason=True) wiki.statistics.update(Log(type=LogType.SCAN_REASON, title=str(reason))) self.wikis.move_to_end(wiki.script_url) diff --git a/src/exceptions.py b/src/exceptions.py index d9a499b..d837e30 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -10,7 +10,8 @@ class WikiServerError(Exception): class WikiNotFoundError(Exception): - pass + def __init__(self, code: int): + self.code = code class WikiRemovedError(Exception): pass diff --git a/src/wiki.py b/src/wiki.py index 852886e..e06e335 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -242,7 +242,7 @@ class Wiki: try: stacked_message: StackedDiscordMessage = pickle.loads(db_record["message_object"]) except ValueError: - logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(row[0])) + logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(db_record)) return except TypeError: logger.error( @@ -373,6 +373,8 @@ class Wiki: logger.critical( "Redirect detected! Either the wiki given in the script settings (wiki field) is incorrect/the wiki got removed or is giving us the false value. Please provide the real URL to the wiki, current URL redirects to {}".format( request.url)) + elif request.status in (410, 404): + raise WikiNotFoundError(request.status) elif 399 < request.status < 500: logger.error("Request returned ClientError status code on {url}".format(url=request.url)) self.statistics.update(Log(type=LogType.HTTP_ERROR, title="{} error".format(request.status), details=str(request.headers) + "\n" + str(request.url)))