From ef3dddfb0c9b4356b4551832842b97910629d642 Mon Sep 17 00:00:00 2001 From: Frisk Date: Tue, 9 Aug 2022 16:08:30 +0200 Subject: [PATCH] More development, improvements in neglected areas --- src/domain.py | 15 +++++++++++++-- src/domain_manager.py | 6 ++++++ src/exceptions.py | 4 +++- src/statistics.py | 18 ++++++++++++++++-- src/wiki.py | 13 ++++++++++--- 5 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/domain.py b/src/domain.py index 8b10e3c..bd307d2 100644 --- a/src/domain.py +++ b/src/domain.py @@ -6,6 +6,8 @@ from src.config import settings from typing import TYPE_CHECKING, Optional from functools import cache from src.discussions import Discussions +from statistics import Log, LogType + logger = logging.getLogger("rcgcdb.domain") if TYPE_CHECKING: @@ -21,7 +23,7 @@ class Domain: self.wikis: OrderedDict[str, src.wiki.Wiki] = OrderedDict() self.rate_limiter: src.wiki_ratelimiter = src.wiki_ratelimiter.RateLimiter() self.irc: Optional[src.irc_feed.AioIRCCat] = None - self.discussions_handler: Optional[Discussions] = None + self.discussions_handler: Optional[Discussions] = Discussions(self.wikis) if name == "fandom.com" else None def __iter__(self): return iter(self.wikis) @@ -32,6 +34,14 @@ class Domain: def __len__(self): return len(self.wikis) + def destroy(self): + if self.irc: + self.irc.connection.disconnect("Leaving") + if self.discussions_handler: + self.discussions_handler.close() + if self.task: + self.task.cancel() + def get_wiki(self, item, default=None) -> Optional[src.wiki.Wiki]: return self.wikis.get(item, default) @@ -64,9 +74,10 @@ class Domain: if first: self.wikis.move_to_end(wiki.script_url, last=False) - async def run_wiki_scan(self, wiki: src.wiki.Wiki): + async def run_wiki_scan(self, wiki: src.wiki.Wiki, reason: Optional[int] = None): await self.rate_limiter.timeout_wait() await wiki.scan() + wiki.statistics.update(Log(type=LogType.SCAN_REASON, title=str(reason))) self.wikis.move_to_end(wiki.script_url) self.rate_limiter.timeout_add(1.0) diff --git a/src/domain_manager.py b/src/domain_manager.py index eea1019..c7424ca 100644 --- a/src/domain_manager.py +++ b/src/domain_manager.py @@ -50,6 +50,10 @@ class DomainManager: new_domain = await self.new_domain(wiki_domain) new_domain.add_wiki(wiki) + def remove_domain(self, domain): + domain.destoy() + del self.domains[domain] + def remove_wiki(self, script_url: str): wiki_domain = self.get_domain(script_url) try: @@ -58,6 +62,8 @@ class DomainManager: raise NoDomain else: domain.remove_wiki(script_url) + if len(domain) == 0: + self.remove_domain(domain) @staticmethod def get_domain(url: str) -> str: diff --git a/src/exceptions.py b/src/exceptions.py index 70b2611..d7c0607 100644 --- a/src/exceptions.py +++ b/src/exceptions.py @@ -2,7 +2,9 @@ class WikiError(Exception): pass class WikiServerError(Exception): - pass + def __init__(self, exception: BaseException): + self.exception = exception + class WikiNotFoundError(Exception): pass diff --git a/src/statistics.py b/src/statistics.py index 379e6a8..b86eb5f 100644 --- a/src/statistics.py +++ b/src/statistics.py @@ -12,6 +12,7 @@ class LogType(Enum): HTTP_ERROR: 2 MEDIAWIKI_ERROR: 3 VALUE_UPDATE: 4 + SCAN_REASON: 5 queue_limit = settings.get("queue_limit", 30) @@ -40,10 +41,23 @@ class Statistics: self.last_action: Optional[int] = rc_id self.last_checked_discussion: Optional[int] = None self.last_post: Optional[int] = discussion_id - self.logs: LimitedList = LimitedList() + self.logs: LimitedList[Log] = LimitedList() def update(self, *args: Log, **kwargs: dict[str, Union[float, int]]): for key, value in kwargs: self.__setattr__(key, value) for log in args: - self.logs.append(log) \ No newline at end of file + self.logs.append(log) + + def filter_by_time(self, time_ago: int, logs: list = None): # cannot have self.logs in here as this is evaluated once + """Returns logs with time between time_ago seconds ago and now""" + time_limit = int(time.time()) - time_ago + return [x for x in (self.logs if logs is None else logs) if x.time > time_limit] + + def filter_by_type(self, log_type: LogType, logs: list = None): + """Returns logs with same type as in log_type""" + return [x for x in (self.logs if logs is None else logs) if x.type == log_type] + + def recent_connection_errors(self) -> int: + """Count how many connection errors there were recently (2 minutes)""" + return len(self.filter_by_type(LogType.CONNECTION_ERROR, logs=self.filter_by_time(120))) # find connection errors from 2 minutes ago diff --git a/src/wiki.py b/src/wiki.py index b89148a..474427b 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -194,18 +194,25 @@ class Wiki: "rclimit": amount, "rctype": "edit|new|log|categorize", "siprop": "namespaces|general"}) try: response = await self.api_request(params=params) - except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError): + except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e: logger.error("A connection error occurred while requesting {}".format(params)) - raise WikiServerError + raise WikiServerError(e) return response async def scan(self, amount=10): + """Fetches recent changes of a wiki + + :raises WikiServerError + """ while True: # Trap event in case there are more changes needed to be fetched try: request = await self.fetch_wiki(amount=amount) self.client.last_request = request except WikiServerError as e: - self.statistics.update(Log(type=LogType.CONNECTION_ERROR, title=e.)) # We need more details in WIkiServerError exception + self.statistics.update(Log(type=LogType.CONNECTION_ERROR, title=str(e.exception))) + if self.statistics.recent_connection_errors() > 1: + raise + await asyncio.sleep(2.0) if not self.mw_messages: # TODO Split into other function mw_messages = request.get("query", {}).get("allmessages", [])