More development, improvements in neglected areas

src/domain.py

@@ -6,6 +6,8 @@ from src.config import settings
 from typing import TYPE_CHECKING, Optional
 from functools import cache
 from src.discussions import Discussions
+from statistics import Log, LogType
+
 logger = logging.getLogger("rcgcdb.domain")
 
 if TYPE_CHECKING:
@@ -21,7 +23,7 @@ class Domain:
         self.wikis: OrderedDict[str, src.wiki.Wiki] = OrderedDict()
         self.rate_limiter: src.wiki_ratelimiter = src.wiki_ratelimiter.RateLimiter()
         self.irc: Optional[src.irc_feed.AioIRCCat] = None
-        self.discussions_handler: Optional[Discussions] = None
+        self.discussions_handler: Optional[Discussions] = Discussions(self.wikis) if name == "fandom.com" else None
 
     def __iter__(self):
         return iter(self.wikis)
@@ -32,6 +34,14 @@ class Domain:
     def __len__(self):
         return len(self.wikis)
 
+    def destroy(self):
+        if self.irc:
+            self.irc.connection.disconnect("Leaving")
+        if self.discussions_handler:
+            self.discussions_handler.close()
+        if self.task:
+            self.task.cancel()
+
     def get_wiki(self, item, default=None) -> Optional[src.wiki.Wiki]:
         return self.wikis.get(item, default)
 
@@ -64,9 +74,10 @@ class Domain:
         if first:
             self.wikis.move_to_end(wiki.script_url, last=False)
 
-    async def run_wiki_scan(self, wiki: src.wiki.Wiki):
+    async def run_wiki_scan(self, wiki: src.wiki.Wiki, reason: Optional[int] = None):
         await self.rate_limiter.timeout_wait()
         await wiki.scan()
+        wiki.statistics.update(Log(type=LogType.SCAN_REASON, title=str(reason)))
         self.wikis.move_to_end(wiki.script_url)
         self.rate_limiter.timeout_add(1.0)
 

src/domain_manager.py

@@ -50,6 +50,10 @@ class DomainManager:
             new_domain = await self.new_domain(wiki_domain)
             new_domain.add_wiki(wiki)
 
+    def remove_domain(self, domain):
+        domain.destoy()
+        del self.domains[domain]
+
     def remove_wiki(self, script_url: str):
         wiki_domain = self.get_domain(script_url)
         try:
@@ -58,6 +62,8 @@ class DomainManager:
             raise NoDomain
         else:
             domain.remove_wiki(script_url)
+            if len(domain) == 0:
+                self.remove_domain(domain)
 
     @staticmethod
     def get_domain(url: str) -> str:

src/exceptions.py

@@ -2,7 +2,9 @@ class WikiError(Exception):
 	pass
 
 class WikiServerError(Exception):
-	pass
+	def __init__(self, exception: BaseException):
+		self.exception = exception
+
 
 class WikiNotFoundError(Exception):
 	pass

src/statistics.py

@@ -12,6 +12,7 @@ class LogType(Enum):
     HTTP_ERROR: 2
     MEDIAWIKI_ERROR: 3
     VALUE_UPDATE: 4
+    SCAN_REASON: 5
 
 queue_limit = settings.get("queue_limit", 30)
 
@@ -40,10 +41,23 @@ class Statistics:
         self.last_action: Optional[int] = rc_id
         self.last_checked_discussion: Optional[int] = None
         self.last_post: Optional[int] = discussion_id
-        self.logs: LimitedList = LimitedList()
+        self.logs: LimitedList[Log] = LimitedList()
 
     def update(self, *args: Log, **kwargs: dict[str, Union[float, int]]):
         for key, value in kwargs:
             self.__setattr__(key, value)
         for log in args:
             self.logs.append(log)
+
+    def filter_by_time(self, time_ago: int, logs: list = None):  # cannot have self.logs in here as this is evaluated once
+        """Returns logs with time between time_ago seconds ago and now"""
+        time_limit = int(time.time()) - time_ago
+        return [x for x in (self.logs if logs is None else logs) if x.time > time_limit]
+
+    def filter_by_type(self, log_type: LogType, logs: list = None):
+        """Returns logs with same type as in log_type"""
+        return [x for x in (self.logs if logs is None else logs) if x.type == log_type]
+
+    def recent_connection_errors(self) -> int:
+        """Count how many connection errors there were recently (2 minutes)"""
+        return len(self.filter_by_type(LogType.CONNECTION_ERROR, logs=self.filter_by_time(120)))   # find connection errors from 2 minutes ago

src/wiki.py

@@ -194,18 +194,25 @@ class Wiki:
 					  "rclimit": amount, "rctype": "edit|new|log|categorize", "siprop": "namespaces|general"})
 		try:
 			response = await self.api_request(params=params)
-		except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError):
+		except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError) as e:
 			logger.error("A connection error occurred while requesting {}".format(params))
-			raise WikiServerError
+			raise WikiServerError(e)
 		return response
 
 	async def scan(self, amount=10):
+		"""Fetches recent changes of a wiki
+
+		:raises WikiServerError
+		"""
 		while True:  # Trap event in case there are more changes needed to be fetched
 			try:
 				request = await self.fetch_wiki(amount=amount)
 				self.client.last_request = request
 			except WikiServerError as e:
-				self.statistics.update(Log(type=LogType.CONNECTION_ERROR, title=e.))  # We need more details in WIkiServerError exception
+				self.statistics.update(Log(type=LogType.CONNECTION_ERROR, title=str(e.exception)))
+				if self.statistics.recent_connection_errors() > 1:
+					raise
+				await asyncio.sleep(2.0)
 			if not self.mw_messages:
 				# TODO Split into other function
 				mw_messages = request.get("query", {}).get("allmessages", [])