Some more work

settings.json.example

@@ -12,6 +12,7 @@
     "pg_db": "rcgcdb",
     "pg_pass": "secret_password",
     "pg_port": "5432",
+    "redis_host": "localhost",
     "irc_servers": {
         "your custom name for the farm": {
             "domains": ["wikipedia.org", "otherwikipedia.org"],

src/domain.py

@@ -1,31 +1,40 @@
 from __future__ import annotations
 import asyncio
-import irc.client_aio
+from collections import OrderedDict
 from typing import TYPE_CHECKING, Optional
 
 
 if TYPE_CHECKING:
     import src.wiki
     import src.wiki_ratelimiter
+    import irc.client_aio
 
 
 class Domain:
-    def __init__(self, name: str, irc_client: Optional[irc.client_aio.AioSimpleIRCClient] = None):
+    def __init__(self, name: str):
         self.name = name  # This should be always in format of topname.extension for example fandom.com
         self.task: asyncio.Task = self.create_task()
-        self.wikis: list[src.wiki.Wiki] = list()
+        self.wikis: OrderedDict[str, src.wiki.Wiki] = OrderedDict()
         self.rate_limiter: src.wiki_ratelimiter = src.wiki_ratelimiter.RateLimiter()
+        self.irc = None
+
+    def __iter__(self):
+        return iter(self.wikis)
+
+    def __getitem__(self, item):
+        return
+
+    def set_irc(self, irc_client: irc.client_aio.AioSimpleIRCClient):
         self.irc = irc_client
 
-    def add_wiki(self, wiki: src.wiki.Wiki, index: int = None):
+    def add_wiki(self, wiki: src.wiki.Wiki, first=False):
         """Adds a wiki to domain list.
 
         :parameter wiki - Wiki object
-        :parameter index (optional) - index at which the wiki should be added, if not specified it's the end of the list"""
-        if index:
-            self.wikis.insert(index, wiki)
-        else:
-            self.wikis.append(wiki)
+        :parameter first (optional) - bool indicating if wikis should be added as first or last in the ordered dict"""
+        self.wikis[wiki.script_url] = wiki
+        if first:
+            self.wikis.move_to_end(wiki.script_url, last=False)
 
     def create_task(self) -> asyncio.Task:
         return asyncio.create_task(self.run_wiki_check())

src/domain_manager.py

@@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Optional
 from urllib.parse import urlparse, urlunparse
 from src.config import settings
 from src.domain import Domain
+from src.irc_feed import AioIRCCat
 
 
 if TYPE_CHECKING:
@@ -29,9 +30,14 @@ class DomainManager:
         parsed_url = urlparse(url)
         return ".".join(urlunparse((*parsed_url[0:2], "", "", "", "")).split(".")[-2:])
 
-    def new_domain(self, name: str) -> Domain:
-        # TODO IRC Part
-        self.domains[name] = Domain(name, irc)
+    async def new_domain(self, name: str) -> Domain:
+        irc = None
+        domain_object = Domain(name)
+        for irc_server in settings["irc_servers"].keys():
+            if name in settings["irc_servers"][irc_server]["domains"]:
+                domain_object.set_irc(AioIRCCat(settings["irc_servers"][irc_server]["irc_channel_mapping"], domain_object))
+                break  # Allow only one IRC for a domain
+        self.domains[name] = domain_object
         return self.domains[name]
 
 

src/irc_feed.py

@@ -1,22 +1,26 @@
+from __future__ import annotations
 import irc.client_aio
 import json
 import logging
+from typing import TYPE_CHECKING
 from urllib.parse import urlparse, quote
 
 logger = logging.getLogger("rcgcdw.irc_feed")
 
+if TYPE_CHECKING:
+	from src.domain import Domain
 
 class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
 	def connect(self, *args, **kwargs):
 		super().connect(*args, **kwargs)
 		self.connection_details = (args, kwargs)
 
-	def __init__(self, targets, all_wikis):
+	def __init__(self, targets: dict[str, str], domain_object: Domain):
 		irc.client_aio.SimpleIRCClient.__init__(self)
 		self.targets = targets
 		self.updated = set()  # Storage for edited wikis
 		self.updated_discussions = set()
-		self.wikis = all_wikis
+		self.domain = domain_object
 		self.connection.buffer_class.errors = "replace"  # Ignore encoding errors
 		self.connection_details = None
 
@@ -46,9 +50,12 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
 		# print(message)
 		url = urlparse(message)
 		full_url = "https://"+url.netloc + recognize_langs(url.path)
-		if full_url in self.wikis and self.wikis[full_url].rc_active != -1:
-			self.updated.add(full_url)
-			logger.debug("New website appended to the list! {}".format(full_url))
+		try:
+			if self.domain[full_url].rc_id != -1:
+				self.updated.add(full_url)
+				logger.debug("New website appended to the list! {}".format(full_url))
+		except KeyError:
+			pass
 
 
 	def parse_fandom_discussion(self, message: str):
@@ -60,7 +67,7 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
 		if post.get('action', 'unknown') != "deleted":  # ignore deletion events
 			url = urlparse(post.get('url'))
 			full_url ="https://"+ url.netloc + recognize_langs(url.path)
-			if full_url in self.wikis:  # POSSIBLE MEMORY LEAK AS WE DON'T HAVE A WAY TO CHECK IF WIKI IS LOOKING FOR DISCUSSIONS OR NOT
+			if full_url in self.domain:  # POSSIBLE MEMORY LEAK AS WE DON'T HAVE A WAY TO CHECK IF WIKI IS LOOKING FOR DISCUSSIONS OR NOT
 				self.updated_discussions.add("https://"+full_url)
 				logger.debug("New website appended to the list (discussions)! {}".format(full_url))
 

src/redis_connector.py

@@ -0,0 +1,14 @@
+import asyncio
+import aioredis
+from src.config import settings
+
+
+class Redis:
+    def __init__(self):
+        self.connection = None
+
+    async def connect(self):
+        self.connection = await aioredis.create_pool("redis://" + settings["redis_host"], encoding="UTF-8")
+
+
+redis = Redis()