Created a skeleton for rewrite

requirements.txt

@@ -5,3 +5,4 @@ nest-asyncio >= 1.4.0
 irc >= 19.0.1
 beautifulsoup4>=4.9.3
 asyncpg>=0.22.0
+aioredis >= 1.3.1

src/bot.py

@@ -20,6 +20,7 @@ from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info, essenti
 from src.discord import DiscordMessage, generic_msg_sender_exception_logger, stack_message_list
 from src.wiki_ratelimiter import RateLimiter
 from src.irc_feed import AioIRCCat
+from src.domain_manager import domains
 
 
 logging.config.dictConfig(settings["logging"])
@@ -43,9 +44,8 @@ main_tasks: dict = {}
 async def populate_allwikis():
     async with db.pool().acquire() as connection:
         async with connection.transaction():
-			async for db_wiki in connection.cursor('SELECT DISTINCT wiki, rcid FROM rcgcdw'):
-				all_wikis[db_wiki["wiki"]] = Wiki()  # populate all_wikis
-				all_wikis[db_wiki["wiki"]].rc_active = db_wiki["rcid"]
+            async for db_wiki in connection.cursor('SELECT DISTINCT wiki, rcid, postid FROM rcgcdw'):
+                domains.new_wiki(Wiki(db_wiki["wiki"], db_wiki["rcid"], db_wiki["postid"]))
 
 queue_limit = settings.get("queue_limit", 30)
 QueuedWiki = namedtuple("QueuedWiki", ['url', 'amount'])
@@ -399,6 +399,7 @@ async def message_sender():
             logger.exception("Exception on DC message sender")
             await generic_msg_sender_exception_logger(traceback.format_exc(), "Message sender exception")
 
+
 async def discussion_handler():
     await asyncio.sleep(3.0)  # Make some time before IRC code is executed, happens only once and saves if inside
     try:

src/domain.py

@@ -10,9 +10,25 @@ if TYPE_CHECKING:
 
 
 class Domain:
-    def __init__(self, task: asyncio.Task, irc_client: Optional[irc.client_aio.AioSimpleIRCClient] = None):
-        self.task = task
+    def __init__(self, name: str, irc_client: Optional[irc.client_aio.AioSimpleIRCClient] = None):
+        self.name = name  # This should be always in format of topname.extension for example fandom.com
+        self.task: asyncio.Task = self.create_task()
         self.wikis: list[src.wiki.Wiki] = list()
         self.rate_limiter: src.wiki_ratelimiter = src.wiki_ratelimiter.RateLimiter()
         self.irc = irc_client
 
+    def add_wiki(self, wiki: src.wiki.Wiki, index: int = None):
+        """Adds a wiki to domain list.
+
+        :parameter wiki - Wiki object
+        :parameter index (optional) - index at which the wiki should be added, if not specified it's the end of the list"""
+        if index:
+            self.wikis.insert(index, wiki)
+        else:
+            self.wikis.append(wiki)
+
+    def create_task(self) -> asyncio.Task:
+        return asyncio.create_task(self.run_wiki_check())
+
+    async def run_wiki_check(self):
+        raise NotImplementedError

src/domain_manager.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+from typing import TYPE_CHECKING, Optional
+from urllib.parse import urlparse, urlunparse
+from src.config import settings
+from src.domain import Domain
+
+
+if TYPE_CHECKING:
+    from src.wiki import Wiki
+
+
+class DomainManager:
+    def __init__(self):
+        self.domains: dict[str, Domain] = {}
+
+    def new_wiki(self, wiki: Wiki):
+        """Finds a domain for the wiki and adds a wiki to the domain object.
+
+        :parameter wiki - Wiki object to be added"""
+        wiki_domain = self.get_domain(wiki.script_url)
+        try:
+            self.domains[wiki_domain].add_wiki(wiki)
+        except KeyError:
+            self.new_domain(wiki_domain).add_wiki(wiki)
+
+    @staticmethod
+    def get_domain(url: str) -> str:
+        """Returns a domain for given URL (for example fandom.com, wikipedia.org)"""
+        parsed_url = urlparse(url)
+        return ".".join(urlunparse((*parsed_url[0:2], "", "", "", "")).split(".")[-2:])
+
+    def new_domain(self, name: str) -> Domain:
+        # TODO IRC Part
+        self.domains[name] = Domain(name, irc)
+        return self.domains[name]
+
+
+domains = DomainManager()

src/wiki.py

@@ -17,9 +17,18 @@ from bs4 import BeautifulSoup
 
 logger = logging.getLogger("rcgcdb.wiki")
 
+class Wiki:
+	def __init__(self, script_url: str, rc_id: int, discussion_id: int):
+		self.script_url = script_url
+		self.session = aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(6.0))
+		self.statistics = Statistics()
+
+	@property
+	def rc_id(self):
+		return self.statistics.rc_id
 
 @dataclass
-class Wiki:
+class Wiki_old:
 	mw_messages: int = None
 	fail_times: int = 0  # corresponding to amount of times connection with wiki failed for client reasons (400-499)
 	session: aiohttp.ClientSession = None