diff --git a/settings.json.example b/settings.json.example index 8fc6340..c81496c 100644 --- a/settings.json.example +++ b/settings.json.example @@ -12,6 +12,7 @@ "pg_db": "rcgcdb", "pg_pass": "secret_password", "pg_port": "5432", + "redis_host": "localhost", "irc_servers": { "your custom name for the farm": { "domains": ["wikipedia.org", "otherwikipedia.org"], diff --git a/src/domain.py b/src/domain.py index 0078a8b..3d2a157 100644 --- a/src/domain.py +++ b/src/domain.py @@ -1,31 +1,40 @@ from __future__ import annotations import asyncio -import irc.client_aio +from collections import OrderedDict from typing import TYPE_CHECKING, Optional if TYPE_CHECKING: import src.wiki import src.wiki_ratelimiter + import irc.client_aio class Domain: - def __init__(self, name: str, irc_client: Optional[irc.client_aio.AioSimpleIRCClient] = None): + def __init__(self, name: str): self.name = name # This should be always in format of topname.extension for example fandom.com self.task: asyncio.Task = self.create_task() - self.wikis: list[src.wiki.Wiki] = list() + self.wikis: OrderedDict[str, src.wiki.Wiki] = OrderedDict() self.rate_limiter: src.wiki_ratelimiter = src.wiki_ratelimiter.RateLimiter() + self.irc = None + + def __iter__(self): + return iter(self.wikis) + + def __getitem__(self, item): + return + + def set_irc(self, irc_client: irc.client_aio.AioSimpleIRCClient): self.irc = irc_client - def add_wiki(self, wiki: src.wiki.Wiki, index: int = None): + def add_wiki(self, wiki: src.wiki.Wiki, first=False): """Adds a wiki to domain list. :parameter wiki - Wiki object - :parameter index (optional) - index at which the wiki should be added, if not specified it's the end of the list""" - if index: - self.wikis.insert(index, wiki) - else: - self.wikis.append(wiki) + :parameter first (optional) - bool indicating if wikis should be added as first or last in the ordered dict""" + self.wikis[wiki.script_url] = wiki + if first: + self.wikis.move_to_end(wiki.script_url, last=False) def create_task(self) -> asyncio.Task: return asyncio.create_task(self.run_wiki_check()) diff --git a/src/domain_manager.py b/src/domain_manager.py index af7f6fd..dfbb0c2 100644 --- a/src/domain_manager.py +++ b/src/domain_manager.py @@ -3,6 +3,7 @@ from typing import TYPE_CHECKING, Optional from urllib.parse import urlparse, urlunparse from src.config import settings from src.domain import Domain +from src.irc_feed import AioIRCCat if TYPE_CHECKING: @@ -29,9 +30,14 @@ class DomainManager: parsed_url = urlparse(url) return ".".join(urlunparse((*parsed_url[0:2], "", "", "", "")).split(".")[-2:]) - def new_domain(self, name: str) -> Domain: - # TODO IRC Part - self.domains[name] = Domain(name, irc) + async def new_domain(self, name: str) -> Domain: + irc = None + domain_object = Domain(name) + for irc_server in settings["irc_servers"].keys(): + if name in settings["irc_servers"][irc_server]["domains"]: + domain_object.set_irc(AioIRCCat(settings["irc_servers"][irc_server]["irc_channel_mapping"], domain_object)) + break # Allow only one IRC for a domain + self.domains[name] = domain_object return self.domains[name] diff --git a/src/irc_feed.py b/src/irc_feed.py index a9a22e4..80205a3 100644 --- a/src/irc_feed.py +++ b/src/irc_feed.py @@ -1,22 +1,26 @@ +from __future__ import annotations import irc.client_aio import json import logging +from typing import TYPE_CHECKING from urllib.parse import urlparse, quote logger = logging.getLogger("rcgcdw.irc_feed") +if TYPE_CHECKING: + from src.domain import Domain class AioIRCCat(irc.client_aio.AioSimpleIRCClient): def connect(self, *args, **kwargs): super().connect(*args, **kwargs) self.connection_details = (args, kwargs) - def __init__(self, targets, all_wikis): + def __init__(self, targets: dict[str, str], domain_object: Domain): irc.client_aio.SimpleIRCClient.__init__(self) self.targets = targets self.updated = set() # Storage for edited wikis self.updated_discussions = set() - self.wikis = all_wikis + self.domain = domain_object self.connection.buffer_class.errors = "replace" # Ignore encoding errors self.connection_details = None @@ -46,9 +50,12 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient): # print(message) url = urlparse(message) full_url = "https://"+url.netloc + recognize_langs(url.path) - if full_url in self.wikis and self.wikis[full_url].rc_active != -1: - self.updated.add(full_url) - logger.debug("New website appended to the list! {}".format(full_url)) + try: + if self.domain[full_url].rc_id != -1: + self.updated.add(full_url) + logger.debug("New website appended to the list! {}".format(full_url)) + except KeyError: + pass def parse_fandom_discussion(self, message: str): @@ -60,7 +67,7 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient): if post.get('action', 'unknown') != "deleted": # ignore deletion events url = urlparse(post.get('url')) full_url ="https://"+ url.netloc + recognize_langs(url.path) - if full_url in self.wikis: # POSSIBLE MEMORY LEAK AS WE DON'T HAVE A WAY TO CHECK IF WIKI IS LOOKING FOR DISCUSSIONS OR NOT + if full_url in self.domain: # POSSIBLE MEMORY LEAK AS WE DON'T HAVE A WAY TO CHECK IF WIKI IS LOOKING FOR DISCUSSIONS OR NOT self.updated_discussions.add("https://"+full_url) logger.debug("New website appended to the list (discussions)! {}".format(full_url)) diff --git a/src/redis_connector.py b/src/redis_connector.py new file mode 100644 index 0000000..215d0b8 --- /dev/null +++ b/src/redis_connector.py @@ -0,0 +1,14 @@ +import asyncio +import aioredis +from src.config import settings + + +class Redis: + def __init__(self): + self.connection = None + + async def connect(self): + self.connection = await aioredis.create_pool("redis://" + settings["redis_host"], encoding="UTF-8") + + +redis = Redis()