diff --git a/src/bot.py b/src/bot.py index e0d0498..3f9f624 100644 --- a/src/bot.py +++ b/src/bot.py @@ -84,13 +84,13 @@ async def wiki_scanner(): if change["rcid"] < db_wiki[6]: for target in targets.items(): await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths) - - await asyncio.sleep(delay=calc_delay) + async def message_sender(): pass + async def main_loop(): task1 = asyncio.create_task(wiki_scanner()) task2 = asyncio.create_task(message_sender()) diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 10099f5..45fe486 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -5,7 +5,7 @@ import time import logging import base64 from config import settings -from src.misc import link_formatter, create_article_path, LinkParser, profile_field_name, ContentParser, DiscordMessage +from src.misc import link_formatter, create_article_path, LinkParser, profile_field_name, ContentParser, DiscordMessage, safe_read from urllib.parse import quote_plus # from html.parser import HTMLParser @@ -20,7 +20,7 @@ from src.i18n import langs logger = logging.getLogger("rcgcdw.rc_formatters") #from src.rcgcdw import recent_changes, ngettext, logger, profile_field_name, LinkParser, pull_comment -async def compact_formatter(action, change, parsed_comment, categories, recent_changes, _, ngettext, paths): +async def compact_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths): WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] @@ -310,16 +310,16 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c else: logger.warning("No entry for {event} with params: {params}".format(event=action, params=change)) return - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content)) + send_to_discord(DiscordMessage("compact", action, target[1], content=content)) -async def embed_formatter(action, change, parsed_comment, categories, recent_changes, _, ngettext, paths): +async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths): WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] WIKI_JUST_DOMAIN = paths[3] LinkParser = LinkParser() - embed = DiscordMessage("embed", action, settings["webhookURL"]) + embed = DiscordMessage("embed", action, target[1]) if parsed_comment is None: parsed_comment = _("No description provided") if action != "suppressed": @@ -352,12 +352,12 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha minor=_("m") if action == "edit" and "minor" in change else "", bot=_('b') if "bot" in change else "", space=" " if "bot" in change or (action == "edit" and "minor" in change) or action == "new" else "") if settings["appearance"]["embed"]["show_edit_changes"]: if action == "new": - changed_content = safe_read(recent_changes.safe_request( + changed_content = await safe_read(await recent_changes.safe_request( "{wiki}?action=compare&format=json&fromtext=&torev={diff}&topst=1&prop=diff".format( wiki=WIKI_API_PATH, diff=change["revid"] )), "compare", "*") else: - changed_content = safe_read(recent_changes.safe_request( + changed_content = await safe_read(await recent_changes.safe_request( "{wiki}?action=compare&format=json&fromrev={oldrev}&torev={diff}&topst=1&prop=diff".format( wiki=WIKI_API_PATH, diff=change["revid"],oldrev=change["old_revid"] )), "compare", "*") @@ -383,7 +383,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha logger.warning("Unable to download data on the edit content!") elif action in ("upload/overwrite", "upload/upload", "upload/revert"): # sending files license = None - urls = safe_read(recent_changes.safe_request( + urls = await safe_read(await recent_changes.safe_request( "{wiki}?action=query&format=json&prop=imageinfo&list=&meta=&titles={filename}&iiprop=timestamp%7Curl%7Carchivename&iilimit=5".format( wiki=WIKI_API_PATH, filename=change["title"])), "query", "pages") link = create_article_path(change["title"].replace(" ", "_"), WIKI_ARTICLE_PATH) @@ -423,7 +423,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha else: embed["title"] = _("Uploaded {name}").format(name=change["title"]) if settings["license_detection"]: - article_content = safe_read(recent_changes.safe_request( + article_content = await safe_read(await recent_changes.safe_request( "{wiki}?action=query&format=json&prop=revisions&titles={article}&rvprop=content".format( wiki=WIKI_API_PATH, article=quote_plus(change["title"], safe=''))), "query", "pages") if article_content is None: diff --git a/src/misc.py b/src/misc.py index 35a0f89..3036cde 100644 --- a/src/misc.py +++ b/src/misc.py @@ -8,12 +8,13 @@ from collections import defaultdict import random from urllib.parse import urlparse, urlunparse import math +import aiohttp profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")} logger = logging.getLogger("rcgcdw.misc") class DiscordMessage(): """A class defining a typical Discord JSON representation of webhook payload.""" - def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): + def __init__(self, message_type: str, event_type: str, webhook_url: list, content=None): self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) self.webhook_url = webhook_url @@ -218,6 +219,22 @@ class ContentParser(HTMLParser): self.current_tag = "" +async def safe_read(request: aiohttp.ClientResponse, *keys): + if request is None: + return None + try: + request = await request.json(encoding="UTF-8") + for item in keys: + request = request[item] + except KeyError: + logger.warning( + "Failure while extracting data from request on key {key} in {change}".format(key=item, change=request)) + return None + except aiohttp.ClientResponseError: + logger.warning("Failure while extracting data from request in {change}".format(change=request)) + return None + return request + # class RecentChangesClass(): # """Store verious data and functions related to wiki and fetching of Recent Changes""" # def __init__(self): diff --git a/src/msgqueue.py b/src/msgqueue.py index 4e4bb5d..73b41e0 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -46,4 +46,8 @@ class MessageQueue: logger.debug(self._queue) -messagequeue = MessageQueue() \ No newline at end of file +messagequeue = MessageQueue() + + +async def send_to_discord(msg): + messagequeue.add_message(msg) diff --git a/src/wiki.py b/src/wiki.py index 9b623e2..bb0eb09 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -13,7 +13,7 @@ logger = logging.getLogger("rcgcdb.wiki") supported_logs = ["protect/protect", "protect/modify", "protect/unprotect", "upload/overwrite", "upload/upload", "delete/delete", "delete/delete_redir", "delete/restore", "delete/revision", "delete/event", "import/upload", "import/interwiki", "merge/merge", "move/move", "move/move_redir", "protect/move_prot", "block/block", "block/unblock", "block/reblock", "rights/rights", "rights/autopromote", "abusefilter/modify", "abusefilter/create", "interwiki/iw_add", "interwiki/iw_edit", "interwiki/iw_delete", "curseprofile/comment-created", "curseprofile/comment-edited", "curseprofile/comment-deleted", "curseprofile/comment-purged", "curseprofile/profile-edited", "curseprofile/comment-replied", "contentmodel/change", "sprite/sprite", "sprite/sheet", "sprite/slice", "managetags/create", "managetags/delete", "managetags/activate", "managetags/deactivate", "tag/update", "cargo/createtable", "cargo/deletetable", "cargo/recreatetable", "cargo/replacetable", "upload/revert"] - + @dataclass class Wiki: @@ -44,6 +44,17 @@ class Wiki: raise WikiServerError return response + @staticmethod + async def safe_request(url): + try: + request = await session.get(url, timeout=5, allow_redirects=False) + request.raise_for_status() + except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError): + logger.exception("Reached connection error for request on link {url}".format(url=url)) + return None + else: + return request + async def check_status(self, wiki_url, status): if 199 < status < 300: self.fail_times = 0 @@ -131,12 +142,12 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ lang = langs[target[0][0]] ngettext = lang.ngettext - recent_changes = RecentChangesClass() # TODO Look into replacing RecentChangesClass with local_wiki + # recent_changes = RecentChangesClass() # TODO Look into replacing RecentChangesClass with local_wiki LinkParser = LinkParser("domain") logger.debug(change) appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter if ("actionhidden" in change or "suppressed" in change): # if event is hidden using suppression - await appearance_mode("suppressed", change, "", changed_categories, recent_changes, target, _, ngettext, paths) + await appearance_mode("suppressed", change, "", changed_categories, local_wiki, target, _, ngettext, paths) return if "commenthidden" not in change: LinkParser.feed(change["parsedcomment"]) @@ -164,4 +175,4 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ else: logger.warning("This event is not implemented in the script. Please make an issue on the tracker attaching the following info: wiki url, time, and this information: {}".format(change)) return - await appearance_mode(identification_string, change, parsed_comment, changed_categories, recent_changes, target, _, ngettext, paths) + await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, _, ngettext, paths)