diff --git a/src/bot.py b/src/bot.py index 3f9f624..5020c5a 100644 --- a/src/bot.py +++ b/src/bot.py @@ -7,7 +7,7 @@ from src.misc import get_paths from src.exceptions import * from src.database import db_cursor from collections import defaultdict -from queue_handler import DBHandler +from src.queue_handler import DBHandler logging.config.dictConfig(settings["logging"]) logger = logging.getLogger("rcgcdb.bot") @@ -35,7 +35,7 @@ def calculate_delay() -> float: return min_delay -def generate_targets(wiki_url: str) -> defaultdict[list]: +def generate_targets(wiki_url: str) -> defaultdict: combinations = defaultdict(list) for webhook in db_cursor.execute('SELECT ROWID, * FROM rcgcdw WHERE wiki = ?', wiki_url): # rowid, guild, configid, webhook, wiki, lang, display, rcid, wikiid, postid @@ -50,13 +50,14 @@ async def wiki_scanner(): for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'): extended = False if db_wiki[3] not in all_wikis: - logger.debug("New wiki: {}".format(wiki[1])) + logger.debug("New wiki: {}".format(db_wiki[3])) all_wikis[db_wiki[3]] = Wiki() local_wiki = all_wikis[db_wiki[3]] # set a reference to a wiki object from memory if local_wiki.mw_messages is None: extended = True + logger.debug("test") try: - wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[0]) + wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3]) await local_wiki.check_status(wiki[3], wiki_response.status) except (WikiServerError, WikiError): continue # ignore this wiki if it throws errors @@ -83,9 +84,13 @@ async def wiki_scanner(): for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up if change["rcid"] < db_wiki[6]: for target in targets.items(): - await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths) + await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp) + if recent_changes: + DBHandler.add(db_wiki[3], change["rcid"]) await asyncio.sleep(delay=calc_delay) + DBHandler.update_db() + async def message_sender(): pass @@ -95,4 +100,5 @@ async def main_loop(): task1 = asyncio.create_task(wiki_scanner()) task2 = asyncio.create_task(message_sender()) + asyncio.run(main_loop()) diff --git a/src/config.py b/src/config.py index b970a39..53646b3 100644 --- a/src/config.py +++ b/src/config.py @@ -1,7 +1,7 @@ import json, sys, logging try: # load settings - with open("../settings.json") as sfile: + with open("settings.json") as sfile: settings = json.load(sfile) if "user-agent" in settings["header"]: settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.0") # set the version in the useragent diff --git a/src/discord.py b/src/discord.py index 4bd04fb..068a069 100644 --- a/src/discord.py +++ b/src/discord.py @@ -5,8 +5,6 @@ from src.database import db_cursor logger = logging.getLogger("rcgcdb.discord") -reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"), 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")} - # General functions class DiscordMessage(): """A class defining a typical Discord JSON representation of webhook payload.""" @@ -71,7 +69,9 @@ class DiscordMessage(): # User facing webhook functions -def wiki_removal(wiki_id, status): +def wiki_removal(wiki_id, status): # TODO Add lang selector + reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"), + 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")} reason = reasons.get(status, _("unknown error")) for observer in db_cursor.execute('SELECT * FROM observers WHERE wiki_id = ?', wiki_id): DiscordMessage("compact", "webhook/remove", webhook_url=observer[4], content=_("The webhook for {} has been removed due to {}.".format(reason))) # TODO diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 45fe486..6af84bf 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -4,9 +4,10 @@ import re import time import logging import base64 -from config import settings +from src.config import settings from src.misc import link_formatter, create_article_path, LinkParser, profile_field_name, ContentParser, DiscordMessage, safe_read from urllib.parse import quote_plus +from src.msgqueue import send_to_discord # from html.parser import HTMLParser from bs4 import BeautifulSoup @@ -20,7 +21,10 @@ from src.i18n import langs logger = logging.getLogger("rcgcdw.rc_formatters") #from src.rcgcdw import recent_changes, ngettext, logger, profile_field_name, LinkParser, pull_comment -async def compact_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths): +async def compact_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, + additional_data=None): + if additional_data is None: + additional_data = {"namespaces": {}, "tags": {}} WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] @@ -121,8 +125,8 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c else: restriction_description = _(" on namespaces: ") for namespace in change["logparams"]["restrictions"]["namespaces"]: - if str(namespace) in recent_changes.namespaces: # if we have cached namespace name for given namespace number, add its name to the list - namespaces.append("*{ns}*".format(ns=recent_changes.namespaces[str(namespace)]["*"])) + if str(namespace) in additional_data.namespaces: # if we have cached namespace name for given namespace number, add its name to the list + namespaces.append("*{ns}*".format(ns=additional_data.namespaces[str(namespace)]["*"])) else: namespaces.append("*{ns}*".format(ns=namespace)) restriction_description = restriction_description + ", ".join(namespaces) @@ -177,7 +181,7 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c content = _("[{author}]({author_url}) edited the {field} on {target} profile. *({desc})*").format(author=author, author_url=author_url, target=target, - field=profile_field_name(change["logparams"]['4:section'], False), + field=profile_field_name(change["logparams"]['4:section'], False, _), desc=BeautifulSoup(change["parsedcomment"], "lxml").get_text()) elif action in ("rights/rights", "rights/autopromote"): link = link_formatter(create_article_path("User:{user}".format(user=change["title"].split(":")[1]), WIKI_ARTICLE_PATH)) @@ -310,16 +314,18 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c else: logger.warning("No entry for {event} with params: {params}".format(event=action, params=change)) return - send_to_discord(DiscordMessage("compact", action, target[1], content=content)) + await send_to_discord(DiscordMessage("compact", action, target[1], content=content, wiki=WIKI_SCRIPT_PATH)) -async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths): +async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None): + if additional_data is None: + additional_data = {"namespaces": {}, "tags": {}} WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] WIKI_JUST_DOMAIN = paths[3] LinkParser = LinkParser() - embed = DiscordMessage("embed", action, target[1]) + embed = DiscordMessage("embed", action, target[1], wiki=WIKI_SCRIPT_PATH) if parsed_comment is None: parsed_comment = _("No description provided") if action != "suppressed": @@ -350,7 +356,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = "{redirect}{article} ({new}{minor}{bot}{space}{editsize})".format(redirect="⤷ " if "redirect" in change else "", article=change["title"], editsize="+" + str( editsize) if editsize > 0 else editsize, new=_("(N!) ") if action == "new" else "", minor=_("m") if action == "edit" and "minor" in change else "", bot=_('b') if "bot" in change else "", space=" " if "bot" in change or (action == "edit" and "minor" in change) or action == "new" else "") - if settings["appearance"]["embed"]["show_edit_changes"]: + if target[1] == 3: if action == "new": changed_content = await safe_read(await recent_changes.safe_request( "{wiki}?action=compare&format=json&fromtext=&torev={diff}&topst=1&prop=diff".format( @@ -362,7 +368,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha wiki=WIKI_API_PATH, diff=change["revid"],oldrev=change["old_revid"] )), "compare", "*") if changed_content: - EditDiff = ContentParser() + EditDiff = ContentParser(_) EditDiff.feed(changed_content) if EditDiff.small_prev_del: if EditDiff.small_prev_del.replace("~~", "").isspace(): @@ -422,37 +428,9 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = _("Reverted a version of {name}").format(name=change["title"]) else: embed["title"] = _("Uploaded {name}").format(name=change["title"]) - if settings["license_detection"]: - article_content = await safe_read(await recent_changes.safe_request( - "{wiki}?action=query&format=json&prop=revisions&titles={article}&rvprop=content".format( - wiki=WIKI_API_PATH, article=quote_plus(change["title"], safe=''))), "query", "pages") - if article_content is None: - logger.warning("Something went wrong when getting license for the image") - return 0 - if "-1" not in article_content: - content = list(article_content.values())[0]['revisions'][0]['*'] - try: - matches = re.search(re.compile(settings["license_regex"], re.IGNORECASE), content) - if matches is not None: - license = matches.group("license") - else: - if re.search(re.compile(settings["license_regex_detect"], re.IGNORECASE), content) is None: - license = _("**No license!**") - else: - license = "?" - except IndexError: - logger.error( - "Given regex for the license detection is incorrect. It does not have a capturing group called \"license\" specified. Please fix license_regex value in the config!") - license = "?" - except re.error: - logger.error( - "Given regex for the license detection is incorrect. Please fix license_regex or license_regex_detect values in the config!") - license = "?" - if license is not None: - parsed_comment += _("\nLicense: {}").format(license) if additional_info_retrieved: embed.add_field(_("Options"), _("([preview]({link}))").format(link=image_direct_url)) - if settings["appearance"]["embed"]["embed_images"]: + if target[1] > 1: embed["image"]["url"] = image_direct_url elif action == "delete/delete": link = create_article_path(change["title"].replace(" ", "_"), WIKI_ARTICLE_PATH) @@ -506,8 +484,8 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha else: restriction_description = _("Blocked from editing pages on following namespaces: ") for namespace in change["logparams"]["restrictions"]["namespaces"]: - if str(namespace) in recent_changes.namespaces: # if we have cached namespace name for given namespace number, add its name to the list - namespaces.append("*{ns}*".format(ns=recent_changes.namespaces[str(namespace)]["*"])) + if str(namespace) in additional_data.namespaces: # if we have cached namespace name for given namespace number, add its name to the list + namespaces.append("*{ns}*".format(ns=additional_data.namespaces[str(namespace)]["*"])) else: namespaces.append("*{ns}*".format(ns=namespace)) restriction_description = restriction_description + ", ".join(namespaces) @@ -527,22 +505,22 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = _("Unblocked {blocked_user}").format(blocked_user=user) elif action == "curseprofile/comment-created": if settings["appearance"]["embed"]["show_edit_changes"]: - parsed_comment = recent_changes.pull_comment(change["logparams"]["4:comment_id"], WIKI_ARTICLE_PATH) - link = create_article_path("Special:CommentPermalink/{commentid}".format(commentid=change["logparams"]["4:comment_id"])) + parsed_comment = await recent_changes.pull_comment(change["logparams"]["4:comment_id"], WIKI_API_PATH) + link = create_article_path("Special:CommentPermalink/{commentid}".format(commentid=change["logparams"]["4:comment_id"]), WIKI_ARTICLE_PATH) embed["title"] = _("Left a comment on {target}'s profile").format(target=change["title"].split(':')[1]) if change["title"].split(':')[1] != \ change["user"] else _( "Left a comment on their own profile") elif action == "curseprofile/comment-replied": if settings["appearance"]["embed"]["show_edit_changes"]: - parsed_comment = recent_changes.pull_comment(change["logparams"]["4:comment_id"], WIKI_ARTICLE_PATH) - link = create_article_path("Special:CommentPermalink/{commentid}".format(commentid=change["logparams"]["4:comment_id"])) + parsed_comment = await recent_changes.pull_comment(change["logparams"]["4:comment_id"], WIKI_API_PATH) + link = create_article_path("Special:CommentPermalink/{commentid}".format(commentid=change["logparams"]["4:comment_id"]), WIKI_ARTICLE_PATH) embed["title"] = _("Replied to a comment on {target}'s profile").format(target=change["title"].split(':')[1]) if change["title"].split(':')[1] != \ change["user"] else _( "Replied to a comment on their own profile") elif action == "curseprofile/comment-edited": if settings["appearance"]["embed"]["show_edit_changes"]: - parsed_comment = recent_changes.pull_comment(change["logparams"]["4:comment_id"], WIKI_ARTICLE_PATH) - link = create_article_path("Special:CommentPermalink/{commentid}".format(commentid=change["logparams"]["4:comment_id"])) + parsed_comment = await recent_changes.pull_comment(change["logparams"]["4:comment_id"], WIKI_API_PATH) + link = create_article_path("Special:CommentPermalink/{commentid}".format(commentid=change["logparams"]["4:comment_id"]), WIKI_ARTICLE_PATH) embed["title"] = _("Edited a comment on {target}'s profile").format(target=change["title"].split(':')[1]) if change["title"].split(':')[1] != \ change["user"] else _( "Edited a comment on their own profile") @@ -716,11 +694,11 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha if "tags" in change and change["tags"]: tag_displayname = [] for tag in change["tags"]: - if tag in recent_changes.tags: - if recent_changes.tags[tag] is None: + if tag in additional_data.tags: + if additional_data.tags[tag] is None: continue # Ignore hidden tags else: - tag_displayname.append(recent_changes.tags[tag]) + tag_displayname.append(additional_data.tags[tag]) else: tag_displayname.append(tag) embed.add_field(_("Tags"), ", ".join(tag_displayname)) @@ -730,4 +708,4 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) embed.finish_embed() - send_to_discord(embed) \ No newline at end of file + await send_to_discord(embed) \ No newline at end of file diff --git a/src/misc.py b/src/misc.py index 3036cde..346ecb5 100644 --- a/src/misc.py +++ b/src/misc.py @@ -9,14 +9,14 @@ import random from urllib.parse import urlparse, urlunparse import math import aiohttp -profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")} logger = logging.getLogger("rcgcdw.misc") class DiscordMessage(): """A class defining a typical Discord JSON representation of webhook payload.""" - def __init__(self, message_type: str, event_type: str, webhook_url: list, content=None): + def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None): self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) self.webhook_url = webhook_url + self.wiki = wiki if message_type == "embed": self.__setup_embed() @@ -76,9 +76,46 @@ class DiscordMessage(): def set_name(self, name): self.webhook_object["username"] = name + +async def send_to_discord_webhook(data: DiscordMessage, session: aiohttp.ClientSession): + header = settings["header"] + header['Content-Type'] = 'application/json' + for webhook in data.webhook_url: + try: + result = await session.post("https://discord.com/api/webhooks/"+webhook, data=repr(data), + headers=header) + except (aiohttp.ClientConnectionError, aiohttp.ServerConnectionError): + logger.exception("Could not send the message to Discord") + return 3 + return await handle_discord_http(result.status, repr(data), await result.text()) + + +async def handle_discord_http(code, formatted_embed, result): + if 300 > code > 199: # message went through + return 0 + elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header + logger.error( + "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") + logger.error(formatted_embed) + logger.error(result.text) + return 1 + elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND + logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") + + return 1 + elif code == 429: + logger.error("We are sending too many requests to the Discord, slowing down...") + return 2 + elif 499 < code < 600: + logger.error( + "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( + code)) + return 3 + + def get_paths(wiki: str, request) -> tuple: parsed_url = urlparse(wiki) - WIKI_API_PATH = wiki + request["query"]["general"]["scriptpath"] + "/api.php" + WIKI_API_PATH = wiki + request["query"]["general"]["scriptpath"] + "api.php" WIKI_SCRIPT_PATH = wiki WIKI_ARTICLE_PATH = urlunparse((*parsed_url[0:2], "", "", "", "")) + request["query"]["general"]["articlepath"] WIKI_JUST_DOMAIN = urlunparse((*parsed_url[0:2], "", "", "", "")) @@ -140,7 +177,15 @@ def create_article_path(article: str, WIKI_ARTICLE_PATH: str) -> str: return WIKI_ARTICLE_PATH.replace("$1", article) -def profile_field_name(name, embed): +def profile_field_name(name, embed, _): + profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), + "profile-link-google": _("Google link"), "profile-link-facebook": _("Facebook link"), + "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), + "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), + "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), + "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), + "profile-link-battlenet": _("Battle.net handle")} + try: return profile_fields[name] except KeyError: @@ -151,14 +196,17 @@ def profile_field_name(name, embed): class ContentParser(HTMLParser): - more = _("\n__And more__") current_tag = "" small_prev_ins = "" small_prev_del = "" - ins_length = len(more) - del_length = len(more) added = False + def __init__(self, _): + super().__init__() + self.more = _("\n__And more__") + self.ins_length = len(self.more) + self.del_length = len(self.more) + def handle_starttag(self, tagname, attribs): if tagname == "ins" or tagname == "del": self.current_tag = tagname diff --git a/src/msgqueue.py b/src/msgqueue.py index 73b41e0..3f177ea 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -1,10 +1,13 @@ -import asyncio, logging +import asyncio, logging, aiohttp +from src.misc import send_to_discord_webhook +from src.config import settings logger = logging.getLogger("rcgcdw.msgqueue") class MessageQueue: """Message queue class for undelivered messages""" def __init__(self): self._queue = [] + self.session = None def __repr__(self): return self._queue @@ -24,7 +27,12 @@ class MessageQueue: def cut_messages(self, item_num): self._queue = self._queue[item_num:] + async def create_session(self): + self.session = aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0)) + async def resend_msgs(self): + if self.session is None: + await self.create_session() if self._queue: logger.info( "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( @@ -33,9 +41,9 @@ class MessageQueue: logger.debug( "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), str(item))) - if send_to_discord_webhook(item) < 2: + if await send_to_discord_webhook(item, self.session) < 2: logger.debug("Sending message succeeded") - await asyncio.sleep(2.5) + await asyncio.sleep(1.5) else: logger.debug("Sending message failed") break diff --git a/src/queue_handler.py b/src/queue_handler.py index 8cc7a3a..41a948e 100644 --- a/src/queue_handler.py +++ b/src/queue_handler.py @@ -1,4 +1,5 @@ import logging +from src.database import db_cursor, db_connection logger = logging.getLogger("rcgcdb.queue_handler") @@ -14,6 +15,9 @@ class UpdateDB(): def update_db(self): for update in self.updated: + db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", update[1], update[0]) + db_connection.commit() + self.clear_list() -DBHandler = UpdateDB() \ No newline at end of file +DBHandler = UpdateDB() diff --git a/src/session.py b/src/session.py index 2b3b2aa..c4f7c56 100644 --- a/src/session.py +++ b/src/session.py @@ -1,4 +1,8 @@ import aiohttp from src.config import settings +session = None -session = aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0)) + +async def start_session(): + global session + session = aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0)) diff --git a/src/wiki.py b/src/wiki.py index bb0eb09..5b66e22 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -1,26 +1,32 @@ from dataclasses import dataclass -from src.session import session import re import logging, aiohttp from src.exceptions import * from src.database import db_cursor, db_connection from src.formatters.rc import embed_formatter, compact_formatter -from src.misc import LinkParser, RecentChangesClass -from i18n import langs +from src.misc import LinkParser +from src.i18n import langs import src.discord +from src.config import settings +from bs4 import BeautifulSoup logger = logging.getLogger("rcgcdb.wiki") supported_logs = ["protect/protect", "protect/modify", "protect/unprotect", "upload/overwrite", "upload/upload", "delete/delete", "delete/delete_redir", "delete/restore", "delete/revision", "delete/event", "import/upload", "import/interwiki", "merge/merge", "move/move", "move/move_redir", "protect/move_prot", "block/block", "block/unblock", "block/reblock", "rights/rights", "rights/autopromote", "abusefilter/modify", "abusefilter/create", "interwiki/iw_add", "interwiki/iw_edit", "interwiki/iw_delete", "curseprofile/comment-created", "curseprofile/comment-edited", "curseprofile/comment-deleted", "curseprofile/comment-purged", "curseprofile/profile-edited", "curseprofile/comment-replied", "contentmodel/change", "sprite/sprite", "sprite/sheet", "sprite/slice", "managetags/create", "managetags/delete", "managetags/activate", "managetags/deactivate", "tag/update", "cargo/createtable", "cargo/deletetable", "cargo/recreatetable", "cargo/replacetable", "upload/revert"] - @dataclass class Wiki: mw_messages: int = None fail_times: int = 0 # corresponding to amount of times connection with wiki failed for client reasons (400-499) + session: aiohttp.ClientSession = None + + async def create_session(self): + self.session = aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0)) async def fetch_wiki(self, extended, script_path) -> aiohttp.ClientResponse: + if self.session is None: + await self.create_session() url_path = script_path + "api.php" amount = 20 if extended: @@ -38,16 +44,15 @@ class Wiki: "rcprop": "title|redirect|timestamp|ids|loginfo|parsedcomment|sizes|flags|tags|user", "rclimit": amount, "rctype": "edit|new|log|external", "siprop": "namespaces|general"} try: - response = await session.get(url_path, params=params) + response = await self.session.get(url_path, params=params) except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError): logger.exception("A connection error occurred while requesting {}".format(url_path)) raise WikiServerError return response - @staticmethod - async def safe_request(url): + async def safe_request(self, url): try: - request = await session.get(url, timeout=5, allow_redirects=False) + request = await self.session.get(url, timeout=5, allow_redirects=False) request.raise_for_status() except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError): logger.exception("Reached connection error for request on link {url}".format(url=url)) @@ -76,6 +81,23 @@ class Wiki: logger.warning("{} rows affected by DELETE FROM rcgcdw WHERE wiki = {}".format(db_cursor.rowcount, wiki_id)) db_connection.commit() + async def pull_comment(self, comment_id, WIKI_API_PATH): + try: + comment = await self.safe_request( + "{wiki}?action=comment&do=getRaw&comment_id={comment}&format=json".format(wiki=WIKI_API_PATH, + comment=comment_id)).json()[ + "text"] + logger.debug("Got the following comment from the API: {}".format(comment)) + except (TypeError, AttributeError): + logger.exception("Could not resolve the comment text.") + except KeyError: + logger.exception("CurseProfile extension API did not respond with a valid comment content.") + else: + if len(comment) > 1000: + comment = comment[0:1000] + "…" + return comment + return "" + async def process_cats(event: dict, local_wiki: Wiki, category_msgs: dict, categorize_events: dict): if event["type"] == "categorize": @@ -134,7 +156,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict): mw_msgs[key] = msgs # it may be a little bit messy for sure, however I don't expect any reason to remove mw_msgs entries by one local_wiki.mw_messages = key -async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_wiki: tuple, target: tuple, paths: tuple): +async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_wiki: tuple, target: tuple, paths: tuple, request: dict): """Prepares essential information for both embed and compact message format.""" def _(string: str) -> str: """Our own translation string to make it compatible with async""" @@ -175,4 +197,10 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ else: logger.warning("This event is not implemented in the script. Please make an issue on the tracker attaching the following info: wiki url, time, and this information: {}".format(change)) return - await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, _, ngettext, paths) + additional_data = {"namespaces": request["query"]["namespaces"], "tags": {}} + for tag in request["query"]["tags"]: + try: + additional_data["tags"][tag["name"]] = (BeautifulSoup(tag["displayname"], "lxml")).get_text() + except KeyError: + additional_data["tags"][tag["name"]] = None # Tags with no displ + await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, _, ngettext, paths, additional_data=additional_data)