From e4ae528e70e22bab16ef93b820c05ba2402cbb7d Mon Sep 17 00:00:00 2001 From: Frisk Date: Thu, 23 Jul 2020 11:46:32 +0200 Subject: [PATCH] Fixed like entire code so it runs like 10 horses on a race --- src/bot.py | 9 +-- src/discord.py | 90 +++++++++++++++++++++--- src/formatters/discussions.py | 3 +- src/formatters/rc.py | 37 +++------- src/misc.py | 125 ++++------------------------------ src/msgqueue.py | 5 +- src/wiki.py | 8 +-- 7 files changed, 116 insertions(+), 161 deletions(-) diff --git a/src/bot.py b/src/bot.py index e72ce4c..48e2fde 100644 --- a/src/bot.py +++ b/src/bot.py @@ -48,8 +48,9 @@ def generate_targets(wiki_url: str) -> defaultdict: async def wiki_scanner(): while True: calc_delay = calculate_delay() - # db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'): - for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'): + fetch_all = db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki') + for db_wiki in fetch_all.fetchall(): + logger.debug("Wiki {}".format(db_wiki[3])) extended = False if db_wiki[3] not in all_wikis: logger.debug("New wiki: {}".format(db_wiki[3])) @@ -86,13 +87,13 @@ async def wiki_scanner(): for change in recent_changes: await process_cats(change, local_wiki, mw_msgs, categorize_events) for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up - if change["rcid"] < db_wiki[6]: + if change["rcid"] > db_wiki[6]: for target in targets.items(): await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp) if recent_changes: DBHandler.add(db_wiki[3], change["rcid"]) + DBHandler.update_db() await asyncio.sleep(delay=calc_delay) - DBHandler.update_db() async def message_sender(): diff --git a/src/discord.py b/src/discord.py index 068a069..c1ac431 100644 --- a/src/discord.py +++ b/src/discord.py @@ -1,16 +1,41 @@ import json, random, math, logging from collections import defaultdict + from src.config import settings from src.database import db_cursor +from src.misc import logger +from src.config import settings +from src.database import db_cursor +from src.i18n import langs +import aiohttp, gettext logger = logging.getLogger("rcgcdb.discord") # General functions + + +# User facing webhook functions +def wiki_removal(wiki_id, status): + for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wikiid = ?', (wiki_id,)): + def _(string: str) -> str: + """Our own translation string to make it compatible with async""" + return langs[observer[4]].gettext(string) + reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"), + 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")} + reason = reasons.get(status, _("unknown error")) + send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[observer[2]], content=_("The webhook for {} has been removed due to {}.".format(wiki_id, reason)), wiki=None)) + +async def webhook_removal_monitor(webhook_url: list, reason: int): + await send_to_discord_webhook_monitoring(DiscordMessage("compact", "webhook/remove", None, content="The webhook {} has been removed due to {}.".format("https://discord.com/api/webhooks/" + webhook_url[0], reason), wiki=None), + aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(4.0))) + + class DiscordMessage(): """A class defining a typical Discord JSON representation of webhook payload.""" - def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): + def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None): self.webhook_object = dict(allowed_mentions={"parse": []}) self.webhook_url = webhook_url + self.wiki = wiki if message_type == "embed": self.__setup_embed() @@ -47,7 +72,10 @@ class DiscordMessage(): def finish_embed(self): if self.embed["color"] is None: - self.embed["color"] = random.randrange(1, 16777215) + if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: + self.embed["color"] = random.randrange(1, 16777215) + else: + self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] else: self.embed["color"] = math.floor(self.embed["color"]) @@ -68,14 +96,54 @@ class DiscordMessage(): self.webhook_object["username"] = name -# User facing webhook functions -def wiki_removal(wiki_id, status): # TODO Add lang selector - reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"), - 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")} - reason = reasons.get(status, _("unknown error")) - for observer in db_cursor.execute('SELECT * FROM observers WHERE wiki_id = ?', wiki_id): - DiscordMessage("compact", "webhook/remove", webhook_url=observer[4], content=_("The webhook for {} has been removed due to {}.".format(reason))) # TODO - # Monitoring webhook functions def wiki_removal_monitor(wiki_id, status): - pass \ No newline at end of file + pass + + +async def send_to_discord_webhook_monitoring(data: DiscordMessage, session: aiohttp.ClientSession): + header = settings["header"] + header['Content-Type'] = 'application/json' + try: + result = await session.post("https://discord.com/api/webhooks/"+settings["monitoring_webhook"], data=repr(data), + headers=header) + except (aiohttp.ClientConnectionError, aiohttp.ServerConnectionError): + logger.exception("Could not send the message to Discord") + return 3 + + +async def send_to_discord_webhook(data: DiscordMessage, session: aiohttp.ClientSession): + header = settings["header"] + header['Content-Type'] = 'application/json' + for webhook in data.webhook_url: + try: + result = await session.post("https://discord.com/api/webhooks/"+webhook, data=repr(data), + headers=header) + except (aiohttp.ClientConnectionError, aiohttp.ServerConnectionError): + logger.exception("Could not send the message to Discord") + return 3 + return await handle_discord_http(result.status, repr(data), await result.text(), data) + + +async def handle_discord_http(code, formatted_embed, result, dmsg): + if 300 > code > 199: # message went through + return 0 + elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header + logger.error( + "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") + logger.error(formatted_embed) + logger.error(result.text) + return 1 + elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND + logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") + db_cursor.execute("DELETE FROM rcgcdw WHERE webhook = ?", (dmsg.webhook_url[0],)) + await webhook_removal_monitor(dmsg.webhook_url, code) + return 1 + elif code == 429: + logger.error("We are sending too many requests to the Discord, slowing down...") + return 2 + elif 499 < code < 600: + logger.error( + "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( + code)) + return 3 \ No newline at end of file diff --git a/src/formatters/discussions.py b/src/formatters/discussions.py index 4db58ec..b12c9c7 100644 --- a/src/formatters/discussions.py +++ b/src/formatters/discussions.py @@ -4,7 +4,8 @@ import gettext from urllib.parse import quote_plus from src.config import settings -from src.misc import DiscordMessage, send_to_discord, escape_formatting +from src.misc import send_to_discord, escape_formatting +from discord import DiscordMessage from src.i18n import disc _ = disc.gettext diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 970e904..09e1331 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -5,7 +5,8 @@ import time import logging import base64 from src.config import settings -from src.misc import link_formatter, create_article_path, LinkParser, profile_field_name, ContentParser, DiscordMessage, safe_read +from src.misc import link_formatter, create_article_path, parse_link, profile_field_name, ContentParser, safe_read +from src.discord import DiscordMessage from urllib.parse import quote_plus from src.msgqueue import send_to_discord # from html.parser import HTMLParser @@ -23,14 +24,12 @@ logger = logging.getLogger("rcgcdw.rc_formatters") async def compact_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None): - global LinkParser if additional_data is None: additional_data = {"namespaces": {}, "tags": {}} WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] WIKI_JUST_DOMAIN = paths[3] - LinkParser = LinkParser(paths[3]) if action != "suppressed": author_url = link_formatter(create_article_path("User:{user}".format(user=change["user"]), WIKI_ARTICLE_PATH)) author = change["user"] @@ -280,21 +279,15 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c link = link_formatter(create_article_path(change["title"], WIKI_ARTICLE_PATH)) content = _("[{author}]({author_url}) edited the slice for [{article}]({article_url})").format(author=author, author_url=author_url, article=change["title"], article_url=link) elif action == "cargo/createtable": - LinkParser.feed(change["logparams"]["0"]) - table = LinkParser.new_string - LinkParser.new_string = "" + table = parse_link(paths[3], change["logparams"]["0"]) content = _("[{author}]({author_url}) created the Cargo table \"{table}\"").format(author=author, author_url=author_url, table=table) elif action == "cargo/deletetable": content = _("[{author}]({author_url}) deleted the Cargo table \"{table}\"").format(author=author, author_url=author_url, table=change["logparams"]["0"]) elif action == "cargo/recreatetable": - LinkParser.feed(change["logparams"]["0"]) - table = LinkParser.new_string - LinkParser.new_string = "" + table = parse_link(paths[3], change["logparams"]["0"]) content = _("[{author}]({author_url}) recreated the Cargo table \"{table}\"").format(author=author, author_url=author_url, table=table) elif action == "cargo/replacetable": - LinkParser.feed(change["logparams"]["0"]) - table = LinkParser.new_string - LinkParser.new_string = "" + table = parse_link(paths[3], change["logparams"]["0"]) content = _("[{author}]({author_url}) replaced the Cargo table \"{table}\"").format(author=author, author_url=author_url, table=table) elif action == "managetags/create": link = link_formatter(create_article_path("Special:Tags", WIKI_ARTICLE_PATH)) @@ -319,14 +312,12 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None): - global LinkParser if additional_data is None: additional_data = {"namespaces": {}, "tags": {}} WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] WIKI_JUST_DOMAIN = paths[3] - LinkParser = LinkParser(paths[3]) embed = DiscordMessage("embed", action, target[1], wiki=WIKI_SCRIPT_PATH) if parsed_comment is None: parsed_comment = _("No description provided") @@ -358,7 +349,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = "{redirect}{article} ({new}{minor}{bot}{space}{editsize})".format(redirect="⤷ " if "redirect" in change else "", article=change["title"], editsize="+" + str( editsize) if editsize > 0 else editsize, new=_("(N!) ") if action == "new" else "", minor=_("m") if action == "edit" and "minor" in change else "", bot=_('b') if "bot" in change else "", space=" " if "bot" in change or (action == "edit" and "minor" in change) or action == "new" else "") - if target[1] == 3: + if target[0][1] == 3: if action == "new": changed_content = await safe_read(await recent_changes.safe_request( "{wiki}?action=compare&format=json&fromtext=&torev={diff}&topst=1&prop=diff".format( @@ -432,7 +423,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = _("Uploaded {name}").format(name=change["title"]) if additional_info_retrieved: embed.add_field(_("Options"), _("([preview]({link}))").format(link=image_direct_url)) - if target[1] > 1: + if target[0][1] > 1: embed["image"]["url"] = image_direct_url elif action == "delete/delete": link = create_article_path(change["title"].replace(" ", "_"), WIKI_ARTICLE_PATH) @@ -643,9 +634,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha link = create_article_path(change["title"].replace(" ", "_"), WIKI_ARTICLE_PATH) embed["title"] = _("Edited the slice for {article}").format(article=change["title"]) elif action == "cargo/createtable": - LinkParser.feed(change["logparams"]["0"]) - table = re.search(r"\[(.*?)\]\(<(.*?)>\)", LinkParser.new_string) - LinkParser.new_string = "" + table = re.search(r"\[(.*?)\]\(<(.*?)>\)", parse_link(paths[3], change["logparams"]["0"])) link = table.group(2) embed["title"] = _("Created the Cargo table \"{table}\"").format(table=table.group(1)) parsed_comment = None @@ -654,16 +643,12 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = _("Deleted the Cargo table \"{table}\"").format(table=change["logparams"]["0"]) parsed_comment = None elif action == "cargo/recreatetable": - LinkParser.feed(change["logparams"]["0"]) - table = re.search(r"\[(.*?)\]\(<(.*?)>\)", LinkParser.new_string) - LinkParser.new_string = "" + table = re.search(r"\[(.*?)\]\(<(.*?)>\)", parse_link(paths[3], change["logparams"]["0"])) link = table.group(2) embed["title"] = _("Recreated the Cargo table \"{table}\"").format(table=table.group(1)) parsed_comment = None elif action == "cargo/replacetable": - LinkParser.feed(change["logparams"]["0"]) - table = re.search(r"\[(.*?)\]\(<(.*?)>\)", LinkParser.new_string) - LinkParser.new_string = "" + table = re.search(r"\[(.*?)\]\(<(.*?)>\)", parse_link(paths[3], change["logparams"]["0"])) link = table.group(2) embed["title"] = _("Replaced the Cargo table \"{table}\"").format(table=table.group(1)) parsed_comment = None @@ -710,4 +695,4 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) embed.finish_embed() - await send_to_discord(embed) \ No newline at end of file + await send_to_discord(embed) diff --git a/src/misc.py b/src/misc.py index f46c24e..f3b714a 100644 --- a/src/misc.py +++ b/src/misc.py @@ -1,116 +1,12 @@ from html.parser import HTMLParser import base64, re -from src.config import settings -import json + import logging -from collections import defaultdict -import random from urllib.parse import urlparse, urlunparse -import math import aiohttp + logger = logging.getLogger("rcgcdw.misc") -class DiscordMessage(): - """A class defining a typical Discord JSON representation of webhook payload.""" - def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None): - self.webhook_object = dict(allowed_mentions={"parse": []}) - self.webhook_url = webhook_url - self.wiki = wiki - - if message_type == "embed": - self.__setup_embed() - elif message_type == "compact": - self.webhook_object["content"] = content - - self.event_type = event_type - - def __setitem__(self, key, value): - """Set item is used only in embeds.""" - try: - self.embed[key] = value - except NameError: - raise TypeError("Tried to assign a value when message type is plain message!") - - def __getitem__(self, item): - return self.embed[item] - - def __repr__(self): - """Return the Discord webhook object ready to be sent""" - return json.dumps(self.webhook_object) - - def __setup_embed(self): - self.embed = defaultdict(dict) - if "embeds" not in self.webhook_object: - self.webhook_object["embeds"] = [self.embed] - else: - self.webhook_object["embeds"].append(self.embed) - self.embed["color"] = None - - def add_embed(self): - self.finish_embed() - self.__setup_embed() - - def finish_embed(self): - if self.embed["color"] is None: - if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: - self.embed["color"] = random.randrange(1, 16777215) - else: - self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] - else: - self.embed["color"] = math.floor(self.embed["color"]) - - def set_author(self, name, url, icon_url=""): - self.embed["author"]["name"] = name - self.embed["author"]["url"] = url - self.embed["author"]["icon_url"] = icon_url - - def add_field(self, name, value, inline=False): - if "fields" not in self.embed: - self.embed["fields"] = [] - self.embed["fields"].append(dict(name=name, value=value, inline=inline)) - - def set_avatar(self, url): - self.webhook_object["avatar_url"] = url - - def set_name(self, name): - self.webhook_object["username"] = name - - -async def send_to_discord_webhook(data: DiscordMessage, session: aiohttp.ClientSession): - header = settings["header"] - header['Content-Type'] = 'application/json' - for webhook in data.webhook_url: - try: - result = await session.post("https://discord.com/api/webhooks/"+webhook, data=repr(data), - headers=header) - except (aiohttp.ClientConnectionError, aiohttp.ServerConnectionError): - logger.exception("Could not send the message to Discord") - return 3 - return await handle_discord_http(result.status, repr(data), await result.text()) - - -async def handle_discord_http(code, formatted_embed, result): - if 300 > code > 199: # message went through - return 0 - elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header - logger.error( - "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") - logger.error(formatted_embed) - logger.error(result.text) - return 1 - elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND - logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") - - return 1 - elif code == 429: - logger.error("We are sending too many requests to the Discord, slowing down...") - return 2 - elif 499 < code < 600: - logger.error( - "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( - code)) - return 3 - def get_paths(wiki: str, request) -> tuple: parsed_url = urlparse(wiki) @@ -125,10 +21,7 @@ class LinkParser(HTMLParser): new_string = "" recent_href = "" - - def __init__(self, domain): - self.WIKI_JUST_DOMAIN = domain - super().__init__() + WIKI_JUST_DOMAIN = "" def handle_starttag(self, tag, attrs): for attr in attrs: @@ -159,6 +52,18 @@ class LinkParser(HTMLParser): pass +LinkParse = LinkParser() + +def parse_link(domain: str, to_parse: str) -> str: + """Because I have strange issues using the LinkParser class myself, this is a helper function + to utilize the LinkParser properly""" + LinkParse.WIKI_JUST_DOMAIN = domain + LinkParse.feed(to_parse) + LinkParse.new_string = "" + LinkParse.recent_href = "" + return LinkParse.new_string + + def link_formatter(link: str) -> str: """Formats a link to not embed it""" return "<" + re.sub(r"([)])", "\\\\\\1", link).replace(" ", "_") + ">" diff --git a/src/msgqueue.py b/src/msgqueue.py index e8327ee..496424c 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -1,5 +1,5 @@ import asyncio, logging, aiohttp -from src.misc import send_to_discord_webhook +from src.discord import send_to_discord_webhook from src.config import settings logger = logging.getLogger("rcgcdw.msgqueue") @@ -35,8 +35,7 @@ class MessageQueue: await self.create_session() if self._queue: logger.info( - "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( - len(self._queue))) + "{} messages waiting to be delivered to Discord.".format(len(self._queue))) for num, item in enumerate(self._queue): logger.debug( "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), diff --git a/src/wiki.py b/src/wiki.py index 6a38706..c86852f 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -4,7 +4,7 @@ import logging, aiohttp from src.exceptions import * from src.database import db_cursor, db_connection from src.formatters.rc import embed_formatter, compact_formatter -from src.misc import LinkParser +from src.misc import parse_link from src.i18n import langs import src.discord from src.config import settings @@ -157,7 +157,6 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict): local_wiki.mw_messages = key async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_wiki: tuple, target: tuple, paths: tuple, request: dict): - global LinkParser """Prepares essential information for both embed and compact message format.""" def _(string: str) -> str: """Our own translation string to make it compatible with async""" @@ -166,16 +165,13 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ lang = langs[target[0][0]] ngettext = lang.ngettext # recent_changes = RecentChangesClass() # TODO Look into replacing RecentChangesClass with local_wiki - LinkParser = LinkParser(paths[3]) logger.debug(change) appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter if ("actionhidden" in change or "suppressed" in change): # if event is hidden using suppression await appearance_mode("suppressed", change, "", changed_categories, local_wiki, target, _, ngettext, paths) return if "commenthidden" not in change: - LinkParser.feed(change["parsedcomment"]) - parsed_comment = LinkParser.new_string - LinkParser.new_string = "" + parsed_comment = parse_link(paths[3], change["parsedcomment"]) parsed_comment = re.sub(r"(`|_|\*|~|{|}|\|\|)", "\\\\\\1", parsed_comment, 0) else: parsed_comment = _("~~hidden~~")