From 54ee888f86cd9d6959e4b244047fbc12ec0a9060 Mon Sep 17 00:00:00 2001 From: Frisk Date: Sat, 6 May 2023 14:29:27 +0200 Subject: [PATCH] Further work on RcGcDb --- extensions/base/discussions.py | 40 +++++++++++++-------------- src/api/formatter.py | 1 + src/discord/message.py | 9 +++++-- src/discord/queue.py | 15 ++++++----- src/discussions.py | 49 +++++++++++++++++----------------- src/domain.py | 6 +++++ src/domain_manager.py | 2 ++ src/irc_feed.py | 3 +++ src/queue_handler.py | 1 + src/wiki.py | 6 ++--- 10 files changed, 77 insertions(+), 55 deletions(-) diff --git a/extensions/base/discussions.py b/extensions/base/discussions.py index f75323e..474a008 100644 --- a/extensions/base/discussions.py +++ b/extensions/base/discussions.py @@ -136,16 +136,16 @@ def common_discussions(post: dict, embed: DiscordMessage, ctx: Context): @formatter.embed(event="discussion/forum") def embed_discussion_forum(ctx: Context, post: dict): - embed = DiscordMessage("embed", "discussion", ctx.settings["fandom_discussions"]["webhookURL"]) + embed = DiscordMessage("embed", "discussion", ctx.webhook_url) common_discussions(post, embed, ctx) author = ctx._("unknown") # Fail safe if post["createdBy"]["name"]: author = post["createdBy"]["name"] - embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.client.WIKI_SCRIPT_PATH, creatorId=post["creatorId"]), icon_url=post["createdBy"]["avatarUrl"]) if not post["isReply"]: - embed["url"] = "{url}f/p/{threadId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + embed["url"] = "{url}f/p/{threadId}".format(url=ctx.client.WIKI_SCRIPT_PATH, threadId=post["threadId"]) embed["title"] = ctx._("Created \"{title}\"").format(title=post["title"]) thread_funnel = post.get("funnel") @@ -189,7 +189,7 @@ def embed_discussion_forum(ctx: Context, post: dict): else: embed.event_type = "discussion/forum/reply" embed["title"] = ctx._("Replied to \"{title}\"").format(title=post["_embedded"]["thread"][0]["title"]) - embed["url"] = "{url}f/p/{threadId}/r/{postId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + embed["url"] = "{url}f/p/{threadId}/r/{postId}".format(url=ctx.client.WIKI_SCRIPT_PATH, threadId=post["threadId"], postId=post["id"]) return embed @@ -200,7 +200,7 @@ def compact_discussion_forum(ctx: Context, post: dict): author = ctx._("unknown") # Fail safe if post["createdBy"]["name"]: author = post["createdBy"]["name"] - author_url = "<{url}f/u/{creatorId}>".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + author_url = "<{url}f/u/{creatorId}>".format(url=ctx.client.WIKI_SCRIPT_PATH, creatorId=post["creatorId"]) if not post["isReply"]: thread_funnel = post.get("funnel") @@ -219,13 +219,13 @@ def compact_discussion_forum(ctx: Context, post: dict): thread_funnel)) event_type = "unknown" message = msg_text.format(author=author, author_url=author_url, title=post["title"], - url=ctx.settings["fandom_discussions"]["wiki_url"], threadId=post["threadId"], + url=ctx.client.WIKI_SCRIPT_PATH, threadId=post["threadId"], forumName=post["forumName"]) else: event_type = "discussion/forum/reply" message = ctx._( "[{author}]({author_url}) created a [reply](<{url}f/p/{threadId}/r/{postId}>) to [{title}](<{url}f/p/{threadId}>) in {forumName}").format( - author=author, author_url=author_url, url=ctx.settings["fandom_discussions"]["wiki_url"], + author=author, author_url=author_url, url=ctx.client.WIKI_SCRIPT_PATH, threadId=post["threadId"], postId=post["id"], title=post["_embedded"]["thread"][0]["title"], forumName=post["forumName"]) return DiscordMessage("compact", event_type, ctx.webhook_url, content=message) @@ -238,14 +238,14 @@ def compact_author_discussions(post: dict, ctx: Context): author = ctx._("unknown") # Fail safe if post["creatorIp"]: author = post["creatorIp"][1:] if ctx.settings.get("hide_ips", False) is False else ctx._("Unregistered user") - author_url = "<{url}wiki/Special:Contributions{creatorIp}>".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + author_url = "<{url}wiki/Special:Contributions{creatorIp}>".format(url=ctx.client.WIKI_SCRIPT_PATH, creatorIp=post["creatorIp"]) else: if post["createdBy"]["name"]: author = post["createdBy"]["name"] author_url = clean_link(ctx.client.create_article_path("User:{user}".format(user=author))) else: - author_url = "<{url}f/u/{creatorId}>".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + author_url = "<{url}f/u/{creatorId}>".format(url=ctx.client.WIKI_SCRIPT_PATH, creatorId=post["creatorId"]) return author, author_url @@ -256,22 +256,22 @@ def embed_author_discussions(post: dict, embed: DiscordMessage, ctx: Context): author = post["creatorIp"][1:] embed.set_author(author if ctx.settings.get("hide_ips", False) is False else ctx._("Unregistered user"), "{url}wiki/Special:Contributions{creatorIp}".format( - url=ctx.settings["fandom_discussions"]["wiki_url"], creatorIp=post["creatorIp"])) + url=ctx.client.WIKI_SCRIPT_PATH, creatorIp=post["creatorIp"])) else: if post["createdBy"]["name"]: author = post["createdBy"]["name"] - embed.set_author(author, "{url}wiki/User:{creator}".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + embed.set_author(author, "{url}wiki/User:{creator}".format(url=ctx.client.WIKI_SCRIPT_PATH, creator=author.replace(" ", "_")), icon_url=post["createdBy"]["avatarUrl"]) else: - embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"], + embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.client.WIKI_SCRIPT_PATH, creatorId=post["creatorId"]), icon_url=post["createdBy"]["avatarUrl"]) @formatter.embed(event="discussion/wall") def embed_discussion_wall(ctx: Context, post: dict): - embed = DiscordMessage("embed", "discussion", ctx.settings["fandom_discussions"]["webhookURL"]) + embed = DiscordMessage("embed", "discussion", ctx.webhook_url) common_discussions(post, embed, ctx) embed_author_discussions(post, embed, ctx) user_wall = ctx._("unknown") # Fail safe @@ -280,13 +280,13 @@ def embed_discussion_wall(ctx: Context, post: dict): if not post["isReply"]: embed.event_type = "discussion/wall/post" embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}".format( - url=ctx.settings["fandom_discussions"]["wiki_url"], user_wall=quote_plus(user_wall.replace(" ", "_")), + url=ctx.client.WIKI_SCRIPT_PATH, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"]) embed["title"] = ctx._("Created \"{title}\" on {user}'s Message Wall").format(title=post["title"], user=user_wall) else: embed.event_type = "discussion/wall/reply" embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}".format( - url=ctx.settings["fandom_discussions"]["wiki_url"], user_wall=quote_plus(user_wall.replace(" ", "_")), + url=ctx.client.WIKI_SCRIPT_PATH, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"]) embed["title"] = ctx._("Replied to \"{title}\" on {user}'s Message Wall").format( title=post["_embedded"]["thread"][0]["title"], user=user_wall) @@ -303,13 +303,13 @@ def compact_discussion_wall(ctx: Context, post: dict): event_type = "discussion/wall/post" message = ctx._( "[{author}]({author_url}) created [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format( - author=author, author_url=author_url, title=post["title"], url=ctx.settings["fandom_discussions"]["wiki_url"], + author=author, author_url=author_url, title=post["title"], url=ctx.client.WIKI_SCRIPT_PATH, user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"]) else: event_type = "discussion/wall/reply" message = ctx._( "[{author}]({author_url}) created a [reply](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}>) to [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format( - author=author, author_url=author_url, url=ctx.settings["fandom_discussions"]["wiki_url"], + author=author, author_url=author_url, url=ctx.client.WIKI_SCRIPT_PATH, title=post["_embedded"]["thread"][0]["title"], user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"]) return DiscordMessage("compact", event_type, ctx.webhook_url, content=message) @@ -319,12 +319,12 @@ def compact_discussion_wall(ctx: Context, post: dict): @formatter.embed(event="discussion/article_comment") def embed_discussion_article_comment(ctx: Context, post: dict): - embed = DiscordMessage("embed", "discussion", ctx.settings["fandom_discussions"]["webhookURL"]) + embed = DiscordMessage("embed", "discussion", ctx.webhook_url) common_discussions(post, embed, ctx) embed_author_discussions(post, embed, ctx) article_paths = ctx.comment_page if article_paths is None: - article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.settings["fandom_discussions"]["wiki_url"]} # No page known + article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.client.WIKI_SCRIPT_PATH} # No page known if not post["isReply"]: embed.event_type = "discussion/comment/post" embed["url"] = "{url}?commentId={commentId}".format(url=article_paths["fullUrl"], commentId=post["threadId"]) @@ -344,7 +344,7 @@ def compact_discussion_article_comment(ctx: Context, post: dict): author, author_url = compact_author_discussions(post, ctx) article_paths = ctx.comment_page if article_paths is None: - article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.settings["fandom_discussions"]["wiki_url"]} # No page known + article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.client.WIKI_SCRIPT_PATH} # No page known article_paths["fullUrl"] = article_paths["fullUrl"].replace(")", "\)").replace("()", "\(") if not post["isReply"]: event_type = "discussion/comment/post" diff --git a/src/api/formatter.py b/src/api/formatter.py index 735a55a..f1a1c90 100644 --- a/src/api/formatter.py +++ b/src/api/formatter.py @@ -40,6 +40,7 @@ def _register_formatter(func, kwargs, formatter_type: str, action_type=None): f"{src.api.hooks.formatter_hooks[formatter_type][act].__module__}! " f"Overwriting it with one from {func.__module__}") src.api.hooks.formatter_hooks[formatter_type][act] = func + logger.debug("Registering {type} hook for {event} event".format(type=formatter_type, event=act)) def embed(**kwargs): diff --git a/src/discord/message.py b/src/discord/message.py index ec3c614..8b0d059 100644 --- a/src/discord/message.py +++ b/src/discord/message.py @@ -183,6 +183,11 @@ class StackedDiscordMessage(): message_structure["embeds"] = [message.embed for message in self.message_list] return json.dumps(message_structure) + def check_for_length(self, message_length: int): + if self.message_type: + return len(self) + message_length > 6000 or len(self.message_list) > 9 + return (len(self) + message_length) > 2000 + def filter(self, params: dict) -> list[tuple[int, DiscordMessage]]: """Filters messages by their metadata""" return [(num, message) for num, message in enumerate(self.message_list) if message.matches(params)] @@ -193,9 +198,9 @@ class StackedDiscordMessage(): self.message_list.pop(message_id) def add_message(self, message: DiscordMessage): - if len(self) + len(message) > 6000 or len(self.message_list) > 9: + if self.check_for_length(len(message)): raise MessageTooBig - self.length += len(message) + self.length += len(message) + (self.message_type == 0) self.message_list.append(message) # self._setup_embed() # self.embed = message.embed diff --git a/src/discord/queue.py b/src/discord/queue.py index 81fd131..b44de5f 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -174,14 +174,16 @@ class MessageQueue: messagequeue = MessageQueue() -def handle_discord_http(code: int, formatted_embed: str, result: ClientResponse): +async def handle_discord_http(code: int, formatted_embed: str, result: ClientResponse): + text = await result.text() + print("HTTP response is {} and response {}".format(code, text)) if 300 > code > 199: # message went through return 0 elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header logger.error( "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") logger.error(formatted_embed) - logger.error(result.text()) + logger.error(text) raise aiohttp.ClientError("Message rejected.") elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND if result.method == "POST": # Ignore not found for DELETE and PATCH requests since the message could already be removed by admin @@ -206,12 +208,13 @@ def handle_discord_http(code: int, formatted_embed: str, result: ClientResponse) async def send_to_discord_webhook(message: [StackedDiscordMessage, DiscordMessageMetadata], webhook_path: str, method: str): + logger.debug("We are at sent_to_discord for {}".format(message)) header = settings["header"] header['Content-Type'] = 'application/json' header['X-RateLimit-Precision'] = "millisecond" async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(total=6)) as session: if isinstance(message, StackedDiscordMessage): - async with session.post(f"https://discord.com/api/webhooks/{webhook_path}?wait=true", data=repr(message)) as resp: + async with session.post(f"https://discord.com/api/webhooks/{webhook_path}?wait=true", data=repr(message)) as resp: # TODO Detect Invalid Webhook Token try: resp_json = await resp.json() # Add Discord Message ID which we can later use to delete/redact messages if we want @@ -222,10 +225,10 @@ async def send_to_discord_webhook(message: [StackedDiscordMessage, DiscordMessag logger.exception("Could not receive message ID from Discord due to invalid MIME type of response.") except ValueError: logger.exception(f"Could not decode JSON response from Discord. Response: {await resp.text()}]") - return handle_discord_http(resp.status, repr(message), resp) + return await handle_discord_http(resp.status, repr(message), resp) elif method == "DELETE": async with session.request(method=message.method, url=f"https://discord.com/api/webhooks/{webhook_path}/messages/{message.discord_callback_message_id}") as resp: - return handle_discord_http(resp.status, repr(message), resp) + return await handle_discord_http(resp.status, repr(message), resp) elif method == "PATCH": async with session.request(method=message.method, url=f"https://discord.com/api/webhooks/{webhook_path}/messages/{message.discord_callback_message_id}", data=repr(message)) as resp: - return handle_discord_http(resp.status, repr(message), resp) + return await handle_discord_http(resp.status, repr(message), resp) diff --git a/src/discussions.py b/src/discussions.py index 1f4ddac..3e3ea02 100644 --- a/src/discussions.py +++ b/src/discussions.py @@ -6,9 +6,9 @@ import logging import time import aiohttp import traceback -from api.context import Context -from api.hooks import formatter_hooks -from api.util import default_message +from src.api.context import Context +from src.api.hooks import formatter_hooks +from src.api.util import default_message from discord.queue import QueueEntry, messagequeue from src.i18n import langs from src.misc import prepare_settings @@ -33,23 +33,24 @@ class Discussions: async def tick_discussions(self): if self.domain_object is None: raise asyncio.CancelledError("fandom.com is not a domain we have any wikis for.") - while True: - try: - wiki_url = self.domain_object.irc.updated_discussions.pop() - except KeyError: - break - wiki = self.domain_object.get_wiki(wiki_url) - if wiki is None: - logger.error(f"Could not find a wiki with URL {wiki_url} in the domain group!") - continue - await self.run_discussion_scan(wiki) - - for wiki in self.filter_and_sort(): - if (int(time.time()) - (wiki.statistics.last_checked_discussion or 0)) > settings.get("irc_overtime", 3600): + while True: + try: + wiki_url = self.domain_object.irc.updated_discussions.pop() + except KeyError: + break + wiki = self.domain_object.get_wiki(wiki_url) + if wiki is None: + logger.error(f"Could not find a wiki with URL {wiki_url} in the domain group!") + continue await self.run_discussion_scan(wiki) - else: - return # Recently scanned wikis will get at the end of the self.wikis, so we assume what is first hasn't been checked for a while + + for wiki in self.filter_and_sort(): + if (int(time.time()) - (wiki.statistics.last_checked_discussion or 0)) > settings.get("irc_overtime", 3600): + await self.run_discussion_scan(wiki) + else: + return # Recently scanned wikis will get at the end of the self.wikis, so we assume what is first hasn't been checked for a while + await asyncio.sleep(5.0) def filter_and_sort(self) -> list[Wiki]: """Filters and sorts wikis from domain to return only the ones that aren't -1 and sorts them from oldest in checking to newest""" @@ -60,9 +61,8 @@ class Discussions: wiki.statistics.last_checked_discussion = int(time.time()) params = {"controller": "DiscussionPost", "method": "getPosts", "includeCounters": "false", "sortDirection": "descending", "sortKey": "creation_date", "limit": 20} - feeds_response = await wiki.fetch_discussions(params) try: - discussion_feed_resp = await feeds_response.json(encoding="UTF-8") + feeds_response, discussion_feed_resp = await wiki.fetch_discussions(params) if "error" in discussion_feed_resp: error = discussion_feed_resp["error"] if error == "NotFoundException": # Discussions disabled @@ -80,8 +80,8 @@ class Discussions: return if wiki.discussion_id is None: # new wiki, just get the last post to not spam the channel if len(discussion_feed) > 0: - dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != -1 OR postid IS NULL )", ( - discussion_feed[-1]["id"], + dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != '-1' OR postid IS NULL )", ( + str(discussion_feed[-1]["id"]), wiki.script_url))) wiki.statistics.update(last_post=discussion_feed[-1]["id"]) else: @@ -130,7 +130,7 @@ class Discussions: messagequeue.add_messages(message_list) if discussion_feed: wiki.statistics.update(last_post=discussion_feed[-1]["id"]) - dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != -1 OR postid IS NULL )", (discussion_feed[-1]["id"], + dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != '-1' OR postid IS NULL )", (str(discussion_feed[-1]["id"]), wiki.script_url))) # If this is not enough for the future, save rcid in message sending function to make sure we always send all of the changes @@ -148,8 +148,9 @@ async def essential_feeds(change: dict, comment_pages: dict, wiki: Wiki, target: context.set_comment_page(comment_page) discord_message: Optional[DiscordMessage] = None try: + discord_message = await asyncio.get_event_loop().run_in_executor( - None, functools.partial(default_message(identification_string, context.message_type, formatter_hooks), context, change)) + None, functools.partial(default_message(f"discussion/{identification_string.lower()}", context.message_type, formatter_hooks), context, change)) except: if settings.get("error_tolerance", 1) > 0: logger.exception("Exception on discord message creation in essential_feeds") diff --git a/src/domain.py b/src/domain.py index c77a77c..cbe435a 100644 --- a/src/domain.py +++ b/src/domain.py @@ -33,6 +33,12 @@ class Domain: def __iter__(self): return iter(self.wikis) + def __str__(self) -> str: + return f"" + + def __repr__(self): + return self.__str__() + def __getitem__(self, item): return diff --git a/src/domain_manager.py b/src/domain_manager.py index 9cb80c6..4fb3991 100644 --- a/src/domain_manager.py +++ b/src/domain_manager.py @@ -43,6 +43,8 @@ class DomainManager: self.remove_wiki(split_payload[1]) elif split_payload[0] == "UPDATE": await self.return_domain(self.get_domain(split_payload[1])).get_wiki(split_payload[1]).update_targets() + elif split_payload[0] == "DEBUG": + logger.info(self.domains) else: raise ValueError("Unknown pub/sub command! Payload: {}".format(payload)) diff --git a/src/irc_feed.py b/src/irc_feed.py index 9c97aca..65bb3dc 100644 --- a/src/irc_feed.py +++ b/src/irc_feed.py @@ -71,11 +71,14 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient): logger.warning("Seems like we have invalid JSON in Discussions part, message: {}".format(message)) return if post.get('action', 'unknown') != "deleted": # ignore deletion events + if isinstance(post.get('url'), bytes): + return url = urlparse(post.get('url')) full_url ="https://"+ url.netloc + recognize_langs(url.path) wiki = self.domain.get_wiki(full_url) if wiki and wiki.discussion_id != -1: self.updated_discussions.add(full_url) + logger.debug("New discussion wiki appended to the list! {}".format(full_url)) # if full_url in self.domain: # self.discussion_callback(full_url) diff --git a/src/queue_handler.py b/src/queue_handler.py index 0bbd8df..88e6674 100644 --- a/src/queue_handler.py +++ b/src/queue_handler.py @@ -33,6 +33,7 @@ class UpdateDB: async with db.pool().acquire() as connection: async with connection.transaction(): for update in self.updated: + logger.debug("Executing: {} {}".format(update[0], update[1])) await connection.execute(update[0], *update[1]) self.clear_list() await asyncio.sleep(10.0) diff --git a/src/wiki.py b/src/wiki.py index d85e853..dfc4428 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -378,7 +378,7 @@ class Wiki: async def remove_wiki_from_db(self, reason: str): raise NotImplementedError # TODO - async def fetch_discussions(self, params: dict) -> aiohttp.ClientResponse: + async def fetch_discussions(self, params: dict) -> tuple[aiohttp.ClientResponse, dict]: header = settings["header"] header["Accept"] = "application/hal+json" async with aiohttp.ClientSession(headers=header, @@ -389,9 +389,9 @@ class Wiki: feeds_response.raise_for_status() except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError, aiohttp.ClientResponseError, aiohttp.TooManyRedirects) as e: - logger.error("A connection error occurred while requesting {}".format(url_path)) + logger.error("A connection error occurred while requesting {}".format(feeds_response.url)) raise WikiServerError(e) - return feeds_response + return feeds_response, await feeds_response.json(encoding="UTF-8") def process_cachable(response: dict, wiki_object: Wiki) -> None: