From 884662b568ed3386cf3c58cefcf9e6cd19b08ec1 Mon Sep 17 00:00:00 2001 From: Frisk Date: Sun, 22 Nov 2020 13:44:15 +0100 Subject: [PATCH] Some more work done on the stacking --- src/bot.py | 8 ++++---- src/discord.py | 25 +++++++++++++++---------- src/formatters/rc.py | 2 -- src/msgqueue.py | 25 +++++++++++++++++++------ src/queue_handler.py | 2 +- src/wiki.py | 6 +++--- 6 files changed, 42 insertions(+), 26 deletions(-) diff --git a/src/bot.py b/src/bot.py index 641a6fa..36864f8 100644 --- a/src/bot.py +++ b/src/bot.py @@ -360,7 +360,7 @@ async def discussion_handler(): local_wiki = all_wikis[db_wiki["wiki"]] = Wiki() local_wiki.rc_active = db_wiki["rcid"] try: - feeds_response = await local_wiki.fetch_feeds(db_wiki["wikiid"], session) + feeds_response = await local_wiki.fetch_feeds(db_wiki["wiki"], session) except (WikiServerError, WikiError): continue # ignore this wiki if it throws errors try: @@ -389,9 +389,9 @@ async def discussion_handler(): continue if db_wiki["postid"] is None: # new wiki, just get the last post to not spam the channel if len(discussion_feed) > 0: - DBHandler.add(db_wiki["wikiid"], discussion_feed[-1]["id"], True) + DBHandler.add(db_wiki["wiki"], discussion_feed[-1]["id"], True) else: - DBHandler.add(db_wiki["wikiid"], "0", True) + DBHandler.add(db_wiki["wiki"], "0", True) DBHandler.update_db() continue comment_events = [] @@ -432,7 +432,7 @@ async def discussion_handler(): logger.exception("Exception on Feeds formatter") await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in feed formatter", Post=str(post)[0:1000], Wiki=db_wiki["wiki"]) if discussion_feed: - DBHandler.add(db_wiki["wikiid"], post["id"], True) + DBHandler.add(db_wiki["wiki"], post["id"], True) await asyncio.sleep(delay=2.0) # hardcoded really doesn't need much more DBHandler.update_db() except asyncio.CancelledError: diff --git a/src/discord.py b/src/discord.py index 79e1ed6..a7d140a 100644 --- a/src/discord.py +++ b/src/discord.py @@ -46,12 +46,17 @@ class DiscordMessage: self.wiki = wiki if message_type == "embed": - self.__setup_embed() + self._setup_embed() elif message_type == "compact": self.webhook_object["content"] = content self.event_type = event_type + def message_type(self): + if "content" in self.webhook_object: + return "compact" + return "embed" + def __setitem__(self, key, value): """Set item is used only in embeds.""" try: @@ -66,15 +71,9 @@ class DiscordMessage: """Return the Discord webhook object ready to be sent""" return json.dumps(self.webhook_object) - def __setup_embed(self): + def _setup_embed(self): """Setup another embed""" self.embed = defaultdict(dict) - if "embeds" not in self.webhook_object: - self.webhook_object["embeds"] = [self.embed] - else: - if len(self.webhook_object["embeds"]) > 9: - raise EmbedListFull - self.webhook_object["embeds"].append(self.embed) self.embed["color"] = None def finish_embed(self): @@ -85,6 +84,12 @@ class DiscordMessage: self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] else: self.embed["color"] = math.floor(self.embed["color"]) + if "embeds" not in self.webhook_object: + self.webhook_object["embeds"] = [self.embed] + else: + if len(self.webhook_object["embeds"]) > 10: + raise EmbedListFull + self.webhook_object["embeds"].append(self.embed) def set_author(self, name, url, icon_url=""): self.embed["author"]["name"] = name @@ -114,9 +119,9 @@ class StackedDiscordMessage(DiscordMessage): self.add_embed(message.embed) def add_embed(self, embed): - self.finish_embed() - self.__setup_embed() + self._setup_embed() self.embed = embed + self.finish_embed() # Monitoring webhook functions diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 60eafd2..4a9c0a7 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -311,11 +311,9 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c elif action == "managetags/create": link = link_formatter(create_article_path("Special:Tags", WIKI_ARTICLE_PATH)) content = "🏷️ "+_("[{author}]({author_url}) created a [tag]({tag_url}) \"{tag}\"").format(author=author, author_url=author_url, tag=change["logparams"]["tag"], tag_url=link) - recent_changes.init_info() elif action == "managetags/delete": link = link_formatter(create_article_path("Special:Tags", WIKI_ARTICLE_PATH)) content = "🏷️ "+_("[{author}]({author_url}) deleted a [tag]({tag_url}) \"{tag}\"").format(author=author, author_url=author_url, tag=change["logparams"]["tag"], tag_url=link) - recent_changes.init_info() elif action == "managetags/activate": link = link_formatter(create_article_path("Special:Tags", WIKI_ARTICLE_PATH)) content = "🏷️ "+_("[{author}]({author_url}) activated a [tag]({tag_url}) \"{tag}\"").format(author=author, author_url=author_url, tag=change["logparams"]["tag"], tag_url=link) diff --git a/src/msgqueue.py b/src/msgqueue.py index 7c3802d..54bf6fd 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -1,6 +1,7 @@ import asyncio, logging, aiohttp from src.discord import send_to_discord_webhook, DiscordMessage, StackedDiscordMessage from src.config import settings +from src.exceptions import EmbedListFull from math import ceil from collections import defaultdict logger = logging.getLogger("rcgcdw.msgqueue") @@ -26,6 +27,12 @@ class MessageQueue: def add_message(self, message): self._queue.append(message) + def replace_message(self, to_replace: DiscordMessage, with_replace: StackedDiscordMessage): + try: + self._queue[self._queue.index(to_replace)] = with_replace + except ValueError: + raise + def cut_messages(self, item_num): self._queue = self._queue[item_num:] @@ -43,18 +50,24 @@ class MessageQueue: async def send_msg_set(self, msg_set: tuple): webhook_url, messages = msg_set # str("daosdkosakda/adkahfwegr34", list(DiscordMessage, DiscordMessage, DiscordMessage) - if len(messages) > 1 and messages[0].message_type == "embed": + if len(messages) > 1 and messages[0].message_type() == "embed": for i, msg in enumerate(messages): if isinstance(msg, DiscordMessage): break - for group_index in range(ceil(len(messages)/10)): - message_group_index = group_index*10+i - stackable = StackedDiscordMessage(messages[message_group_index]) #TODO Find a way to replace item on the list with stacked message + removed_msgs = 0 + for group_index in range(ceil((len(messages)-i)/10)): + message_group_index = group_index*10+i-removed_msgs + stackable = StackedDiscordMessage(messages[message_group_index]) for message in messages[message_group_index+1:message_group_index+9]: - stackable.add_embed(message.embed) + try: + stackable.add_embed(message.embed) + except EmbedListFull: + break self._queue.remove(message) messages.remove(message) - + self.replace_message(messages[message_group_index], stackable) + messages[message_group_index] = stackable + removed_msgs += 1 for msg in messages: if self.global_rate_limit: return # if we are globally rate limited just wait for first gblocked request to finish diff --git a/src/queue_handler.py b/src/queue_handler.py index 4ab68c6..e7853e4 100644 --- a/src/queue_handler.py +++ b/src/queue_handler.py @@ -19,7 +19,7 @@ class UpdateDB: if update[2] is None: sql = "UPDATE rcgcdw SET rcid = ? WHERE wiki = ? AND ( rcid != -1 OR rcid IS NULL )" else: - sql = "UPDATE rcgcdw SET postid = ? WHERE wikiid = ?" + sql = "UPDATE rcgcdw SET postid = ? WHERE wiki = ?" db_cursor.execute(sql, (update[1], update[0],)) db_connection.commit() self.clear_list() diff --git a/src/wiki.py b/src/wiki.py index 0418e9e..50557c6 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -67,9 +67,9 @@ class Wiki: return response @staticmethod - async def fetch_feeds(wiki_id, session: aiohttp.ClientSession) -> aiohttp.ClientResponse: - url_path = "https://services.fandom.com/discussion/{wikiid}/posts".format(wikiid=wiki_id) - params = {"sortDirection": "descending", "sortKey": "creation_date", "limit": 20} + async def fetch_feeds(wiki, session: aiohttp.ClientSession) -> aiohttp.ClientResponse: + url_path = "{wiki}wikia.php".format(wiki=wiki) + params = {"controller": "DiscussionPost", "method": "getPosts", "sortDirection": "descending", "sortKey": "creation_date", "limit": 20} try: response = await session.get(url_path, params=params) response.raise_for_status()