From 59d2869f4f567e325f47807eb44186651a7eb7dc Mon Sep 17 00:00:00 2001 From: Frisk Date: Sat, 28 Nov 2020 14:08:37 +0100 Subject: [PATCH] Stacking mechanic change --- src/bot.py | 15 ++++++++++++--- src/discord.py | 45 +++++++++++++++++++++++++++++++++++++++++++- src/formatters/rc.py | 8 ++++---- src/msgqueue.py | 33 ++++++-------------------------- src/wiki.py | 4 ++-- 5 files changed, 68 insertions(+), 37 deletions(-) diff --git a/src/bot.py b/src/bot.py index cc7a6f0..ba59d0a 100644 --- a/src/bot.py +++ b/src/bot.py @@ -13,12 +13,13 @@ from src.config import settings from src.database import db_cursor, db_connection from src.exceptions import * from src.misc import get_paths, get_domain -from src.msgqueue import messagequeue +from src.msgqueue import messagequeue, send_to_discord from src.queue_handler import DBHandler from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info, essential_feeds -from src.discord import DiscordMessage, generic_msg_sender_exception_logger +from src.discord import DiscordMessage, generic_msg_sender_exception_logger, stack_message_list from src.wiki_ratelimiter import RateLimiter + logging.config.dictConfig(settings["logging"]) logger = logging.getLogger("rcgcdb.bot") logger.debug("Current settings: {settings}".format(settings=settings)) @@ -281,14 +282,17 @@ async def scan_group(group: str): await process_cats(change, local_wiki, mw_msgs, categorize_events) else: # If we broke from previous loop (too many changes) don't execute sending messages here highest_rc = local_wiki.rc_active # setup var for later use + message_list = defaultdict(list) for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up if change["rcid"] > local_wiki.rc_active: if highest_rc is None or change["rcid"] > highest_rc: # make sure that the highest_rc is really highest rcid but do allow other entries with potentially lesser rcids come after without breaking the cycle highest_rc = change["rcid"] for target in targets.items(): try: - await essential_info(change, categorize_events, local_wiki, target, paths, + message = await essential_info(change, categorize_events, local_wiki, target, paths, recent_changes_resp, rate_limiter) + if message is not None: + message_list[target[0]].append(message) except asyncio.CancelledError: raise except: @@ -298,6 +302,11 @@ async def scan_group(group: str): else: logger.exception("Exception on RC formatter") await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in RC formatter", Wiki=queued_wiki.url, Change=str(change)[0:1000]) + # Lets stack the messages + for messages in message_list.values(): + messages = stack_message_list(messages) + for message in messages: + await send_to_discord(message) if recent_changes: # we don't have to test for highest_rc being null, because if there are no RC entries recent_changes will be an empty list which will result in false in here and DO NOT save the value local_wiki.rc_active = highest_rc DBHandler.add(queued_wiki.url, highest_rc) diff --git a/src/discord.py b/src/discord.py index 0e9ed32..47a0f70 100644 --- a/src/discord.py +++ b/src/discord.py @@ -7,6 +7,7 @@ from src.database import db_cursor from src.i18n import langs from src.exceptions import EmbedListFull from asyncio import TimeoutError +from math import ceil import aiohttp @@ -44,11 +45,13 @@ class DiscordMessage: self.webhook_object = dict(allowed_mentions={"parse": []}) self.webhook_url = webhook_url self.wiki = wiki + self.length = 0 if message_type == "embed": self._setup_embed() elif message_type == "compact": self.webhook_object["content"] = content + self.length = len(content) self.event_type = event_type @@ -60,6 +63,8 @@ class DiscordMessage: def __setitem__(self, key, value): """Set item is used only in embeds.""" try: + if key in ('title', 'description'): + self.length += len(value) - len(self.embed.get(key, "")) self.embed[key] = value except NameError: raise TypeError("Tried to assign a value when message type is plain message!") @@ -76,6 +81,9 @@ class DiscordMessage: self.embed = defaultdict(dict) self.embed["color"] = None + def __len__(self): + return self.length + def finish_embed(self): if self.embed["color"] is None: if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: @@ -91,7 +99,8 @@ class DiscordMessage: raise EmbedListFull self.webhook_object["embeds"].append(self.embed) - def set_author(self, name, url, icon_url=""): + def set_author(self, name: str, url: str, icon_url=""): + self.length += len(name) self.embed["author"]["name"] = name self.embed["author"]["url"] = url self.embed["author"]["icon_url"] = icon_url @@ -99,6 +108,7 @@ class DiscordMessage: def add_field(self, name, value, inline=False): if "fields" not in self.embed: self.embed["fields"] = [] + self.length += len(name) + len(value) self.embed["fields"].append(dict(name=name, value=value, inline=inline)) def set_avatar(self, url): @@ -107,6 +117,37 @@ class DiscordMessage: def set_name(self, name): self.webhook_object["username"] = name +def stack_message_list(messages: list) -> list: + if len(messages) > 1: + if messages[0].message_type() == "embed": + # for i, msg in enumerate(messages): + # if not isinstance(msg, StackedDiscordMessage): + # break + # else: # all messages in messages are stacked, exit this if + # i += 1 + removed_msgs = 0 + for group_index in range(ceil((len(messages)) / 10)): + message_group_index = group_index * 10 - removed_msgs + stackable = StackedDiscordMessage(messages[message_group_index]) + for message in messages[message_group_index + 1:message_group_index + 10]: + try: + stackable.add_embed(message.embed) + except EmbedListFull: + break + messages.remove(message) + removed_msgs += 1 + messages[message_group_index] = stackable + elif messages[0].message_type() == "compact": + message_index = 0 + while len(messages) > message_index+1: + if (len(messages[message_index]) + len(messages[message_index+1])) < 2000: + messages[message_index].webhook_object["content"] = messages[message_index].webhook_object["content"] + "\n" + messages[message_index + 1].webhook_object["content"] + messages[message_index].length += (len(messages[message_index + 1]) + 1) + messages.remove(messages[message_index + 1]) + else: + message_index += 1 + return messages + class StackedDiscordMessage(DiscordMessage): def __init__(self, discordmessage: DiscordMessage): @@ -119,6 +160,8 @@ class StackedDiscordMessage(DiscordMessage): self.add_embed(message.embed) def add_embed(self, embed): + if len(self) + len(embed) > 6000: + raise EmbedListFull self._setup_embed() self.embed = embed self.finish_embed() diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 86e5577..3726479 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -20,7 +20,7 @@ if 1 == 2: # additional translation strings in unreachable code _("autoreview"), _("autopatrol"), _("wiki_guardian"), ngettext("second", "seconds", 1), ngettext("minute", "minutes", 1), ngettext("hour", "hours", 1), ngettext("day", "days", 1), ngettext("week", "weeks", 1), ngettext("month", "months",1), ngettext("year", "years", 1), ngettext("millennium", "millennia", 1), ngettext("decade", "decades", 1), ngettext("century", "centuries", 1)) async def compact_formatter(action, change, parsed_comment, categories, recent_changes, message_target, paths, rate_limiter, - additional_data=None): + additional_data=None) -> DiscordMessage: """Recent Changes compact formatter, part of RcGcDw""" _ = langs[message_target[0][0]]["rc_formatters"].gettext ngettext = langs[message_target[0][0]]["rc_formatters"].ngettext @@ -361,10 +361,10 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c return else: content = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format(event=action, author=author, author_url=author_url, support=settings["support"]) - await send_to_discord(DiscordMessage("compact", action, message_target[1], content=content, wiki=WIKI_SCRIPT_PATH)) + return DiscordMessage("compact", action, message_target[1], content=content, wiki=WIKI_SCRIPT_PATH) -async def embed_formatter(action, change, parsed_comment, categories, recent_changes, message_target, paths, rate_limiter, additional_data=None): +async def embed_formatter(action, change, parsed_comment, categories, recent_changes, message_target, paths, rate_limiter, additional_data=None) -> DiscordMessage: """Recent Changes embed formatter, part of RcGcDw""" _ = langs[message_target[0][0]]["rc_formatters"].gettext ngettext = langs[message_target[0][0]]["rc_formatters"].ngettext @@ -805,4 +805,4 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) embed.finish_embed() - await send_to_discord(embed) + return embed diff --git a/src/msgqueue.py b/src/msgqueue.py index b55a300..4b74d69 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -2,7 +2,6 @@ import asyncio, logging, aiohttp from src.discord import send_to_discord_webhook, DiscordMessage, StackedDiscordMessage from src.config import settings from src.exceptions import EmbedListFull -from math import ceil from collections import defaultdict logger = logging.getLogger("rcgcdw.msgqueue") @@ -26,12 +25,12 @@ class MessageQueue: def add_message(self, message): self._queue.append(message) - - def replace_message(self, to_replace: DiscordMessage, with_replace: StackedDiscordMessage): - try: - self._queue[self._queue.index(to_replace)] = with_replace - except ValueError: - raise + # + # def replace_message(self, to_replace: DiscordMessage, with_replace: StackedDiscordMessage): + # try: + # self._queue[self._queue.index(to_replace)] = with_replace + # except ValueError: + # raise def cut_messages(self, item_num): self._queue = self._queue[item_num:] @@ -50,26 +49,6 @@ class MessageQueue: async def send_msg_set(self, msg_set: tuple): webhook_url, messages = msg_set # str("daosdkosakda/adkahfwegr34", list(DiscordMessage, DiscordMessage, DiscordMessage) - if len(messages) > 1 and messages[0].message_type() == "embed": - for i, msg in enumerate(messages): - if not isinstance(msg, StackedDiscordMessage): - break - else: #all messages in messages are stacked, exit this if - i += 1 - removed_msgs = 0 - for group_index in range(ceil((len(messages)-i)/10)): - message_group_index = group_index*10+i-removed_msgs - stackable = StackedDiscordMessage(messages[message_group_index]) - for message in messages[message_group_index+1:message_group_index+10]: - try: - stackable.add_embed(message.embed) - except EmbedListFull: - break - self._queue.remove(message) - messages.remove(message) - removed_msgs += 1 - self.replace_message(messages[message_group_index], stackable) - messages[message_group_index] = stackable for msg in messages: if self.global_rate_limit: return # if we are globally rate limited just wait for first gblocked request to finish diff --git a/src/wiki.py b/src/wiki.py index a8758f5..388dd4a 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -205,7 +205,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict): # db_wiki: webhook, wiki, lang, display, rcid, postid async def essential_info(change: dict, changed_categories, local_wiki: Wiki, target: tuple, paths: tuple, request: dict, - rate_limiter: RateLimiter): + rate_limiter: RateLimiter) -> src.discord.DiscordMessage: """Prepares essential information for both embed and compact message format.""" _ = langs[target[0][0]]["wiki"].gettext changed_categories = changed_categories.get(change["revid"], None) @@ -236,7 +236,7 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, tar additional_data["tags"][tag["name"]] = (BeautifulSoup(tag["displayname"], "lxml")).get_text() except KeyError: additional_data["tags"][tag["name"]] = None # Tags with no displ - await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, paths, rate_limiter, additional_data=additional_data) + return await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, paths, rate_limiter, additional_data=additional_data) async def essential_feeds(change: dict, comment_pages: dict, db_wiki: sqlite3.Row, target: tuple):