Stacking mechanic change

src/bot.py

@@ -13,12 +13,13 @@ from src.config import settings
 from src.database import db_cursor, db_connection
 from src.exceptions import *
 from src.misc import get_paths, get_domain
-from src.msgqueue import messagequeue
+from src.msgqueue import messagequeue, send_to_discord
 from src.queue_handler import DBHandler
 from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info, essential_feeds
-from src.discord import DiscordMessage, generic_msg_sender_exception_logger
+from src.discord import DiscordMessage, generic_msg_sender_exception_logger, stack_message_list
 from src.wiki_ratelimiter import RateLimiter
 
+
 logging.config.dictConfig(settings["logging"])
 logger = logging.getLogger("rcgcdb.bot")
 logger.debug("Current settings: {settings}".format(settings=settings))
@@ -281,14 +282,17 @@ async def scan_group(group: str):
 					await process_cats(change, local_wiki, mw_msgs, categorize_events)
 				else:  # If we broke from previous loop (too many changes) don't execute sending messages here
 					highest_rc = local_wiki.rc_active  # setup var for later use
+					message_list = defaultdict(list)
 					for change in recent_changes:  # Yeah, second loop since the categories require to be all loaded up
 						if change["rcid"] > local_wiki.rc_active:
 							if highest_rc is None or change["rcid"] > highest_rc:  # make sure that the highest_rc is really highest rcid but do allow other entries with potentially lesser rcids come after without breaking the cycle
 								highest_rc = change["rcid"]
 							for target in targets.items():
 								try:
-									await essential_info(change, categorize_events, local_wiki, target, paths,
+									message = await essential_info(change, categorize_events, local_wiki, target, paths,
 									                     recent_changes_resp, rate_limiter)
+									if message is not None:
+										message_list[target[0]].append(message)
 								except asyncio.CancelledError:
 									raise
 								except:
@@ -298,6 +302,11 @@ async def scan_group(group: str):
 									else:
 										logger.exception("Exception on RC formatter")
 										await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in RC formatter", Wiki=queued_wiki.url, Change=str(change)[0:1000])
+					# Lets stack the messages
+					for messages in message_list.values():
+						messages = stack_message_list(messages)
+						for message in messages:
+							await send_to_discord(message)
 					if recent_changes:  # we don't have to test for highest_rc being null, because if there are no RC entries recent_changes will be an empty list which will result in false in here and DO NOT save the value
 						local_wiki.rc_active = highest_rc
 						DBHandler.add(queued_wiki.url, highest_rc)

src/discord.py

@@ -7,6 +7,7 @@ from src.database import db_cursor
 from src.i18n import langs
 from src.exceptions import EmbedListFull
 from asyncio import TimeoutError
+from math import ceil
 
 import aiohttp
 
@@ -44,11 +45,13 @@ class DiscordMessage:
 		self.webhook_object = dict(allowed_mentions={"parse": []})
 		self.webhook_url = webhook_url
 		self.wiki = wiki
+		self.length = 0
 
 		if message_type == "embed":
 			self._setup_embed()
 		elif message_type == "compact":
 			self.webhook_object["content"] = content
+			self.length = len(content)
 
 		self.event_type = event_type
 
@@ -60,6 +63,8 @@ class DiscordMessage:
 	def __setitem__(self, key, value):
 		"""Set item is used only in embeds."""
 		try:
+			if key in ('title', 'description'):
+				self.length += len(value) - len(self.embed.get(key, ""))
 			self.embed[key] = value
 		except NameError:
 			raise TypeError("Tried to assign a value when message type is plain message!")
@@ -76,6 +81,9 @@ class DiscordMessage:
 		self.embed = defaultdict(dict)
 		self.embed["color"] = None
 
+	def __len__(self):
+		return self.length
+
 	def finish_embed(self):
 		if self.embed["color"] is None:
 			if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None:
@@ -91,7 +99,8 @@ class DiscordMessage:
 				raise EmbedListFull
 			self.webhook_object["embeds"].append(self.embed)
 
-	def set_author(self, name, url, icon_url=""):
+	def set_author(self, name: str, url: str, icon_url=""):
+		self.length += len(name)
 		self.embed["author"]["name"] = name
 		self.embed["author"]["url"] = url
 		self.embed["author"]["icon_url"] = icon_url
@@ -99,6 +108,7 @@ class DiscordMessage:
 	def add_field(self, name, value, inline=False):
 		if "fields" not in self.embed:
 			self.embed["fields"] = []
+		self.length += len(name) + len(value)
 		self.embed["fields"].append(dict(name=name, value=value, inline=inline))
 
 	def set_avatar(self, url):
@@ -107,6 +117,37 @@ class DiscordMessage:
 	def set_name(self, name):
 		self.webhook_object["username"] = name
 
+def stack_message_list(messages: list) -> list:
+	if len(messages) > 1:
+		if messages[0].message_type() == "embed":
+			# for i, msg in enumerate(messages):
+			# 	if not isinstance(msg, StackedDiscordMessage):
+			# 		break
+			# else:  # all messages in messages are stacked, exit this if
+			# 	i += 1
+			removed_msgs = 0
+			for group_index in range(ceil((len(messages)) / 10)):
+				message_group_index = group_index * 10 - removed_msgs
+				stackable = StackedDiscordMessage(messages[message_group_index])
+				for message in messages[message_group_index + 1:message_group_index + 10]:
+					try:
+						stackable.add_embed(message.embed)
+					except EmbedListFull:
+						break
+					messages.remove(message)
+					removed_msgs += 1
+				messages[message_group_index] = stackable
+		elif messages[0].message_type() == "compact":
+			message_index = 0
+			while len(messages) > message_index+1:
+				if (len(messages[message_index]) + len(messages[message_index+1])) < 2000:
+					messages[message_index].webhook_object["content"] = messages[message_index].webhook_object["content"] + "\n" + messages[message_index + 1].webhook_object["content"]
+					messages[message_index].length += (len(messages[message_index + 1]) + 1)
+					messages.remove(messages[message_index + 1])
+				else:
+					message_index += 1
+	return messages
+
 
 class StackedDiscordMessage(DiscordMessage):
 	def __init__(self, discordmessage: DiscordMessage):
@@ -119,6 +160,8 @@ class StackedDiscordMessage(DiscordMessage):
 			self.add_embed(message.embed)
 
 	def add_embed(self, embed):
+		if len(self) + len(embed) > 6000:
+			raise EmbedListFull
 		self._setup_embed()
 		self.embed = embed
 		self.finish_embed()

src/formatters/rc.py

@@ -20,7 +20,7 @@ if 1 == 2: # additional translation strings in unreachable code
 	      _("autoreview"), _("autopatrol"), _("wiki_guardian"), ngettext("second", "seconds", 1), ngettext("minute", "minutes", 1), ngettext("hour", "hours", 1), ngettext("day", "days", 1), ngettext("week", "weeks", 1), ngettext("month", "months",1), ngettext("year", "years", 1), ngettext("millennium", "millennia", 1), ngettext("decade", "decades", 1), ngettext("century", "centuries", 1))
 
 async def compact_formatter(action, change, parsed_comment, categories, recent_changes, message_target, paths, rate_limiter,
-                            additional_data=None):
+                            additional_data=None) -> DiscordMessage:
 	"""Recent Changes compact formatter, part of RcGcDw"""
 	_ = langs[message_target[0][0]]["rc_formatters"].gettext
 	ngettext = langs[message_target[0][0]]["rc_formatters"].ngettext
@@ -361,10 +361,10 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c
 			return
 		else:
 			content = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format(event=action, author=author, author_url=author_url, support=settings["support"])
-	await send_to_discord(DiscordMessage("compact", action, message_target[1], content=content, wiki=WIKI_SCRIPT_PATH))
+	return DiscordMessage("compact", action, message_target[1], content=content, wiki=WIKI_SCRIPT_PATH)
 
 
-async def embed_formatter(action, change, parsed_comment, categories, recent_changes, message_target, paths, rate_limiter, additional_data=None):
+async def embed_formatter(action, change, parsed_comment, categories, recent_changes, message_target, paths, rate_limiter, additional_data=None) -> DiscordMessage:
 	"""Recent Changes embed formatter, part of RcGcDw"""
 	_ = langs[message_target[0][0]]["rc_formatters"].gettext
 	ngettext = langs[message_target[0][0]]["rc_formatters"].ngettext
@@ -805,4 +805,4 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
 		del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else ""
 		embed.add_field(_("Changed categories"), new_cat + del_cat)
 	embed.finish_embed()
-	await send_to_discord(embed)
+	return embed

src/msgqueue.py

@@ -2,7 +2,6 @@ import asyncio, logging, aiohttp
 from src.discord import send_to_discord_webhook, DiscordMessage, StackedDiscordMessage
 from src.config import settings
 from src.exceptions import EmbedListFull
-from math import ceil
 from collections import defaultdict
 logger = logging.getLogger("rcgcdw.msgqueue")
 
@@ -26,12 +25,12 @@ class MessageQueue:
 
 	def add_message(self, message):
 		self._queue.append(message)
-
-	def replace_message(self, to_replace: DiscordMessage, with_replace: StackedDiscordMessage):
-		try:
-			self._queue[self._queue.index(to_replace)] = with_replace
-		except ValueError:
-			raise
+	#
+	# def replace_message(self, to_replace: DiscordMessage, with_replace: StackedDiscordMessage):
+	# 	try:
+	# 		self._queue[self._queue.index(to_replace)] = with_replace
+	# 	except ValueError:
+	# 		raise
 
 	def cut_messages(self, item_num):
 		self._queue = self._queue[item_num:]
@@ -50,26 +49,6 @@ class MessageQueue:
 
 	async def send_msg_set(self, msg_set: tuple):
 		webhook_url, messages = msg_set  #  str("daosdkosakda/adkahfwegr34", list(DiscordMessage, DiscordMessage, DiscordMessage)
-		if len(messages) > 1 and messages[0].message_type() == "embed":
-			for i, msg in enumerate(messages):
-				if not isinstance(msg, StackedDiscordMessage):
-					break
-			else:  #all messages in messages are stacked, exit this if
-				i += 1
-			removed_msgs = 0
-			for group_index in range(ceil((len(messages)-i)/10)):
-				message_group_index = group_index*10+i-removed_msgs
-				stackable = StackedDiscordMessage(messages[message_group_index])
-				for message in messages[message_group_index+1:message_group_index+10]:
-					try:
-						stackable.add_embed(message.embed)
-					except EmbedListFull:
-						break
-					self._queue.remove(message)
-					messages.remove(message)
-					removed_msgs += 1
-				self.replace_message(messages[message_group_index], stackable)
-				messages[message_group_index] = stackable
 		for msg in messages:
 			if self.global_rate_limit:
 				return  # if we are globally rate limited just wait for first gblocked request to finish

src/wiki.py

@@ -205,7 +205,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict):
 
 # db_wiki: webhook, wiki, lang, display, rcid, postid
 async def essential_info(change: dict, changed_categories, local_wiki: Wiki, target: tuple, paths: tuple, request: dict,
-                         rate_limiter: RateLimiter):
+                         rate_limiter: RateLimiter) -> src.discord.DiscordMessage:
 	"""Prepares essential information for both embed and compact message format."""
 	_ = langs[target[0][0]]["wiki"].gettext
 	changed_categories = changed_categories.get(change["revid"], None)
@@ -236,7 +236,7 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, tar
 			additional_data["tags"][tag["name"]] = (BeautifulSoup(tag["displayname"], "lxml")).get_text()
 		except KeyError:
 			additional_data["tags"][tag["name"]] = None  # Tags with no displ
-	await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, paths, rate_limiter, additional_data=additional_data)
+	return await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, paths, rate_limiter, additional_data=additional_data)
 
 
 async def essential_feeds(change: dict, comment_pages: dict, db_wiki: sqlite3.Row, target: tuple):