Works on formatting the Fandom discussions, fix to daily overview message handling

discussions.py

@@ -19,7 +19,7 @@
 import logging, gettext, schedule, requests, json, datetime
 from collections import defaultdict
 from configloader import settings
-from misc import datafile, send_to_discord, DiscordMessage
+from misc import datafile, send_to_discord, DiscordMessage, WIKI_SCRIPT_PATH, escape_formatting
 from session import session
 
 # Initialize translation
@@ -48,6 +48,7 @@ def embed_formatter(post, post_type):
 	embed.set_author(post["createdBy"]["name"], "{wikiurl}f/u/{creatorId}".format(
 		wikiurl=settings["fandom_discussions"]["wiki_url"], creatorId=post["creatorId"]), icon_url=post["createdBy"]["avatarUrl"])
 	if post_type == "TEXT":  # TODO
+		npost = DiscussionsFromHellParser(post)
 		if post["isReply"]:
 			embed["title"] = _("Replied to \"{title}\"").format(title=post["_embedded"]["thread"][0]["title"])
 			embed["url"] = "{wikiurl}f/p/{threadId}/r/{postId}".format(
@@ -57,8 +58,7 @@ def embed_formatter(post, post_type):
 			embed["url"] = "{wikiurl}f/p/{threadId}".format(wikiurl=settings["fandom_discussions"]["wiki_url"],
 			                                                threadId=post["threadId"])
 		if settings["fandom_discussions"]["appearance"]["embed"]["show_content"]:
-			embed["description"] = post["rawContent"] if len(post["rawContent"]) < 2000 else post["rawContent"][
+			embed["description"] = npost.parse()
-			                                                                                 0:2000] + "…"
 	elif post_type == "POLL":
 		poll = post["poll"]
 		embed["title"] = _("Created a poll titled \"{}\"").format(poll["question"])
@@ -119,12 +119,81 @@ def parse_discussion_post(post):
 	else:
 		discussion_logger.warning("The type of {} is an unknown discussion post type. Please post an issue on the project page to have it added https://gitlab.com/piotrex43/RcGcDw/-/issues.")
 
+class DiscussionsFromHellParser:
+	"""This class converts fairly convoluted Fandom jsonModal of a discussion post into Markdown formatted usable thing. Takes string, returns string.
+		Kudos to MarkusRost for allowing me to implement this formatter based on his code in Wiki-Bot."""
+	def __init__(self, post):
+		self.post = post
+		self.jsonModal = json.loads(post.get("jsonModel", "{}"))
+		self.markdown_text = ""
+		self.item_num = 1
 
-def format_discussion_text(modal):
+	def parse(self):
-	"""This function converts fairly convoluted Fandom jsonModal of a discussion post into Markdown formatted usable thing. Takes string, returns string.
+		"""Main parsing logic"""
-	Kudos to MarkusRost for allowing me to implement this formatter based on his code in Wiki-Bot."""
+		for root_item in self.jsonModal["content"]:
-	description = ""
+			if "content" in root_item:
-	discussion_modal = json.loads(modal)
+				self.parse_content(root_item["content"])
+			if len(self.markdown_text) > 2000:
+				break
+		images = {}
+		for num, image in enumerate(self.post["_embedded"]["contentImages"]):
+			images["img-{}".format(num)] = image["url"]
+		self.markdown_text = self.markdown_text.format(images)
+		self.markdown_text = self.markdown_text[0:2000] + "…"
+		return self.markdown_text
+
+	def parse_content(self, content, ctype=None):
+		for item in content:
+			if ctype == "bulletList":
+				self.markdown_text += "\t• "
+			if ctype == "orderedList":
+				self.markdown_text += "\t{num}. ".format(num=self.item_num)
+				self.item_num += 1
+			if item["type"] == "text":
+				if "marks" in item:
+					prefix, suffix = self.convert_marks(item["marks"])
+					self.markdown_text = "{old}{pre}{text}{suf}".format(old=self.markdown_text, pre=prefix, text=escape_formatting(item["text"]), suf=suffix)
+				else:
+					self.markdown_text += escape_formatting(item["text"])
+			elif item["type"] == "paragraph":
+				if "content" in item:
+					self.parse_content(item, item["type"])
+				self.markdown_text += "\n"
+			elif item["type"] == "openGraph":
+				if not item["attrs"]["wasAddedWithInlineLink"]:
+					self.markdown_text = "{old}{link}\n".format(old=self.markdown_text, link=item["attrs"]["url"])
+			elif item["type"] == "image":
+				self.markdown_text = "{old}{img-{img}}\n".format(old=self.markdown_text, img=item["attrs"]["id"])
+			elif item["type"] == "code_block":
+				self.markdown_text += "```\n"
+				if "content" in item:
+					self.parse_content(item, item["type"])
+				self.markdown_text += "\n```\n"
+			elif item["type"] == "bulletList":
+				if "content" in item:
+					self.parse_content(item, item["type"])
+			elif item["type"] == "orderedList":
+				self.item_num = 1
+				if "content" in item:
+					self.parse_content(item, item["type"])
+
+	def convert_marks(self, marks):
+		prefix = ""
+		suffix = ""
+		for mark in marks:
+			if mark["type"] == "mention":
+				prefix += "["
+				suffix = "]({wiki}f/u/{userid}){suffix}".format(wiki=WIKI_SCRIPT_PATH, userid=mark["attrs"]["userId"], suffix=suffix)
+			elif mark["type"] == "strong":
+				prefix += "**"
+				suffix = "**{suffix}".format(suffix=suffix)
+			elif mark["type"] == "link":
+				prefix += "["
+				suffix = "]({link}){suffix}".format(link=mark["attrs"]["href"], suffix=suffix)
+			elif mark["type"] == "em":
+				prefix += "_"
+				suffix = "_" + suffix
+		return prefix, suffix
 
 
 def safe_request(url):

misc.py

@@ -115,6 +115,9 @@ def link_formatter(link):
 	"""Formats a link to not embed it"""
 	return "<" + re.sub(r"([)])", "\\\\\\1", link).replace(" ", "_") + ">"
 
+def escape_formatting(data):
+	"""Escape Discord formatting"""
+	return re.sub(r"([`_*~<>{}@/|\\])", "\\\\\\1", data, 0)
 
 class ContentParser(HTMLParser):
 	more = _("\n__And more__")

rcgcdw.py

@@ -997,14 +997,11 @@ def day_overview():
 			if not settings["send_empty_overview"]:
 				return  # no changes in this day
 			else:
-				embed = defaultdict(dict)
+				embed = DiscordMessage("embed", "daily_overview")
 				embed["title"] = _("Daily overview")
 				embed["url"] = create_article_path("Special:Statistics")
 				embed["description"] = _("No activity")
-				embed["color"] = settings["appearance"]["embed"]["daily_overview"]["color"]
+				embed.set_author(settings["wikiname"], create_article_path(""), icon_url=settings["appearance"]["embed"]["daily_overview"]["icon"])
-				embed["author"]["icon_url"] = settings["appearance"]["embed"]["daily_overview"]["icon"]
-				embed["author"]["name"] = settings["wikiname"]
-				embed["author"]["url"] = create_article_path("")
 		else:
 			for item in result[0]:
 				if "actionhidden" in item or "suppressed" in item or "userhidden" in item:
@@ -1025,13 +1022,10 @@ def day_overview():
 					admin = admin + 1 if item["logtype"] in ["delete", "merge", "block", "protect", "import", "rights",
 					                                         "abusefilter", "interwiki", "managetags"] else admin
 			overall = round(new_articles + edits * 0.1 + files * 0.3 + admin * 0.1 + math.fabs(changed_bytes * 0.001), 2)
-			embed = defaultdict(dict)
+			embed = DiscordMessage("embed", "daily_overview")
 			embed["title"] = _("Daily overview")
 			embed["url"] = create_article_path("Special:Statistics")
-			embed["color"] = settings["appearance"]["embed"]["daily_overview"]["color"]
+			embed.set_author(settings["wikiname"], create_article_path(""), icon_url=settings["appearance"]["embed"]["daily_overview"]["icon"])
-			embed["author"]["icon_url"] = settings["appearance"]["embed"]["daily_overview"]["icon"]
-			embed["author"]["name"] = settings["wikiname"]
-			embed["author"]["url"] = create_article_path("")
 			if activity:
 				active_users = []
 				for user, numberu in Counter(activity).most_common(3):  # find most active users
@@ -1050,7 +1044,6 @@ def day_overview():
 				houramount = ""
 			if not active_articles:
 				active_articles = [_("But nobody came")]
-			embed["fields"] = []
 			edits, files, admin, changed_bytes, new_articles, unique_contributors, overall = daily_overview_sync(edits, files, admin, changed_bytes, new_articles, len(activity), overall)
 			fields = (
 			(ngettext("Most active user", "Most active users", len(active_users)), ', '.join(active_users)),
@@ -1061,10 +1054,9 @@ def day_overview():
 			(ngettext("Most active hour", "Most active hours", len(active_hours)), ', '.join(active_hours) + houramount),
 			(_("Day score"), overall))
 			for name, value in fields:
-				embed["fields"].append({"name": name, "value": value, "inline": True})
+				embed.add_field(name, value, inline=True)
-		data = {"embeds": [dict(embed)]}
+		embed.finish_embed()
-		formatted_embed = json.dumps(data, indent=4)
+		send_to_discord(embed)
-		send_to_discord(formatted_embed)
 	else:
 		logger.debug("function requesting changes for day overview returned with error code")