Further work on RcGcDb

extensions/base/discussions.py

@@ -136,16 +136,16 @@ def common_discussions(post: dict, embed: DiscordMessage, ctx: Context):
 
 @formatter.embed(event="discussion/forum")
 def embed_discussion_forum(ctx: Context, post: dict):
-    embed = DiscordMessage("embed", "discussion", ctx.settings["fandom_discussions"]["webhookURL"])
+    embed = DiscordMessage("embed", "discussion", ctx.webhook_url)
     common_discussions(post, embed, ctx)
     author = ctx._("unknown")  # Fail safe
     if post["createdBy"]["name"]:
         author = post["createdBy"]["name"]
-    embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+    embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                            creatorId=post["creatorId"]),
                                                            icon_url=post["createdBy"]["avatarUrl"])
     if not post["isReply"]:
-        embed["url"] = "{url}f/p/{threadId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+        embed["url"] = "{url}f/p/{threadId}".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                     threadId=post["threadId"])
         embed["title"] = ctx._("Created \"{title}\"").format(title=post["title"])
         thread_funnel = post.get("funnel")
@@ -189,7 +189,7 @@ def embed_discussion_forum(ctx: Context, post: dict):
     else:
         embed.event_type = "discussion/forum/reply"
         embed["title"] = ctx._("Replied to \"{title}\"").format(title=post["_embedded"]["thread"][0]["title"])
-        embed["url"] = "{url}f/p/{threadId}/r/{postId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+        embed["url"] = "{url}f/p/{threadId}/r/{postId}".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                                threadId=post["threadId"], postId=post["id"])
     return embed
 
@@ -200,7 +200,7 @@ def compact_discussion_forum(ctx: Context, post: dict):
     author = ctx._("unknown")  # Fail safe
     if post["createdBy"]["name"]:
         author = post["createdBy"]["name"]
-    author_url = "<{url}f/u/{creatorId}>".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+    author_url = "<{url}f/u/{creatorId}>".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                  creatorId=post["creatorId"])
     if not post["isReply"]:
         thread_funnel = post.get("funnel")
@@ -219,13 +219,13 @@ def compact_discussion_forum(ctx: Context, post: dict):
                     thread_funnel))
             event_type = "unknown"
         message = msg_text.format(author=author, author_url=author_url, title=post["title"],
-                                  url=ctx.settings["fandom_discussions"]["wiki_url"], threadId=post["threadId"],
+                                  url=ctx.client.WIKI_SCRIPT_PATH, threadId=post["threadId"],
                                   forumName=post["forumName"])
     else:
         event_type = "discussion/forum/reply"
         message = ctx._(
             "[{author}]({author_url}) created a [reply](<{url}f/p/{threadId}/r/{postId}>) to [{title}](<{url}f/p/{threadId}>) in {forumName}").format(
-            author=author, author_url=author_url, url=ctx.settings["fandom_discussions"]["wiki_url"],
+            author=author, author_url=author_url, url=ctx.client.WIKI_SCRIPT_PATH,
             threadId=post["threadId"], postId=post["id"], title=post["_embedded"]["thread"][0]["title"],
             forumName=post["forumName"])
     return DiscordMessage("compact", event_type, ctx.webhook_url, content=message)
@@ -238,14 +238,14 @@ def compact_author_discussions(post: dict, ctx: Context):
     author = ctx._("unknown")  # Fail safe
     if post["creatorIp"]:
         author = post["creatorIp"][1:] if ctx.settings.get("hide_ips", False) is False else ctx._("Unregistered user")
-        author_url = "<{url}wiki/Special:Contributions{creatorIp}>".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+        author_url = "<{url}wiki/Special:Contributions{creatorIp}>".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                                            creatorIp=post["creatorIp"])
     else:
         if post["createdBy"]["name"]:
             author = post["createdBy"]["name"]
             author_url = clean_link(ctx.client.create_article_path("User:{user}".format(user=author)))
         else:
-            author_url = "<{url}f/u/{creatorId}>".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+            author_url = "<{url}f/u/{creatorId}>".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                      creatorId=post["creatorId"])
     return author, author_url
 
@@ -256,22 +256,22 @@ def embed_author_discussions(post: dict, embed: DiscordMessage, ctx: Context):
         author = post["creatorIp"][1:]
         embed.set_author(author if ctx.settings.get("hide_ips", False) is False else ctx._("Unregistered user"),
                          "{url}wiki/Special:Contributions{creatorIp}".format(
-                             url=ctx.settings["fandom_discussions"]["wiki_url"], creatorIp=post["creatorIp"]))
+                             url=ctx.client.WIKI_SCRIPT_PATH, creatorIp=post["creatorIp"]))
     else:
         if post["createdBy"]["name"]:
             author = post["createdBy"]["name"]
-            embed.set_author(author, "{url}wiki/User:{creator}".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+            embed.set_author(author, "{url}wiki/User:{creator}".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                                        creator=author.replace(" ", "_")),
                              icon_url=post["createdBy"]["avatarUrl"])
         else:
-            embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.settings["fandom_discussions"]["wiki_url"],
+            embed.set_author(author, "{url}f/u/{creatorId}".format(url=ctx.client.WIKI_SCRIPT_PATH,
                                                                    creatorId=post["creatorId"]),
                              icon_url=post["createdBy"]["avatarUrl"])
 
 
 @formatter.embed(event="discussion/wall")
 def embed_discussion_wall(ctx: Context, post: dict):
-    embed = DiscordMessage("embed", "discussion", ctx.settings["fandom_discussions"]["webhookURL"])
+    embed = DiscordMessage("embed", "discussion", ctx.webhook_url)
     common_discussions(post, embed, ctx)
     embed_author_discussions(post, embed, ctx)
     user_wall = ctx._("unknown")  # Fail safe
@@ -280,13 +280,13 @@ def embed_discussion_wall(ctx: Context, post: dict):
     if not post["isReply"]:
         embed.event_type = "discussion/wall/post"
         embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}".format(
-            url=ctx.settings["fandom_discussions"]["wiki_url"], user_wall=quote_plus(user_wall.replace(" ", "_")),
+            url=ctx.client.WIKI_SCRIPT_PATH, user_wall=quote_plus(user_wall.replace(" ", "_")),
             threadId=post["threadId"])
         embed["title"] = ctx._("Created \"{title}\" on {user}'s Message Wall").format(title=post["title"], user=user_wall)
     else:
         embed.event_type = "discussion/wall/reply"
         embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}".format(
-            url=ctx.settings["fandom_discussions"]["wiki_url"], user_wall=quote_plus(user_wall.replace(" ", "_")),
+            url=ctx.client.WIKI_SCRIPT_PATH, user_wall=quote_plus(user_wall.replace(" ", "_")),
             threadId=post["threadId"], replyId=post["id"])
         embed["title"] = ctx._("Replied to \"{title}\" on {user}'s Message Wall").format(
             title=post["_embedded"]["thread"][0]["title"], user=user_wall)
@@ -303,13 +303,13 @@ def compact_discussion_wall(ctx: Context, post: dict):
         event_type = "discussion/wall/post"
         message = ctx._(
             "[{author}]({author_url}) created [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(
-            author=author, author_url=author_url, title=post["title"], url=ctx.settings["fandom_discussions"]["wiki_url"],
+            author=author, author_url=author_url, title=post["title"], url=ctx.client.WIKI_SCRIPT_PATH,
             user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"])
     else:
         event_type = "discussion/wall/reply"
         message = ctx._(
             "[{author}]({author_url}) created a [reply](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}>) to [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(
-            author=author, author_url=author_url, url=ctx.settings["fandom_discussions"]["wiki_url"],
+            author=author, author_url=author_url, url=ctx.client.WIKI_SCRIPT_PATH,
             title=post["_embedded"]["thread"][0]["title"], user=user_wall,
             user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"])
     return DiscordMessage("compact", event_type, ctx.webhook_url, content=message)
@@ -319,12 +319,12 @@ def compact_discussion_wall(ctx: Context, post: dict):
 
 @formatter.embed(event="discussion/article_comment")
 def embed_discussion_article_comment(ctx: Context, post: dict):
-    embed = DiscordMessage("embed", "discussion", ctx.settings["fandom_discussions"]["webhookURL"])
+    embed = DiscordMessage("embed", "discussion", ctx.webhook_url)
     common_discussions(post, embed, ctx)
     embed_author_discussions(post, embed, ctx)
     article_paths = ctx.comment_page
     if article_paths is None:
-        article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.settings["fandom_discussions"]["wiki_url"]}  # No page known
+        article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.client.WIKI_SCRIPT_PATH}  # No page known
     if not post["isReply"]:
         embed.event_type = "discussion/comment/post"
         embed["url"] = "{url}?commentId={commentId}".format(url=article_paths["fullUrl"], commentId=post["threadId"])
@@ -344,7 +344,7 @@ def compact_discussion_article_comment(ctx: Context, post: dict):
     author, author_url = compact_author_discussions(post, ctx)
     article_paths = ctx.comment_page
     if article_paths is None:
-        article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.settings["fandom_discussions"]["wiki_url"]}  # No page known
+        article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.client.WIKI_SCRIPT_PATH}  # No page known
     article_paths["fullUrl"] = article_paths["fullUrl"].replace(")", "\)").replace("()", "\(")
     if not post["isReply"]:
         event_type = "discussion/comment/post"

src/api/formatter.py

@@ -40,6 +40,7 @@ def _register_formatter(func, kwargs, formatter_type: str, action_type=None):
 						   f"{src.api.hooks.formatter_hooks[formatter_type][act].__module__}! "
 						   f"Overwriting it with one from {func.__module__}")
 		src.api.hooks.formatter_hooks[formatter_type][act] = func
+		logger.debug("Registering {type} hook for {event} event".format(type=formatter_type, event=act))
 
 
 def embed(**kwargs):

src/discord/message.py

@@ -183,6 +183,11 @@ class StackedDiscordMessage():
 			message_structure["embeds"] = [message.embed for message in self.message_list]
 		return json.dumps(message_structure)
 
+	def check_for_length(self, message_length: int):
+		if self.message_type:
+			return len(self) + message_length > 6000 or len(self.message_list) > 9
+		return (len(self) + message_length) > 2000
+
 	def filter(self, params: dict) -> list[tuple[int, DiscordMessage]]:
 		"""Filters messages by their metadata"""
 		return [(num, message) for num, message in enumerate(self.message_list) if message.matches(params)]
@@ -193,9 +198,9 @@ class StackedDiscordMessage():
 			self.message_list.pop(message_id)
 
 	def add_message(self, message: DiscordMessage):
-		if len(self) + len(message) > 6000 or len(self.message_list) > 9:
+		if self.check_for_length(len(message)):
 			raise MessageTooBig
-		self.length += len(message)
+		self.length += len(message) + (self.message_type == 0)
 		self.message_list.append(message)
 		# self._setup_embed()
 		# self.embed = message.embed

src/discord/queue.py

@@ -174,14 +174,16 @@ class MessageQueue:
 messagequeue = MessageQueue()
 
 
-def handle_discord_http(code: int, formatted_embed: str, result: ClientResponse):
+async def handle_discord_http(code: int, formatted_embed: str, result: ClientResponse):
+	text = await result.text()
+	print("HTTP response is {} and response {}".format(code, text))
 	if 300 > code > 199:  # message went through
 		return 0
 	elif code == 400:  # HTTP BAD REQUEST result.status_code, data, result, header
 		logger.error(
 			"Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:")
 		logger.error(formatted_embed)
-		logger.error(result.text())
+		logger.error(text)
 		raise aiohttp.ClientError("Message rejected.")
 	elif code == 401 or code == 404:  # HTTP UNAUTHORIZED AND NOT FOUND
 		if result.method == "POST":  # Ignore not found for DELETE and PATCH requests since the message could already be removed by admin
@@ -206,12 +208,13 @@ def handle_discord_http(code: int, formatted_embed: str, result: ClientResponse)
 
 
 async def send_to_discord_webhook(message: [StackedDiscordMessage, DiscordMessageMetadata], webhook_path: str, method: str):
+	logger.debug("We are at sent_to_discord for {}".format(message))
 	header = settings["header"]
 	header['Content-Type'] = 'application/json'
 	header['X-RateLimit-Precision'] = "millisecond"
 	async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(total=6)) as session:
 		if isinstance(message, StackedDiscordMessage):
-			async with session.post(f"https://discord.com/api/webhooks/{webhook_path}?wait=true", data=repr(message)) as resp:
+			async with session.post(f"https://discord.com/api/webhooks/{webhook_path}?wait=true", data=repr(message)) as resp:  # TODO Detect Invalid Webhook Token
 				try:
 					resp_json = await resp.json()
 					# Add Discord Message ID which we can later use to delete/redact messages if we want
@@ -222,10 +225,10 @@ async def send_to_discord_webhook(message: [StackedDiscordMessage, DiscordMessag
 					logger.exception("Could not receive message ID from Discord due to invalid MIME type of response.")
 				except ValueError:
 					logger.exception(f"Could not decode JSON response from Discord. Response: {await resp.text()}]")
-				return handle_discord_http(resp.status, repr(message), resp)
+				return await handle_discord_http(resp.status, repr(message), resp)
 		elif method == "DELETE":
 			async with session.request(method=message.method, url=f"https://discord.com/api/webhooks/{webhook_path}/messages/{message.discord_callback_message_id}") as resp:
-				return handle_discord_http(resp.status, repr(message), resp)
+				return await handle_discord_http(resp.status, repr(message), resp)
 		elif method == "PATCH":
 			async with session.request(method=message.method, url=f"https://discord.com/api/webhooks/{webhook_path}/messages/{message.discord_callback_message_id}", data=repr(message)) as resp:
-				return handle_discord_http(resp.status, repr(message), resp)
+				return await handle_discord_http(resp.status, repr(message), resp)

src/discussions.py

@@ -6,9 +6,9 @@ import logging
 import time
 import aiohttp
 import traceback
-from api.context import Context
+from src.api.context import Context
-from api.hooks import formatter_hooks
+from src.api.hooks import formatter_hooks
-from api.util import default_message
+from src.api.util import default_message
 from discord.queue import QueueEntry, messagequeue
 from src.i18n import langs
 from src.misc import prepare_settings
@@ -33,23 +33,24 @@ class Discussions:
     async def tick_discussions(self):
         if self.domain_object is None:
             raise asyncio.CancelledError("fandom.com is not a domain we have any wikis for.")
-
         while True:
-            try:
+            while True:
-                wiki_url = self.domain_object.irc.updated_discussions.pop()
+                try:
-            except KeyError:
+                    wiki_url = self.domain_object.irc.updated_discussions.pop()
-                break
+                except KeyError:
-            wiki = self.domain_object.get_wiki(wiki_url)
+                    break
-            if wiki is None:
+                wiki = self.domain_object.get_wiki(wiki_url)
-                logger.error(f"Could not find a wiki with URL {wiki_url} in the domain group!")
+                if wiki is None:
-                continue
+                    logger.error(f"Could not find a wiki with URL {wiki_url} in the domain group!")
-            await self.run_discussion_scan(wiki)
+                    continue
-
-        for wiki in self.filter_and_sort():
-            if (int(time.time()) - (wiki.statistics.last_checked_discussion or 0)) > settings.get("irc_overtime", 3600):
                 await self.run_discussion_scan(wiki)
-            else:
+
-                return  # Recently scanned wikis will get at the end of the self.wikis, so we assume what is first hasn't been checked for a while
+            for wiki in self.filter_and_sort():
+                if (int(time.time()) - (wiki.statistics.last_checked_discussion or 0)) > settings.get("irc_overtime", 3600):
+                    await self.run_discussion_scan(wiki)
+                else:
+                    return  # Recently scanned wikis will get at the end of the self.wikis, so we assume what is first hasn't been checked for a while
+            await asyncio.sleep(5.0)
 
     def filter_and_sort(self) -> list[Wiki]:
         """Filters and sorts wikis from domain to return only the ones that aren't -1 and sorts them from oldest in checking to newest"""
@@ -60,9 +61,8 @@ class Discussions:
         wiki.statistics.last_checked_discussion = int(time.time())
         params = {"controller": "DiscussionPost", "method": "getPosts", "includeCounters": "false",
                   "sortDirection": "descending", "sortKey": "creation_date", "limit": 20}
-        feeds_response = await wiki.fetch_discussions(params)
         try:
-            discussion_feed_resp = await feeds_response.json(encoding="UTF-8")
+            feeds_response, discussion_feed_resp = await wiki.fetch_discussions(params)
             if "error" in discussion_feed_resp:
                 error = discussion_feed_resp["error"]
                 if error == "NotFoundException":  # Discussions disabled
@@ -80,8 +80,8 @@ class Discussions:
             return
         if wiki.discussion_id is None:  # new wiki, just get the last post to not spam the channel
             if len(discussion_feed) > 0:
-                dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != -1 OR postid IS NULL )", (
+                dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != '-1' OR postid IS NULL )", (
-                    discussion_feed[-1]["id"],
+                    str(discussion_feed[-1]["id"]),
                     wiki.script_url)))
                 wiki.statistics.update(last_post=discussion_feed[-1]["id"])
             else:
@@ -130,7 +130,7 @@ class Discussions:
         messagequeue.add_messages(message_list)
         if discussion_feed:
             wiki.statistics.update(last_post=discussion_feed[-1]["id"])
-            dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != -1 OR postid IS NULL )", (discussion_feed[-1]["id"],
+            dbmanager.add(("UPDATE rcgcdw SET postid = $1 WHERE wiki = $2 AND ( postid != '-1' OR postid IS NULL )", (str(discussion_feed[-1]["id"]),
                                                                                                           wiki.script_url)))  # If this is not enough for the future, save rcid in message sending function to make sure we always send all of the changes
 
 
@@ -148,8 +148,9 @@ async def essential_feeds(change: dict, comment_pages: dict, wiki: Wiki, target:
     context.set_comment_page(comment_page)
     discord_message: Optional[DiscordMessage] = None
     try:
+
         discord_message = await asyncio.get_event_loop().run_in_executor(
-                None, functools.partial(default_message(identification_string, context.message_type, formatter_hooks), context, change))
+                None, functools.partial(default_message(f"discussion/{identification_string.lower()}", context.message_type, formatter_hooks), context, change))
     except:
         if settings.get("error_tolerance", 1) > 0:
             logger.exception("Exception on discord message creation in essential_feeds")

src/domain.py

@@ -33,6 +33,12 @@ class Domain:
     def __iter__(self):
         return iter(self.wikis)
 
+    def __str__(self) -> str:
+        return f"<Domain name='{self.name}' task='{self.task}' wikis='{self.wikis}' irc='{self.irc.connection.connected}' failures={self.failures}>"
+
+    def __repr__(self):
+        return self.__str__()
+
     def __getitem__(self, item):
         return
 

src/domain_manager.py

@@ -43,6 +43,8 @@ class DomainManager:
             self.remove_wiki(split_payload[1])
         elif split_payload[0] == "UPDATE":
             await self.return_domain(self.get_domain(split_payload[1])).get_wiki(split_payload[1]).update_targets()
+        elif split_payload[0] == "DEBUG":
+            logger.info(self.domains)
         else:
             raise ValueError("Unknown pub/sub command! Payload: {}".format(payload))
 

src/irc_feed.py

@@ -71,11 +71,14 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
 			logger.warning("Seems like we have invalid JSON in Discussions part, message: {}".format(message))
 			return
 		if post.get('action', 'unknown') != "deleted":  # ignore deletion events
+			if isinstance(post.get('url'), bytes):
+				return
 			url = urlparse(post.get('url'))
 			full_url ="https://"+ url.netloc + recognize_langs(url.path)
 			wiki = self.domain.get_wiki(full_url)
 			if wiki and wiki.discussion_id != -1:
 				self.updated_discussions.add(full_url)
+				logger.debug("New discussion wiki appended to the list! {}".format(full_url))
 			# if full_url in self.domain:
 			# 	self.discussion_callback(full_url)
 

src/queue_handler.py

@@ -33,6 +33,7 @@ class UpdateDB:
 					async with db.pool().acquire() as connection:
 						async with connection.transaction():
 							for update in self.updated:
+								logger.debug("Executing: {} {}".format(update[0], update[1]))
 								await connection.execute(update[0], *update[1])
 							self.clear_list()
 				await asyncio.sleep(10.0)

src/wiki.py

@@ -378,7 +378,7 @@ class Wiki:
 	async def remove_wiki_from_db(self, reason: str):
 		raise NotImplementedError  # TODO
 
-	async def fetch_discussions(self, params: dict) -> aiohttp.ClientResponse:
+	async def fetch_discussions(self, params: dict) -> tuple[aiohttp.ClientResponse, dict]:
 		header = settings["header"]
 		header["Accept"] = "application/hal+json"
 		async with aiohttp.ClientSession(headers=header,
@@ -389,9 +389,9 @@ class Wiki:
 				feeds_response.raise_for_status()
 			except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError,
 					aiohttp.ClientResponseError, aiohttp.TooManyRedirects) as e:
-				logger.error("A connection error occurred while requesting {}".format(url_path))
+				logger.error("A connection error occurred while requesting {}".format(feeds_response.url))
 				raise WikiServerError(e)
-		return feeds_response
+			return feeds_response, await feeds_response.json(encoding="UTF-8")
 
 
 def process_cachable(response: dict, wiki_object: Wiki) -> None: