Experimental support for time to send metric

src/discord/message.py

@@ -13,6 +13,8 @@
 # You should have received a copy of the GNU General Public License
 # along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.
 from __future__ import annotations
+
+import datetime
 import json
 import math
 import random
@@ -24,19 +26,23 @@ from src.exceptions import EmbedListFull
 
 if TYPE_CHECKING:
 	from wiki import Wiki
+	from domain import Domain
 
 with open("src/api/template_settings.json", "r") as template_json:
 	settings: dict = json.load(template_json)
 
 
 class DiscordMessageMetadata:
-	def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, message_display = None):
+	def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, message_display = None,
+				 time_of_change: Optional[datetime.datetime] = None, domain: Domain = None):
 		self.method = method  # unused, remains for compatibility reasons
 		self.page_id = page_id
 		self.log_id = log_id
 		self.rev_id = rev_id
 		self.webhook_url = webhook_url
 		self.message_display = message_display
+		self.time_of_change = time_of_change.replace(tzinfo=datetime.timezone.utc)  # We are assuming all wikis use UTC as server time (this is terrible, we need to do this better)
+		self.domain = domain
 
 	def matches(self, other: dict):
 		for key, value in other.items():
@@ -182,7 +188,7 @@ class StackedDiscordMessage():
 			message_structure["content"] = "\n".join([message.return_content() for message in self.message_list])
 		elif self.message_type == 1:
 			message_structure["embeds"] = [message.embed for message in self.message_list]
-		if self.message_list and "components" in self.message_list[0].webhook_object:
+		if self.message_list and "components" in self.message_list[0].webhook_object:  # use components from the first message
 			message_structure["components"] = self.message_list[0].webhook_object["components"]
 		return json.dumps(message_structure)
 

src/discord/queue.py

@@ -163,6 +163,11 @@ class MessageQueue:
 					self.global_rate_limit = True
 				await asyncio.sleep(e.remaining / 1000)
 				return
+			else:
+				if status == 0:
+					for message in msg.message_list:
+						if message.metadata.domain is not None and message.metadata.time_of_change is not None:
+							message.metadata.domain.register_message_timing_report(message.metadata.time_of_change)
 			for queue_message in messages[max(index-len(msg.message_list), 0):index+1]:  # mark messages as delivered
 				queue_message.confirm_sent_status(webhook_url)
 			if client_error is False:
@@ -204,7 +209,7 @@ async def handle_discord_http(code: int, formatted_embed: str, result: ClientRes
 			# TODO remove_webhook_maybe()
 			raise aiohttp.ClientError("Message sent to bad webhook.")
 		else:
-			return 0
+			return 1
 	elif code == 429:
 		logger.error("We are sending too many requests to the Discord, slowing down...")
 		if "x-ratelimit-global" in result.headers.keys():

src/domain.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 import asyncio
+import datetime
 import logging
 import time
 import traceback
@@ -10,6 +11,7 @@ import sys
 
 import aiohttp
 
+from misc import LimitedList
 from src.discord.message import DiscordMessage
 from src.config import settings
 from src.argparser import command_line_args
@@ -30,16 +32,23 @@ class Domain:
         self.wikis: OrderedDict[str, src.wiki.Wiki] = OrderedDict()
         self.irc: Optional[src.irc_feed.AioIRCCat] = None
         self.last_failure_report = 0
-        self.failure_data = [0, 0, 0]  # Amount of failures, timestamp of last failure info, count of announcements
+        self.message_timings: LimitedList = LimitedList(limit=100)
         # self.discussions_handler: Optional[Discussions] = Discussions(self.wikis) if name == "fandom.com" else None
 
     def __iter__(self):
         return iter(self.wikis)
 
     def __str__(self) -> str:
+        if len(self.message_timings) > 0:  # min throws exception when used on empty iterable
+            tmin, avg, tmax = (self.convert_seconds_to_readable(min(self.message_timings)),
+                             self.convert_seconds_to_readable(int(sum(self.message_timings)/len(self.message_timings))),
+                             self.convert_seconds_to_readable(max(self.message_timings)))
+        else:
+            tmin, avg, tmax = 0, 0, 0
         return (f"<Domain name='{self.name}' task='{self.task}' wikis='{self.wikis}' "
                 f"irc='{self.irc.connection.connected if self.irc else False}' "
-                f"calculated_delay={self.calculate_sleep_time(len(self)) if not self.irc else 'handled by IRC scheduler'}>")
+                f"calculated_delay={self.calculate_sleep_time(len(self)) if not self.irc else 'handled by IRC scheduler'} "
+                f"msgdelays=(min={tmin}, avg={avg}, max={tmax})>")
 
     def __repr__(self):
         return self.__str__()
@@ -50,6 +59,11 @@ class Domain:
     def __len__(self):
         return len(self.wikis)
 
+    @staticmethod
+    def convert_seconds_to_readable(seconds: int) -> str:
+        """Helper function to prepare human readable times for domain report"""
+        return f"{int(seconds/60)}m{seconds % 60}s"
+
     def destroy(self):
         """Destroy the domain – do all of the tasks that should make sure there is no leftovers before being collected by GC"""
         if self.irc:
@@ -129,6 +143,13 @@ class Domain:
             if affected:
                 return affected
 
+    def register_message_timing_report(self, initial_time: datetime.datetime, send_time: Optional[datetime.datetime] = None) -> None:
+        """This function registers time between edit being made and message with given edit being sent on Discord
+        For metrics and debugging"""
+        if send_time is None:
+            send_time = datetime.datetime.now(tz=datetime.timezone.utc)
+        self.message_timings.append((send_time - initial_time).seconds)
+
     async def irc_scheduler(self):
         try:
             while True:

src/misc.py

@@ -12,7 +12,6 @@ from src.config import settings
 
 logger = logging.getLogger("rcgcdw.misc")
 
-
 def get_paths(wiki: str, request) -> tuple:
 	"""Prepares wiki paths for the functions"""
 	parsed_url = urlparse(wiki)
@@ -210,3 +209,17 @@ def prepare_settings(display_mode: int) -> dict:
 	template["appearance"]["embed"]["embed_images"] = True if display_mode > 1 else False
 	template["appearance"]["embed"]["show_edit_changes"] = True if display_mode > 2 else False
 	return template
+
+
+class LimitedList(list):
+	def __init__(self, *args, limit=settings.get("queue_limit", 30)):
+		list.__init__(self, *args)
+		self.queue_limit = limit
+
+	def append(self, obj) -> None:
+		if len(self) > self.queue_limit:
+			self.pop(0)
+		super(LimitedList, self).append(obj)
+
+	def __repr__(self):
+		return "\n".join([str(x) for x in self])

src/statistics.py

@@ -2,6 +2,7 @@ import time
 from datetime import datetime
 import aiohttp.web_request
 
+from misc import LimitedList
 from src.config import settings
 from typing import Union, Optional, List
 from enum import Enum
@@ -15,9 +16,6 @@ class LogType(Enum):
     SCAN_REASON = 5
 
 
-queue_limit = settings.get("queue_limit", 30)
-
-
 class Log:
     """Log class represents an event that happened to a wiki fetch. Main purpose of those logs is debug and error-tracking."""
     def __init__(self, **kwargs):
@@ -33,19 +31,6 @@ class Log:
         return f"<Log {self.type.name} at {datetime.fromtimestamp(float(self.time)).isoformat()} on {self.title} with details: {self.details}>"
 
 
-class LimitedList(list):
-    def __init__(self, *args):
-        list.__init__(self, *args)
-
-    def append(self, obj: Log) -> None:
-        if len(self) > queue_limit:
-            self.pop(0)
-        super(LimitedList, self).append(obj)
-
-    def __repr__(self):
-        return "\n".join([str(x) for x in self])
-
-
 class Statistics:
     def __init__(self, rc_id: Optional[int], discussion_id: Optional[str]):
         self.last_request: Optional[aiohttp.web_request.Request] = None

src/wiki.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import datetime
 import functools
 import time
 import re
@@ -460,7 +461,9 @@ async def rc_processor(wiki: Wiki, change: dict, changed_categories: dict, displ
 	from src.misc import LinkParser
 	LinkParser = LinkParser(wiki.client.WIKI_JUST_DOMAIN)
 	metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None),
-													page_id=change.get("pageid", None), message_display=display_options.display)
+													page_id=change.get("pageid", None), message_display=display_options.display,
+													time_of_change=datetime.datetime.strptime(change["timestamp"], '%Y-%m-%dT%H:%M:%SZ'),
+									  				domain=wiki.domain)
 	context = Context("embed" if display_options.display > 0 else "compact", "recentchanges", webhooks, wiki.client,
 					  langs[display_options.lang]["formatters"], prepare_settings(display_options.display), display_options.buttons)
 	if ("actionhidden" in change or "suppressed" in change) and "suppressed" not in settings["ignored"]:  # if event is hidden using suppression