diff --git a/src/discord/message.py b/src/discord/message.py
index 77d73d1..d4b434b 100644
--- a/src/discord/message.py
+++ b/src/discord/message.py
@@ -13,6 +13,8 @@
# You should have received a copy of the GNU General Public License
# along with RcGcDw. If not, see .
from __future__ import annotations
+
+import datetime
import json
import math
import random
@@ -24,19 +26,23 @@ from src.exceptions import EmbedListFull
if TYPE_CHECKING:
from wiki import Wiki
+ from domain import Domain
with open("src/api/template_settings.json", "r") as template_json:
settings: dict = json.load(template_json)
class DiscordMessageMetadata:
- def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, message_display = None):
+ def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, message_display = None,
+ time_of_change: Optional[datetime.datetime] = None, domain: Domain = None):
self.method = method # unused, remains for compatibility reasons
self.page_id = page_id
self.log_id = log_id
self.rev_id = rev_id
self.webhook_url = webhook_url
self.message_display = message_display
+ self.time_of_change = time_of_change.replace(tzinfo=datetime.timezone.utc) # We are assuming all wikis use UTC as server time (this is terrible, we need to do this better)
+ self.domain = domain
def matches(self, other: dict):
for key, value in other.items():
@@ -182,7 +188,7 @@ class StackedDiscordMessage():
message_structure["content"] = "\n".join([message.return_content() for message in self.message_list])
elif self.message_type == 1:
message_structure["embeds"] = [message.embed for message in self.message_list]
- if self.message_list and "components" in self.message_list[0].webhook_object:
+ if self.message_list and "components" in self.message_list[0].webhook_object: # use components from the first message
message_structure["components"] = self.message_list[0].webhook_object["components"]
return json.dumps(message_structure)
diff --git a/src/discord/queue.py b/src/discord/queue.py
index 056c77c..01c2e4b 100644
--- a/src/discord/queue.py
+++ b/src/discord/queue.py
@@ -163,6 +163,11 @@ class MessageQueue:
self.global_rate_limit = True
await asyncio.sleep(e.remaining / 1000)
return
+ else:
+ if status == 0:
+ for message in msg.message_list:
+ if message.metadata.domain is not None and message.metadata.time_of_change is not None:
+ message.metadata.domain.register_message_timing_report(message.metadata.time_of_change)
for queue_message in messages[max(index-len(msg.message_list), 0):index+1]: # mark messages as delivered
queue_message.confirm_sent_status(webhook_url)
if client_error is False:
@@ -204,7 +209,7 @@ async def handle_discord_http(code: int, formatted_embed: str, result: ClientRes
# TODO remove_webhook_maybe()
raise aiohttp.ClientError("Message sent to bad webhook.")
else:
- return 0
+ return 1
elif code == 429:
logger.error("We are sending too many requests to the Discord, slowing down...")
if "x-ratelimit-global" in result.headers.keys():
diff --git a/src/domain.py b/src/domain.py
index 9d9ec49..e8314a6 100644
--- a/src/domain.py
+++ b/src/domain.py
@@ -1,5 +1,6 @@
from __future__ import annotations
import asyncio
+import datetime
import logging
import time
import traceback
@@ -10,6 +11,7 @@ import sys
import aiohttp
+from misc import LimitedList
from src.discord.message import DiscordMessage
from src.config import settings
from src.argparser import command_line_args
@@ -30,16 +32,23 @@ class Domain:
self.wikis: OrderedDict[str, src.wiki.Wiki] = OrderedDict()
self.irc: Optional[src.irc_feed.AioIRCCat] = None
self.last_failure_report = 0
- self.failure_data = [0, 0, 0] # Amount of failures, timestamp of last failure info, count of announcements
+ self.message_timings: LimitedList = LimitedList(limit=100)
# self.discussions_handler: Optional[Discussions] = Discussions(self.wikis) if name == "fandom.com" else None
def __iter__(self):
return iter(self.wikis)
def __str__(self) -> str:
+ if len(self.message_timings) > 0: # min throws exception when used on empty iterable
+ tmin, avg, tmax = (self.convert_seconds_to_readable(min(self.message_timings)),
+ self.convert_seconds_to_readable(int(sum(self.message_timings)/len(self.message_timings))),
+ self.convert_seconds_to_readable(max(self.message_timings)))
+ else:
+ tmin, avg, tmax = 0, 0, 0
return (f"")
+ f"calculated_delay={self.calculate_sleep_time(len(self)) if not self.irc else 'handled by IRC scheduler'} "
+ f"msgdelays=(min={tmin}, avg={avg}, max={tmax})>")
def __repr__(self):
return self.__str__()
@@ -50,6 +59,11 @@ class Domain:
def __len__(self):
return len(self.wikis)
+ @staticmethod
+ def convert_seconds_to_readable(seconds: int) -> str:
+ """Helper function to prepare human readable times for domain report"""
+ return f"{int(seconds/60)}m{seconds % 60}s"
+
def destroy(self):
"""Destroy the domain – do all of the tasks that should make sure there is no leftovers before being collected by GC"""
if self.irc:
@@ -129,6 +143,13 @@ class Domain:
if affected:
return affected
+ def register_message_timing_report(self, initial_time: datetime.datetime, send_time: Optional[datetime.datetime] = None) -> None:
+ """This function registers time between edit being made and message with given edit being sent on Discord
+ For metrics and debugging"""
+ if send_time is None:
+ send_time = datetime.datetime.now(tz=datetime.timezone.utc)
+ self.message_timings.append((send_time - initial_time).seconds)
+
async def irc_scheduler(self):
try:
while True:
diff --git a/src/misc.py b/src/misc.py
index f62a5b2..6303787 100644
--- a/src/misc.py
+++ b/src/misc.py
@@ -12,7 +12,6 @@ from src.config import settings
logger = logging.getLogger("rcgcdw.misc")
-
def get_paths(wiki: str, request) -> tuple:
"""Prepares wiki paths for the functions"""
parsed_url = urlparse(wiki)
@@ -210,3 +209,17 @@ def prepare_settings(display_mode: int) -> dict:
template["appearance"]["embed"]["embed_images"] = True if display_mode > 1 else False
template["appearance"]["embed"]["show_edit_changes"] = True if display_mode > 2 else False
return template
+
+
+class LimitedList(list):
+ def __init__(self, *args, limit=settings.get("queue_limit", 30)):
+ list.__init__(self, *args)
+ self.queue_limit = limit
+
+ def append(self, obj) -> None:
+ if len(self) > self.queue_limit:
+ self.pop(0)
+ super(LimitedList, self).append(obj)
+
+ def __repr__(self):
+ return "\n".join([str(x) for x in self])
diff --git a/src/statistics.py b/src/statistics.py
index 01b2029..403117f 100644
--- a/src/statistics.py
+++ b/src/statistics.py
@@ -2,6 +2,7 @@ import time
from datetime import datetime
import aiohttp.web_request
+from misc import LimitedList
from src.config import settings
from typing import Union, Optional, List
from enum import Enum
@@ -15,9 +16,6 @@ class LogType(Enum):
SCAN_REASON = 5
-queue_limit = settings.get("queue_limit", 30)
-
-
class Log:
"""Log class represents an event that happened to a wiki fetch. Main purpose of those logs is debug and error-tracking."""
def __init__(self, **kwargs):
@@ -33,19 +31,6 @@ class Log:
return f""
-class LimitedList(list):
- def __init__(self, *args):
- list.__init__(self, *args)
-
- def append(self, obj: Log) -> None:
- if len(self) > queue_limit:
- self.pop(0)
- super(LimitedList, self).append(obj)
-
- def __repr__(self):
- return "\n".join([str(x) for x in self])
-
-
class Statistics:
def __init__(self, rc_id: Optional[int], discussion_id: Optional[str]):
self.last_request: Optional[aiohttp.web_request.Request] = None
diff --git a/src/wiki.py b/src/wiki.py
index baafc33..70ef612 100644
--- a/src/wiki.py
+++ b/src/wiki.py
@@ -1,5 +1,6 @@
from __future__ import annotations
+import datetime
import functools
import time
import re
@@ -460,7 +461,9 @@ async def rc_processor(wiki: Wiki, change: dict, changed_categories: dict, displ
from src.misc import LinkParser
LinkParser = LinkParser(wiki.client.WIKI_JUST_DOMAIN)
metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None),
- page_id=change.get("pageid", None), message_display=display_options.display)
+ page_id=change.get("pageid", None), message_display=display_options.display,
+ time_of_change=datetime.datetime.strptime(change["timestamp"], '%Y-%m-%dT%H:%M:%SZ'),
+ domain=wiki.domain)
context = Context("embed" if display_options.display > 0 else "compact", "recentchanges", webhooks, wiki.client,
langs[display_options.lang]["formatters"], prepare_settings(display_options.display), display_options.buttons)
if ("actionhidden" in change or "suppressed" in change) and "suppressed" not in settings["ignored"]: # if event is hidden using suppression