diff --git a/src/discord/message.py b/src/discord/message.py index 77d73d1..d4b434b 100644 --- a/src/discord/message.py +++ b/src/discord/message.py @@ -13,6 +13,8 @@ # You should have received a copy of the GNU General Public License # along with RcGcDw. If not, see . from __future__ import annotations + +import datetime import json import math import random @@ -24,19 +26,23 @@ from src.exceptions import EmbedListFull if TYPE_CHECKING: from wiki import Wiki + from domain import Domain with open("src/api/template_settings.json", "r") as template_json: settings: dict = json.load(template_json) class DiscordMessageMetadata: - def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, message_display = None): + def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, message_display = None, + time_of_change: Optional[datetime.datetime] = None, domain: Domain = None): self.method = method # unused, remains for compatibility reasons self.page_id = page_id self.log_id = log_id self.rev_id = rev_id self.webhook_url = webhook_url self.message_display = message_display + self.time_of_change = time_of_change.replace(tzinfo=datetime.timezone.utc) # We are assuming all wikis use UTC as server time (this is terrible, we need to do this better) + self.domain = domain def matches(self, other: dict): for key, value in other.items(): @@ -182,7 +188,7 @@ class StackedDiscordMessage(): message_structure["content"] = "\n".join([message.return_content() for message in self.message_list]) elif self.message_type == 1: message_structure["embeds"] = [message.embed for message in self.message_list] - if self.message_list and "components" in self.message_list[0].webhook_object: + if self.message_list and "components" in self.message_list[0].webhook_object: # use components from the first message message_structure["components"] = self.message_list[0].webhook_object["components"] return json.dumps(message_structure) diff --git a/src/discord/queue.py b/src/discord/queue.py index 056c77c..01c2e4b 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -163,6 +163,11 @@ class MessageQueue: self.global_rate_limit = True await asyncio.sleep(e.remaining / 1000) return + else: + if status == 0: + for message in msg.message_list: + if message.metadata.domain is not None and message.metadata.time_of_change is not None: + message.metadata.domain.register_message_timing_report(message.metadata.time_of_change) for queue_message in messages[max(index-len(msg.message_list), 0):index+1]: # mark messages as delivered queue_message.confirm_sent_status(webhook_url) if client_error is False: @@ -204,7 +209,7 @@ async def handle_discord_http(code: int, formatted_embed: str, result: ClientRes # TODO remove_webhook_maybe() raise aiohttp.ClientError("Message sent to bad webhook.") else: - return 0 + return 1 elif code == 429: logger.error("We are sending too many requests to the Discord, slowing down...") if "x-ratelimit-global" in result.headers.keys(): diff --git a/src/domain.py b/src/domain.py index 9d9ec49..e8314a6 100644 --- a/src/domain.py +++ b/src/domain.py @@ -1,5 +1,6 @@ from __future__ import annotations import asyncio +import datetime import logging import time import traceback @@ -10,6 +11,7 @@ import sys import aiohttp +from misc import LimitedList from src.discord.message import DiscordMessage from src.config import settings from src.argparser import command_line_args @@ -30,16 +32,23 @@ class Domain: self.wikis: OrderedDict[str, src.wiki.Wiki] = OrderedDict() self.irc: Optional[src.irc_feed.AioIRCCat] = None self.last_failure_report = 0 - self.failure_data = [0, 0, 0] # Amount of failures, timestamp of last failure info, count of announcements + self.message_timings: LimitedList = LimitedList(limit=100) # self.discussions_handler: Optional[Discussions] = Discussions(self.wikis) if name == "fandom.com" else None def __iter__(self): return iter(self.wikis) def __str__(self) -> str: + if len(self.message_timings) > 0: # min throws exception when used on empty iterable + tmin, avg, tmax = (self.convert_seconds_to_readable(min(self.message_timings)), + self.convert_seconds_to_readable(int(sum(self.message_timings)/len(self.message_timings))), + self.convert_seconds_to_readable(max(self.message_timings))) + else: + tmin, avg, tmax = 0, 0, 0 return (f"") + f"calculated_delay={self.calculate_sleep_time(len(self)) if not self.irc else 'handled by IRC scheduler'} " + f"msgdelays=(min={tmin}, avg={avg}, max={tmax})>") def __repr__(self): return self.__str__() @@ -50,6 +59,11 @@ class Domain: def __len__(self): return len(self.wikis) + @staticmethod + def convert_seconds_to_readable(seconds: int) -> str: + """Helper function to prepare human readable times for domain report""" + return f"{int(seconds/60)}m{seconds % 60}s" + def destroy(self): """Destroy the domain – do all of the tasks that should make sure there is no leftovers before being collected by GC""" if self.irc: @@ -129,6 +143,13 @@ class Domain: if affected: return affected + def register_message_timing_report(self, initial_time: datetime.datetime, send_time: Optional[datetime.datetime] = None) -> None: + """This function registers time between edit being made and message with given edit being sent on Discord + For metrics and debugging""" + if send_time is None: + send_time = datetime.datetime.now(tz=datetime.timezone.utc) + self.message_timings.append((send_time - initial_time).seconds) + async def irc_scheduler(self): try: while True: diff --git a/src/misc.py b/src/misc.py index f62a5b2..6303787 100644 --- a/src/misc.py +++ b/src/misc.py @@ -12,7 +12,6 @@ from src.config import settings logger = logging.getLogger("rcgcdw.misc") - def get_paths(wiki: str, request) -> tuple: """Prepares wiki paths for the functions""" parsed_url = urlparse(wiki) @@ -210,3 +209,17 @@ def prepare_settings(display_mode: int) -> dict: template["appearance"]["embed"]["embed_images"] = True if display_mode > 1 else False template["appearance"]["embed"]["show_edit_changes"] = True if display_mode > 2 else False return template + + +class LimitedList(list): + def __init__(self, *args, limit=settings.get("queue_limit", 30)): + list.__init__(self, *args) + self.queue_limit = limit + + def append(self, obj) -> None: + if len(self) > self.queue_limit: + self.pop(0) + super(LimitedList, self).append(obj) + + def __repr__(self): + return "\n".join([str(x) for x in self]) diff --git a/src/statistics.py b/src/statistics.py index 01b2029..403117f 100644 --- a/src/statistics.py +++ b/src/statistics.py @@ -2,6 +2,7 @@ import time from datetime import datetime import aiohttp.web_request +from misc import LimitedList from src.config import settings from typing import Union, Optional, List from enum import Enum @@ -15,9 +16,6 @@ class LogType(Enum): SCAN_REASON = 5 -queue_limit = settings.get("queue_limit", 30) - - class Log: """Log class represents an event that happened to a wiki fetch. Main purpose of those logs is debug and error-tracking.""" def __init__(self, **kwargs): @@ -33,19 +31,6 @@ class Log: return f"" -class LimitedList(list): - def __init__(self, *args): - list.__init__(self, *args) - - def append(self, obj: Log) -> None: - if len(self) > queue_limit: - self.pop(0) - super(LimitedList, self).append(obj) - - def __repr__(self): - return "\n".join([str(x) for x in self]) - - class Statistics: def __init__(self, rc_id: Optional[int], discussion_id: Optional[str]): self.last_request: Optional[aiohttp.web_request.Request] = None diff --git a/src/wiki.py b/src/wiki.py index baafc33..70ef612 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -1,5 +1,6 @@ from __future__ import annotations +import datetime import functools import time import re @@ -460,7 +461,9 @@ async def rc_processor(wiki: Wiki, change: dict, changed_categories: dict, displ from src.misc import LinkParser LinkParser = LinkParser(wiki.client.WIKI_JUST_DOMAIN) metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), - page_id=change.get("pageid", None), message_display=display_options.display) + page_id=change.get("pageid", None), message_display=display_options.display, + time_of_change=datetime.datetime.strptime(change["timestamp"], '%Y-%m-%dT%H:%M:%SZ'), + domain=wiki.domain) context = Context("embed" if display_options.display > 0 else "compact", "recentchanges", webhooks, wiki.client, langs[display_options.lang]["formatters"], prepare_settings(display_options.display), display_options.buttons) if ("actionhidden" in change or "suppressed" in change) and "suppressed" not in settings["ignored"]: # if event is hidden using suppression