Frisk/RcGcDb
1
0
Fork 0
mirror of https://gitlab.com/chicken-riders/RcGcDb.git synced 2025-02-23 00:54:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Finished DUMP functionality

Browse source
This commit is contained in:
Frisk 2024-07-16 20:51:45 +02:00
parent 0ac5f8b824
commit d5d0c0a6e6
6 changed files with 34 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
extensions/base/discussions.py
View file

@ -23,7 +23,6 @@ from src.api.context import Context
from src.discord.message import DiscordMessage, DiscordMessageMetadata
from src.api import formatter
logger = logging.getLogger("rcgcdw.discussion_formatter")
@ -62,7 +61,8 @@ class DiscussionsFromHellParser:
suf=suffix)
else:
if ctype == "code_block":
self.markdown_text += item["text"] # ignore formatting on preformatted text which cannot have additional formatting anyways
self.markdown_text += item[
"text"] # ignore formatting on preformatted text which cannot have additional formatting anyways
else:
self.markdown_text += sanitize_to_markdown(item["text"])
elif item["type"] == "paragraph":
@ -132,6 +132,7 @@ def common_discussions(post: dict, embed: DiscordMessage, ctx: Context):
embed["timestamp"] = datetime.datetime.fromtimestamp(post["creationDate"]["epochSecond"],
tz=datetime.timezone.utc).isoformat()
# discussion/forum - Discussions on the "forum" available via "Discuss" button
@formatter.embed(event="discussion/forum")
@ -180,10 +181,13 @@ def embed_discussion_forum(ctx: Context, post: dict):
if post["_embedded"]["thread"][0]["tags"]:
tag_displayname = []
for tag in post["_embedded"]["thread"][0]["tags"]:
tag_displayname.append("[{title}]({url})".format(title=tag["articleTitle"], url=ctx.client.create_article_path(
tag_displayname.append(
"[{title}]({url})".format(title=tag["articleTitle"], url=ctx.client.create_article_path(
quote_plus(tag["articleTitle"].replace(" ", "_"), "/:?=&"))))
if len(", ".join(tag_displayname)) > 1000:
embed.add_field(ctx.pgettext("Fandom discussions Tags/Forums", "Tags"), ctx.pgettext("Fandom discussions amount of Tags/Forums", "{} tags").format(len(post["_embedded"]["thread"][0]["tags"])))
embed.add_field(ctx.pgettext("Fandom discussions Tags/Forums", "Tags"),
ctx.pgettext("Fandom discussions amount of Tags/Forums", "{} tags").format(
len(post["_embedded"]["thread"][0]["tags"])))
else:
embed.add_field(ctx.pgettext("Fandom discussions Tags/Forums", "Tags"), ", ".join(tag_displayname))
else:
@ -230,6 +234,7 @@ def compact_discussion_forum(ctx: Context, post: dict):
forumName=post["forumName"])
return DiscordMessage("compact", event_type, ctx.webhook_url, content=message)
# discussion/wall - Wall posts/replies
@ -282,7 +287,8 @@ def embed_discussion_wall(ctx: Context, post: dict):
embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}".format(
url=ctx.client.WIKI_SCRIPT_PATH, user_wall=quote_plus(user_wall.replace(" ", "_")),
threadId=post["threadId"])
embed["title"] = ctx._("Created \"{title}\" on {user}'s Message Wall").format(title=post["title"], user=user_wall)
embed["title"] = ctx._("Created \"{title}\" on {user}'s Message Wall").format(title=post["title"],
user=user_wall)
else:
embed.event_type = "discussion/wall/reply"
embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}".format(
@ -314,6 +320,7 @@ def compact_discussion_wall(ctx: Context, post: dict):
user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"])
return DiscordMessage("compact", event_type, ctx.webhook_url, content=message)
# discussion/article_comment - Article comments
@ -345,7 +352,7 @@ def compact_discussion_article_comment(ctx: Context, post: dict):
article_paths = ctx.comment_page
if article_paths is None:
article_paths = {"title": ctx._("unknown"), "fullUrl": ctx.client.WIKI_SCRIPT_PATH} # No page known
article_paths["fullUrl"] = article_paths["fullUrl"].replace(")", "\)").replace("()", "\(")
article_paths["fullUrl"] = article_paths["fullUrl"].replace(")", "\\)").replace("()", "\\(")
if not post["isReply"]:
event_type = "discussion/comment/post"
message = ctx._(

3
src/discord/message.py
View file

@ -44,6 +44,9 @@ class DiscordMessageMetadata:
self.time_of_change = time_of_change
self.domain = domain
def __str__(self):
return f"<DiscordMessageMetadata page_id={self.page_id} log_id={self.log_id} rev_id={self.rev_id}>"
def matches(self, other: dict):
for key, value in other.items():
if self.__dict__[key] != value:

3
src/discord/queue.py
View file

@ -1,5 +1,5 @@
# This file is part of Recent changes Goat compatible Discord webhook (RcGcDw).
import datetime
# RcGcDw is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
@ -168,6 +168,7 @@ class MessageQueue:
message = None
for message in msg.message_list:
if message.metadata.domain is not None and message.metadata.time_of_change is not None:
logger.debug(f"Initial time {message.metadata.time_of_change} and send time {datetime.datetime.now(tz=datetime.timezone.utc)} for diff {message.metadata.rev_id} on {message.wiki.script_url}")
message.metadata.domain.register_message_timing_report(message.metadata.time_of_change)
if message and message.metadata.domain is not None:
message.metadata.domain.discord_message_registration()

5
src/domain.py
View file

@ -56,11 +56,12 @@ class Domain:
"wikis": [x for x in self.wikis.keys()],
"irc": self.irc.connection.connected if self.irc else False,
"delay": self.calculate_sleep_time(len(self)) if not self.irc else 'handled by IRC scheduler',
"msgdelay": {"min": min(self.message_timings), "avg": int(sum(self.message_timings)/len(self.message_timings)),
"max": max(self.message_timings)},
"msgdelay": {"min": min(self.message_timings or [0]), "avg": int(sum(self.message_timings)/(len(self.message_timings) or 1)),
"max": max(self.message_timings or [0])},
"discord_messages": self.total_discord_messages_sent,
"last_failure_report": self.last_failure_report
}
return dict_obj
def __repr__(self):
return self.__str__()

13
src/domain_manager.py
View file

@ -7,6 +7,8 @@ from urllib.parse import urlparse, urlunparse
import logging
import asyncpg
import asyncio
from discord.queue import messagequeue
from src.exceptions import NoDomain
from src.config import settings
from src.domain import Domain
@ -77,15 +79,18 @@ class DomainManager:
"wiki_count": sum([len(x.wikis) for x in self.domains.values()]),
"tasks": {},
"domains": {},
"queued_messages": [],
"total_discord_messages_sent": sum([x.total_discord_messages_sent for x in self.domains.values()])
}
for task in asyncio.all_tasks():
json_object["tasks"][task.get_name()] = {"done": task.done(), "result": task.result() if task.done() else None}
for name, domain in self.domains.items():
json_object[name] = domain.json()
await connection.execute("""select pg_notify('webhookupdates', %(jsondump)s);""", {'jsondump': json.dumps(json_object)})
# we need: dict/list of tasks, dict of domains,
json_object["domains"][name] = domain.json()
for message in messagequeue._queue:
json_object["queued_messages"].append({"metadata": str(message.discord_message.metadata), "url": message.wiki.script_url})
await connection.execute("select pg_notify('webhookupdates', 'DEBUG RESPONSE ' || $1);", json.dumps(json_object))
elif split_payload[1] == "RESPONSE":
return
else:
raise ValueError("Unknown pub/sub command! Payload: {}".format(payload))

2
src/irc_feed.py
View file

@ -39,7 +39,7 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
self.connection.buffer_class.errors = "replace" # Ignore encoding errors
self.connection_details = None
self.last_msg = time.time()
self.activity_tester = asyncio.get_event_loop().create_task(self.testactivity())
self.activity_tester = asyncio.get_event_loop().create_task(self.testactivity(), name="{}_IRC_activity_tester".format(domain_object.name))
def __str__(self):
return self.__repr__()