From 8a6c8ae28e9a87d5e2dbe8a4f2d62db518eaeab0 Mon Sep 17 00:00:00 2001 From: Frisk Date: Mon, 26 Oct 2020 13:25:14 +0100 Subject: [PATCH] Fixed #154 and added more to message deletion code --- src/discussion_formatters.py | 4 +- src/message_redaction.py | 45 +++++++++++---- src/misc.py | 105 +++++++++++++++++++---------------- src/rc_formatters.py | 9 +-- src/rcgcdw.py | 6 +- 5 files changed, 102 insertions(+), 67 deletions(-) diff --git a/src/discussion_formatters.py b/src/discussion_formatters.py index 8dce2ac..1a6071b 100644 --- a/src/discussion_formatters.py +++ b/src/discussion_formatters.py @@ -67,7 +67,7 @@ def compact_formatter(post_type, post, article_paths): else: message = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=post_type, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message)) + send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message), meta={"request_type": "POST"}) def embed_formatter(post_type, post, article_paths): @@ -168,7 +168,7 @@ def embed_formatter(post_type, post, article_paths): else: embed.add_field(_("Report this on the support server"), change_params) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) class DiscussionsFromHellParser: diff --git a/src/message_redaction.py b/src/message_redaction.py index 03507b0..b27a338 100644 --- a/src/message_redaction.py +++ b/src/message_redaction.py @@ -1,16 +1,17 @@ from src.configloader import settings +from src.misc import send_to_discord import logging logger = logging.getLogger("rcgcdw.message_redaction") import sqlite3 -def create_schema(cursor: sqlite3.Cursor): +def create_schema(): logger.info("Creating database schema...") - cursor.executescript( + db_cursor.executescript( """BEGIN TRANSACTION; CREATE TABLE IF NOT EXISTS "messages" ( "message_id" TEXT, - "content" INTEGER + "content" TEXT ); CREATE TABLE IF NOT EXISTS "event" ( "pageid" INTEGER, @@ -22,14 +23,36 @@ def create_schema(cursor: sqlite3.Cursor): COMMIT;""") logger.info("Database schema has been recreated.") -def create_connection(): - db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) - db_cursor = db_connection.cursor() - return db_connection, db_cursor -def check_tables(cursor: sqlite3.Cursor): - rep = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") +def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): + _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) + _db_connection.row_factory = sqlite3.Row + _db_cursor = db_connection.cursor() + return _db_connection, _db_cursor + + +def check_tables(): + rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") if not rep.fetchone(): - create_schema(cursor) + create_schema() -def add_entry(pageid, revid, logid, message): + +def add_entry(pageid: int, revid: int, logid: int, message): + db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("message_id"), message)) + db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("message_id"))) + + +def delete_messages(pageid: int): + to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid)) + for message in to_delete: + webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) + send_to_discord(None, {"request_type": "DELETE", "webhook_url": webhook_url}) + db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", list_of_messageids) + + +def redact_messages(rev_ids: list, to_censor: dict): + raise NotImplemented + + +db_connection, db_cursor = create_connection() +check_tables() diff --git a/src/misc.py b/src/misc.py index 8025c8d..b8ddf63 100644 --- a/src/misc.py +++ b/src/misc.py @@ -23,6 +23,7 @@ import requests from collections import defaultdict from src.configloader import settings from src.i18n import misc +from typing import Optional _ = misc.gettext @@ -123,7 +124,7 @@ class MessageQueue: misc_logger.debug( "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), str(item))) - if send_to_discord_webhook(item) < 2: + if send_to_discord_webhook(item[0], metadata=item[1]) < 2: misc_logger.debug("Sending message succeeded") else: misc_logger.debug("Sending message failed") @@ -325,7 +326,7 @@ def send_simple(msgtype, message, name, avatar): discord_msg.set_avatar(avatar) discord_msg.set_name(name) messagequeue.resend_msgs() - send_to_discord(discord_msg) + send_to_discord(discord_msg, meta={"request_type": "POST"}) def update_ratelimit(request): @@ -336,51 +337,8 @@ def update_ratelimit(request): rate_limit += settings.get("discord_message_cooldown", 0) -def send_to_discord_webhook(data): - global rate_limit - header = settings["header"] - header['Content-Type'] = 'application/json' - try: - time.sleep(rate_limit) - rate_limit = 0 - result = requests.post(data.webhook_url, data=repr(data), - headers=header, timeout=10) - update_ratelimit(result) - except requests.exceptions.Timeout: - misc_logger.warning("Timeouted while sending data to the webhook.") - return 3 - except requests.exceptions.ConnectionError: - misc_logger.warning("Connection error while sending the data to a webhook") - return 3 - else: - return handle_discord_http(result.status_code, data, result) - -def send_to_discord(data): - for regex in settings["disallow_regexes"]: - if data.webhook_object.get("content", None): - if re.search(re.compile(regex), data.webhook_object["content"]): - misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) - return # discard the message without anything - else: - for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: - if re.search(re.compile(regex), to_check): - misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( - to_check, regex)) - return # discard the message without anything - if messagequeue: - messagequeue.add_message(data) - else: - code = send_to_discord_webhook(data) - if code == 3: - messagequeue.add_message(data) - elif code == 2: - time.sleep(5.0) - messagequeue.add_message(data) - elif code < 2: - pass - -class DiscordMessage(): +class DiscordMessage: """A class defining a typical Discord JSON representation of webhook payload.""" def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) @@ -455,6 +413,59 @@ def profile_field_name(name, embed): return _("unknown") +def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: dict = None): + global rate_limit + header = settings["header"] + header['Content-Type'] = 'application/json' + standard_args = dict(headers=header, timeout=10) + if metadata["request_type"] == "POST": + req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if settings.get("auto_suppression", {"enabled": True}).get("enabled") else "false", data=repr(data), **standard_args) + elif metadata["request_type"] == "DELETE": + req = requests.Request("DELETE", metadata["webhook_url"], **standard_args) + elif metadata["request_type"] == "PATCH": + req = requests.Request("PATCH", metadata["webhook_url"], data=metadata["new_data"], **standard_args) + try: + time.sleep(rate_limit) + rate_limit = 0 + req = req.prepare() + result = req.send() + update_ratelimit(result) + except requests.exceptions.Timeout: + misc_logger.warning("Timeouted while sending data to the webhook.") + return 3 + except requests.exceptions.ConnectionError: + misc_logger.warning("Connection error while sending the data to a webhook") + return 3 + else: + return handle_discord_http(result.status_code, data, result) + + +def send_to_discord(data: Optional[DiscordMessage], meta: dict): + if data is not None: + for regex in settings["disallow_regexes"]: + if data.webhook_object.get("content", None): + if re.search(re.compile(regex), data.webhook_object["content"]): + misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) + return # discard the message without anything + else: + for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: + if re.search(re.compile(regex), to_check): + misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( + to_check, regex)) + return # discard the message without anything + if messagequeue: + messagequeue.add_message((data, meta)) + else: + code = send_to_discord_webhook(data, metadata=meta) + if code == 3: + messagequeue.add_message((data, meta)) + elif code == 2: + time.sleep(5.0) + messagequeue.add_message((data, meta)) + elif code < 2: + pass + + class LinkParser(HTMLParser): new_string = "" recent_href = "" @@ -484,4 +495,4 @@ class LinkParser(HTMLParser): self.new_string = self.new_string + data.replace("//", "/\\/") def handle_endtag(self, tag): - misc_logger.debug(self.new_string) \ No newline at end of file + misc_logger.debug(self.new_string) diff --git a/src/rc_formatters.py b/src/rc_formatters.py index 001192f..f952c56 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -12,6 +12,7 @@ from bs4 import BeautifulSoup from src.configloader import settings from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ WIKI_API_PATH, ContentParser, profile_field_name, LinkParser +from src.message_redaction import delete_messages, redact_messages from src.i18n import rc_formatters #from src.rc import recent_changes, pull_comment _ = rc_formatters.gettext @@ -62,7 +63,7 @@ def compact_abuselog_formatter(change, recent_changes): action=abusefilter_actions.get(change["action"], _("Unknown")), target=change.get("title", _("Unknown")), target_url=create_article_path(change.get("title", _("Unknown"))), result=abusefilter_results.get(change["result"], _("Unknown"))) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message)) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message), meta={"request_type": "POST"}) def compact_formatter(action, change, parsed_comment, categories, recent_changes): @@ -403,7 +404,7 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "❓ "+_( "Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=action, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content)) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content), meta={"request_type": "POST"}) def embed_abuselog_formatter(change, recent_changes): action = "abuselog/{}".format(change["result"]) @@ -415,7 +416,7 @@ def embed_abuselog_formatter(change, recent_changes): embed.add_field(_("Action taken"), abusefilter_results.get(change["result"], _("Unknown"))) embed.add_field(_("Title"), change.get("title", _("Unknown"))) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) def embed_formatter(action, change, parsed_comment, categories, recent_changes): @@ -889,4 +890,4 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 3d17407..2a52d33 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -161,10 +161,10 @@ def day_overview(): if item["type"] == "edit": edits += 1 changed_bytes += item["newlen"] - item["oldlen"] - if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]: + if (recent_changes.namespaces is not None and "content" in recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0: articles = add_to_dict(articles, item["title"]) elif item["type"] == "new": - if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]: + if "content" in (recent_changes.namespaces is not None and recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0: new_articles += 1 changed_bytes += item["newlen"] elif item["type"] == "log": @@ -202,7 +202,7 @@ def day_overview(): for name, value in fields: embed.add_field(name, value, inline=True) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) else: logger.debug("function requesting changes for day overview returned with error code")