From 69ff24fc99dea2fa2722a3ddd40ed49e21e71b42 Mon Sep 17 00:00:00 2001 From: Frisk Date: Sat, 24 Oct 2020 13:41:33 +0200 Subject: [PATCH 01/17] Added initial file --- src/message_redaction.py | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 src/message_redaction.py diff --git a/src/message_redaction.py b/src/message_redaction.py new file mode 100644 index 0000000..b509e1f --- /dev/null +++ b/src/message_redaction.py @@ -0,0 +1,8 @@ +from src.configloader import settings +import sqlite3 + + +def create_schema(): + + +def create_connection(): From a8eeba8831093fc6df0abe270e076fcce0868499 Mon Sep 17 00:00:00 2001 From: Frisk Date: Sun, 25 Oct 2020 00:33:00 +0200 Subject: [PATCH 02/17] Added some functions --- src/message_redaction.py | 31 +++++++++++++++++++++++++++++-- src/rc.py | 4 ++-- 2 files changed, 31 insertions(+), 4 deletions(-) diff --git a/src/message_redaction.py b/src/message_redaction.py index b509e1f..03507b0 100644 --- a/src/message_redaction.py +++ b/src/message_redaction.py @@ -1,8 +1,35 @@ from src.configloader import settings +import logging +logger = logging.getLogger("rcgcdw.message_redaction") import sqlite3 -def create_schema(): - +def create_schema(cursor: sqlite3.Cursor): + logger.info("Creating database schema...") + cursor.executescript( + """BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS "messages" ( + "message_id" TEXT, + "content" INTEGER + ); + CREATE TABLE IF NOT EXISTS "event" ( + "pageid" INTEGER, + "revid" INTEGER, + "logid" INTEGER, + "msg_id" TEXT NOT NULL, + FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE + ); + COMMIT;""") + logger.info("Database schema has been recreated.") def create_connection(): + db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) + db_cursor = db_connection.cursor() + return db_connection, db_cursor + +def check_tables(cursor: sqlite3.Cursor): + rep = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") + if not rep.fetchone(): + create_schema(cursor) + +def add_entry(pageid, revid, logid, message): diff --git a/src/rc.py b/src/rc.py index f9c7593..beaa94c 100644 --- a/src/rc.py +++ b/src/rc.py @@ -397,10 +397,10 @@ def essential_info(change, changed_categories): parsed_comment = _("~~hidden~~") if not parsed_comment: parsed_comment = None + if "userhidden" in change: + change["user"] = _("hidden") if change["type"] in ["edit", "new"]: logger.debug("List of categories in essential_info: {}".format(changed_categories)) - if "userhidden" in change: - change["user"] = _("hidden") identification_string = change["type"] if change.get("ns", -1) in settings.get("ignored_namespaces", ()): return From 8a6c8ae28e9a87d5e2dbe8a4f2d62db518eaeab0 Mon Sep 17 00:00:00 2001 From: Frisk Date: Mon, 26 Oct 2020 13:25:14 +0100 Subject: [PATCH 03/17] Fixed #154 and added more to message deletion code --- src/discussion_formatters.py | 4 +- src/message_redaction.py | 45 +++++++++++---- src/misc.py | 105 +++++++++++++++++++---------------- src/rc_formatters.py | 9 +-- src/rcgcdw.py | 6 +- 5 files changed, 102 insertions(+), 67 deletions(-) diff --git a/src/discussion_formatters.py b/src/discussion_formatters.py index 8dce2ac..1a6071b 100644 --- a/src/discussion_formatters.py +++ b/src/discussion_formatters.py @@ -67,7 +67,7 @@ def compact_formatter(post_type, post, article_paths): else: message = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=post_type, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message)) + send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message), meta={"request_type": "POST"}) def embed_formatter(post_type, post, article_paths): @@ -168,7 +168,7 @@ def embed_formatter(post_type, post, article_paths): else: embed.add_field(_("Report this on the support server"), change_params) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) class DiscussionsFromHellParser: diff --git a/src/message_redaction.py b/src/message_redaction.py index 03507b0..b27a338 100644 --- a/src/message_redaction.py +++ b/src/message_redaction.py @@ -1,16 +1,17 @@ from src.configloader import settings +from src.misc import send_to_discord import logging logger = logging.getLogger("rcgcdw.message_redaction") import sqlite3 -def create_schema(cursor: sqlite3.Cursor): +def create_schema(): logger.info("Creating database schema...") - cursor.executescript( + db_cursor.executescript( """BEGIN TRANSACTION; CREATE TABLE IF NOT EXISTS "messages" ( "message_id" TEXT, - "content" INTEGER + "content" TEXT ); CREATE TABLE IF NOT EXISTS "event" ( "pageid" INTEGER, @@ -22,14 +23,36 @@ def create_schema(cursor: sqlite3.Cursor): COMMIT;""") logger.info("Database schema has been recreated.") -def create_connection(): - db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) - db_cursor = db_connection.cursor() - return db_connection, db_cursor -def check_tables(cursor: sqlite3.Cursor): - rep = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") +def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): + _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) + _db_connection.row_factory = sqlite3.Row + _db_cursor = db_connection.cursor() + return _db_connection, _db_cursor + + +def check_tables(): + rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") if not rep.fetchone(): - create_schema(cursor) + create_schema() -def add_entry(pageid, revid, logid, message): + +def add_entry(pageid: int, revid: int, logid: int, message): + db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("message_id"), message)) + db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("message_id"))) + + +def delete_messages(pageid: int): + to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid)) + for message in to_delete: + webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) + send_to_discord(None, {"request_type": "DELETE", "webhook_url": webhook_url}) + db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", list_of_messageids) + + +def redact_messages(rev_ids: list, to_censor: dict): + raise NotImplemented + + +db_connection, db_cursor = create_connection() +check_tables() diff --git a/src/misc.py b/src/misc.py index 8025c8d..b8ddf63 100644 --- a/src/misc.py +++ b/src/misc.py @@ -23,6 +23,7 @@ import requests from collections import defaultdict from src.configloader import settings from src.i18n import misc +from typing import Optional _ = misc.gettext @@ -123,7 +124,7 @@ class MessageQueue: misc_logger.debug( "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), str(item))) - if send_to_discord_webhook(item) < 2: + if send_to_discord_webhook(item[0], metadata=item[1]) < 2: misc_logger.debug("Sending message succeeded") else: misc_logger.debug("Sending message failed") @@ -325,7 +326,7 @@ def send_simple(msgtype, message, name, avatar): discord_msg.set_avatar(avatar) discord_msg.set_name(name) messagequeue.resend_msgs() - send_to_discord(discord_msg) + send_to_discord(discord_msg, meta={"request_type": "POST"}) def update_ratelimit(request): @@ -336,51 +337,8 @@ def update_ratelimit(request): rate_limit += settings.get("discord_message_cooldown", 0) -def send_to_discord_webhook(data): - global rate_limit - header = settings["header"] - header['Content-Type'] = 'application/json' - try: - time.sleep(rate_limit) - rate_limit = 0 - result = requests.post(data.webhook_url, data=repr(data), - headers=header, timeout=10) - update_ratelimit(result) - except requests.exceptions.Timeout: - misc_logger.warning("Timeouted while sending data to the webhook.") - return 3 - except requests.exceptions.ConnectionError: - misc_logger.warning("Connection error while sending the data to a webhook") - return 3 - else: - return handle_discord_http(result.status_code, data, result) - -def send_to_discord(data): - for regex in settings["disallow_regexes"]: - if data.webhook_object.get("content", None): - if re.search(re.compile(regex), data.webhook_object["content"]): - misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) - return # discard the message without anything - else: - for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: - if re.search(re.compile(regex), to_check): - misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( - to_check, regex)) - return # discard the message without anything - if messagequeue: - messagequeue.add_message(data) - else: - code = send_to_discord_webhook(data) - if code == 3: - messagequeue.add_message(data) - elif code == 2: - time.sleep(5.0) - messagequeue.add_message(data) - elif code < 2: - pass - -class DiscordMessage(): +class DiscordMessage: """A class defining a typical Discord JSON representation of webhook payload.""" def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) @@ -455,6 +413,59 @@ def profile_field_name(name, embed): return _("unknown") +def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: dict = None): + global rate_limit + header = settings["header"] + header['Content-Type'] = 'application/json' + standard_args = dict(headers=header, timeout=10) + if metadata["request_type"] == "POST": + req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if settings.get("auto_suppression", {"enabled": True}).get("enabled") else "false", data=repr(data), **standard_args) + elif metadata["request_type"] == "DELETE": + req = requests.Request("DELETE", metadata["webhook_url"], **standard_args) + elif metadata["request_type"] == "PATCH": + req = requests.Request("PATCH", metadata["webhook_url"], data=metadata["new_data"], **standard_args) + try: + time.sleep(rate_limit) + rate_limit = 0 + req = req.prepare() + result = req.send() + update_ratelimit(result) + except requests.exceptions.Timeout: + misc_logger.warning("Timeouted while sending data to the webhook.") + return 3 + except requests.exceptions.ConnectionError: + misc_logger.warning("Connection error while sending the data to a webhook") + return 3 + else: + return handle_discord_http(result.status_code, data, result) + + +def send_to_discord(data: Optional[DiscordMessage], meta: dict): + if data is not None: + for regex in settings["disallow_regexes"]: + if data.webhook_object.get("content", None): + if re.search(re.compile(regex), data.webhook_object["content"]): + misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) + return # discard the message without anything + else: + for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: + if re.search(re.compile(regex), to_check): + misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( + to_check, regex)) + return # discard the message without anything + if messagequeue: + messagequeue.add_message((data, meta)) + else: + code = send_to_discord_webhook(data, metadata=meta) + if code == 3: + messagequeue.add_message((data, meta)) + elif code == 2: + time.sleep(5.0) + messagequeue.add_message((data, meta)) + elif code < 2: + pass + + class LinkParser(HTMLParser): new_string = "" recent_href = "" @@ -484,4 +495,4 @@ class LinkParser(HTMLParser): self.new_string = self.new_string + data.replace("//", "/\\/") def handle_endtag(self, tag): - misc_logger.debug(self.new_string) \ No newline at end of file + misc_logger.debug(self.new_string) diff --git a/src/rc_formatters.py b/src/rc_formatters.py index 001192f..f952c56 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -12,6 +12,7 @@ from bs4 import BeautifulSoup from src.configloader import settings from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ WIKI_API_PATH, ContentParser, profile_field_name, LinkParser +from src.message_redaction import delete_messages, redact_messages from src.i18n import rc_formatters #from src.rc import recent_changes, pull_comment _ = rc_formatters.gettext @@ -62,7 +63,7 @@ def compact_abuselog_formatter(change, recent_changes): action=abusefilter_actions.get(change["action"], _("Unknown")), target=change.get("title", _("Unknown")), target_url=create_article_path(change.get("title", _("Unknown"))), result=abusefilter_results.get(change["result"], _("Unknown"))) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message)) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message), meta={"request_type": "POST"}) def compact_formatter(action, change, parsed_comment, categories, recent_changes): @@ -403,7 +404,7 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "❓ "+_( "Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=action, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content)) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content), meta={"request_type": "POST"}) def embed_abuselog_formatter(change, recent_changes): action = "abuselog/{}".format(change["result"]) @@ -415,7 +416,7 @@ def embed_abuselog_formatter(change, recent_changes): embed.add_field(_("Action taken"), abusefilter_results.get(change["result"], _("Unknown"))) embed.add_field(_("Title"), change.get("title", _("Unknown"))) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) def embed_formatter(action, change, parsed_comment, categories, recent_changes): @@ -889,4 +890,4 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 3d17407..2a52d33 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -161,10 +161,10 @@ def day_overview(): if item["type"] == "edit": edits += 1 changed_bytes += item["newlen"] - item["oldlen"] - if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]: + if (recent_changes.namespaces is not None and "content" in recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0: articles = add_to_dict(articles, item["title"]) elif item["type"] == "new": - if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]: + if "content" in (recent_changes.namespaces is not None and recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0: new_articles += 1 changed_bytes += item["newlen"] elif item["type"] == "log": @@ -202,7 +202,7 @@ def day_overview(): for name, value in fields: embed.add_field(name, value, inline=True) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta={"request_type": "POST"}) else: logger.debug("function requesting changes for day overview returned with error code") From cdb4a1f99a2fca9f6c147ee6f54fcd8f31b45a26 Mon Sep 17 00:00:00 2001 From: Frisk Date: Thu, 5 Nov 2020 22:20:35 +0100 Subject: [PATCH 04/17] Further work on the message redaction function --- src/discussion_formatters.py | 6 +++--- src/message_redaction.py | 8 +++++--- src/misc.py | 40 ++++++++++++++++++++++++++++-------- src/rc_formatters.py | 12 ++++++----- src/rcgcdw.py | 4 ++-- 5 files changed, 48 insertions(+), 22 deletions(-) diff --git a/src/discussion_formatters.py b/src/discussion_formatters.py index 1a6071b..dd4aa81 100644 --- a/src/discussion_formatters.py +++ b/src/discussion_formatters.py @@ -4,7 +4,7 @@ import gettext from urllib.parse import quote_plus from src.configloader import settings -from src.misc import link_formatter, create_article_path, DiscordMessage, send_to_discord, escape_formatting +from src.misc import link_formatter, create_article_path, DiscordMessage, send_to_discord, escape_formatting, DiscordMessageMetadata from src.i18n import discussion_formatters _ = discussion_formatters.gettext @@ -67,7 +67,7 @@ def compact_formatter(post_type, post, article_paths): else: message = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=post_type, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message), meta={"request_type": "POST"}) + send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message), meta=DiscordMessageMetadata("POST")) def embed_formatter(post_type, post, article_paths): @@ -168,7 +168,7 @@ def embed_formatter(post_type, post, article_paths): else: embed.add_field(_("Report this on the support server"), change_params) embed.finish_embed() - send_to_discord(embed, meta={"request_type": "POST"}) + send_to_discord(embed, meta=DiscordMessageMetadata("POST")) class DiscussionsFromHellParser: diff --git a/src/message_redaction.py b/src/message_redaction.py index b27a338..df6e157 100644 --- a/src/message_redaction.py +++ b/src/message_redaction.py @@ -1,5 +1,5 @@ from src.configloader import settings -from src.misc import send_to_discord +from src.misc import send_to_discord, DiscordMessageMetadata import logging logger = logging.getLogger("rcgcdw.message_redaction") import sqlite3 @@ -44,10 +44,12 @@ def add_entry(pageid: int, revid: int, logid: int, message): def delete_messages(pageid: int): to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid)) + msg_to_remove = [] for message in to_delete: webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) - send_to_discord(None, {"request_type": "DELETE", "webhook_url": webhook_url}) - db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", list_of_messageids) + msg_to_remove.append(message[0]) + send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) + db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", msg_to_remove) def redact_messages(rev_ids: list, to_censor: dict): diff --git a/src/misc.py b/src/misc.py index b8ddf63..041c795 100644 --- a/src/misc.py +++ b/src/misc.py @@ -25,6 +25,10 @@ from src.configloader import settings from src.i18n import misc from typing import Optional +AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": True}).get("enabled") +if AUTO_SUPPRESSION_ENABLED: + from src.message_redaction import add_entry as add_message_redaction_entry + _ = misc.gettext # Create a custom logger @@ -326,7 +330,7 @@ def send_simple(msgtype, message, name, avatar): discord_msg.set_avatar(avatar) discord_msg.set_name(name) messagequeue.resend_msgs() - send_to_discord(discord_msg, meta={"request_type": "POST"}) + send_to_discord(discord_msg, meta=DiscordMessageMetadata("POST")) def update_ratelimit(request): @@ -413,23 +417,41 @@ def profile_field_name(name, embed): return _("unknown") -def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: dict = None): +class DiscordMessageMetadata: + def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): + self.method = method + self.page_id = page_id + self.log_id = log_id + self.rev_id = rev_id + self.webhook_url = webhook_url + self.new_data = new_data + + def dump_ids(self): + return self.page_id, self.rev_id, self.log_id + +def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMessageMetadata): global rate_limit header = settings["header"] header['Content-Type'] = 'application/json' standard_args = dict(headers=header, timeout=10) - if metadata["request_type"] == "POST": - req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if settings.get("auto_suppression", {"enabled": True}).get("enabled") else "false", data=repr(data), **standard_args) - elif metadata["request_type"] == "DELETE": - req = requests.Request("DELETE", metadata["webhook_url"], **standard_args) - elif metadata["request_type"] == "PATCH": - req = requests.Request("PATCH", metadata["webhook_url"], data=metadata["new_data"], **standard_args) + if metadata.method == "POST": + req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if AUTO_SUPPRESSION_ENABLED else "false", data=repr(data), **standard_args) + elif metadata.method == "DELETE": + req = requests.Request("DELETE", metadata.webhook_url, **standard_args) + elif metadata.method == "PATCH": + req = requests.Request("PATCH", metadata.webhook_url, data=metadata.new_data, **standard_args) try: time.sleep(rate_limit) rate_limit = 0 req = req.prepare() result = req.send() update_ratelimit(result) + if AUTO_SUPPRESSION_ENABLED: + # TODO Prepare request with all of safety checks + try: + add_message_redaction_entry(*metadata.dump_ids(), result.json()) + except ValueError: + misc_logger.error("Couldn't get json of result of sending Discord message.") except requests.exceptions.Timeout: misc_logger.warning("Timeouted while sending data to the webhook.") return 3 @@ -440,7 +462,7 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: dict = Non return handle_discord_http(result.status_code, data, result) -def send_to_discord(data: Optional[DiscordMessage], meta: dict): +def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata): if data is not None: for regex in settings["disallow_regexes"]: if data.webhook_object.get("content", None): diff --git a/src/rc_formatters.py b/src/rc_formatters.py index f952c56..fa68065 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -11,7 +11,7 @@ from bs4 import BeautifulSoup from src.configloader import settings from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ - WIKI_API_PATH, ContentParser, profile_field_name, LinkParser + WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, DiscordMessageMetadata from src.message_redaction import delete_messages, redact_messages from src.i18n import rc_formatters #from src.rc import recent_changes, pull_comment @@ -63,10 +63,11 @@ def compact_abuselog_formatter(change, recent_changes): action=abusefilter_actions.get(change["action"], _("Unknown")), target=change.get("title", _("Unknown")), target_url=create_article_path(change.get("title", _("Unknown"))), result=abusefilter_results.get(change["result"], _("Unknown"))) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message), meta={"request_type": "POST"}) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message), meta=DiscordMessageMetadata("POST")) def compact_formatter(action, change, parsed_comment, categories, recent_changes): + request_metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), page_id=change.get("pageid", None)) if action != "suppressed": author_url = link_formatter(create_article_path("User:{user}".format(user=change["user"]))) author = change["user"] @@ -404,7 +405,7 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "❓ "+_( "Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=action, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content), meta={"request_type": "POST"}) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content), meta=request_metadata) def embed_abuselog_formatter(change, recent_changes): action = "abuselog/{}".format(change["result"]) @@ -416,11 +417,12 @@ def embed_abuselog_formatter(change, recent_changes): embed.add_field(_("Action taken"), abusefilter_results.get(change["result"], _("Unknown"))) embed.add_field(_("Title"), change.get("title", _("Unknown"))) embed.finish_embed() - send_to_discord(embed, meta={"request_type": "POST"}) + send_to_discord(embed, meta=DiscordMessageMetadata("POST")) def embed_formatter(action, change, parsed_comment, categories, recent_changes): embed = DiscordMessage("embed", action, settings["webhookURL"]) + request_metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), page_id=change.get("pageid", None)) if parsed_comment is None: parsed_comment = _("No description provided") if action != "suppressed": @@ -890,4 +892,4 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) embed.finish_embed() - send_to_discord(embed, meta={"request_type": "POST"}) + send_to_discord(embed, meta=request_metadata) diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 2a52d33..1b6025e 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -27,7 +27,7 @@ from collections import defaultdict, Counter from src.configloader import settings from src.misc import add_to_dict, datafile, \ WIKI_API_PATH, create_article_path, send_to_discord, \ - DiscordMessage + DiscordMessage, DiscordMessageMetadata from src.rc import recent_changes from src.exceptions import MWError from src.i18n import rcgcdw @@ -202,7 +202,7 @@ def day_overview(): for name, value in fields: embed.add_field(name, value, inline=True) embed.finish_embed() - send_to_discord(embed, meta={"request_type": "POST"}) + send_to_discord(embed, meta=DiscordMessageMetadata("POST")) else: logger.debug("function requesting changes for day overview returned with error code") From 780f88c6ac0467bdbdb1bbdd07d60d58498569d4 Mon Sep 17 00:00:00 2001 From: Frisk Date: Sat, 7 Nov 2020 14:54:10 +0100 Subject: [PATCH 05/17] add conditional import --- src/rc_formatters.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/src/rc_formatters.py b/src/rc_formatters.py index fa68065..0a9399f 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -11,8 +11,10 @@ from bs4 import BeautifulSoup from src.configloader import settings from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ - WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, DiscordMessageMetadata -from src.message_redaction import delete_messages, redact_messages + WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, DiscordMessageMetadata, AUTO_SUPPRESSION_ENABLED +if AUTO_SUPPRESSION_ENABLED: + from src.message_redaction import delete_messages, redact_messages + from src.i18n import rc_formatters #from src.rc import recent_changes, pull_comment _ = rc_formatters.gettext @@ -106,10 +108,12 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes page_link = link_formatter(create_article_path(change["title"])) content = "🗑️ "+_("[{author}]({author_url}) deleted [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) + delete_messages(change.get("pageid")) elif action == "delete/delete_redir": page_link = link_formatter(create_article_path(change["title"])) content = "🗑️ "+_("[{author}]({author_url}) deleted redirect by overwriting [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) + delete_messages(change.get("pageid")) elif action == "move/move": link = link_formatter(create_article_path(change["logparams"]['target_title'])) redirect_status = _("without making a redirect") if "suppressredirect" in change["logparams"] else _("with a redirect") @@ -557,9 +561,11 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): elif action == "delete/delete": link = create_article_path(change["title"]) embed["title"] = _("Deleted page {article}").format(article=change["title"]) + delete_messages(change.get("pageid")) elif action == "delete/delete_redir": link = create_article_path(change["title"]) embed["title"] = _("Deleted redirect {article} by overwriting").format(article=change["title"]) + delete_messages(change.get("pageid")) elif action == "move/move": link = create_article_path(change["logparams"]['target_title']) parsed_comment = "{supress}. {desc}".format(desc=parsed_comment, From 99a1aa782796f7c33f414bf7bd664c1172534a6f Mon Sep 17 00:00:00 2001 From: Frisk Date: Sun, 8 Nov 2020 01:36:41 +0100 Subject: [PATCH 06/17] Added debug messages --- src/message_redaction.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/message_redaction.py b/src/message_redaction.py index df6e157..6c5af41 100644 --- a/src/message_redaction.py +++ b/src/message_redaction.py @@ -28,26 +28,31 @@ def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) _db_connection.row_factory = sqlite3.Row _db_cursor = db_connection.cursor() + logger.debug("Database connection created") return _db_connection, _db_cursor def check_tables(): rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") if not rep.fetchone(): + logger.debug("No schema detected, creating schema!") create_schema() def add_entry(pageid: int, revid: int, logid: int, message): db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("message_id"), message)) db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("message_id"))) + logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) def delete_messages(pageid: int): to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid)) msg_to_remove = [] + logger.debug("Deleting messages for pageid: {}".format(pageid)) for message in to_delete: webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) msg_to_remove.append(message[0]) + logger.debug("Removing following message: {}".format(message)) send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", msg_to_remove) From 9bc01153f883d2503e641147a6386139fc5c0be4 Mon Sep 17 00:00:00 2001 From: Frisk Date: Sun, 8 Nov 2020 22:29:15 +0100 Subject: [PATCH 07/17] Structurize the code, fixed some issues (like not recognizing edit and new events) --- src/__init__.py | 0 src/discord/__init__.py | 0 src/discord/message.py | 84 +++++++++++++ src/discord/queue.py | 156 ++++++++++++++++++++++++ src/discord/redaction.py | 27 +++++ src/discussion_formatters.py | 4 +- src/discussions.py | 3 +- src/fileio/__init__.py | 0 src/fileio/database.py | 51 ++++++++ src/message_redaction.py | 62 ---------- src/misc.py | 222 +---------------------------------- src/rc.py | 7 +- src/rc_formatters.py | 9 +- src/rcgcdw.py | 5 +- 14 files changed, 340 insertions(+), 290 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/discord/__init__.py create mode 100644 src/discord/message.py create mode 100644 src/discord/queue.py create mode 100644 src/discord/redaction.py create mode 100644 src/fileio/__init__.py create mode 100644 src/fileio/database.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord/__init__.py b/src/discord/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord/message.py b/src/discord/message.py new file mode 100644 index 0000000..f918657 --- /dev/null +++ b/src/discord/message.py @@ -0,0 +1,84 @@ +import json +import math +import random +from collections import defaultdict + +from src.configloader import settings + + +class DiscordMessage: + """A class defining a typical Discord JSON representation of webhook payload.""" + def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): + self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) + self.webhook_url = webhook_url + + if message_type == "embed": + self.__setup_embed() + elif message_type == "compact": + self.webhook_object["content"] = content + + self.event_type = event_type + + def __setitem__(self, key, value): + """Set item is used only in embeds.""" + try: + self.embed[key] = value + except NameError: + raise TypeError("Tried to assign a value when message type is plain message!") + + def __getitem__(self, item): + return self.embed[item] + + def __repr__(self): + """Return the Discord webhook object ready to be sent""" + return json.dumps(self.webhook_object) + + def __setup_embed(self): + self.embed = defaultdict(dict) + if "embeds" not in self.webhook_object: + self.webhook_object["embeds"] = [self.embed] + else: + self.webhook_object["embeds"].append(self.embed) + self.embed["color"] = None + + def add_embed(self): + self.finish_embed() + self.__setup_embed() + + def finish_embed(self): + if self.embed["color"] is None: + if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: + self.embed["color"] = random.randrange(1, 16777215) + else: + self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] + else: + self.embed["color"] = math.floor(self.embed["color"]) + + def set_author(self, name, url, icon_url=""): + self.embed["author"]["name"] = name + self.embed["author"]["url"] = url + self.embed["author"]["icon_url"] = icon_url + + def add_field(self, name, value, inline=False): + if "fields" not in self.embed: + self.embed["fields"] = [] + self.embed["fields"].append(dict(name=name, value=value, inline=inline)) + + def set_avatar(self, url): + self.webhook_object["avatar_url"] = url + + def set_name(self, name): + self.webhook_object["username"] = name + + +class DiscordMessageMetadata: + def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): + self.method = method + self.page_id = page_id + self.log_id = log_id + self.rev_id = rev_id + self.webhook_url = webhook_url + self.new_data = new_data + + def dump_ids(self): + return self.page_id, self.rev_id, self.log_id \ No newline at end of file diff --git a/src/discord/queue.py b/src/discord/queue.py new file mode 100644 index 0000000..51eb42d --- /dev/null +++ b/src/discord/queue.py @@ -0,0 +1,156 @@ +import re +import sys +import time +import logging +from typing import Optional + +import requests + +from src.configloader import settings +from src.discord.message import DiscordMessage, DiscordMessageMetadata + +AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": True}).get("enabled") +if AUTO_SUPPRESSION_ENABLED: + from src.fileio.database import add_entry as add_message_redaction_entry + +rate_limit = 0 + +logger = logging.getLogger("rcgcdw.discord.queue") + +class MessageQueue: + """Message queue class for undelivered messages""" + def __init__(self): + self._queue = [] + + def __repr__(self): + return self._queue + + def __len__(self): + return len(self._queue) + + def __iter__(self): + return iter(self._queue) + + def clear(self): + self._queue.clear() + + def add_message(self, message): + self._queue.append(message) + + def cut_messages(self, item_num): + self._queue = self._queue[item_num:] + + def resend_msgs(self): + if self._queue: + logger.info( + "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( + len(self._queue))) + for num, item in enumerate(self._queue): + logger.debug( + "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), + str(item))) + if send_to_discord_webhook(item[0], metadata=item[1]) < 2: + logger.debug("Sending message succeeded") + else: + logger.debug("Sending message failed") + break + else: + self.clear() + logger.debug("Queue emptied, all messages delivered") + self.cut_messages(num) + logger.debug(self._queue) + + +messagequeue = MessageQueue() + + +def handle_discord_http(code, formatted_embed, result): + if 300 > code > 199: # message went through + return 0 + elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header + logger.error( + "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") + logger.error(formatted_embed) + logger.error(result.text) + return 1 + elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND + if result.request.method == "POST": # Ignore not found for DELETE and PATCH requests since the message could already be removed by admin + logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") + sys.exit(1) + else: + return 0 + elif code == 429: + logger.error("We are sending too many requests to the Discord, slowing down...") + return 2 + elif 499 < code < 600: + logger.error( + "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( + code)) + return 3 + + +def update_ratelimit(request): + """Updates rate limit time""" + global rate_limit + rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get( + 'x-ratelimit-reset-after', 0)) + rate_limit += settings.get("discord_message_cooldown", 0) + + +def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMessageMetadata): + global rate_limit + header = settings["header"] + header['Content-Type'] = 'application/json' + standard_args = dict(headers=header) + if metadata.method == "POST": + req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if AUTO_SUPPRESSION_ENABLED else "false", data=repr(data), **standard_args) + elif metadata.method == "DELETE": + req = requests.Request("DELETE", metadata.webhook_url, **standard_args) + elif metadata.method == "PATCH": + req = requests.Request("PATCH", metadata.webhook_url, data=metadata.new_data, **standard_args) + try: + time.sleep(rate_limit) + rate_limit = 0 + req = req.prepare() + result = requests.Session().send(req, timeout=10) + update_ratelimit(result) + if AUTO_SUPPRESSION_ENABLED and metadata.method == "POST": + # TODO Prepare request with all of safety checks + try: + add_message_redaction_entry(*metadata.dump_ids(), result.json()) + except ValueError: + logger.error("Couldn't get json of result of sending Discord message.") + except requests.exceptions.Timeout: + logger.warning("Timeouted while sending data to the webhook.") + return 3 + except requests.exceptions.ConnectionError: + logger.warning("Connection error while sending the data to a webhook") + return 3 + else: + return handle_discord_http(result.status_code, data, result) + + +def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata): + if data is not None: + for regex in settings["disallow_regexes"]: + if data.webhook_object.get("content", None): + if re.search(re.compile(regex), data.webhook_object["content"]): + logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) + return # discard the message without anything + else: + for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: + if re.search(re.compile(regex), to_check): + logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( + to_check, regex)) + return # discard the message without anything + if messagequeue: + messagequeue.add_message((data, meta)) + else: + code = send_to_discord_webhook(data, metadata=meta) + if code == 3: + messagequeue.add_message((data, meta)) + elif code == 2: + time.sleep(5.0) + messagequeue.add_message((data, meta)) + elif code < 2: + pass \ No newline at end of file diff --git a/src/discord/redaction.py b/src/discord/redaction.py new file mode 100644 index 0000000..941a8bd --- /dev/null +++ b/src/discord/redaction.py @@ -0,0 +1,27 @@ +import logging + +from src.configloader import settings +from src.discord.message import DiscordMessageMetadata +from src.discord.queue import send_to_discord +from src.fileio.database import db_cursor, db_connection + +logger = logging.getLogger("rcgcdw.discord.redaction") + + +def delete_messages(pageid: int): + """Delete messages that match that pageid""" + logger.debug(type(pageid)) + to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid,)) + msg_to_remove = [] + logger.debug("Deleting messages for pageid: {}".format(pageid)) + for message in to_delete: + webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) + msg_to_remove.append(message[0]) + logger.debug("Removing following message: {}".format(message[0])) + send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) + db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (message[0],)) + db_connection.commit() + + +def redact_messages(rev_ids: list, to_censor: dict): + raise NotImplemented \ No newline at end of file diff --git a/src/discussion_formatters.py b/src/discussion_formatters.py index dd4aa81..3706c84 100644 --- a/src/discussion_formatters.py +++ b/src/discussion_formatters.py @@ -4,7 +4,9 @@ import gettext from urllib.parse import quote_plus from src.configloader import settings -from src.misc import link_formatter, create_article_path, DiscordMessage, send_to_discord, escape_formatting, DiscordMessageMetadata +from src.misc import link_formatter, create_article_path, escape_formatting +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata from src.i18n import discussion_formatters _ = discussion_formatters.gettext diff --git a/src/discussions.py b/src/discussions.py index 579ffaa..4d8ca37 100644 --- a/src/discussions.py +++ b/src/discussions.py @@ -22,7 +22,8 @@ from typing import Dict, Any from src.configloader import settings from src.discussion_formatters import embed_formatter, compact_formatter -from src.misc import datafile, messagequeue, prepare_paths +from src.misc import datafile, prepare_paths +from src.discord.queue import messagequeue from src.session import session from src.exceptions import ArticleCommentError diff --git a/src/fileio/__init__.py b/src/fileio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fileio/database.py b/src/fileio/database.py new file mode 100644 index 0000000..a2870b8 --- /dev/null +++ b/src/fileio/database.py @@ -0,0 +1,51 @@ +import sqlite3 +import logging +from src.configloader import settings + +logger = logging.getLogger("rcgcdw.fileio.database") + + +def create_schema(): + logger.info("Creating database schema...") + db_cursor.executescript( + """BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS "messages" ( + "message_id" TEXT, + "content" TEXT + ); + CREATE TABLE IF NOT EXISTS "event" ( + "pageid" INTEGER, + "revid" INTEGER, + "logid" INTEGER, + "msg_id" TEXT NOT NULL, + FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE + ); + COMMIT;""") + logger.info("Database schema has been recreated.") + + +def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): + _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) + _db_connection.row_factory = sqlite3.Row + _db_cursor = _db_connection.cursor() + logger.debug("Database connection created") + return _db_connection, _db_cursor + + +def check_tables(): + """Check if tables exist, if not, create schema""" + rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") + if not rep.fetchone(): + logger.debug("No schema detected, creating schema!") + create_schema() + + +def add_entry(pageid: int, revid: int, logid: int, message): + """Add an edit or log entry to the DB""" + db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("id"), str(message))) + db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("id"))) + logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) + db_connection.commit() + +db_connection, db_cursor = create_connection() +check_tables() diff --git a/src/message_redaction.py b/src/message_redaction.py index 6c5af41..b28b04f 100644 --- a/src/message_redaction.py +++ b/src/message_redaction.py @@ -1,65 +1,3 @@ -from src.configloader import settings -from src.misc import send_to_discord, DiscordMessageMetadata -import logging -logger = logging.getLogger("rcgcdw.message_redaction") -import sqlite3 -def create_schema(): - logger.info("Creating database schema...") - db_cursor.executescript( - """BEGIN TRANSACTION; - CREATE TABLE IF NOT EXISTS "messages" ( - "message_id" TEXT, - "content" TEXT - ); - CREATE TABLE IF NOT EXISTS "event" ( - "pageid" INTEGER, - "revid" INTEGER, - "logid" INTEGER, - "msg_id" TEXT NOT NULL, - FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE - ); - COMMIT;""") - logger.info("Database schema has been recreated.") - -def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): - _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) - _db_connection.row_factory = sqlite3.Row - _db_cursor = db_connection.cursor() - logger.debug("Database connection created") - return _db_connection, _db_cursor - - -def check_tables(): - rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") - if not rep.fetchone(): - logger.debug("No schema detected, creating schema!") - create_schema() - - -def add_entry(pageid: int, revid: int, logid: int, message): - db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("message_id"), message)) - db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("message_id"))) - logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) - - -def delete_messages(pageid: int): - to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid)) - msg_to_remove = [] - logger.debug("Deleting messages for pageid: {}".format(pageid)) - for message in to_delete: - webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) - msg_to_remove.append(message[0]) - logger.debug("Removing following message: {}".format(message)) - send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) - db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", msg_to_remove) - - -def redact_messages(rev_ids: list, to_censor: dict): - raise NotImplemented - - -db_connection, db_cursor = create_connection() -check_tables() diff --git a/src/misc.py b/src/misc.py index 041c795..d988d5f 100644 --- a/src/misc.py +++ b/src/misc.py @@ -16,18 +16,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . import base64 -import json, logging, sys, re, time, random, math +import json, logging, sys, re from html.parser import HTMLParser from urllib.parse import urlparse, urlunparse, quote import requests -from collections import defaultdict from src.configloader import settings +from src.discord.message import DiscordMessage, DiscordMessageMetadata +from src.discord.queue import messagequeue, send_to_discord from src.i18n import misc -from typing import Optional AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": True}).get("enabled") if AUTO_SUPPRESSION_ENABLED: - from src.message_redaction import add_entry as add_message_redaction_entry + pass _ = misc.gettext @@ -43,7 +43,6 @@ WIKI_API_PATH: str = "" WIKI_ARTICLE_PATH: str = "" WIKI_SCRIPT_PATH: str = "" WIKI_JUST_DOMAIN: str = "" -rate_limit = 0 profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")} @@ -95,52 +94,6 @@ class DataFile: return self.data[item] - -class MessageQueue: - """Message queue class for undelivered messages""" - def __init__(self): - self._queue = [] - - def __repr__(self): - return self._queue - - def __len__(self): - return len(self._queue) - - def __iter__(self): - return iter(self._queue) - - def clear(self): - self._queue.clear() - - def add_message(self, message): - self._queue.append(message) - - def cut_messages(self, item_num): - self._queue = self._queue[item_num:] - - def resend_msgs(self): - if self._queue: - misc_logger.info( - "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( - len(self._queue))) - for num, item in enumerate(self._queue): - misc_logger.debug( - "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), - str(item))) - if send_to_discord_webhook(item[0], metadata=item[1]) < 2: - misc_logger.debug("Sending message succeeded") - else: - misc_logger.debug("Sending message failed") - break - else: - self.clear() - misc_logger.debug("Queue emptied, all messages delivered") - self.cut_messages(num) - misc_logger.debug(self._queue) - - -messagequeue = MessageQueue() datafile = DataFile() @@ -245,28 +198,6 @@ def safe_read(request, *keys): return request -def handle_discord_http(code, formatted_embed, result): - if 300 > code > 199: # message went through - return 0 - elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header - misc_logger.error( - "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") - misc_logger.error(formatted_embed) - misc_logger.error(result.text) - return 1 - elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND - misc_logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") - sys.exit(1) - elif code == 429: - misc_logger.error("We are sending too many requests to the Discord, slowing down...") - return 2 - elif 499 < code < 600: - misc_logger.error( - "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( - code)) - return 3 - - def add_to_dict(dictionary, key): if key in dictionary: dictionary[key] += 1 @@ -333,80 +264,6 @@ def send_simple(msgtype, message, name, avatar): send_to_discord(discord_msg, meta=DiscordMessageMetadata("POST")) -def update_ratelimit(request): - """Updates rate limit time""" - global rate_limit - rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get( - 'x-ratelimit-reset-after', 0)) - rate_limit += settings.get("discord_message_cooldown", 0) - - - -class DiscordMessage: - """A class defining a typical Discord JSON representation of webhook payload.""" - def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): - self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) - self.webhook_url = webhook_url - - if message_type == "embed": - self.__setup_embed() - elif message_type == "compact": - self.webhook_object["content"] = content - - self.event_type = event_type - - def __setitem__(self, key, value): - """Set item is used only in embeds.""" - try: - self.embed[key] = value - except NameError: - raise TypeError("Tried to assign a value when message type is plain message!") - - def __getitem__(self, item): - return self.embed[item] - - def __repr__(self): - """Return the Discord webhook object ready to be sent""" - return json.dumps(self.webhook_object) - - def __setup_embed(self): - self.embed = defaultdict(dict) - if "embeds" not in self.webhook_object: - self.webhook_object["embeds"] = [self.embed] - else: - self.webhook_object["embeds"].append(self.embed) - self.embed["color"] = None - - def add_embed(self): - self.finish_embed() - self.__setup_embed() - - def finish_embed(self): - if self.embed["color"] is None: - if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: - self.embed["color"] = random.randrange(1, 16777215) - else: - self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] - else: - self.embed["color"] = math.floor(self.embed["color"]) - - def set_author(self, name, url, icon_url=""): - self.embed["author"]["name"] = name - self.embed["author"]["url"] = url - self.embed["author"]["icon_url"] = icon_url - - def add_field(self, name, value, inline=False): - if "fields" not in self.embed: - self.embed["fields"] = [] - self.embed["fields"].append(dict(name=name, value=value, inline=inline)) - - def set_avatar(self, url): - self.webhook_object["avatar_url"] = url - - def set_name(self, name): - self.webhook_object["username"] = name - - def profile_field_name(name, embed): try: return profile_fields[name] @@ -417,77 +274,6 @@ def profile_field_name(name, embed): return _("unknown") -class DiscordMessageMetadata: - def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): - self.method = method - self.page_id = page_id - self.log_id = log_id - self.rev_id = rev_id - self.webhook_url = webhook_url - self.new_data = new_data - - def dump_ids(self): - return self.page_id, self.rev_id, self.log_id - -def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMessageMetadata): - global rate_limit - header = settings["header"] - header['Content-Type'] = 'application/json' - standard_args = dict(headers=header, timeout=10) - if metadata.method == "POST": - req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if AUTO_SUPPRESSION_ENABLED else "false", data=repr(data), **standard_args) - elif metadata.method == "DELETE": - req = requests.Request("DELETE", metadata.webhook_url, **standard_args) - elif metadata.method == "PATCH": - req = requests.Request("PATCH", metadata.webhook_url, data=metadata.new_data, **standard_args) - try: - time.sleep(rate_limit) - rate_limit = 0 - req = req.prepare() - result = req.send() - update_ratelimit(result) - if AUTO_SUPPRESSION_ENABLED: - # TODO Prepare request with all of safety checks - try: - add_message_redaction_entry(*metadata.dump_ids(), result.json()) - except ValueError: - misc_logger.error("Couldn't get json of result of sending Discord message.") - except requests.exceptions.Timeout: - misc_logger.warning("Timeouted while sending data to the webhook.") - return 3 - except requests.exceptions.ConnectionError: - misc_logger.warning("Connection error while sending the data to a webhook") - return 3 - else: - return handle_discord_http(result.status_code, data, result) - - -def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata): - if data is not None: - for regex in settings["disallow_regexes"]: - if data.webhook_object.get("content", None): - if re.search(re.compile(regex), data.webhook_object["content"]): - misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) - return # discard the message without anything - else: - for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: - if re.search(re.compile(regex), to_check): - misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( - to_check, regex)) - return # discard the message without anything - if messagequeue: - messagequeue.add_message((data, meta)) - else: - code = send_to_discord_webhook(data, metadata=meta) - if code == 3: - messagequeue.add_message((data, meta)) - elif code == 2: - time.sleep(5.0) - messagequeue.add_message((data, meta)) - elif code < 2: - pass - - class LinkParser(HTMLParser): new_string = "" recent_href = "" diff --git a/src/rc.py b/src/rc.py index beaa94c..51e3b6d 100644 --- a/src/rc.py +++ b/src/rc.py @@ -6,7 +6,8 @@ import requests from bs4 import BeautifulSoup from src.configloader import settings -from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, messagequeue, datafile, send_simple, safe_read, LinkParser +from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, LinkParser +from src.discord.queue import messagequeue from src.exceptions import MWError from src.session import session from src.rc_formatters import compact_formatter, embed_formatter, compact_abuselog_formatter, embed_abuselog_formatter @@ -399,11 +400,11 @@ def essential_info(change, changed_categories): parsed_comment = None if "userhidden" in change: change["user"] = _("hidden") + if change.get("ns", -1) in settings.get("ignored_namespaces", ()): + return if change["type"] in ["edit", "new"]: logger.debug("List of categories in essential_info: {}".format(changed_categories)) identification_string = change["type"] - if change.get("ns", -1) in settings.get("ignored_namespaces", ()): - return elif change["type"] == "log": identification_string = "{logtype}/{logaction}".format(logtype=change["logtype"], logaction=change["logaction"]) if identification_string not in supported_logs: diff --git a/src/rc_formatters.py b/src/rc_formatters.py index 0a9399f..684ef21 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -10,10 +10,13 @@ from urllib.parse import quote_plus, quote from bs4 import BeautifulSoup from src.configloader import settings -from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ - WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, DiscordMessageMetadata, AUTO_SUPPRESSION_ENABLED +from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, safe_read, \ + WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, AUTO_SUPPRESSION_ENABLED +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata + if AUTO_SUPPRESSION_ENABLED: - from src.message_redaction import delete_messages, redact_messages + from src.discord.redaction import delete_messages, redact_messages from src.i18n import rc_formatters #from src.rc import recent_changes, pull_comment diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 1b6025e..e19622a 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -26,8 +26,9 @@ import src.misc from collections import defaultdict, Counter from src.configloader import settings from src.misc import add_to_dict, datafile, \ - WIKI_API_PATH, create_article_path, send_to_discord, \ - DiscordMessage, DiscordMessageMetadata + WIKI_API_PATH, create_article_path +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata from src.rc import recent_changes from src.exceptions import MWError from src.i18n import rcgcdw From baa8eeead9317b6ef9c8514eb287a6661807a41f Mon Sep 17 00:00:00 2001 From: Frisk Date: Mon, 9 Nov 2020 02:22:31 +0100 Subject: [PATCH 08/17] Just some cleanup code and deleting messages from queue --- src/discord/queue.py | 17 ++++++++++++++++- src/discord/redaction.py | 4 +++- src/fileio/database.py | 15 ++++++++++++++- src/rc.py | 5 ++++- 4 files changed, 37 insertions(+), 4 deletions(-) diff --git a/src/discord/queue.py b/src/discord/queue.py index 51eb42d..a2794ed 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -9,7 +9,7 @@ import requests from src.configloader import settings from src.discord.message import DiscordMessage, DiscordMessageMetadata -AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": True}).get("enabled") +AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": False}).get("enabled") if AUTO_SUPPRESSION_ENABLED: from src.fileio.database import add_entry as add_message_redaction_entry @@ -40,6 +40,21 @@ class MessageQueue: def cut_messages(self, item_num): self._queue = self._queue[item_num:] + @staticmethod + def compare_message_to_dict(metadata: DiscordMessageMetadata, to_match: dict): + """Compare DiscordMessageMetadata fields and match them against dictionary""" + for name, val in to_match.items(): + if getattr(metadata, name, None) != val: + return False + return True + + def delete_all_with_matching_metadata(self, **properties): + """Deletes all of the messages that have matching metadata properties (useful for message redaction)""" + for message in messagequeue: + if self.compare_message_to_dict(message[1], properties): + # TODO Delete messages from the queue + raise NotImplemented + def resend_msgs(self): if self._queue: logger.info( diff --git a/src/discord/redaction.py b/src/discord/redaction.py index 941a8bd..499a241 100644 --- a/src/discord/redaction.py +++ b/src/discord/redaction.py @@ -2,7 +2,7 @@ import logging from src.configloader import settings from src.discord.message import DiscordMessageMetadata -from src.discord.queue import send_to_discord +from src.discord.queue import send_to_discord, messagequeue from src.fileio.database import db_cursor, db_connection logger = logging.getLogger("rcgcdw.discord.redaction") @@ -12,6 +12,8 @@ def delete_messages(pageid: int): """Delete messages that match that pageid""" logger.debug(type(pageid)) to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid,)) + if len(messagequeue) > 0: + messagequeue.delete_all_with_matching_metadata(pageid=pageid) msg_to_remove = [] logger.debug("Deleting messages for pageid: {}".format(pageid)) for message in to_delete: diff --git a/src/fileio/database.py b/src/fileio/database.py index a2870b8..4593f00 100644 --- a/src/fileio/database.py +++ b/src/fileio/database.py @@ -11,13 +11,15 @@ def create_schema(): """BEGIN TRANSACTION; CREATE TABLE IF NOT EXISTS "messages" ( "message_id" TEXT, - "content" TEXT + "content" TEXT, + PRIMARY KEY("message_id") ); CREATE TABLE IF NOT EXISTS "event" ( "pageid" INTEGER, "revid" INTEGER, "logid" INTEGER, "msg_id" TEXT NOT NULL, + PRIMARY KEY("msg_id"), FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE ); COMMIT;""") @@ -47,5 +49,16 @@ def add_entry(pageid: int, revid: int, logid: int, message): logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) db_connection.commit() +def clean_entries(): + """Cleans entries that are 50+""" + cleanup = db_cursor.execute( + "SELECT message_id FROM messages WHERE message_id NOT IN (SELECT message_id FROM messages ORDER BY message_id asc LIMIT 50);") + for row in cleanup: + db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (cleanup[0])) + cleanup = db_cursor.execute("SELECT msg_id FROM event WHERE msg_id NOT IN (SELECT msg_id FROM event ORDER BY msg_id asc LIMIT 50);") + for row in cleanup: + db_cursor.execute("DELETE FROM event WHERE msg_id = ?", (cleanup[0])) + db_connection.commit() + db_connection, db_cursor = create_connection() check_tables() diff --git a/src/rc.py b/src/rc.py index 51e3b6d..1d4b5f6 100644 --- a/src/rc.py +++ b/src/rc.py @@ -6,7 +6,7 @@ import requests from bs4 import BeautifulSoup from src.configloader import settings -from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, LinkParser +from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, LinkParser, AUTO_SUPPRESSION_ENABLED from src.discord.queue import messagequeue from src.exceptions import MWError from src.session import session @@ -333,6 +333,9 @@ class Recent_Changes_Class(object): def clear_cache(self): self.map_ips = {} + if AUTO_SUPPRESSION_ENABLED: + from src.fileio.database import clean_entries + clean_entries() def init_info(self): startup_info = safe_read(self.safe_request( From fac70ba7a45dabc789a2ccef5de84fb07600e264 Mon Sep 17 00:00:00 2001 From: Frisk Date: Mon, 9 Nov 2020 14:58:05 +0100 Subject: [PATCH 09/17] Added queue clearing out of messages and periodic DB cleanup for old entries --- src/discord/queue.py | 7 +++---- src/fileio/database.py | 8 ++++---- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/src/discord/queue.py b/src/discord/queue.py index a2794ed..f1a0907 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -50,10 +50,9 @@ class MessageQueue: def delete_all_with_matching_metadata(self, **properties): """Deletes all of the messages that have matching metadata properties (useful for message redaction)""" - for message in messagequeue: - if self.compare_message_to_dict(message[1], properties): - # TODO Delete messages from the queue - raise NotImplemented + for index, item in reversed(list(enumerate(self._queue))): + if self.compare_message_to_dict(item[1], properties): + self._queue.pop(index) def resend_msgs(self): if self._queue: diff --git a/src/fileio/database.py b/src/fileio/database.py index 4593f00..99126f9 100644 --- a/src/fileio/database.py +++ b/src/fileio/database.py @@ -52,12 +52,12 @@ def add_entry(pageid: int, revid: int, logid: int, message): def clean_entries(): """Cleans entries that are 50+""" cleanup = db_cursor.execute( - "SELECT message_id FROM messages WHERE message_id NOT IN (SELECT message_id FROM messages ORDER BY message_id asc LIMIT 50);") + "SELECT message_id FROM messages WHERE message_id NOT IN (SELECT message_id FROM messages ORDER BY message_id desc LIMIT 50);") for row in cleanup: - db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (cleanup[0])) - cleanup = db_cursor.execute("SELECT msg_id FROM event WHERE msg_id NOT IN (SELECT msg_id FROM event ORDER BY msg_id asc LIMIT 50);") + db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (row[0])) + cleanup = db_cursor.execute("SELECT msg_id FROM event WHERE msg_id NOT IN (SELECT msg_id FROM event ORDER BY msg_id desc LIMIT 50);") for row in cleanup: - db_cursor.execute("DELETE FROM event WHERE msg_id = ?", (cleanup[0])) + db_cursor.execute("DELETE FROM event WHERE msg_id = ?", (row[0])) db_connection.commit() db_connection, db_cursor = create_connection() From 8d7da1d23753eeb64c0687d20047b4cdc4de14b9 Mon Sep 17 00:00:00 2001 From: Frisk Date: Wed, 11 Nov 2020 00:48:11 +0100 Subject: [PATCH 10/17] Small improvements --- src/discord/redaction.py | 9 +++++++-- src/misc.py | 4 +--- src/rc_formatters.py | 33 +++++++++++++++++++++++++++++---- 3 files changed, 37 insertions(+), 9 deletions(-) diff --git a/src/discord/redaction.py b/src/discord/redaction.py index 499a241..9c8e765 100644 --- a/src/discord/redaction.py +++ b/src/discord/redaction.py @@ -25,5 +25,10 @@ def delete_messages(pageid: int): db_connection.commit() -def redact_messages(rev_ids: list, to_censor: dict): - raise NotImplemented \ No newline at end of file +def redact_messages(ids: list, entry_type: int, to_censor: dict): + """Redact past Discord messages + + ids: list of ints + entry_type: int - 0 for revdel, 1 for logdel + to_censor: dict - logparams of message parts to censor""" + raise NotImplemented diff --git a/src/misc.py b/src/misc.py index d988d5f..2c1273f 100644 --- a/src/misc.py +++ b/src/misc.py @@ -25,9 +25,7 @@ from src.discord.message import DiscordMessage, DiscordMessageMetadata from src.discord.queue import messagequeue, send_to_discord from src.i18n import misc -AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": True}).get("enabled") -if AUTO_SUPPRESSION_ENABLED: - pass +AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": False}).get("enabled") _ = misc.gettext diff --git a/src/rc_formatters.py b/src/rc_formatters.py index 684ef21..72c0e0e 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -111,12 +111,14 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes page_link = link_formatter(create_article_path(change["title"])) content = "🗑️ "+_("[{author}]({author_url}) deleted [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) - delete_messages(change.get("pageid")) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(change.get("pageid")) elif action == "delete/delete_redir": page_link = link_formatter(create_article_path(change["title"])) content = "🗑️ "+_("[{author}]({author_url}) deleted redirect by overwriting [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) - delete_messages(change.get("pageid")) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(change.get("pageid")) elif action == "move/move": link = link_formatter(create_article_path(change["logparams"]['target_title'])) redirect_status = _("without making a redirect") if "suppressredirect" in change["logparams"] else _("with a redirect") @@ -273,6 +275,13 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "👁️ "+ngettext("[{author}]({author_url}) changed visibility of revision on page [{article}]({article_url}){comment}", "[{author}]({author_url}) changed visibility of {amount} revisions on page [{article}]({article_url}){comment}", amount).format(author=author, author_url=author_url, article=change["title"], article_url=link, amount=amount, comment=parsed_comment) + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + except KeyError: + pass + else: + # TODO Get pageid elif action == "import/upload": link = link_formatter(create_article_path(change["title"])) content = "📥 "+ngettext("[{author}]({author_url}) imported [{article}]({article_url}) with {count} revision{comment}", @@ -283,6 +292,13 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "♻️ "+_("[{author}]({author_url}) restored [{article}]({article_url}){comment}").format(author=author, author_url=author_url, article=change["title"], article_url=link, comment=parsed_comment) elif action == "delete/event": content = "👁️ "+_("[{author}]({author_url}) changed visibility of log events{comment}").format(author=author, author_url=author_url, comment=parsed_comment) + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + except KeyError: + pass + else: + delete_messages(logparams.get("ids", []), 1, logparams.get("new", {})) # TODO Check validity elif action == "import/interwiki": content = "📥 "+_("[{author}]({author_url}) imported interwiki{comment}").format(author=author, author_url=author_url, comment=parsed_comment) elif action == "abusefilter/modify": @@ -564,11 +580,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): elif action == "delete/delete": link = create_article_path(change["title"]) embed["title"] = _("Deleted page {article}").format(article=change["title"]) - delete_messages(change.get("pageid")) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(change.get("pageid")) elif action == "delete/delete_redir": link = create_article_path(change["title"]) embed["title"] = _("Deleted redirect {article} by overwriting").format(article=change["title"]) - delete_messages(change.get("pageid")) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(change.get("pageid")) elif action == "move/move": link = create_article_path(change["logparams"]['target_title']) parsed_comment = "{supress}. {desc}".format(desc=parsed_comment, @@ -729,6 +747,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): elif action == "delete/event": link = create_article_path("Special:RecentChanges") embed["title"] = _("Changed visibility of log events") + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + except KeyError: + pass + else: + redact_messages(logparams.get("ids", []), 1, logparams.get("new", {})) elif action == "import/interwiki": link = create_article_path("Special:RecentChanges") embed["title"] = _("Imported interwiki") From 0cdc0cef3271fa450d0a133257055478d883cfcc Mon Sep 17 00:00:00 2001 From: Frisk Date: Sun, 15 Nov 2020 01:43:34 +0100 Subject: [PATCH 11/17] Progress --- src/discord/redaction.py | 43 +++++++++++++++++++++++++++++++++------- src/rc_formatters.py | 14 +++++++------ 2 files changed, 44 insertions(+), 13 deletions(-) diff --git a/src/discord/redaction.py b/src/discord/redaction.py index 9c8e765..c51605c 100644 --- a/src/discord/redaction.py +++ b/src/discord/redaction.py @@ -1,21 +1,27 @@ import logging from src.configloader import settings -from src.discord.message import DiscordMessageMetadata +from src.discord.message import DiscordMessageMetadata, DiscordMessage from src.discord.queue import send_to_discord, messagequeue from src.fileio.database import db_cursor, db_connection logger = logging.getLogger("rcgcdw.discord.redaction") -def delete_messages(pageid: int): - """Delete messages that match that pageid""" - logger.debug(type(pageid)) - to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid,)) +def delete_messages(matching_data: dict): + """Delete messages that match given data""" + sql_conditions = "" + args = [] + for data in matching_data.items(): + sql_conditions += "? = ? AND" + args.extend(data) + else: + sql_conditions = sql_conditions[0:-4] # remove last AND statement + to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE {CON}".format(CON=sql_conditions), args) if len(messagequeue) > 0: - messagequeue.delete_all_with_matching_metadata(pageid=pageid) + messagequeue.delete_all_with_matching_metadata(**matching_data) msg_to_remove = [] - logger.debug("Deleting messages for pageid: {}".format(pageid)) + logger.debug("Deleting messages for data: {}".format(matching_data)) for message in to_delete: webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) msg_to_remove.append(message[0]) @@ -31,4 +37,27 @@ def redact_messages(ids: list, entry_type: int, to_censor: dict): ids: list of ints entry_type: int - 0 for revdel, 1 for logdel to_censor: dict - logparams of message parts to censor""" + for event_id in ids: + if entry_type == 0: # TODO check if queries are proper + message = db_cursor.execute("SELECT content FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.revid = ?;", event_id) + else: + message = db_cursor.execute( + "SELECT content FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.logid = ?;", + event_id) + if settings["appearance"]["mode"] == "embed": + if message is not None: + message = message.fetchone() + new_embed = message["embeds"][0] + if "user" in to_censor: + new_embed["author"]["name"] = _("Removed") + del new_embed["author"]["url"] + if "action" in to_censor: + new_embed["title"] = _("Removed") + del new_embed["url"] + if "comment" in to_censor: + new_embed["description"] = _("Removed") + message["embeds"][0] = new_embed + # TODO somehow send nly important data as PATCH? + send_to_discord() + raise NotImplemented diff --git a/src/rc_formatters.py b/src/rc_formatters.py index 72c0e0e..a4e003c 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -112,13 +112,13 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "🗑️ "+_("[{author}]({author_url}) deleted [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) if AUTO_SUPPRESSION_ENABLED: - delete_messages(change.get("pageid")) + delete_messages(dict(pageid=change.get("pageid"))) elif action == "delete/delete_redir": page_link = link_formatter(create_article_path(change["title"])) content = "🗑️ "+_("[{author}]({author_url}) deleted redirect by overwriting [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) if AUTO_SUPPRESSION_ENABLED: - delete_messages(change.get("pageid")) + delete_messages(dict(pageid=change.get("pageid"))) elif action == "move/move": link = link_formatter(create_article_path(change["logparams"]['target_title'])) redirect_status = _("without making a redirect") if "suppressredirect" in change["logparams"] else _("with a redirect") @@ -278,10 +278,11 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes if AUTO_SUPPRESSION_ENABLED: try: logparams = change["logparams"] + pageid = change["pageid"] except KeyError: pass else: - # TODO Get pageid + delete_messages(dict(pageid=pageid)) elif action == "import/upload": link = link_formatter(create_article_path(change["title"])) content = "📥 "+ngettext("[{author}]({author_url}) imported [{article}]({article_url}) with {count} revision{comment}", @@ -298,7 +299,8 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes except KeyError: pass else: - delete_messages(logparams.get("ids", []), 1, logparams.get("new", {})) # TODO Check validity + for revid in logparams.get("ids", []): + delete_messages(dict(revid=revid)) elif action == "import/interwiki": content = "📥 "+_("[{author}]({author_url}) imported interwiki{comment}").format(author=author, author_url=author_url, comment=parsed_comment) elif action == "abusefilter/modify": @@ -581,12 +583,12 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): link = create_article_path(change["title"]) embed["title"] = _("Deleted page {article}").format(article=change["title"]) if AUTO_SUPPRESSION_ENABLED: - delete_messages(change.get("pageid")) + delete_messages(dict(pageid=change.get("pageid"))) elif action == "delete/delete_redir": link = create_article_path(change["title"]) embed["title"] = _("Deleted redirect {article} by overwriting").format(article=change["title"]) if AUTO_SUPPRESSION_ENABLED: - delete_messages(change.get("pageid")) + delete_messages(dict(pageid=change.get("pageid"))) elif action == "move/move": link = create_article_path(change["logparams"]['target_title']) parsed_comment = "{supress}. {desc}".format(desc=parsed_comment, From 5032716226eedd54c7ef9c0f6d1964ee386bc97d Mon Sep 17 00:00:00 2001 From: Frisk Date: Sun, 15 Nov 2020 11:17:04 +0100 Subject: [PATCH 12/17] Fixes --- src/discord/message.py | 5 +++++ src/discord/queue.py | 2 +- src/discord/redaction.py | 20 +++++++++----------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/discord/message.py b/src/discord/message.py index f918657..4bd2b2a 100644 --- a/src/discord/message.py +++ b/src/discord/message.py @@ -71,6 +71,11 @@ class DiscordMessage: self.webhook_object["username"] = name +class DiscordMessageRaw(DiscordMessage): + def __init__(self, content: dict): + self.webhook_object = content + + class DiscordMessageMetadata: def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): self.method = method diff --git a/src/discord/queue.py b/src/discord/queue.py index f1a0907..4eb0013 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -121,7 +121,7 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMes elif metadata.method == "DELETE": req = requests.Request("DELETE", metadata.webhook_url, **standard_args) elif metadata.method == "PATCH": - req = requests.Request("PATCH", metadata.webhook_url, data=metadata.new_data, **standard_args) + req = requests.Request("PATCH", metadata.webhook_url, data=data, **standard_args) try: time.sleep(rate_limit) rate_limit = 0 diff --git a/src/discord/redaction.py b/src/discord/redaction.py index c51605c..48c248a 100644 --- a/src/discord/redaction.py +++ b/src/discord/redaction.py @@ -1,7 +1,7 @@ import logging from src.configloader import settings -from src.discord.message import DiscordMessageMetadata, DiscordMessage +from src.discord.message import DiscordMessageMetadata, DiscordMessage, DiscordMessageRaw from src.discord.queue import send_to_discord, messagequeue from src.fileio.database import db_cursor, db_connection @@ -11,13 +11,11 @@ logger = logging.getLogger("rcgcdw.discord.redaction") def delete_messages(matching_data: dict): """Delete messages that match given data""" sql_conditions = "" - args = [] - for data in matching_data.items(): - sql_conditions += "? = ? AND" - args.extend(data) + for key, value in matching_data.items(): + sql_conditions += "{} = ? AND".format(key) else: sql_conditions = sql_conditions[0:-4] # remove last AND statement - to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE {CON}".format(CON=sql_conditions), args) + to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE {CON}".format(CON=sql_conditions), list(matching_data.values())) if len(messagequeue) > 0: messagequeue.delete_all_with_matching_metadata(**matching_data) msg_to_remove = [] @@ -27,7 +25,8 @@ def delete_messages(matching_data: dict): msg_to_remove.append(message[0]) logger.debug("Removing following message: {}".format(message[0])) send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) - db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (message[0],)) + for msg in msg_to_remove: + db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (msg,)) db_connection.commit() @@ -50,14 +49,13 @@ def redact_messages(ids: list, entry_type: int, to_censor: dict): new_embed = message["embeds"][0] if "user" in to_censor: new_embed["author"]["name"] = _("Removed") - del new_embed["author"]["url"] + new_embed["author"].pop("url") if "action" in to_censor: new_embed["title"] = _("Removed") - del new_embed["url"] + new_embed.pop("url") if "comment" in to_censor: new_embed["description"] = _("Removed") message["embeds"][0] = new_embed - # TODO somehow send nly important data as PATCH? - send_to_discord() + send_to_discord(DiscordMessageRaw(message), DiscordMessageMetadata("PATCH")) raise NotImplemented From b2be691cccca2f16768ecf6b870e91a303dc6f4d Mon Sep 17 00:00:00 2001 From: Frisk Date: Mon, 16 Nov 2020 23:09:34 +0100 Subject: [PATCH 13/17] Kinda finished working on message deletion? Needs testing and fix for the PATCH requests --- scripts/generate-translations.sh | 2 +- src/discord/message.py | 4 ++-- src/discord/queue.py | 2 +- src/discord/redaction.py | 25 +++++++++++++++++-------- src/fileio/database.py | 3 ++- src/i18n.py | 3 ++- src/message_redaction.py | 3 --- src/rc_formatters.py | 7 +++++++ 8 files changed, 32 insertions(+), 17 deletions(-) delete mode 100644 src/message_redaction.py diff --git a/scripts/generate-translations.sh b/scripts/generate-translations.sh index c67368e..e98ae14 100644 --- a/scripts/generate-translations.sh +++ b/scripts/generate-translations.sh @@ -1,5 +1,5 @@ cd .. -declare -a StringArray=("discussion_formatters" "rc_formatters" "rcgcdw" "rc" "misc") +declare -a StringArray=("discussion_formatters" "rc_formatters" "rcgcdw" "rc" "misc", "redaction") for file in ${StringArray[@]}; do xgettext -L Python --package-name=RcGcDw -o "locale/templates/$file.pot" src/$file.py done diff --git a/src/discord/message.py b/src/discord/message.py index 4bd2b2a..077c58b 100644 --- a/src/discord/message.py +++ b/src/discord/message.py @@ -72,9 +72,9 @@ class DiscordMessage: class DiscordMessageRaw(DiscordMessage): - def __init__(self, content: dict): + def __init__(self, content: dict, webhook_url: str): self.webhook_object = content - + self.webhook_url = webhook_url class DiscordMessageMetadata: def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): diff --git a/src/discord/queue.py b/src/discord/queue.py index 4eb0013..4bc42dd 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -121,7 +121,7 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMes elif metadata.method == "DELETE": req = requests.Request("DELETE", metadata.webhook_url, **standard_args) elif metadata.method == "PATCH": - req = requests.Request("PATCH", metadata.webhook_url, data=data, **standard_args) + req = requests.Request("PATCH", data.webhook_url, data=repr(data), **standard_args) try: time.sleep(rate_limit) rate_limit = 0 diff --git a/src/discord/redaction.py b/src/discord/redaction.py index 48c248a..47232f8 100644 --- a/src/discord/redaction.py +++ b/src/discord/redaction.py @@ -1,11 +1,14 @@ import logging - +import json from src.configloader import settings from src.discord.message import DiscordMessageMetadata, DiscordMessage, DiscordMessageRaw from src.discord.queue import send_to_discord, messagequeue from src.fileio.database import db_cursor, db_connection +from src.i18n import redaction as redaction_translation -logger = logging.getLogger("rcgcdw.discord.redaction") +logger = logging.getLogger("rcgcdw.discord.redaction") # TODO Figure out why does this logger do not work +_ = redaction_translation.gettext +#ngettext = redaction_translation.ngettext def delete_messages(matching_data: dict): @@ -38,24 +41,30 @@ def redact_messages(ids: list, entry_type: int, to_censor: dict): to_censor: dict - logparams of message parts to censor""" for event_id in ids: if entry_type == 0: # TODO check if queries are proper - message = db_cursor.execute("SELECT content FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.revid = ?;", event_id) + message = db_cursor.execute("SELECT content FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.revid = ?;", (event_id, )) else: message = db_cursor.execute( "SELECT content FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.logid = ?;", - event_id) + (event_id,)) if settings["appearance"]["mode"] == "embed": if message is not None: message = message.fetchone() - new_embed = message["embeds"][0] + try: + message = json.loads(message[0]) + new_embed = message["embeds"][0] + except ValueError: + logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(message[0])) + return if "user" in to_censor: new_embed["author"]["name"] = _("Removed") new_embed["author"].pop("url") if "action" in to_censor: new_embed["title"] = _("Removed") new_embed.pop("url") + if "content" in to_censor: + new_embed.pop("fields") if "comment" in to_censor: new_embed["description"] = _("Removed") message["embeds"][0] = new_embed - send_to_discord(DiscordMessageRaw(message), DiscordMessageMetadata("PATCH")) - - raise NotImplemented + logger.debug(message) + send_to_discord(DiscordMessageRaw(message, settings["webhookURL"]), DiscordMessageMetadata("PATCH")) diff --git a/src/fileio/database.py b/src/fileio/database.py index 99126f9..36dac2d 100644 --- a/src/fileio/database.py +++ b/src/fileio/database.py @@ -1,5 +1,6 @@ import sqlite3 import logging +import json from src.configloader import settings logger = logging.getLogger("rcgcdw.fileio.database") @@ -44,7 +45,7 @@ def check_tables(): def add_entry(pageid: int, revid: int, logid: int, message): """Add an edit or log entry to the DB""" - db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("id"), str(message))) + db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("id"), json.dumps(message))) db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("id"))) logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) db_connection.commit() diff --git a/src/i18n.py b/src/i18n.py index ec08ca8..1f43c4c 100644 --- a/src/i18n.py +++ b/src/i18n.py @@ -11,8 +11,9 @@ try: rc = gettext.translation('rc', localedir='locale', languages=[settings["lang"]]) rc_formatters = gettext.translation('rc_formatters', localedir='locale', languages=[settings["lang"]]) misc = gettext.translation('misc', localedir='locale', languages=[settings["lang"]]) + redaction = gettext.translation('redaction', localedir='locale', languages=[settings["lang"]]) else: - rcgcdw, discussion_formatters, rc, rc_formatters, misc = gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations() + rcgcdw, discussion_formatters, rc, rc_formatters, misc, redaction = gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations() except FileNotFoundError: logger.critical("No language files have been found. Make sure locale folder is located in the directory.") sys.exit(1) diff --git a/src/message_redaction.py b/src/message_redaction.py deleted file mode 100644 index b28b04f..0000000 --- a/src/message_redaction.py +++ /dev/null @@ -1,3 +0,0 @@ - - - diff --git a/src/rc_formatters.py b/src/rc_formatters.py index a4e003c..7973cde 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -738,6 +738,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): embed["title"] = ngettext("Changed visibility of revision on page {article} ", "Changed visibility of {amount} revisions on page {article} ", amount).format( article=change["title"], amount=amount) + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + except KeyError: + pass + else: + redact_messages(logparams.get("ids", []), 0, logparams.get("new", {})) elif action == "import/upload": link = create_article_path(change["title"]) embed["title"] = ngettext("Imported {article} with {count} revision", From 4bff2d2038408cddbe6a2b07529bfb838e28a8a1 Mon Sep 17 00:00:00 2001 From: Frisk Date: Tue, 17 Nov 2020 23:43:45 +0100 Subject: [PATCH 14/17] Finished working on #150 --- src/discord/message.py | 2 +- src/discord/queue.py | 2 +- src/discord/redaction.py | 18 +++++++++++------- src/fileio/database.py | 10 ++++++---- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/src/discord/message.py b/src/discord/message.py index 077c58b..ed1d357 100644 --- a/src/discord/message.py +++ b/src/discord/message.py @@ -85,5 +85,5 @@ class DiscordMessageMetadata: self.webhook_url = webhook_url self.new_data = new_data - def dump_ids(self): + def dump_ids(self) -> (int, int, int): return self.page_id, self.rev_id, self.log_id \ No newline at end of file diff --git a/src/discord/queue.py b/src/discord/queue.py index 4bc42dd..e58d704 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -131,7 +131,7 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMes if AUTO_SUPPRESSION_ENABLED and metadata.method == "POST": # TODO Prepare request with all of safety checks try: - add_message_redaction_entry(*metadata.dump_ids(), result.json()) + add_message_redaction_entry(*metadata.dump_ids(), repr(data), result.json().get("id")) except ValueError: logger.error("Couldn't get json of result of sending Discord message.") except requests.exceptions.Timeout: diff --git a/src/discord/redaction.py b/src/discord/redaction.py index 47232f8..68562f3 100644 --- a/src/discord/redaction.py +++ b/src/discord/redaction.py @@ -6,7 +6,7 @@ from src.discord.queue import send_to_discord, messagequeue from src.fileio.database import db_cursor, db_connection from src.i18n import redaction as redaction_translation -logger = logging.getLogger("rcgcdw.discord.redaction") # TODO Figure out why does this logger do not work +logger = logging.getLogger("rcgcdw.discord.redaction") # TODO Figure out why does this logger do not work _ = redaction_translation.gettext #ngettext = redaction_translation.ngettext @@ -41,19 +41,19 @@ def redact_messages(ids: list, entry_type: int, to_censor: dict): to_censor: dict - logparams of message parts to censor""" for event_id in ids: if entry_type == 0: # TODO check if queries are proper - message = db_cursor.execute("SELECT content FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.revid = ?;", (event_id, )) + message = db_cursor.execute("SELECT content, message_id FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.revid = ?;", (event_id, )) else: message = db_cursor.execute( - "SELECT content FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.logid = ?;", + "SELECT content, message_id FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.logid = ?;", (event_id,)) if settings["appearance"]["mode"] == "embed": if message is not None: - message = message.fetchone() + row = message.fetchone() try: - message = json.loads(message[0]) + message = json.loads(row[0]) new_embed = message["embeds"][0] except ValueError: - logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(message[0])) + logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(row[0])) return if "user" in to_censor: new_embed["author"]["name"] = _("Removed") @@ -66,5 +66,9 @@ def redact_messages(ids: list, entry_type: int, to_censor: dict): if "comment" in to_censor: new_embed["description"] = _("Removed") message["embeds"][0] = new_embed + db_cursor.execute("UPDATE messages SET content = ? WHERE message_id = ?;", (json.dumps(message), row[1],)) + db_connection.commit() logger.debug(message) - send_to_discord(DiscordMessageRaw(message, settings["webhookURL"]), DiscordMessageMetadata("PATCH")) + send_to_discord(DiscordMessageRaw(message, settings["webhookURL"]+"/messages/"+str(row[1])), DiscordMessageMetadata("PATCH")) + else: + logger.debug("Could not find message in the database.") diff --git a/src/fileio/database.py b/src/fileio/database.py index 36dac2d..0fea678 100644 --- a/src/fileio/database.py +++ b/src/fileio/database.py @@ -43,10 +43,12 @@ def check_tables(): create_schema() -def add_entry(pageid: int, revid: int, logid: int, message): - """Add an edit or log entry to the DB""" - db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("id"), json.dumps(message))) - db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("id"))) +def add_entry(pageid: int, revid: int, logid: int, message, message_id: str): + """Add an edit or log entry to the DB + :param message_id: + """ + db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message_id, message)) + db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message_id)) logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) db_connection.commit() From f15525c5427731b27e65158d8a9748c39c42c471 Mon Sep 17 00:00:00 2001 From: Frisk Date: Wed, 18 Nov 2020 00:04:42 +0100 Subject: [PATCH 15/17] Trying to work out why pipeline fails --- src/discord/queue.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/discord/queue.py b/src/discord/queue.py index e58d704..65e60ef 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -140,6 +140,9 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMes except requests.exceptions.ConnectionError: logger.warning("Connection error while sending the data to a webhook") return 3 + except requests.exceptions.MissingSchema: + logger.debug(repr(data)) + sys.exit(1) else: return handle_discord_http(result.status_code, data, result) From 863125db791ca47c80e090eb6da034aed40b0500 Mon Sep 17 00:00:00 2001 From: Frisk Date: Wed, 18 Nov 2020 00:12:20 +0100 Subject: [PATCH 16/17] Fix when AUTO_SUPPRESSION_ENABLED is false --- src/discord/queue.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/src/discord/queue.py b/src/discord/queue.py index 65e60ef..47d4c8a 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -117,7 +117,7 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMes header['Content-Type'] = 'application/json' standard_args = dict(headers=header) if metadata.method == "POST": - req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if AUTO_SUPPRESSION_ENABLED else "false", data=repr(data), **standard_args) + req = requests.Request("POST", data.webhook_url+"?wait=" + ("true" if AUTO_SUPPRESSION_ENABLED else "false"), data=repr(data), **standard_args) elif metadata.method == "DELETE": req = requests.Request("DELETE", metadata.webhook_url, **standard_args) elif metadata.method == "PATCH": @@ -140,9 +140,6 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMes except requests.exceptions.ConnectionError: logger.warning("Connection error while sending the data to a webhook") return 3 - except requests.exceptions.MissingSchema: - logger.debug(repr(data)) - sys.exit(1) else: return handle_discord_http(result.status_code, data, result) From 3865db42b864107b4a1fa6a419289014f27151c1 Mon Sep 17 00:00:00 2001 From: Frisk Date: Wed, 18 Nov 2020 00:13:54 +0100 Subject: [PATCH 17/17] Added new options in settings.json --- settings.json.example | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/settings.json.example b/settings.json.example index d104706..3f9dc52 100644 --- a/settings.json.example +++ b/settings.json.example @@ -33,6 +33,10 @@ "show_bots": false, "show_abuselog": false, "discord_message_cooldown": 0, + "auto_suppression": { + "enabled": false, + "db_location": ":memory:" + }, "logging": { "version": 1, "disable_existing_loggers": false,