diff --git a/scripts/generate-translations.sh b/scripts/generate-translations.sh index c67368e..e98ae14 100644 --- a/scripts/generate-translations.sh +++ b/scripts/generate-translations.sh @@ -1,5 +1,5 @@ cd .. -declare -a StringArray=("discussion_formatters" "rc_formatters" "rcgcdw" "rc" "misc") +declare -a StringArray=("discussion_formatters" "rc_formatters" "rcgcdw" "rc" "misc", "redaction") for file in ${StringArray[@]}; do xgettext -L Python --package-name=RcGcDw -o "locale/templates/$file.pot" src/$file.py done diff --git a/settings.json.example b/settings.json.example index d104706..3f9dc52 100644 --- a/settings.json.example +++ b/settings.json.example @@ -33,6 +33,10 @@ "show_bots": false, "show_abuselog": false, "discord_message_cooldown": 0, + "auto_suppression": { + "enabled": false, + "db_location": ":memory:" + }, "logging": { "version": 1, "disable_existing_loggers": false, diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord/__init__.py b/src/discord/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord/message.py b/src/discord/message.py new file mode 100644 index 0000000..ed1d357 --- /dev/null +++ b/src/discord/message.py @@ -0,0 +1,89 @@ +import json +import math +import random +from collections import defaultdict + +from src.configloader import settings + + +class DiscordMessage: + """A class defining a typical Discord JSON representation of webhook payload.""" + def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): + self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) + self.webhook_url = webhook_url + + if message_type == "embed": + self.__setup_embed() + elif message_type == "compact": + self.webhook_object["content"] = content + + self.event_type = event_type + + def __setitem__(self, key, value): + """Set item is used only in embeds.""" + try: + self.embed[key] = value + except NameError: + raise TypeError("Tried to assign a value when message type is plain message!") + + def __getitem__(self, item): + return self.embed[item] + + def __repr__(self): + """Return the Discord webhook object ready to be sent""" + return json.dumps(self.webhook_object) + + def __setup_embed(self): + self.embed = defaultdict(dict) + if "embeds" not in self.webhook_object: + self.webhook_object["embeds"] = [self.embed] + else: + self.webhook_object["embeds"].append(self.embed) + self.embed["color"] = None + + def add_embed(self): + self.finish_embed() + self.__setup_embed() + + def finish_embed(self): + if self.embed["color"] is None: + if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: + self.embed["color"] = random.randrange(1, 16777215) + else: + self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] + else: + self.embed["color"] = math.floor(self.embed["color"]) + + def set_author(self, name, url, icon_url=""): + self.embed["author"]["name"] = name + self.embed["author"]["url"] = url + self.embed["author"]["icon_url"] = icon_url + + def add_field(self, name, value, inline=False): + if "fields" not in self.embed: + self.embed["fields"] = [] + self.embed["fields"].append(dict(name=name, value=value, inline=inline)) + + def set_avatar(self, url): + self.webhook_object["avatar_url"] = url + + def set_name(self, name): + self.webhook_object["username"] = name + + +class DiscordMessageRaw(DiscordMessage): + def __init__(self, content: dict, webhook_url: str): + self.webhook_object = content + self.webhook_url = webhook_url + +class DiscordMessageMetadata: + def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): + self.method = method + self.page_id = page_id + self.log_id = log_id + self.rev_id = rev_id + self.webhook_url = webhook_url + self.new_data = new_data + + def dump_ids(self) -> (int, int, int): + return self.page_id, self.rev_id, self.log_id \ No newline at end of file diff --git a/src/discord/queue.py b/src/discord/queue.py new file mode 100644 index 0000000..47d4c8a --- /dev/null +++ b/src/discord/queue.py @@ -0,0 +1,170 @@ +import re +import sys +import time +import logging +from typing import Optional + +import requests + +from src.configloader import settings +from src.discord.message import DiscordMessage, DiscordMessageMetadata + +AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": False}).get("enabled") +if AUTO_SUPPRESSION_ENABLED: + from src.fileio.database import add_entry as add_message_redaction_entry + +rate_limit = 0 + +logger = logging.getLogger("rcgcdw.discord.queue") + +class MessageQueue: + """Message queue class for undelivered messages""" + def __init__(self): + self._queue = [] + + def __repr__(self): + return self._queue + + def __len__(self): + return len(self._queue) + + def __iter__(self): + return iter(self._queue) + + def clear(self): + self._queue.clear() + + def add_message(self, message): + self._queue.append(message) + + def cut_messages(self, item_num): + self._queue = self._queue[item_num:] + + @staticmethod + def compare_message_to_dict(metadata: DiscordMessageMetadata, to_match: dict): + """Compare DiscordMessageMetadata fields and match them against dictionary""" + for name, val in to_match.items(): + if getattr(metadata, name, None) != val: + return False + return True + + def delete_all_with_matching_metadata(self, **properties): + """Deletes all of the messages that have matching metadata properties (useful for message redaction)""" + for index, item in reversed(list(enumerate(self._queue))): + if self.compare_message_to_dict(item[1], properties): + self._queue.pop(index) + + def resend_msgs(self): + if self._queue: + logger.info( + "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( + len(self._queue))) + for num, item in enumerate(self._queue): + logger.debug( + "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), + str(item))) + if send_to_discord_webhook(item[0], metadata=item[1]) < 2: + logger.debug("Sending message succeeded") + else: + logger.debug("Sending message failed") + break + else: + self.clear() + logger.debug("Queue emptied, all messages delivered") + self.cut_messages(num) + logger.debug(self._queue) + + +messagequeue = MessageQueue() + + +def handle_discord_http(code, formatted_embed, result): + if 300 > code > 199: # message went through + return 0 + elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header + logger.error( + "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") + logger.error(formatted_embed) + logger.error(result.text) + return 1 + elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND + if result.request.method == "POST": # Ignore not found for DELETE and PATCH requests since the message could already be removed by admin + logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") + sys.exit(1) + else: + return 0 + elif code == 429: + logger.error("We are sending too many requests to the Discord, slowing down...") + return 2 + elif 499 < code < 600: + logger.error( + "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( + code)) + return 3 + + +def update_ratelimit(request): + """Updates rate limit time""" + global rate_limit + rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get( + 'x-ratelimit-reset-after', 0)) + rate_limit += settings.get("discord_message_cooldown", 0) + + +def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMessageMetadata): + global rate_limit + header = settings["header"] + header['Content-Type'] = 'application/json' + standard_args = dict(headers=header) + if metadata.method == "POST": + req = requests.Request("POST", data.webhook_url+"?wait=" + ("true" if AUTO_SUPPRESSION_ENABLED else "false"), data=repr(data), **standard_args) + elif metadata.method == "DELETE": + req = requests.Request("DELETE", metadata.webhook_url, **standard_args) + elif metadata.method == "PATCH": + req = requests.Request("PATCH", data.webhook_url, data=repr(data), **standard_args) + try: + time.sleep(rate_limit) + rate_limit = 0 + req = req.prepare() + result = requests.Session().send(req, timeout=10) + update_ratelimit(result) + if AUTO_SUPPRESSION_ENABLED and metadata.method == "POST": + # TODO Prepare request with all of safety checks + try: + add_message_redaction_entry(*metadata.dump_ids(), repr(data), result.json().get("id")) + except ValueError: + logger.error("Couldn't get json of result of sending Discord message.") + except requests.exceptions.Timeout: + logger.warning("Timeouted while sending data to the webhook.") + return 3 + except requests.exceptions.ConnectionError: + logger.warning("Connection error while sending the data to a webhook") + return 3 + else: + return handle_discord_http(result.status_code, data, result) + + +def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata): + if data is not None: + for regex in settings["disallow_regexes"]: + if data.webhook_object.get("content", None): + if re.search(re.compile(regex), data.webhook_object["content"]): + logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) + return # discard the message without anything + else: + for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: + if re.search(re.compile(regex), to_check): + logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( + to_check, regex)) + return # discard the message without anything + if messagequeue: + messagequeue.add_message((data, meta)) + else: + code = send_to_discord_webhook(data, metadata=meta) + if code == 3: + messagequeue.add_message((data, meta)) + elif code == 2: + time.sleep(5.0) + messagequeue.add_message((data, meta)) + elif code < 2: + pass \ No newline at end of file diff --git a/src/discord/redaction.py b/src/discord/redaction.py new file mode 100644 index 0000000..68562f3 --- /dev/null +++ b/src/discord/redaction.py @@ -0,0 +1,74 @@ +import logging +import json +from src.configloader import settings +from src.discord.message import DiscordMessageMetadata, DiscordMessage, DiscordMessageRaw +from src.discord.queue import send_to_discord, messagequeue +from src.fileio.database import db_cursor, db_connection +from src.i18n import redaction as redaction_translation + +logger = logging.getLogger("rcgcdw.discord.redaction") # TODO Figure out why does this logger do not work +_ = redaction_translation.gettext +#ngettext = redaction_translation.ngettext + + +def delete_messages(matching_data: dict): + """Delete messages that match given data""" + sql_conditions = "" + for key, value in matching_data.items(): + sql_conditions += "{} = ? AND".format(key) + else: + sql_conditions = sql_conditions[0:-4] # remove last AND statement + to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE {CON}".format(CON=sql_conditions), list(matching_data.values())) + if len(messagequeue) > 0: + messagequeue.delete_all_with_matching_metadata(**matching_data) + msg_to_remove = [] + logger.debug("Deleting messages for data: {}".format(matching_data)) + for message in to_delete: + webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) + msg_to_remove.append(message[0]) + logger.debug("Removing following message: {}".format(message[0])) + send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) + for msg in msg_to_remove: + db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (msg,)) + db_connection.commit() + + +def redact_messages(ids: list, entry_type: int, to_censor: dict): + """Redact past Discord messages + + ids: list of ints + entry_type: int - 0 for revdel, 1 for logdel + to_censor: dict - logparams of message parts to censor""" + for event_id in ids: + if entry_type == 0: # TODO check if queries are proper + message = db_cursor.execute("SELECT content, message_id FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.revid = ?;", (event_id, )) + else: + message = db_cursor.execute( + "SELECT content, message_id FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.logid = ?;", + (event_id,)) + if settings["appearance"]["mode"] == "embed": + if message is not None: + row = message.fetchone() + try: + message = json.loads(row[0]) + new_embed = message["embeds"][0] + except ValueError: + logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(row[0])) + return + if "user" in to_censor: + new_embed["author"]["name"] = _("Removed") + new_embed["author"].pop("url") + if "action" in to_censor: + new_embed["title"] = _("Removed") + new_embed.pop("url") + if "content" in to_censor: + new_embed.pop("fields") + if "comment" in to_censor: + new_embed["description"] = _("Removed") + message["embeds"][0] = new_embed + db_cursor.execute("UPDATE messages SET content = ? WHERE message_id = ?;", (json.dumps(message), row[1],)) + db_connection.commit() + logger.debug(message) + send_to_discord(DiscordMessageRaw(message, settings["webhookURL"]+"/messages/"+str(row[1])), DiscordMessageMetadata("PATCH")) + else: + logger.debug("Could not find message in the database.") diff --git a/src/discussion_formatters.py b/src/discussion_formatters.py index 11f1aa5..9a58262 100644 --- a/src/discussion_formatters.py +++ b/src/discussion_formatters.py @@ -4,7 +4,9 @@ import gettext from urllib.parse import quote_plus from src.configloader import settings -from src.misc import link_formatter, create_article_path, DiscordMessage, send_to_discord, escape_formatting +from src.misc import link_formatter, create_article_path, escape_formatting +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata from src.i18n import discussion_formatters _ = discussion_formatters.gettext @@ -67,7 +69,7 @@ def compact_formatter(post_type, post, article_paths): else: message = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=post_type, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message)) + send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message), meta=DiscordMessageMetadata("POST")) def embed_formatter(post_type, post, article_paths): @@ -167,7 +169,7 @@ def embed_formatter(post_type, post, article_paths): else: embed.add_field(_("Report this on the support server"), change_params) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta=DiscordMessageMetadata("POST")) class DiscussionsFromHellParser: diff --git a/src/discussions.py b/src/discussions.py index 579ffaa..4d8ca37 100644 --- a/src/discussions.py +++ b/src/discussions.py @@ -22,7 +22,8 @@ from typing import Dict, Any from src.configloader import settings from src.discussion_formatters import embed_formatter, compact_formatter -from src.misc import datafile, messagequeue, prepare_paths +from src.misc import datafile, prepare_paths +from src.discord.queue import messagequeue from src.session import session from src.exceptions import ArticleCommentError diff --git a/src/fileio/__init__.py b/src/fileio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fileio/database.py b/src/fileio/database.py new file mode 100644 index 0000000..0fea678 --- /dev/null +++ b/src/fileio/database.py @@ -0,0 +1,67 @@ +import sqlite3 +import logging +import json +from src.configloader import settings + +logger = logging.getLogger("rcgcdw.fileio.database") + + +def create_schema(): + logger.info("Creating database schema...") + db_cursor.executescript( + """BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS "messages" ( + "message_id" TEXT, + "content" TEXT, + PRIMARY KEY("message_id") + ); + CREATE TABLE IF NOT EXISTS "event" ( + "pageid" INTEGER, + "revid" INTEGER, + "logid" INTEGER, + "msg_id" TEXT NOT NULL, + PRIMARY KEY("msg_id"), + FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE + ); + COMMIT;""") + logger.info("Database schema has been recreated.") + + +def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): + _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) + _db_connection.row_factory = sqlite3.Row + _db_cursor = _db_connection.cursor() + logger.debug("Database connection created") + return _db_connection, _db_cursor + + +def check_tables(): + """Check if tables exist, if not, create schema""" + rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") + if not rep.fetchone(): + logger.debug("No schema detected, creating schema!") + create_schema() + + +def add_entry(pageid: int, revid: int, logid: int, message, message_id: str): + """Add an edit or log entry to the DB + :param message_id: + """ + db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message_id, message)) + db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message_id)) + logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) + db_connection.commit() + +def clean_entries(): + """Cleans entries that are 50+""" + cleanup = db_cursor.execute( + "SELECT message_id FROM messages WHERE message_id NOT IN (SELECT message_id FROM messages ORDER BY message_id desc LIMIT 50);") + for row in cleanup: + db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (row[0])) + cleanup = db_cursor.execute("SELECT msg_id FROM event WHERE msg_id NOT IN (SELECT msg_id FROM event ORDER BY msg_id desc LIMIT 50);") + for row in cleanup: + db_cursor.execute("DELETE FROM event WHERE msg_id = ?", (row[0])) + db_connection.commit() + +db_connection, db_cursor = create_connection() +check_tables() diff --git a/src/i18n.py b/src/i18n.py index ec08ca8..1f43c4c 100644 --- a/src/i18n.py +++ b/src/i18n.py @@ -11,8 +11,9 @@ try: rc = gettext.translation('rc', localedir='locale', languages=[settings["lang"]]) rc_formatters = gettext.translation('rc_formatters', localedir='locale', languages=[settings["lang"]]) misc = gettext.translation('misc', localedir='locale', languages=[settings["lang"]]) + redaction = gettext.translation('redaction', localedir='locale', languages=[settings["lang"]]) else: - rcgcdw, discussion_formatters, rc, rc_formatters, misc = gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations() + rcgcdw, discussion_formatters, rc, rc_formatters, misc, redaction = gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations() except FileNotFoundError: logger.critical("No language files have been found. Make sure locale folder is located in the directory.") sys.exit(1) diff --git a/src/misc.py b/src/misc.py index 78a05f9..25fb039 100644 --- a/src/misc.py +++ b/src/misc.py @@ -16,14 +16,17 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . import base64 -import json, logging, sys, re, time, random, math +import json, logging, sys, re from html.parser import HTMLParser from urllib.parse import urlparse, urlunparse, quote import requests -from collections import defaultdict from src.configloader import settings +from src.discord.message import DiscordMessage, DiscordMessageMetadata +from src.discord.queue import messagequeue, send_to_discord from src.i18n import misc +AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": False}).get("enabled") + _ = misc.gettext # Create a custom logger @@ -38,7 +41,6 @@ WIKI_API_PATH: str = "" WIKI_ARTICLE_PATH: str = "" WIKI_SCRIPT_PATH: str = "" WIKI_JUST_DOMAIN: str = "" -rate_limit = 0 profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")} @@ -90,52 +92,6 @@ class DataFile: return self.data[item] - -class MessageQueue: - """Message queue class for undelivered messages""" - def __init__(self): - self._queue = [] - - def __repr__(self): - return self._queue - - def __len__(self): - return len(self._queue) - - def __iter__(self): - return iter(self._queue) - - def clear(self): - self._queue.clear() - - def add_message(self, message): - self._queue.append(message) - - def cut_messages(self, item_num): - self._queue = self._queue[item_num:] - - def resend_msgs(self): - if self._queue: - misc_logger.info( - "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( - len(self._queue))) - for num, item in enumerate(self._queue): - misc_logger.debug( - "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), - str(item))) - if send_to_discord_webhook(item) < 2: - misc_logger.debug("Sending message succeeded") - else: - misc_logger.debug("Sending message failed") - break - else: - self.clear() - misc_logger.debug("Queue emptied, all messages delivered") - self.cut_messages(num) - misc_logger.debug(self._queue) - - -messagequeue = MessageQueue() datafile = DataFile() @@ -241,28 +197,6 @@ def safe_read(request, *keys): return request -def handle_discord_http(code, formatted_embed, result): - if 300 > code > 199: # message went through - return 0 - elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header - misc_logger.error( - "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") - misc_logger.error(formatted_embed) - misc_logger.error(result.text) - return 1 - elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND - misc_logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") - sys.exit(1) - elif code == 429: - misc_logger.error("We are sending too many requests to the Discord, slowing down...") - return 2 - elif 499 < code < 600: - misc_logger.error( - "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( - code)) - return 3 - - def add_to_dict(dictionary, key): if key in dictionary: dictionary[key] += 1 @@ -326,124 +260,7 @@ def send_simple(msgtype, message, name, avatar): discord_msg.set_avatar(avatar) discord_msg.set_name(name) messagequeue.resend_msgs() - send_to_discord(discord_msg) - - -def update_ratelimit(request): - """Updates rate limit time""" - global rate_limit - rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get( - 'x-ratelimit-reset-after', 0)) - rate_limit += settings.get("discord_message_cooldown", 0) - - -def send_to_discord_webhook(data): - global rate_limit - header = settings["header"] - header['Content-Type'] = 'application/json' - try: - time.sleep(rate_limit) - rate_limit = 0 - result = requests.post(data.webhook_url, data=repr(data), - headers=header, timeout=10) - update_ratelimit(result) - except requests.exceptions.Timeout: - misc_logger.warning("Timeouted while sending data to the webhook.") - return 3 - except requests.exceptions.ConnectionError: - misc_logger.warning("Connection error while sending the data to a webhook") - return 3 - else: - return handle_discord_http(result.status_code, data, result) - - -def send_to_discord(data): - for regex in settings["disallow_regexes"]: - if data.webhook_object.get("content", None): - if re.search(re.compile(regex), data.webhook_object["content"]): - misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) - return # discard the message without anything - else: - for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: - if re.search(re.compile(regex), to_check): - misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( - to_check, regex)) - return # discard the message without anything - if messagequeue: - messagequeue.add_message(data) - else: - code = send_to_discord_webhook(data) - if code == 3: - messagequeue.add_message(data) - elif code == 2: - time.sleep(5.0) - messagequeue.add_message(data) - elif code < 2: - pass - -class DiscordMessage(): - """A class defining a typical Discord JSON representation of webhook payload.""" - def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): - self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) - self.webhook_url = webhook_url - - if message_type == "embed": - self.__setup_embed() - elif message_type == "compact": - self.webhook_object["content"] = content - - self.event_type = event_type - - def __setitem__(self, key, value): - """Set item is used only in embeds.""" - try: - self.embed[key] = value - except NameError: - raise TypeError("Tried to assign a value when message type is plain message!") - - def __getitem__(self, item): - return self.embed[item] - - def __repr__(self): - """Return the Discord webhook object ready to be sent""" - return json.dumps(self.webhook_object) - - def __setup_embed(self): - self.embed = defaultdict(dict) - if "embeds" not in self.webhook_object: - self.webhook_object["embeds"] = [self.embed] - else: - self.webhook_object["embeds"].append(self.embed) - self.embed["color"] = None - - def add_embed(self): - self.finish_embed() - self.__setup_embed() - - def finish_embed(self): - if self.embed["color"] is None: - if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: - self.embed["color"] = random.randrange(1, 16777215) - else: - self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] - else: - self.embed["color"] = math.floor(self.embed["color"]) - - def set_author(self, name, url, icon_url=""): - self.embed["author"]["name"] = name - self.embed["author"]["url"] = url - self.embed["author"]["icon_url"] = icon_url - - def add_field(self, name, value, inline=False): - if "fields" not in self.embed: - self.embed["fields"] = [] - self.embed["fields"].append(dict(name=name, value=value, inline=inline)) - - def set_avatar(self, url): - self.webhook_object["avatar_url"] = url - - def set_name(self, name): - self.webhook_object["username"] = name + send_to_discord(discord_msg, meta=DiscordMessageMetadata("POST")) def profile_field_name(name, embed): @@ -485,4 +302,4 @@ class LinkParser(HTMLParser): self.new_string = self.new_string + data.replace("//", "/\\/") def handle_endtag(self, tag): - misc_logger.debug(self.new_string) \ No newline at end of file + misc_logger.debug(self.new_string) diff --git a/src/rc.py b/src/rc.py index f9c7593..1d4b5f6 100644 --- a/src/rc.py +++ b/src/rc.py @@ -6,7 +6,8 @@ import requests from bs4 import BeautifulSoup from src.configloader import settings -from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, messagequeue, datafile, send_simple, safe_read, LinkParser +from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, LinkParser, AUTO_SUPPRESSION_ENABLED +from src.discord.queue import messagequeue from src.exceptions import MWError from src.session import session from src.rc_formatters import compact_formatter, embed_formatter, compact_abuselog_formatter, embed_abuselog_formatter @@ -332,6 +333,9 @@ class Recent_Changes_Class(object): def clear_cache(self): self.map_ips = {} + if AUTO_SUPPRESSION_ENABLED: + from src.fileio.database import clean_entries + clean_entries() def init_info(self): startup_info = safe_read(self.safe_request( @@ -397,13 +401,13 @@ def essential_info(change, changed_categories): parsed_comment = _("~~hidden~~") if not parsed_comment: parsed_comment = None - if change["type"] in ["edit", "new"]: - logger.debug("List of categories in essential_info: {}".format(changed_categories)) - if "userhidden" in change: - change["user"] = _("hidden") - identification_string = change["type"] + if "userhidden" in change: + change["user"] = _("hidden") if change.get("ns", -1) in settings.get("ignored_namespaces", ()): return + if change["type"] in ["edit", "new"]: + logger.debug("List of categories in essential_info: {}".format(changed_categories)) + identification_string = change["type"] elif change["type"] == "log": identification_string = "{logtype}/{logaction}".format(logtype=change["logtype"], logaction=change["logaction"]) if identification_string not in supported_logs: diff --git a/src/rc_formatters.py b/src/rc_formatters.py index 958daf0..5ffa1ec 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -10,8 +10,14 @@ from urllib.parse import quote_plus, quote from bs4 import BeautifulSoup from src.configloader import settings -from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ - WIKI_API_PATH, ContentParser, profile_field_name, LinkParser +from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, safe_read, \ + WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, AUTO_SUPPRESSION_ENABLED +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata + +if AUTO_SUPPRESSION_ENABLED: + from src.discord.redaction import delete_messages, redact_messages + from src.i18n import rc_formatters #from src.rc import recent_changes, pull_comment _ = rc_formatters.gettext @@ -62,10 +68,11 @@ def compact_abuselog_formatter(change, recent_changes): action=abusefilter_actions.get(change["action"], _("Unknown")), target=change.get("title", _("Unknown")), target_url=create_article_path(change.get("title", _("Unknown"))), result=abusefilter_results.get(change["result"], _("Unknown"))) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message)) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message), meta=DiscordMessageMetadata("POST")) def compact_formatter(action, change, parsed_comment, categories, recent_changes): + request_metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), page_id=change.get("pageid", None)) if action != "suppressed": author_url = link_formatter(create_article_path("User:{user}".format(user=change["user"]))) author = change["user"] @@ -104,10 +111,14 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes page_link = link_formatter(create_article_path(change["title"])) content = "🗑️ "+_("[{author}]({author_url}) deleted [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(dict(pageid=change.get("pageid"))) elif action == "delete/delete_redir": page_link = link_formatter(create_article_path(change["title"])) content = "🗑️ "+_("[{author}]({author_url}) deleted redirect by overwriting [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link, comment=parsed_comment) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(dict(pageid=change.get("pageid"))) elif action == "move/move": link = link_formatter(create_article_path(change["logparams"]['target_title'])) redirect_status = _("without making a redirect") if "suppressredirect" in change["logparams"] else _("with a redirect") @@ -264,6 +275,14 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "👁️ "+ngettext("[{author}]({author_url}) changed visibility of revision on page [{article}]({article_url}){comment}", "[{author}]({author_url}) changed visibility of {amount} revisions on page [{article}]({article_url}){comment}", amount).format(author=author, author_url=author_url, article=change["title"], article_url=link, amount=amount, comment=parsed_comment) + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + pageid = change["pageid"] + except KeyError: + pass + else: + delete_messages(dict(pageid=pageid)) elif action == "import/upload": link = link_formatter(create_article_path(change["title"])) content = "📥 "+ngettext("[{author}]({author_url}) imported [{article}]({article_url}) with {count} revision{comment}", @@ -274,6 +293,14 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "♻️ "+_("[{author}]({author_url}) restored [{article}]({article_url}){comment}").format(author=author, author_url=author_url, article=change["title"], article_url=link, comment=parsed_comment) elif action == "delete/event": content = "👁️ "+_("[{author}]({author_url}) changed visibility of log events{comment}").format(author=author, author_url=author_url, comment=parsed_comment) + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + except KeyError: + pass + else: + for revid in logparams.get("ids", []): + delete_messages(dict(revid=revid)) elif action == "import/interwiki": content = "📥 "+_("[{author}]({author_url}) imported interwiki{comment}").format(author=author, author_url=author_url, comment=parsed_comment) elif action == "abusefilter/modify": @@ -403,7 +430,7 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes content = "❓ "+_( "Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format( event=action, author=author, author_url=author_url, support=settings["support"]) - send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content)) + send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content), meta=request_metadata) def embed_abuselog_formatter(change, recent_changes): action = "abuselog/{}".format(change["result"]) @@ -415,11 +442,12 @@ def embed_abuselog_formatter(change, recent_changes): embed.add_field(_("Action taken"), abusefilter_results.get(change["result"], _("Unknown"))) embed.add_field(_("Title"), change.get("title", _("Unknown"))) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta=DiscordMessageMetadata("POST")) def embed_formatter(action, change, parsed_comment, categories, recent_changes): embed = DiscordMessage("embed", action, settings["webhookURL"]) + request_metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), page_id=change.get("pageid", None)) if parsed_comment is None: parsed_comment = _("No description provided") if action != "suppressed": @@ -554,9 +582,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): elif action == "delete/delete": link = create_article_path(change["title"]) embed["title"] = _("Deleted page {article}").format(article=change["title"]) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(dict(pageid=change.get("pageid"))) elif action == "delete/delete_redir": link = create_article_path(change["title"]) embed["title"] = _("Deleted redirect {article} by overwriting").format(article=change["title"]) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(dict(pageid=change.get("pageid"))) elif action == "move/move": link = create_article_path(change["logparams"]['target_title']) parsed_comment = "{supress}. {desc}".format(desc=parsed_comment, @@ -709,6 +741,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): embed["title"] = ngettext("Changed visibility of revision on page {article} ", "Changed visibility of {amount} revisions on page {article} ", amount).format( article=change["title"], amount=amount) + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + except KeyError: + pass + else: + redact_messages(logparams.get("ids", []), 0, logparams.get("new", {})) elif action == "import/upload": link = create_article_path(change["title"]) embed["title"] = ngettext("Imported {article} with {count} revision", @@ -720,6 +759,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): elif action == "delete/event": link = create_article_path("Special:RecentChanges") embed["title"] = _("Changed visibility of log events") + if AUTO_SUPPRESSION_ENABLED: + try: + logparams = change["logparams"] + except KeyError: + pass + else: + redact_messages(logparams.get("ids", []), 1, logparams.get("new", {})) elif action == "import/interwiki": link = create_article_path("Special:RecentChanges") embed["title"] = _("Imported interwiki") @@ -886,4 +932,4 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta=request_metadata) diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 3d17407..e19622a 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -26,8 +26,9 @@ import src.misc from collections import defaultdict, Counter from src.configloader import settings from src.misc import add_to_dict, datafile, \ - WIKI_API_PATH, create_article_path, send_to_discord, \ - DiscordMessage + WIKI_API_PATH, create_article_path +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata from src.rc import recent_changes from src.exceptions import MWError from src.i18n import rcgcdw @@ -161,10 +162,10 @@ def day_overview(): if item["type"] == "edit": edits += 1 changed_bytes += item["newlen"] - item["oldlen"] - if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]: + if (recent_changes.namespaces is not None and "content" in recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0: articles = add_to_dict(articles, item["title"]) elif item["type"] == "new": - if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]: + if "content" in (recent_changes.namespaces is not None and recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0: new_articles += 1 changed_bytes += item["newlen"] elif item["type"] == "log": @@ -202,7 +203,7 @@ def day_overview(): for name, value in fields: embed.add_field(name, value, inline=True) embed.finish_embed() - send_to_discord(embed) + send_to_discord(embed, meta=DiscordMessageMetadata("POST")) else: logger.debug("function requesting changes for day overview returned with error code")