Merge branch '150-delete-messages-for-deleted-edits' into 'testing'

Resolve "Delete messages for deleted edits"

See merge request piotrex43/RcGcDw!77

scripts/generate-translations.sh

@@ -1,5 +1,5 @@
 cd ..
-declare -a StringArray=("discussion_formatters" "rc_formatters" "rcgcdw" "rc" "misc")
+declare -a StringArray=("discussion_formatters" "rc_formatters" "rcgcdw" "rc" "misc", "redaction")
 for file in ${StringArray[@]}; do
   xgettext -L Python --package-name=RcGcDw -o "locale/templates/$file.pot" src/$file.py
 done

settings.json.example

@@ -33,6 +33,10 @@
    "show_bots": false,
    "show_abuselog": false,
    "discord_message_cooldown": 0,
+   "auto_suppression": {
+      "enabled": false,
+      "db_location": ":memory:"
+   },
    "logging": {
        "version": 1,
        "disable_existing_loggers": false,

src/discord/message.py

@@ -0,0 +1,89 @@
+import json
+import math
+import random
+from collections import defaultdict
+
+from src.configloader import settings
+
+
+class DiscordMessage:
+	"""A class defining a typical Discord JSON representation of webhook payload."""
+	def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None):
+		self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, ""))
+		self.webhook_url = webhook_url
+
+		if message_type == "embed":
+			self.__setup_embed()
+		elif message_type == "compact":
+			self.webhook_object["content"] = content
+
+		self.event_type = event_type
+
+	def __setitem__(self, key, value):
+		"""Set item is used only in embeds."""
+		try:
+			self.embed[key] = value
+		except NameError:
+			raise TypeError("Tried to assign a value when message type is plain message!")
+
+	def __getitem__(self, item):
+		return self.embed[item]
+
+	def __repr__(self):
+		"""Return the Discord webhook object ready to be sent"""
+		return json.dumps(self.webhook_object)
+
+	def __setup_embed(self):
+		self.embed = defaultdict(dict)
+		if "embeds" not in self.webhook_object:
+			self.webhook_object["embeds"] = [self.embed]
+		else:
+			self.webhook_object["embeds"].append(self.embed)
+		self.embed["color"] = None
+
+	def add_embed(self):
+		self.finish_embed()
+		self.__setup_embed()
+
+	def finish_embed(self):
+		if self.embed["color"] is None:
+			if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None:
+				self.embed["color"] = random.randrange(1, 16777215)
+			else:
+				self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"]
+		else:
+			self.embed["color"] = math.floor(self.embed["color"])
+
+	def set_author(self, name, url, icon_url=""):
+		self.embed["author"]["name"] = name
+		self.embed["author"]["url"] = url
+		self.embed["author"]["icon_url"] = icon_url
+
+	def add_field(self, name, value, inline=False):
+		if "fields" not in self.embed:
+			self.embed["fields"] = []
+		self.embed["fields"].append(dict(name=name, value=value, inline=inline))
+
+	def set_avatar(self, url):
+		self.webhook_object["avatar_url"] = url
+
+	def set_name(self, name):
+		self.webhook_object["username"] = name
+
+
+class DiscordMessageRaw(DiscordMessage):
+	def __init__(self, content: dict, webhook_url: str):
+		self.webhook_object = content
+		self.webhook_url = webhook_url
+
+class DiscordMessageMetadata:
+	def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None):
+		self.method = method
+		self.page_id = page_id
+		self.log_id = log_id
+		self.rev_id = rev_id
+		self.webhook_url = webhook_url
+		self.new_data = new_data
+
+	def dump_ids(self) -> (int, int, int):
+		return self.page_id, self.rev_id, self.log_id

src/discord/queue.py

@@ -0,0 +1,170 @@
+import re
+import sys
+import time
+import logging
+from typing import Optional
+
+import requests
+
+from src.configloader import settings
+from src.discord.message import DiscordMessage, DiscordMessageMetadata
+
+AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": False}).get("enabled")
+if AUTO_SUPPRESSION_ENABLED:
+	from src.fileio.database import add_entry as add_message_redaction_entry
+
+rate_limit = 0
+
+logger = logging.getLogger("rcgcdw.discord.queue")
+
+class MessageQueue:
+	"""Message queue class for undelivered messages"""
+	def __init__(self):
+		self._queue = []
+
+	def __repr__(self):
+		return self._queue
+
+	def __len__(self):
+		return len(self._queue)
+
+	def __iter__(self):
+		return iter(self._queue)
+
+	def clear(self):
+		self._queue.clear()
+
+	def add_message(self, message):
+		self._queue.append(message)
+
+	def cut_messages(self, item_num):
+		self._queue = self._queue[item_num:]
+
+	@staticmethod
+	def compare_message_to_dict(metadata: DiscordMessageMetadata, to_match: dict):
+		"""Compare DiscordMessageMetadata fields and match them against dictionary"""
+		for name, val in to_match.items():
+			if getattr(metadata, name, None) != val:
+				return False
+		return True
+
+	def delete_all_with_matching_metadata(self, **properties):
+		"""Deletes all of the messages that have matching metadata properties (useful for message redaction)"""
+		for index, item in reversed(list(enumerate(self._queue))):
+			if self.compare_message_to_dict(item[1], properties):
+				self._queue.pop(index)
+
+	def resend_msgs(self):
+		if self._queue:
+			logger.info(
+				"{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format(
+					len(self._queue)))
+			for num, item in enumerate(self._queue):
+				logger.debug(
+					"Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num),
+					                                                                                         str(item)))
+				if send_to_discord_webhook(item[0], metadata=item[1]) < 2:
+					logger.debug("Sending message succeeded")
+				else:
+					logger.debug("Sending message failed")
+					break
+			else:
+				self.clear()
+				logger.debug("Queue emptied, all messages delivered")
+			self.cut_messages(num)
+			logger.debug(self._queue)
+
+
+messagequeue = MessageQueue()
+
+
+def handle_discord_http(code, formatted_embed, result):
+	if 300 > code > 199:  # message went through
+		return 0
+	elif code == 400:  # HTTP BAD REQUEST result.status_code, data, result, header
+		logger.error(
+			"Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:")
+		logger.error(formatted_embed)
+		logger.error(result.text)
+		return 1
+	elif code == 401 or code == 404:  # HTTP UNAUTHORIZED AND NOT FOUND
+		if result.request.method == "POST":  # Ignore not found for DELETE and PATCH requests since the message could already be removed by admin
+			logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.")
+			sys.exit(1)
+		else:
+			return 0
+	elif code == 429:
+		logger.error("We are sending too many requests to the Discord, slowing down...")
+		return 2
+	elif 499 < code < 600:
+		logger.error(
+			"Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format(
+				code))
+		return 3
+
+
+def update_ratelimit(request):
+	"""Updates rate limit time"""
+	global rate_limit
+	rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get(
+		'x-ratelimit-reset-after', 0))
+	rate_limit += settings.get("discord_message_cooldown", 0)
+
+
+def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMessageMetadata):
+	global rate_limit
+	header = settings["header"]
+	header['Content-Type'] = 'application/json'
+	standard_args = dict(headers=header)
+	if metadata.method == "POST":
+		req = requests.Request("POST", data.webhook_url+"?wait=" + ("true" if AUTO_SUPPRESSION_ENABLED else "false"), data=repr(data), **standard_args)
+	elif metadata.method == "DELETE":
+		req = requests.Request("DELETE", metadata.webhook_url, **standard_args)
+	elif metadata.method == "PATCH":
+		req = requests.Request("PATCH", data.webhook_url, data=repr(data), **standard_args)
+	try:
+		time.sleep(rate_limit)
+		rate_limit = 0
+		req = req.prepare()
+		result = requests.Session().send(req, timeout=10)
+		update_ratelimit(result)
+		if AUTO_SUPPRESSION_ENABLED and metadata.method == "POST":
+			# TODO Prepare request with all of safety checks
+			try:
+				add_message_redaction_entry(*metadata.dump_ids(), repr(data), result.json().get("id"))
+			except ValueError:
+				logger.error("Couldn't get json of result of sending Discord message.")
+	except requests.exceptions.Timeout:
+		logger.warning("Timeouted while sending data to the webhook.")
+		return 3
+	except requests.exceptions.ConnectionError:
+		logger.warning("Connection error while sending the data to a webhook")
+		return 3
+	else:
+		return handle_discord_http(result.status_code, data, result)
+
+
+def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata):
+	if data is not None:
+		for regex in settings["disallow_regexes"]:
+			if data.webhook_object.get("content", None):
+				if re.search(re.compile(regex), data.webhook_object["content"]):
+					logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex))
+					return  # discard the message without anything
+			else:
+				for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]:
+					if re.search(re.compile(regex), to_check):
+						logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format(
+							to_check, regex))
+						return  # discard the message without anything
+	if messagequeue:
+		messagequeue.add_message((data, meta))
+	else:
+		code = send_to_discord_webhook(data, metadata=meta)
+		if code == 3:
+			messagequeue.add_message((data, meta))
+		elif code == 2:
+			time.sleep(5.0)
+			messagequeue.add_message((data, meta))
+		elif code < 2:
+			pass

src/discord/redaction.py

@@ -0,0 +1,74 @@
+import logging
+import json
+from src.configloader import settings
+from src.discord.message import DiscordMessageMetadata, DiscordMessage, DiscordMessageRaw
+from src.discord.queue import send_to_discord, messagequeue
+from src.fileio.database import db_cursor, db_connection
+from src.i18n import redaction as redaction_translation
+
+logger = logging.getLogger("rcgcdw.discord.redaction")  # TODO Figure out why does this logger do not work
+_ = redaction_translation.gettext
+#ngettext = redaction_translation.ngettext
+
+
+def delete_messages(matching_data: dict):
+	"""Delete messages that match given data"""
+	sql_conditions = ""
+	for key, value in matching_data.items():
+		sql_conditions += "{} = ? AND".format(key)
+	else:
+		sql_conditions = sql_conditions[0:-4]  # remove last AND statement
+	to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE {CON}".format(CON=sql_conditions), list(matching_data.values()))
+	if len(messagequeue) > 0:
+		messagequeue.delete_all_with_matching_metadata(**matching_data)
+	msg_to_remove = []
+	logger.debug("Deleting messages for data: {}".format(matching_data))
+	for message in to_delete:
+		webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0])
+		msg_to_remove.append(message[0])
+		logger.debug("Removing following message: {}".format(message[0]))
+		send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url))
+	for msg in msg_to_remove:
+		db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (msg,))
+	db_connection.commit()
+
+
+def redact_messages(ids: list, entry_type: int, to_censor: dict):
+	"""Redact past Discord messages
+
+	ids: list of ints
+	entry_type: int - 0 for revdel, 1 for logdel
+	to_censor: dict - logparams of message parts to censor"""
+	for event_id in ids:
+		if entry_type == 0:  # TODO check if queries are proper
+			message = db_cursor.execute("SELECT content, message_id FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.revid = ?;", (event_id, ))
+		else:
+			message = db_cursor.execute(
+				"SELECT content, message_id FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.logid = ?;",
+				(event_id,))
+		if settings["appearance"]["mode"] == "embed":
+			if message is not None:
+				row = message.fetchone()
+				try:
+					message = json.loads(row[0])
+					new_embed = message["embeds"][0]
+				except ValueError:
+					logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(row[0]))
+					return
+				if "user" in to_censor:
+					new_embed["author"]["name"] = _("Removed")
+					new_embed["author"].pop("url")
+				if "action" in to_censor:
+					new_embed["title"] = _("Removed")
+					new_embed.pop("url")
+				if "content" in to_censor:
+					new_embed.pop("fields")
+				if "comment" in to_censor:
+					new_embed["description"] = _("Removed")
+				message["embeds"][0] = new_embed
+				db_cursor.execute("UPDATE messages SET content = ? WHERE message_id = ?;", (json.dumps(message), row[1],))
+				db_connection.commit()
+				logger.debug(message)
+				send_to_discord(DiscordMessageRaw(message, settings["webhookURL"]+"/messages/"+str(row[1])), DiscordMessageMetadata("PATCH"))
+			else:
+				logger.debug("Could not find message in the database.")

src/discussion_formatters.py

@@ -4,7 +4,9 @@ import gettext
 from urllib.parse import quote_plus
 
 from src.configloader import settings
-from src.misc import link_formatter, create_article_path, DiscordMessage, send_to_discord, escape_formatting
+from src.misc import link_formatter, create_article_path, escape_formatting
+from src.discord.queue import send_to_discord
+from src.discord.message import DiscordMessage, DiscordMessageMetadata
 from src.i18n import discussion_formatters
 
 _ = discussion_formatters.gettext
@@ -67,7 +69,7 @@ def compact_formatter(post_type, post, article_paths):
 		else:
 			message = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format(
 				event=post_type, author=author, author_url=author_url, support=settings["support"])
-	send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message))
+	send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message), meta=DiscordMessageMetadata("POST"))
 
 
 def embed_formatter(post_type, post, article_paths):
@@ -167,7 +169,7 @@ def embed_formatter(post_type, post, article_paths):
 			else:
 				embed.add_field(_("Report this on the support server"), change_params)
 	embed.finish_embed()
-	send_to_discord(embed)
+	send_to_discord(embed, meta=DiscordMessageMetadata("POST"))
 
 
 class DiscussionsFromHellParser:

src/discussions.py

@@ -22,7 +22,8 @@ from typing import Dict, Any
 from src.configloader import settings
 
 from src.discussion_formatters import embed_formatter, compact_formatter
-from src.misc import datafile, messagequeue, prepare_paths
+from src.misc import datafile, prepare_paths
+from src.discord.queue import messagequeue
 from src.session import session
 from src.exceptions import ArticleCommentError
 

src/fileio/database.py

@@ -0,0 +1,67 @@
+import sqlite3
+import logging
+import json
+from src.configloader import settings
+
+logger = logging.getLogger("rcgcdw.fileio.database")
+
+
+def create_schema():
+	logger.info("Creating database schema...")
+	db_cursor.executescript(
+	"""BEGIN TRANSACTION;
+	CREATE TABLE IF NOT EXISTS "messages" (
+		"message_id"	TEXT,
+		"content"	TEXT,
+		PRIMARY KEY("message_id")
+	);
+	CREATE TABLE IF NOT EXISTS "event" (
+		"pageid"	INTEGER,
+		"revid"	INTEGER,
+		"logid"	INTEGER,
+		"msg_id"	TEXT NOT NULL,
+		PRIMARY KEY("msg_id"),
+		FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE
+	);
+	COMMIT;""")
+	logger.info("Database schema has been recreated.")
+
+
+def create_connection() -> (sqlite3.Connection, sqlite3.Cursor):
+	_db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:'))
+	_db_connection.row_factory = sqlite3.Row
+	_db_cursor = _db_connection.cursor()
+	logger.debug("Database connection created")
+	return _db_connection, _db_cursor
+
+
+def check_tables():
+	"""Check if tables exist, if not, create schema"""
+	rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';")
+	if not rep.fetchone():
+		logger.debug("No schema detected, creating schema!")
+		create_schema()
+
+
+def add_entry(pageid: int, revid: int, logid: int, message, message_id: str):
+	"""Add an edit or log entry to the DB
+	:param message_id:
+	"""
+	db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message_id, message))
+	db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message_id))
+	logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message))
+	db_connection.commit()
+
+def clean_entries():
+	"""Cleans entries that are 50+"""
+	cleanup = db_cursor.execute(
+		"SELECT message_id FROM messages WHERE message_id NOT IN (SELECT message_id FROM messages ORDER BY message_id desc LIMIT 50);")
+	for row in cleanup:
+		db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (row[0]))
+	cleanup = db_cursor.execute("SELECT msg_id FROM event WHERE msg_id NOT IN (SELECT msg_id FROM event ORDER BY msg_id desc LIMIT 50);")
+	for row in cleanup:
+		db_cursor.execute("DELETE FROM event WHERE msg_id = ?", (row[0]))
+	db_connection.commit()
+
+db_connection, db_cursor = create_connection()
+check_tables()

src/i18n.py

@@ -11,8 +11,9 @@ try:
 		rc = gettext.translation('rc', localedir='locale', languages=[settings["lang"]])
 		rc_formatters = gettext.translation('rc_formatters', localedir='locale', languages=[settings["lang"]])
 		misc = gettext.translation('misc', localedir='locale', languages=[settings["lang"]])
+		redaction = gettext.translation('redaction', localedir='locale', languages=[settings["lang"]])
 	else:
-		rcgcdw, discussion_formatters, rc, rc_formatters, misc = gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations()
+		rcgcdw, discussion_formatters, rc, rc_formatters, misc, redaction = gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations(), gettext.NullTranslations()
 except FileNotFoundError:
 	logger.critical("No language files have been found. Make sure locale folder is located in the directory.")
 	sys.exit(1)

src/misc.py

@@ -16,14 +16,17 @@
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
 import base64
-import json, logging, sys, re, time, random, math
+import json, logging, sys, re
 from html.parser import HTMLParser
 from urllib.parse import urlparse, urlunparse, quote
 import requests
-from collections import defaultdict
 from src.configloader import settings
+from src.discord.message import DiscordMessage, DiscordMessageMetadata
+from src.discord.queue import messagequeue, send_to_discord
 from src.i18n import misc
 
+AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": False}).get("enabled")
+
 _ = misc.gettext
 
 # Create a custom logger
@@ -38,7 +41,6 @@ WIKI_API_PATH: str = ""
 WIKI_ARTICLE_PATH: str = ""
 WIKI_SCRIPT_PATH: str = ""
 WIKI_JUST_DOMAIN: str = ""
-rate_limit = 0
 
 profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")}
 
@@ -90,52 +92,6 @@ class DataFile:
 		return self.data[item]
 
 
-
-class MessageQueue:
-	"""Message queue class for undelivered messages"""
-	def __init__(self):
-		self._queue = []
-
-	def __repr__(self):
-		return self._queue
-
-	def __len__(self):
-		return len(self._queue)
-
-	def __iter__(self):
-		return iter(self._queue)
-
-	def clear(self):
-		self._queue.clear()
-
-	def add_message(self, message):
-		self._queue.append(message)
-
-	def cut_messages(self, item_num):
-		self._queue = self._queue[item_num:]
-
-	def resend_msgs(self):
-		if self._queue:
-			misc_logger.info(
-				"{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format(
-					len(self._queue)))
-			for num, item in enumerate(self._queue):
-				misc_logger.debug(
-					"Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num),
-					                                                                                         str(item)))
-				if send_to_discord_webhook(item) < 2:
-					misc_logger.debug("Sending message succeeded")
-				else:
-					misc_logger.debug("Sending message failed")
-					break
-			else:
-				self.clear()
-				misc_logger.debug("Queue emptied, all messages delivered")
-			self.cut_messages(num)
-			misc_logger.debug(self._queue)
-
-
-messagequeue = MessageQueue()
 datafile = DataFile()
 
 
@@ -241,28 +197,6 @@ def safe_read(request, *keys):
 	return request
 
 
-def handle_discord_http(code, formatted_embed, result):
-	if 300 > code > 199:  # message went through
-		return 0
-	elif code == 400:  # HTTP BAD REQUEST result.status_code, data, result, header
-		misc_logger.error(
-			"Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:")
-		misc_logger.error(formatted_embed)
-		misc_logger.error(result.text)
-		return 1
-	elif code == 401 or code == 404:  # HTTP UNAUTHORIZED AND NOT FOUND
-		misc_logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.")
-		sys.exit(1)
-	elif code == 429:
-		misc_logger.error("We are sending too many requests to the Discord, slowing down...")
-		return 2
-	elif 499 < code < 600:
-		misc_logger.error(
-			"Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format(
-				code))
-		return 3
-
-
 def add_to_dict(dictionary, key):
 	if key in dictionary:
 		dictionary[key] += 1
@@ -326,124 +260,7 @@ def send_simple(msgtype, message, name, avatar):
 	discord_msg.set_avatar(avatar)
 	discord_msg.set_name(name)
 	messagequeue.resend_msgs()
-	send_to_discord(discord_msg)
+	send_to_discord(discord_msg, meta=DiscordMessageMetadata("POST"))
-
-
-def update_ratelimit(request):
-	"""Updates rate limit time"""
-	global rate_limit
-	rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get(
-		'x-ratelimit-reset-after', 0))
-	rate_limit += settings.get("discord_message_cooldown", 0)
-
-
-def send_to_discord_webhook(data):
-	global rate_limit
-	header = settings["header"]
-	header['Content-Type'] = 'application/json'
-	try:
-		time.sleep(rate_limit)
-		rate_limit = 0
-		result = requests.post(data.webhook_url, data=repr(data),
-		                       headers=header, timeout=10)
-		update_ratelimit(result)
-	except requests.exceptions.Timeout:
-		misc_logger.warning("Timeouted while sending data to the webhook.")
-		return 3
-	except requests.exceptions.ConnectionError:
-		misc_logger.warning("Connection error while sending the data to a webhook")
-		return 3
-	else:
-		return handle_discord_http(result.status_code, data, result)
-
-
-def send_to_discord(data):
-	for regex in settings["disallow_regexes"]:
-		if data.webhook_object.get("content", None):
-			if re.search(re.compile(regex), data.webhook_object["content"]):
-				misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex))
-				return  # discard the message without anything
-		else:
-			for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]:
-				if re.search(re.compile(regex), to_check):
-					misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format(
-						to_check, regex))
-					return  # discard the message without anything
-	if messagequeue:
-		messagequeue.add_message(data)
-	else:
-		code = send_to_discord_webhook(data)
-		if code == 3:
-			messagequeue.add_message(data)
-		elif code == 2:
-			time.sleep(5.0)
-			messagequeue.add_message(data)
-		elif code < 2:
-			pass
-
-class DiscordMessage():
-	"""A class defining a typical Discord JSON representation of webhook payload."""
-	def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None):
-		self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, ""))
-		self.webhook_url = webhook_url
-
-		if message_type == "embed":
-			self.__setup_embed()
-		elif message_type == "compact":
-			self.webhook_object["content"] = content
-
-		self.event_type = event_type
-
-	def __setitem__(self, key, value):
-		"""Set item is used only in embeds."""
-		try:
-			self.embed[key] = value
-		except NameError:
-			raise TypeError("Tried to assign a value when message type is plain message!")
-
-	def __getitem__(self, item):
-		return self.embed[item]
-
-	def __repr__(self):
-		"""Return the Discord webhook object ready to be sent"""
-		return json.dumps(self.webhook_object)
-
-	def __setup_embed(self):
-		self.embed = defaultdict(dict)
-		if "embeds" not in self.webhook_object:
-			self.webhook_object["embeds"] = [self.embed]
-		else:
-			self.webhook_object["embeds"].append(self.embed)
-		self.embed["color"] = None
-
-	def add_embed(self):
-		self.finish_embed()
-		self.__setup_embed()
-
-	def finish_embed(self):
-		if self.embed["color"] is None:
-			if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None:
-				self.embed["color"] = random.randrange(1, 16777215)
-			else:
-				self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"]
-		else:
-			self.embed["color"] = math.floor(self.embed["color"])
-
-	def set_author(self, name, url, icon_url=""):
-		self.embed["author"]["name"] = name
-		self.embed["author"]["url"] = url
-		self.embed["author"]["icon_url"] = icon_url
-
-	def add_field(self, name, value, inline=False):
-		if "fields" not in self.embed:
-			self.embed["fields"] = []
-		self.embed["fields"].append(dict(name=name, value=value, inline=inline))
-
-	def set_avatar(self, url):
-		self.webhook_object["avatar_url"] = url
-
-	def set_name(self, name):
-		self.webhook_object["username"] = name
 
 
 def profile_field_name(name, embed):
@@ -485,4 +302,4 @@ class LinkParser(HTMLParser):
 		self.new_string = self.new_string + data.replace("//", "/\\/")
 
 	def handle_endtag(self, tag):
-		misc_logger.debug(self.new_string)
+		misc_logger.debug(self.new_string)

src/rc.py

@@ -6,7 +6,8 @@ import requests
 from bs4 import BeautifulSoup
 
 from src.configloader import settings
-from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, messagequeue, datafile, send_simple, safe_read, LinkParser
+from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, LinkParser, AUTO_SUPPRESSION_ENABLED
+from src.discord.queue import messagequeue
 from src.exceptions import MWError
 from src.session import session
 from src.rc_formatters import compact_formatter, embed_formatter, compact_abuselog_formatter, embed_abuselog_formatter
@@ -332,6 +333,9 @@ class Recent_Changes_Class(object):
 
 	def clear_cache(self):
 		self.map_ips = {}
+		if AUTO_SUPPRESSION_ENABLED:
+			from src.fileio.database import clean_entries
+			clean_entries()
 
 	def init_info(self):
 		startup_info = safe_read(self.safe_request(
@@ -397,13 +401,13 @@ def essential_info(change, changed_categories):
 		parsed_comment = _("~~hidden~~")
 	if not parsed_comment:
 		parsed_comment = None
-	if change["type"] in ["edit", "new"]:
+	if "userhidden" in change:
-		logger.debug("List of categories in essential_info: {}".format(changed_categories))
+		change["user"] = _("hidden")
-		if "userhidden" in change:
-			change["user"] = _("hidden")
-		identification_string = change["type"]
 	if change.get("ns", -1) in settings.get("ignored_namespaces", ()):
 		return
+	if change["type"] in ["edit", "new"]:
+		logger.debug("List of categories in essential_info: {}".format(changed_categories))
+		identification_string = change["type"]
 	elif change["type"] == "log":
 		identification_string = "{logtype}/{logaction}".format(logtype=change["logtype"], logaction=change["logaction"])
 		if identification_string not in supported_logs:

src/rc_formatters.py

@@ -10,8 +10,14 @@ from urllib.parse import quote_plus, quote
 from bs4 import BeautifulSoup
 
 from src.configloader import settings
-from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \
+from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, safe_read, \
-	WIKI_API_PATH, ContentParser, profile_field_name, LinkParser
+	WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, AUTO_SUPPRESSION_ENABLED
+from src.discord.queue import send_to_discord
+from src.discord.message import DiscordMessage, DiscordMessageMetadata
+
+if AUTO_SUPPRESSION_ENABLED:
+	from src.discord.redaction import delete_messages, redact_messages
+
 from src.i18n import rc_formatters
 #from src.rc import recent_changes, pull_comment
 _ = rc_formatters.gettext
@@ -62,10 +68,11 @@ def compact_abuselog_formatter(change, recent_changes):
 		action=abusefilter_actions.get(change["action"], _("Unknown")), target=change.get("title", _("Unknown")),
 		target_url=create_article_path(change.get("title", _("Unknown"))),
 		result=abusefilter_results.get(change["result"], _("Unknown")))
-	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message))
+	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message), meta=DiscordMessageMetadata("POST"))
 
 
 def compact_formatter(action, change, parsed_comment, categories, recent_changes):
+	request_metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), page_id=change.get("pageid", None))
 	if action != "suppressed":
 		author_url = link_formatter(create_article_path("User:{user}".format(user=change["user"])))
 		author = change["user"]
@@ -104,10 +111,14 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes
 		page_link = link_formatter(create_article_path(change["title"]))
 		content = "🗑️ "+_("[{author}]({author_url}) deleted [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link,
 		                                                  comment=parsed_comment)
+		if AUTO_SUPPRESSION_ENABLED:
+			delete_messages(dict(pageid=change.get("pageid")))
 	elif action == "delete/delete_redir":
 		page_link = link_formatter(create_article_path(change["title"]))
 		content = "🗑️ "+_("[{author}]({author_url}) deleted redirect by overwriting [{page}]({page_link}){comment}").format(author=author, author_url=author_url, page=change["title"], page_link=page_link,
 		                                                   comment=parsed_comment)
+		if AUTO_SUPPRESSION_ENABLED:
+			delete_messages(dict(pageid=change.get("pageid")))
 	elif action == "move/move":
 		link = link_formatter(create_article_path(change["logparams"]['target_title']))
 		redirect_status = _("without making a redirect") if "suppressredirect" in change["logparams"] else _("with a redirect")
@@ -264,6 +275,14 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes
 		content = "👁️ "+ngettext("[{author}]({author_url}) changed visibility of revision on page [{article}]({article_url}){comment}",
 		                          "[{author}]({author_url}) changed visibility of {amount} revisions on page [{article}]({article_url}){comment}", amount).format(author=author, author_url=author_url,
 			article=change["title"], article_url=link, amount=amount, comment=parsed_comment)
+		if AUTO_SUPPRESSION_ENABLED:
+			try:
+				logparams = change["logparams"]
+				pageid = change["pageid"]
+			except KeyError:
+				pass
+			else:
+				delete_messages(dict(pageid=pageid))
 	elif action == "import/upload":
 		link = link_formatter(create_article_path(change["title"]))
 		content = "📥 "+ngettext("[{author}]({author_url}) imported [{article}]({article_url}) with {count} revision{comment}",
@@ -274,6 +293,14 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes
 		content = "♻️ "+_("[{author}]({author_url}) restored [{article}]({article_url}){comment}").format(author=author, author_url=author_url, article=change["title"], article_url=link, comment=parsed_comment)
 	elif action == "delete/event":
 		content = "👁️ "+_("[{author}]({author_url}) changed visibility of log events{comment}").format(author=author, author_url=author_url, comment=parsed_comment)
+		if AUTO_SUPPRESSION_ENABLED:
+			try:
+				logparams = change["logparams"]
+			except KeyError:
+				pass
+			else:
+				for revid in logparams.get("ids", []):
+					delete_messages(dict(revid=revid))
 	elif action == "import/interwiki":
 		content = "📥 "+_("[{author}]({author_url}) imported interwiki{comment}").format(author=author, author_url=author_url, comment=parsed_comment)
 	elif action == "abusefilter/modify":
@@ -403,7 +430,7 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes
 			content = "❓ "+_(
 				"Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format(
 				event=action, author=author, author_url=author_url, support=settings["support"])
-	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content))
+	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content), meta=request_metadata)
 
 def embed_abuselog_formatter(change, recent_changes):
 	action = "abuselog/{}".format(change["result"])
@@ -415,11 +442,12 @@ def embed_abuselog_formatter(change, recent_changes):
 	embed.add_field(_("Action taken"), abusefilter_results.get(change["result"], _("Unknown")))
 	embed.add_field(_("Title"), change.get("title", _("Unknown")))
 	embed.finish_embed()
-	send_to_discord(embed)
+	send_to_discord(embed, meta=DiscordMessageMetadata("POST"))
 
 
 def embed_formatter(action, change, parsed_comment, categories, recent_changes):
 	embed = DiscordMessage("embed", action, settings["webhookURL"])
+	request_metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), page_id=change.get("pageid", None))
 	if parsed_comment is None:
 		parsed_comment = _("No description provided")
 	if action != "suppressed":
@@ -554,9 +582,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes):
 	elif action == "delete/delete":
 		link = create_article_path(change["title"])
 		embed["title"] = _("Deleted page {article}").format(article=change["title"])
+		if AUTO_SUPPRESSION_ENABLED:
+			delete_messages(dict(pageid=change.get("pageid")))
 	elif action == "delete/delete_redir":
 		link = create_article_path(change["title"])
 		embed["title"] = _("Deleted redirect {article} by overwriting").format(article=change["title"])
+		if AUTO_SUPPRESSION_ENABLED:
+			delete_messages(dict(pageid=change.get("pageid")))
 	elif action == "move/move":
 		link = create_article_path(change["logparams"]['target_title'])
 		parsed_comment = "{supress}. {desc}".format(desc=parsed_comment,
@@ -709,6 +741,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes):
 		embed["title"] = ngettext("Changed visibility of revision on page {article} ",
 		                          "Changed visibility of {amount} revisions on page {article} ", amount).format(
 			article=change["title"], amount=amount)
+		if AUTO_SUPPRESSION_ENABLED:
+			try:
+				logparams = change["logparams"]
+			except KeyError:
+				pass
+			else:
+				redact_messages(logparams.get("ids", []), 0, logparams.get("new", {}))
 	elif action == "import/upload":
 		link = create_article_path(change["title"])
 		embed["title"] = ngettext("Imported {article} with {count} revision",
@@ -720,6 +759,13 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes):
 	elif action == "delete/event":
 		link = create_article_path("Special:RecentChanges")
 		embed["title"] = _("Changed visibility of log events")
+		if AUTO_SUPPRESSION_ENABLED:
+			try:
+				logparams = change["logparams"]
+			except KeyError:
+				pass
+			else:
+				redact_messages(logparams.get("ids", []), 1, logparams.get("new", {}))
 	elif action == "import/interwiki":
 		link = create_article_path("Special:RecentChanges")
 		embed["title"] = _("Imported interwiki")
@@ -886,4 +932,4 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes):
 		del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else ""
 		embed.add_field(_("Changed categories"), new_cat + del_cat)
 	embed.finish_embed()
-	send_to_discord(embed)
+	send_to_discord(embed, meta=request_metadata)

src/rcgcdw.py

@@ -26,8 +26,9 @@ import src.misc
 from collections import defaultdict, Counter
 from src.configloader import settings
 from src.misc import add_to_dict, datafile, \
-	WIKI_API_PATH, create_article_path, send_to_discord, \
+	WIKI_API_PATH, create_article_path
-	DiscordMessage
+from src.discord.queue import send_to_discord
+from src.discord.message import DiscordMessage, DiscordMessageMetadata
 from src.rc import recent_changes
 from src.exceptions import MWError
 from src.i18n import rcgcdw
@@ -161,10 +162,10 @@ def day_overview():
 				if item["type"] == "edit":
 					edits += 1
 					changed_bytes += item["newlen"] - item["oldlen"]
-					if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]:
+					if (recent_changes.namespaces is not None and "content" in recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0:
 						articles = add_to_dict(articles, item["title"])
 				elif item["type"] == "new":
-					if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]:
+					if "content" in (recent_changes.namespaces is not None and recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0:
 						new_articles += 1
 					changed_bytes += item["newlen"]
 				elif item["type"] == "log":
@@ -202,7 +203,7 @@ def day_overview():
 			for name, value in fields:
 				embed.add_field(name, value, inline=True)
 		embed.finish_embed()
-		send_to_discord(embed)
+		send_to_discord(embed, meta=DiscordMessageMetadata("POST"))
 	else:
 		logger.debug("function requesting changes for day overview returned with error code")