Fixed #154 and added more to message deletion code

src/discussion_formatters.py

@@ -67,7 +67,7 @@ def compact_formatter(post_type, post, article_paths):
 		else:
 			message = "❓ "+_("Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format(
 				event=post_type, author=author, author_url=author_url, support=settings["support"])
-	send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message))
+	send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message), meta={"request_type": "POST"})
 
 
 def embed_formatter(post_type, post, article_paths):
@@ -168,7 +168,7 @@ def embed_formatter(post_type, post, article_paths):
 			else:
 				embed.add_field(_("Report this on the support server"), change_params)
 	embed.finish_embed()
-	send_to_discord(embed)
+	send_to_discord(embed, meta={"request_type": "POST"})
 
 
 class DiscussionsFromHellParser:

src/message_redaction.py

@@ -1,16 +1,17 @@
 from src.configloader import settings
+from src.misc import send_to_discord
 import logging
 logger = logging.getLogger("rcgcdw.message_redaction")
 import sqlite3
 
 
-def create_schema(cursor: sqlite3.Cursor):
+def create_schema():
 	logger.info("Creating database schema...")
-	cursor.executescript(
+	db_cursor.executescript(
 	"""BEGIN TRANSACTION;
 	CREATE TABLE IF NOT EXISTS "messages" (
 		"message_id"	TEXT,
-		"content"	INTEGER
+		"content"	TEXT
 	);
 	CREATE TABLE IF NOT EXISTS "event" (
 		"pageid"	INTEGER,
@@ -22,14 +23,36 @@ def create_schema(cursor: sqlite3.Cursor):
 	COMMIT;""")
 	logger.info("Database schema has been recreated.")
 
-def create_connection():
-	db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:'))
-	db_cursor = db_connection.cursor()
-	return db_connection, db_cursor
 
-def check_tables(cursor: sqlite3.Cursor):
+def create_connection() -> (sqlite3.Connection, sqlite3.Cursor):
-	rep = cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';")
+	_db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:'))
+	_db_connection.row_factory = sqlite3.Row
+	_db_cursor = db_connection.cursor()
+	return _db_connection, _db_cursor
+
+
+def check_tables():
+	rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';")
 	if not rep.fetchone():
-		create_schema(cursor)
+		create_schema()
 
-def add_entry(pageid, revid, logid, message):
+
+def add_entry(pageid: int, revid: int, logid: int, message):
+	db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("message_id"), message))
+	db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("message_id")))
+
+
+def delete_messages(pageid: int):
+	to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid))
+	for message in to_delete:
+		webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0])
+		send_to_discord(None, {"request_type": "DELETE", "webhook_url": webhook_url})
+	db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", list_of_messageids)
+
+
+def redact_messages(rev_ids: list, to_censor: dict):
+	raise NotImplemented
+
+
+db_connection, db_cursor = create_connection()
+check_tables()

src/misc.py

@@ -23,6 +23,7 @@ import requests
 from collections import defaultdict
 from src.configloader import settings
 from src.i18n import misc
+from typing import Optional
 
 _ = misc.gettext
 
@@ -123,7 +124,7 @@ class MessageQueue:
 				misc_logger.debug(
 					"Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num),
 					                                                                                         str(item)))
-				if send_to_discord_webhook(item) < 2:
+				if send_to_discord_webhook(item[0], metadata=item[1]) < 2:
 					misc_logger.debug("Sending message succeeded")
 				else:
 					misc_logger.debug("Sending message failed")
@@ -325,7 +326,7 @@ def send_simple(msgtype, message, name, avatar):
 	discord_msg.set_avatar(avatar)
 	discord_msg.set_name(name)
 	messagequeue.resend_msgs()
-	send_to_discord(discord_msg)
+	send_to_discord(discord_msg, meta={"request_type": "POST"})
 
 
 def update_ratelimit(request):
@@ -336,51 +337,8 @@ def update_ratelimit(request):
 	rate_limit += settings.get("discord_message_cooldown", 0)
 
 
-def send_to_discord_webhook(data):
-	global rate_limit
-	header = settings["header"]
-	header['Content-Type'] = 'application/json'
-	try:
-		time.sleep(rate_limit)
-		rate_limit = 0
-		result = requests.post(data.webhook_url, data=repr(data),
-		                       headers=header, timeout=10)
-		update_ratelimit(result)
-	except requests.exceptions.Timeout:
-		misc_logger.warning("Timeouted while sending data to the webhook.")
-		return 3
-	except requests.exceptions.ConnectionError:
-		misc_logger.warning("Connection error while sending the data to a webhook")
-		return 3
-	else:
-		return handle_discord_http(result.status_code, data, result)
 
-
+class DiscordMessage:
-def send_to_discord(data):
-	for regex in settings["disallow_regexes"]:
-		if data.webhook_object.get("content", None):
-			if re.search(re.compile(regex), data.webhook_object["content"]):
-				misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex))
-				return  # discard the message without anything
-		else:
-			for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]:
-				if re.search(re.compile(regex), to_check):
-					misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format(
-						to_check, regex))
-					return  # discard the message without anything
-	if messagequeue:
-		messagequeue.add_message(data)
-	else:
-		code = send_to_discord_webhook(data)
-		if code == 3:
-			messagequeue.add_message(data)
-		elif code == 2:
-			time.sleep(5.0)
-			messagequeue.add_message(data)
-		elif code < 2:
-			pass
-
-class DiscordMessage():
 	"""A class defining a typical Discord JSON representation of webhook payload."""
 	def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None):
 		self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, ""))
@@ -455,6 +413,59 @@ def profile_field_name(name, embed):
 			return _("unknown")
 
 
+def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: dict = None):
+	global rate_limit
+	header = settings["header"]
+	header['Content-Type'] = 'application/json'
+	standard_args = dict(headers=header, timeout=10)
+	if metadata["request_type"] == "POST":
+		req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if settings.get("auto_suppression", {"enabled": True}).get("enabled") else "false", data=repr(data), **standard_args)
+	elif metadata["request_type"] == "DELETE":
+		req = requests.Request("DELETE", metadata["webhook_url"], **standard_args)
+	elif metadata["request_type"] == "PATCH":
+		req = requests.Request("PATCH", metadata["webhook_url"], data=metadata["new_data"], **standard_args)
+	try:
+		time.sleep(rate_limit)
+		rate_limit = 0
+		req = req.prepare()
+		result = req.send()
+		update_ratelimit(result)
+	except requests.exceptions.Timeout:
+		misc_logger.warning("Timeouted while sending data to the webhook.")
+		return 3
+	except requests.exceptions.ConnectionError:
+		misc_logger.warning("Connection error while sending the data to a webhook")
+		return 3
+	else:
+		return handle_discord_http(result.status_code, data, result)
+
+
+def send_to_discord(data: Optional[DiscordMessage], meta: dict):
+	if data is not None:
+		for regex in settings["disallow_regexes"]:
+			if data.webhook_object.get("content", None):
+				if re.search(re.compile(regex), data.webhook_object["content"]):
+					misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex))
+					return  # discard the message without anything
+			else:
+				for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]:
+					if re.search(re.compile(regex), to_check):
+						misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format(
+							to_check, regex))
+						return  # discard the message without anything
+	if messagequeue:
+		messagequeue.add_message((data, meta))
+	else:
+		code = send_to_discord_webhook(data, metadata=meta)
+		if code == 3:
+			messagequeue.add_message((data, meta))
+		elif code == 2:
+			time.sleep(5.0)
+			messagequeue.add_message((data, meta))
+		elif code < 2:
+			pass
+
+
 class LinkParser(HTMLParser):
 	new_string = ""
 	recent_href = ""
@@ -484,4 +495,4 @@ class LinkParser(HTMLParser):
 		self.new_string = self.new_string + data.replace("//", "/\\/")
 
 	def handle_endtag(self, tag):
-		misc_logger.debug(self.new_string)
+		misc_logger.debug(self.new_string)

src/rc_formatters.py

@@ -12,6 +12,7 @@ from bs4 import BeautifulSoup
 from src.configloader import settings
 from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \
 	WIKI_API_PATH, ContentParser, profile_field_name, LinkParser
+from src.message_redaction import delete_messages, redact_messages
 from src.i18n import rc_formatters
 #from src.rc import recent_changes, pull_comment
 _ = rc_formatters.gettext
@@ -62,7 +63,7 @@ def compact_abuselog_formatter(change, recent_changes):
 		action=abusefilter_actions.get(change["action"], _("Unknown")), target=change.get("title", _("Unknown")),
 		target_url=create_article_path(change.get("title", _("Unknown"))),
 		result=abusefilter_results.get(change["result"], _("Unknown")))
-	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message))
+	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=message), meta={"request_type": "POST"})
 
 
 def compact_formatter(action, change, parsed_comment, categories, recent_changes):
@@ -403,7 +404,7 @@ def compact_formatter(action, change, parsed_comment, categories, recent_changes
 			content = "❓ "+_(
 				"Unknown event `{event}` by [{author}]({author_url}), report it on the [support server](<{support}>).").format(
 				event=action, author=author, author_url=author_url, support=settings["support"])
-	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content))
+	send_to_discord(DiscordMessage("compact", action, settings["webhookURL"], content=content), meta={"request_type": "POST"})
 
 def embed_abuselog_formatter(change, recent_changes):
 	action = "abuselog/{}".format(change["result"])
@@ -415,7 +416,7 @@ def embed_abuselog_formatter(change, recent_changes):
 	embed.add_field(_("Action taken"), abusefilter_results.get(change["result"], _("Unknown")))
 	embed.add_field(_("Title"), change.get("title", _("Unknown")))
 	embed.finish_embed()
-	send_to_discord(embed)
+	send_to_discord(embed, meta={"request_type": "POST"})
 
 
 def embed_formatter(action, change, parsed_comment, categories, recent_changes):
@@ -889,4 +890,4 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes):
 		del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else ""
 		embed.add_field(_("Changed categories"), new_cat + del_cat)
 	embed.finish_embed()
-	send_to_discord(embed)
+	send_to_discord(embed, meta={"request_type": "POST"})

src/rcgcdw.py

@@ -161,10 +161,10 @@ def day_overview():
 				if item["type"] == "edit":
 					edits += 1
 					changed_bytes += item["newlen"] - item["oldlen"]
-					if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]:
+					if (recent_changes.namespaces is not None and "content" in recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0:
 						articles = add_to_dict(articles, item["title"])
 				elif item["type"] == "new":
-					if "content" in recent_changes.namespaces.get(str(item["ns"]), {}) or not item["ns"]:
+					if "content" in (recent_changes.namespaces is not None and recent_changes.namespaces.get(str(item["ns"]), {})) or item["ns"] == 0:
 						new_articles += 1
 					changed_bytes += item["newlen"]
 				elif item["type"] == "log":
@@ -202,7 +202,7 @@ def day_overview():
 			for name, value in fields:
 				embed.add_field(name, value, inline=True)
 		embed.finish_embed()
-		send_to_discord(embed)
+		send_to_discord(embed, meta={"request_type": "POST"})
 	else:
 		logger.debug("function requesting changes for day overview returned with error code")