From 9bc01153f883d2503e641147a6386139fc5c0be4 Mon Sep 17 00:00:00 2001 From: Frisk Date: Sun, 8 Nov 2020 22:29:15 +0100 Subject: [PATCH] Structurize the code, fixed some issues (like not recognizing edit and new events) --- src/__init__.py | 0 src/discord/__init__.py | 0 src/discord/message.py | 84 +++++++++++++ src/discord/queue.py | 156 ++++++++++++++++++++++++ src/discord/redaction.py | 27 +++++ src/discussion_formatters.py | 4 +- src/discussions.py | 3 +- src/fileio/__init__.py | 0 src/fileio/database.py | 51 ++++++++ src/message_redaction.py | 62 ---------- src/misc.py | 222 +---------------------------------- src/rc.py | 7 +- src/rc_formatters.py | 9 +- src/rcgcdw.py | 5 +- 14 files changed, 340 insertions(+), 290 deletions(-) create mode 100644 src/__init__.py create mode 100644 src/discord/__init__.py create mode 100644 src/discord/message.py create mode 100644 src/discord/queue.py create mode 100644 src/discord/redaction.py create mode 100644 src/fileio/__init__.py create mode 100644 src/fileio/database.py diff --git a/src/__init__.py b/src/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord/__init__.py b/src/discord/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/discord/message.py b/src/discord/message.py new file mode 100644 index 0000000..f918657 --- /dev/null +++ b/src/discord/message.py @@ -0,0 +1,84 @@ +import json +import math +import random +from collections import defaultdict + +from src.configloader import settings + + +class DiscordMessage: + """A class defining a typical Discord JSON representation of webhook payload.""" + def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): + self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) + self.webhook_url = webhook_url + + if message_type == "embed": + self.__setup_embed() + elif message_type == "compact": + self.webhook_object["content"] = content + + self.event_type = event_type + + def __setitem__(self, key, value): + """Set item is used only in embeds.""" + try: + self.embed[key] = value + except NameError: + raise TypeError("Tried to assign a value when message type is plain message!") + + def __getitem__(self, item): + return self.embed[item] + + def __repr__(self): + """Return the Discord webhook object ready to be sent""" + return json.dumps(self.webhook_object) + + def __setup_embed(self): + self.embed = defaultdict(dict) + if "embeds" not in self.webhook_object: + self.webhook_object["embeds"] = [self.embed] + else: + self.webhook_object["embeds"].append(self.embed) + self.embed["color"] = None + + def add_embed(self): + self.finish_embed() + self.__setup_embed() + + def finish_embed(self): + if self.embed["color"] is None: + if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: + self.embed["color"] = random.randrange(1, 16777215) + else: + self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] + else: + self.embed["color"] = math.floor(self.embed["color"]) + + def set_author(self, name, url, icon_url=""): + self.embed["author"]["name"] = name + self.embed["author"]["url"] = url + self.embed["author"]["icon_url"] = icon_url + + def add_field(self, name, value, inline=False): + if "fields" not in self.embed: + self.embed["fields"] = [] + self.embed["fields"].append(dict(name=name, value=value, inline=inline)) + + def set_avatar(self, url): + self.webhook_object["avatar_url"] = url + + def set_name(self, name): + self.webhook_object["username"] = name + + +class DiscordMessageMetadata: + def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): + self.method = method + self.page_id = page_id + self.log_id = log_id + self.rev_id = rev_id + self.webhook_url = webhook_url + self.new_data = new_data + + def dump_ids(self): + return self.page_id, self.rev_id, self.log_id \ No newline at end of file diff --git a/src/discord/queue.py b/src/discord/queue.py new file mode 100644 index 0000000..51eb42d --- /dev/null +++ b/src/discord/queue.py @@ -0,0 +1,156 @@ +import re +import sys +import time +import logging +from typing import Optional + +import requests + +from src.configloader import settings +from src.discord.message import DiscordMessage, DiscordMessageMetadata + +AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": True}).get("enabled") +if AUTO_SUPPRESSION_ENABLED: + from src.fileio.database import add_entry as add_message_redaction_entry + +rate_limit = 0 + +logger = logging.getLogger("rcgcdw.discord.queue") + +class MessageQueue: + """Message queue class for undelivered messages""" + def __init__(self): + self._queue = [] + + def __repr__(self): + return self._queue + + def __len__(self): + return len(self._queue) + + def __iter__(self): + return iter(self._queue) + + def clear(self): + self._queue.clear() + + def add_message(self, message): + self._queue.append(message) + + def cut_messages(self, item_num): + self._queue = self._queue[item_num:] + + def resend_msgs(self): + if self._queue: + logger.info( + "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( + len(self._queue))) + for num, item in enumerate(self._queue): + logger.debug( + "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), + str(item))) + if send_to_discord_webhook(item[0], metadata=item[1]) < 2: + logger.debug("Sending message succeeded") + else: + logger.debug("Sending message failed") + break + else: + self.clear() + logger.debug("Queue emptied, all messages delivered") + self.cut_messages(num) + logger.debug(self._queue) + + +messagequeue = MessageQueue() + + +def handle_discord_http(code, formatted_embed, result): + if 300 > code > 199: # message went through + return 0 + elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header + logger.error( + "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") + logger.error(formatted_embed) + logger.error(result.text) + return 1 + elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND + if result.request.method == "POST": # Ignore not found for DELETE and PATCH requests since the message could already be removed by admin + logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") + sys.exit(1) + else: + return 0 + elif code == 429: + logger.error("We are sending too many requests to the Discord, slowing down...") + return 2 + elif 499 < code < 600: + logger.error( + "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( + code)) + return 3 + + +def update_ratelimit(request): + """Updates rate limit time""" + global rate_limit + rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get( + 'x-ratelimit-reset-after', 0)) + rate_limit += settings.get("discord_message_cooldown", 0) + + +def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMessageMetadata): + global rate_limit + header = settings["header"] + header['Content-Type'] = 'application/json' + standard_args = dict(headers=header) + if metadata.method == "POST": + req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if AUTO_SUPPRESSION_ENABLED else "false", data=repr(data), **standard_args) + elif metadata.method == "DELETE": + req = requests.Request("DELETE", metadata.webhook_url, **standard_args) + elif metadata.method == "PATCH": + req = requests.Request("PATCH", metadata.webhook_url, data=metadata.new_data, **standard_args) + try: + time.sleep(rate_limit) + rate_limit = 0 + req = req.prepare() + result = requests.Session().send(req, timeout=10) + update_ratelimit(result) + if AUTO_SUPPRESSION_ENABLED and metadata.method == "POST": + # TODO Prepare request with all of safety checks + try: + add_message_redaction_entry(*metadata.dump_ids(), result.json()) + except ValueError: + logger.error("Couldn't get json of result of sending Discord message.") + except requests.exceptions.Timeout: + logger.warning("Timeouted while sending data to the webhook.") + return 3 + except requests.exceptions.ConnectionError: + logger.warning("Connection error while sending the data to a webhook") + return 3 + else: + return handle_discord_http(result.status_code, data, result) + + +def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata): + if data is not None: + for regex in settings["disallow_regexes"]: + if data.webhook_object.get("content", None): + if re.search(re.compile(regex), data.webhook_object["content"]): + logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) + return # discard the message without anything + else: + for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: + if re.search(re.compile(regex), to_check): + logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( + to_check, regex)) + return # discard the message without anything + if messagequeue: + messagequeue.add_message((data, meta)) + else: + code = send_to_discord_webhook(data, metadata=meta) + if code == 3: + messagequeue.add_message((data, meta)) + elif code == 2: + time.sleep(5.0) + messagequeue.add_message((data, meta)) + elif code < 2: + pass \ No newline at end of file diff --git a/src/discord/redaction.py b/src/discord/redaction.py new file mode 100644 index 0000000..941a8bd --- /dev/null +++ b/src/discord/redaction.py @@ -0,0 +1,27 @@ +import logging + +from src.configloader import settings +from src.discord.message import DiscordMessageMetadata +from src.discord.queue import send_to_discord +from src.fileio.database import db_cursor, db_connection + +logger = logging.getLogger("rcgcdw.discord.redaction") + + +def delete_messages(pageid: int): + """Delete messages that match that pageid""" + logger.debug(type(pageid)) + to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid,)) + msg_to_remove = [] + logger.debug("Deleting messages for pageid: {}".format(pageid)) + for message in to_delete: + webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) + msg_to_remove.append(message[0]) + logger.debug("Removing following message: {}".format(message[0])) + send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) + db_cursor.execute("DELETE FROM messages WHERE message_id = ?", (message[0],)) + db_connection.commit() + + +def redact_messages(rev_ids: list, to_censor: dict): + raise NotImplemented \ No newline at end of file diff --git a/src/discussion_formatters.py b/src/discussion_formatters.py index dd4aa81..3706c84 100644 --- a/src/discussion_formatters.py +++ b/src/discussion_formatters.py @@ -4,7 +4,9 @@ import gettext from urllib.parse import quote_plus from src.configloader import settings -from src.misc import link_formatter, create_article_path, DiscordMessage, send_to_discord, escape_formatting, DiscordMessageMetadata +from src.misc import link_formatter, create_article_path, escape_formatting +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata from src.i18n import discussion_formatters _ = discussion_formatters.gettext diff --git a/src/discussions.py b/src/discussions.py index 579ffaa..4d8ca37 100644 --- a/src/discussions.py +++ b/src/discussions.py @@ -22,7 +22,8 @@ from typing import Dict, Any from src.configloader import settings from src.discussion_formatters import embed_formatter, compact_formatter -from src.misc import datafile, messagequeue, prepare_paths +from src.misc import datafile, prepare_paths +from src.discord.queue import messagequeue from src.session import session from src.exceptions import ArticleCommentError diff --git a/src/fileio/__init__.py b/src/fileio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/src/fileio/database.py b/src/fileio/database.py new file mode 100644 index 0000000..a2870b8 --- /dev/null +++ b/src/fileio/database.py @@ -0,0 +1,51 @@ +import sqlite3 +import logging +from src.configloader import settings + +logger = logging.getLogger("rcgcdw.fileio.database") + + +def create_schema(): + logger.info("Creating database schema...") + db_cursor.executescript( + """BEGIN TRANSACTION; + CREATE TABLE IF NOT EXISTS "messages" ( + "message_id" TEXT, + "content" TEXT + ); + CREATE TABLE IF NOT EXISTS "event" ( + "pageid" INTEGER, + "revid" INTEGER, + "logid" INTEGER, + "msg_id" TEXT NOT NULL, + FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE + ); + COMMIT;""") + logger.info("Database schema has been recreated.") + + +def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): + _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) + _db_connection.row_factory = sqlite3.Row + _db_cursor = _db_connection.cursor() + logger.debug("Database connection created") + return _db_connection, _db_cursor + + +def check_tables(): + """Check if tables exist, if not, create schema""" + rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") + if not rep.fetchone(): + logger.debug("No schema detected, creating schema!") + create_schema() + + +def add_entry(pageid: int, revid: int, logid: int, message): + """Add an edit or log entry to the DB""" + db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("id"), str(message))) + db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("id"))) + logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) + db_connection.commit() + +db_connection, db_cursor = create_connection() +check_tables() diff --git a/src/message_redaction.py b/src/message_redaction.py index 6c5af41..b28b04f 100644 --- a/src/message_redaction.py +++ b/src/message_redaction.py @@ -1,65 +1,3 @@ -from src.configloader import settings -from src.misc import send_to_discord, DiscordMessageMetadata -import logging -logger = logging.getLogger("rcgcdw.message_redaction") -import sqlite3 -def create_schema(): - logger.info("Creating database schema...") - db_cursor.executescript( - """BEGIN TRANSACTION; - CREATE TABLE IF NOT EXISTS "messages" ( - "message_id" TEXT, - "content" TEXT - ); - CREATE TABLE IF NOT EXISTS "event" ( - "pageid" INTEGER, - "revid" INTEGER, - "logid" INTEGER, - "msg_id" TEXT NOT NULL, - FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE - ); - COMMIT;""") - logger.info("Database schema has been recreated.") - -def create_connection() -> (sqlite3.Connection, sqlite3.Cursor): - _db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:')) - _db_connection.row_factory = sqlite3.Row - _db_cursor = db_connection.cursor() - logger.debug("Database connection created") - return _db_connection, _db_cursor - - -def check_tables(): - rep = db_cursor.execute("SELECT name FROM sqlite_master WHERE type='table' AND name='messages';") - if not rep.fetchone(): - logger.debug("No schema detected, creating schema!") - create_schema() - - -def add_entry(pageid: int, revid: int, logid: int, message): - db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message.get("message_id"), message)) - db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", (pageid, revid, logid, message.get("message_id"))) - logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) - - -def delete_messages(pageid: int): - to_delete = db_cursor.execute("SELECT msg_id FROM event WHERE pageid = ?", (pageid)) - msg_to_remove = [] - logger.debug("Deleting messages for pageid: {}".format(pageid)) - for message in to_delete: - webhook_url = "{main_webhook}/messages/{message_id}".format(main_webhook=settings["webhookURL"], message_id=message[0]) - msg_to_remove.append(message[0]) - logger.debug("Removing following message: {}".format(message)) - send_to_discord(None, DiscordMessageMetadata("DELETE", webhook_url=webhook_url)) - db_cursor.executemany("DELETE FROM messages WHERE message_id = ?", msg_to_remove) - - -def redact_messages(rev_ids: list, to_censor: dict): - raise NotImplemented - - -db_connection, db_cursor = create_connection() -check_tables() diff --git a/src/misc.py b/src/misc.py index 041c795..d988d5f 100644 --- a/src/misc.py +++ b/src/misc.py @@ -16,18 +16,18 @@ # You should have received a copy of the GNU General Public License # along with this program. If not, see . import base64 -import json, logging, sys, re, time, random, math +import json, logging, sys, re from html.parser import HTMLParser from urllib.parse import urlparse, urlunparse, quote import requests -from collections import defaultdict from src.configloader import settings +from src.discord.message import DiscordMessage, DiscordMessageMetadata +from src.discord.queue import messagequeue, send_to_discord from src.i18n import misc -from typing import Optional AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": True}).get("enabled") if AUTO_SUPPRESSION_ENABLED: - from src.message_redaction import add_entry as add_message_redaction_entry + pass _ = misc.gettext @@ -43,7 +43,6 @@ WIKI_API_PATH: str = "" WIKI_ARTICLE_PATH: str = "" WIKI_SCRIPT_PATH: str = "" WIKI_JUST_DOMAIN: str = "" -rate_limit = 0 profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")} @@ -95,52 +94,6 @@ class DataFile: return self.data[item] - -class MessageQueue: - """Message queue class for undelivered messages""" - def __init__(self): - self._queue = [] - - def __repr__(self): - return self._queue - - def __len__(self): - return len(self._queue) - - def __iter__(self): - return iter(self._queue) - - def clear(self): - self._queue.clear() - - def add_message(self, message): - self._queue.append(message) - - def cut_messages(self, item_num): - self._queue = self._queue[item_num:] - - def resend_msgs(self): - if self._queue: - misc_logger.info( - "{} messages waiting to be delivered to Discord due to Discord throwing errors/no connection to Discord servers.".format( - len(self._queue))) - for num, item in enumerate(self._queue): - misc_logger.debug( - "Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num), - str(item))) - if send_to_discord_webhook(item[0], metadata=item[1]) < 2: - misc_logger.debug("Sending message succeeded") - else: - misc_logger.debug("Sending message failed") - break - else: - self.clear() - misc_logger.debug("Queue emptied, all messages delivered") - self.cut_messages(num) - misc_logger.debug(self._queue) - - -messagequeue = MessageQueue() datafile = DataFile() @@ -245,28 +198,6 @@ def safe_read(request, *keys): return request -def handle_discord_http(code, formatted_embed, result): - if 300 > code > 199: # message went through - return 0 - elif code == 400: # HTTP BAD REQUEST result.status_code, data, result, header - misc_logger.error( - "Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:") - misc_logger.error(formatted_embed) - misc_logger.error(result.text) - return 1 - elif code == 401 or code == 404: # HTTP UNAUTHORIZED AND NOT FOUND - misc_logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.") - sys.exit(1) - elif code == 429: - misc_logger.error("We are sending too many requests to the Discord, slowing down...") - return 2 - elif 499 < code < 600: - misc_logger.error( - "Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format( - code)) - return 3 - - def add_to_dict(dictionary, key): if key in dictionary: dictionary[key] += 1 @@ -333,80 +264,6 @@ def send_simple(msgtype, message, name, avatar): send_to_discord(discord_msg, meta=DiscordMessageMetadata("POST")) -def update_ratelimit(request): - """Updates rate limit time""" - global rate_limit - rate_limit = 0 if int(request.headers.get('x-ratelimit-remaining', "-1")) > 0 else int(request.headers.get( - 'x-ratelimit-reset-after', 0)) - rate_limit += settings.get("discord_message_cooldown", 0) - - - -class DiscordMessage: - """A class defining a typical Discord JSON representation of webhook payload.""" - def __init__(self, message_type: str, event_type: str, webhook_url: str, content=None): - self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) - self.webhook_url = webhook_url - - if message_type == "embed": - self.__setup_embed() - elif message_type == "compact": - self.webhook_object["content"] = content - - self.event_type = event_type - - def __setitem__(self, key, value): - """Set item is used only in embeds.""" - try: - self.embed[key] = value - except NameError: - raise TypeError("Tried to assign a value when message type is plain message!") - - def __getitem__(self, item): - return self.embed[item] - - def __repr__(self): - """Return the Discord webhook object ready to be sent""" - return json.dumps(self.webhook_object) - - def __setup_embed(self): - self.embed = defaultdict(dict) - if "embeds" not in self.webhook_object: - self.webhook_object["embeds"] = [self.embed] - else: - self.webhook_object["embeds"].append(self.embed) - self.embed["color"] = None - - def add_embed(self): - self.finish_embed() - self.__setup_embed() - - def finish_embed(self): - if self.embed["color"] is None: - if settings["appearance"]["embed"].get(self.event_type, {"color": None})["color"] is None: - self.embed["color"] = random.randrange(1, 16777215) - else: - self.embed["color"] = settings["appearance"]["embed"][self.event_type]["color"] - else: - self.embed["color"] = math.floor(self.embed["color"]) - - def set_author(self, name, url, icon_url=""): - self.embed["author"]["name"] = name - self.embed["author"]["url"] = url - self.embed["author"]["icon_url"] = icon_url - - def add_field(self, name, value, inline=False): - if "fields" not in self.embed: - self.embed["fields"] = [] - self.embed["fields"].append(dict(name=name, value=value, inline=inline)) - - def set_avatar(self, url): - self.webhook_object["avatar_url"] = url - - def set_name(self, name): - self.webhook_object["username"] = name - - def profile_field_name(name, embed): try: return profile_fields[name] @@ -417,77 +274,6 @@ def profile_field_name(name, embed): return _("unknown") -class DiscordMessageMetadata: - def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, new_data = None): - self.method = method - self.page_id = page_id - self.log_id = log_id - self.rev_id = rev_id - self.webhook_url = webhook_url - self.new_data = new_data - - def dump_ids(self): - return self.page_id, self.rev_id, self.log_id - -def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMessageMetadata): - global rate_limit - header = settings["header"] - header['Content-Type'] = 'application/json' - standard_args = dict(headers=header, timeout=10) - if metadata.method == "POST": - req = requests.Request("POST", data.webhook_url+"?wait=" + "true" if AUTO_SUPPRESSION_ENABLED else "false", data=repr(data), **standard_args) - elif metadata.method == "DELETE": - req = requests.Request("DELETE", metadata.webhook_url, **standard_args) - elif metadata.method == "PATCH": - req = requests.Request("PATCH", metadata.webhook_url, data=metadata.new_data, **standard_args) - try: - time.sleep(rate_limit) - rate_limit = 0 - req = req.prepare() - result = req.send() - update_ratelimit(result) - if AUTO_SUPPRESSION_ENABLED: - # TODO Prepare request with all of safety checks - try: - add_message_redaction_entry(*metadata.dump_ids(), result.json()) - except ValueError: - misc_logger.error("Couldn't get json of result of sending Discord message.") - except requests.exceptions.Timeout: - misc_logger.warning("Timeouted while sending data to the webhook.") - return 3 - except requests.exceptions.ConnectionError: - misc_logger.warning("Connection error while sending the data to a webhook") - return 3 - else: - return handle_discord_http(result.status_code, data, result) - - -def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata): - if data is not None: - for regex in settings["disallow_regexes"]: - if data.webhook_object.get("content", None): - if re.search(re.compile(regex), data.webhook_object["content"]): - misc_logger.info("Message {} has been rejected due to matching filter ({}).".format(data.webhook_object["content"], regex)) - return # discard the message without anything - else: - for to_check in [data.webhook_object.get("description", ""), data.webhook_object.get("title", ""), *[x["value"] for x in data["fields"]], data.webhook_object.get("author", {"name": ""}).get("name", "")]: - if re.search(re.compile(regex), to_check): - misc_logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format( - to_check, regex)) - return # discard the message without anything - if messagequeue: - messagequeue.add_message((data, meta)) - else: - code = send_to_discord_webhook(data, metadata=meta) - if code == 3: - messagequeue.add_message((data, meta)) - elif code == 2: - time.sleep(5.0) - messagequeue.add_message((data, meta)) - elif code < 2: - pass - - class LinkParser(HTMLParser): new_string = "" recent_href = "" diff --git a/src/rc.py b/src/rc.py index beaa94c..51e3b6d 100644 --- a/src/rc.py +++ b/src/rc.py @@ -6,7 +6,8 @@ import requests from bs4 import BeautifulSoup from src.configloader import settings -from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, messagequeue, datafile, send_simple, safe_read, LinkParser +from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, LinkParser +from src.discord.queue import messagequeue from src.exceptions import MWError from src.session import session from src.rc_formatters import compact_formatter, embed_formatter, compact_abuselog_formatter, embed_abuselog_formatter @@ -399,11 +400,11 @@ def essential_info(change, changed_categories): parsed_comment = None if "userhidden" in change: change["user"] = _("hidden") + if change.get("ns", -1) in settings.get("ignored_namespaces", ()): + return if change["type"] in ["edit", "new"]: logger.debug("List of categories in essential_info: {}".format(changed_categories)) identification_string = change["type"] - if change.get("ns", -1) in settings.get("ignored_namespaces", ()): - return elif change["type"] == "log": identification_string = "{logtype}/{logaction}".format(logtype=change["logtype"], logaction=change["logaction"]) if identification_string not in supported_logs: diff --git a/src/rc_formatters.py b/src/rc_formatters.py index 0a9399f..684ef21 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -10,10 +10,13 @@ from urllib.parse import quote_plus, quote from bs4 import BeautifulSoup from src.configloader import settings -from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ - WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, DiscordMessageMetadata, AUTO_SUPPRESSION_ENABLED +from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, safe_read, \ + WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, AUTO_SUPPRESSION_ENABLED +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata + if AUTO_SUPPRESSION_ENABLED: - from src.message_redaction import delete_messages, redact_messages + from src.discord.redaction import delete_messages, redact_messages from src.i18n import rc_formatters #from src.rc import recent_changes, pull_comment diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 1b6025e..e19622a 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -26,8 +26,9 @@ import src.misc from collections import defaultdict, Counter from src.configloader import settings from src.misc import add_to_dict, datafile, \ - WIKI_API_PATH, create_article_path, send_to_discord, \ - DiscordMessage, DiscordMessageMetadata + WIKI_API_PATH, create_article_path +from src.discord.queue import send_to_discord +from src.discord.message import DiscordMessage, DiscordMessageMetadata from src.rc import recent_changes from src.exceptions import MWError from src.i18n import rcgcdw