diff --git a/src/discord/message.py b/src/discord/message.py index d4a5499..e0f2a39 100644 --- a/src/discord/message.py +++ b/src/discord/message.py @@ -105,12 +105,14 @@ class DiscordMessageRaw(DiscordMessage): self.webhook_object = content self.webhook_url = webhook_url + class DiscordMessageMetadata: - def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None): + def __init__(self, method, log_id = None, page_id = None, rev_id = None, webhook_url = None, timestamp = None): self.method = method self.page_id = page_id self.log_id = log_id self.rev_id = rev_id + self.timestamp = timestamp self.webhook_url = webhook_url def dump_ids(self) -> (int, int, int): diff --git a/src/discord/queue.py b/src/discord/queue.py index f50f5d8..428fc95 100644 --- a/src/discord/queue.py +++ b/src/discord/queue.py @@ -149,7 +149,7 @@ def send_to_discord_webhook(data: Optional[DiscordMessage], metadata: DiscordMes if AUTO_SUPPRESSION_ENABLED and metadata.method == "POST": if 199 < result.status_code < 300: # check if positive error log try: - add_message_redaction_entry(*metadata.dump_ids(), repr(data), result.json().get("id")) + add_message_redaction_entry(*metadata.dump_ids(), repr(data), result.json().get("id"), metadata.timestamp) except ValueError: logger.error("Couldn't get json of result of sending Discord message.") else: diff --git a/src/discord/redaction.py b/src/discord/redaction.py index 0048ac0..a6b3c5b 100644 --- a/src/discord/redaction.py +++ b/src/discord/redaction.py @@ -51,6 +51,41 @@ def delete_messages(matching_data: dict): db_connection.commit() +def redact_image_files(ids, to_censor, page_id): + for event_id in ids: + previous_image = db_cursor.execute("SELECT content, message_id FROM messages INNER JOIN event ON event.msg_id = messages.message_id WHERE event.timestamp < ? AND event.pageid = ? ORDER BY timestamp DESC LIMIT 1;", (event_id, page_id)) + if previous_image is not None: + row = previous_image.fetchone() + try: + message = json.loads(row[0]) + new_embed = message["embeds"][0] + except ValueError: + logger.error("Couldn't loads JSON for message data. What happened? Data: {}".format(row[0])) + return + except TypeError: + logger.error( + "Couldn't find entry in the database for RevDel to censor information. This is probably because the script has been recently restarted or cache cleared.") + return + if "user" in to_censor and "url" in new_embed["author"]: + new_embed["author"]["name"] = _("hidden") + new_embed["author"].pop("url") + if "content" in to_censor and "image" in new_embed: + new_embed.pop("image") + if "comment" in to_censor: + new_embed["description"] = _("~~hidden~~") + message["embeds"][0] = new_embed + db_cursor.execute("UPDATE messages SET content = ? WHERE message_id = ?;", (json.dumps(message), row[1],)) + db_connection.commit() + logger.debug(message) + main_webhook = settings["webhookURL"].split("?", 1) + webhook_url = "{main_webhook}/messages/{message_id}{thread_id}".format(main_webhook=main_webhook[0], + message_id=str(row[1]), thread_id=( + "?" + main_webhook[1] if len(main_webhook) > 1 else "")) + send_to_discord(DiscordMessageRaw(message, webhook_url), DiscordMessageMetadata("PATCH")) + else: + logger.debug("Could not find message in the database.") + + def redact_messages(ids, entry_type: int, to_censor: dict): # : Union[List[Union[str, int]], set[Union[int, str]]] """Redact past Discord messages diff --git a/src/fileio/database.py b/src/fileio/database.py index 8b63a96..af2d223 100644 --- a/src/fileio/database.py +++ b/src/fileio/database.py @@ -54,6 +54,7 @@ def create_schema(): "pageid" INTEGER, "revid" INTEGER, "logid" INTEGER, + "timestamp" TEXT, "msg_id" TEXT NOT NULL, PRIMARY KEY("msg_id"), FOREIGN KEY("msg_id") REFERENCES "messages"("message_id") ON DELETE CASCADE @@ -80,17 +81,18 @@ def check_tables(): @catch_db_OperationalError -def add_entry(pageid: int, revid: int, logid: int, message, message_id: str): +def add_entry(pageid: int, revid: int, logid: int, message, message_id: str, timestamp: str): """Add an edit or log entry to the DB :param message: :param logid: :param revid: :param pageid: :param message_id: + :param timestamp: """ db_cursor.execute("INSERT INTO messages (message_id, content) VALUES (?, ?)", (message_id, message)) - db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id) VALUES (?, ?, ?, ?)", - (pageid, revid, logid, message_id)) + db_cursor.execute("INSERT INTO event (pageid, revid, logid, msg_id, timestamp) VALUES (?, ?, ?, ?, ?)", + (pageid, revid, logid, message_id, timestamp)) logger.debug( "Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message)) diff --git a/src/migrations/__init__.py b/src/migrations/__init__.py index 129d8a8..a63a08d 100644 --- a/src/migrations/__init__.py +++ b/src/migrations/__init__.py @@ -1 +1 @@ -__all__ = ["11311", "falsytypes"] +__all__ = ["11311", "falsytypes", "imgtimestampdb"] diff --git a/src/migrations/imgtimestampdb.py b/src/migrations/imgtimestampdb.py new file mode 100644 index 0000000..40a07bc --- /dev/null +++ b/src/migrations/imgtimestampdb.py @@ -0,0 +1,45 @@ +# This file is part of Recent changes Goat compatible Discord webhook (RcGcDw). +# +# RcGcDw is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# RcGcDw is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with RcGcDw. If not, see . + +from src.configloader import settings +from src.fileio.database import create_connection +import logging + +logger = logging.getLogger("rcgcdw.migrations.imgtimestampdb") + + +def run(): + as_settings = settings.get("auto_suppression", {}) + if as_settings.get("enabled", False) and as_settings.get("db_location", ":memory:") != ":memory:" and settings.get("appearance", {}).get("mode") == "embed" and settings.get("appearance", {}).get("embed", {}).get("embed_images", False): + import sqlite3 + db_connection, db_cursor = create_connection() + try: + rep = db_cursor.execute("SELECT COUNT(*) AS IMGIDPRESENT FROM pragma_table_info('event') WHERE name='timestamp';") + except sqlite3.OperationalError as db_ex: + logger.debug("sqlite3 thrown OperationalError in imgtimestampdb. Either the database doesn't exist or there is another issue.") + return + if rep.fetchone()[0] == 0: + logger.info("Running migration imgtimestampdb") + db_cursor.execute("ALTER TABLE event ADD COLUMN 'timestamp' TEXT") + db_connection.commit() + db_connection.close() + logger.info("Migration imgtimestampdb has been successful.") + else: + logger.debug("Ignoring migration imgtimestampdb") + else: + logger.debug("Ignoring migration imgtimestampdb") + + +run() \ No newline at end of file diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 1debb77..6f69758 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -45,7 +45,7 @@ TESTING = command_args.test # debug mode, pipeline testing AUTO_SUPPRESSION_ENABLED = settings.get("auto_suppression", {"enabled": False}).get("enabled") if AUTO_SUPPRESSION_ENABLED: - from src.discord.redaction import delete_messages, redact_messages, find_middle_next + from src.discord.redaction import delete_messages, redact_messages, find_middle_next, redact_image_files # Prepare logging logging.config.dictConfig(settings["logging"]) @@ -200,7 +200,7 @@ def rc_processor(change, changed_categories): from src.misc import LinkParser LinkParser = LinkParser(client.WIKI_JUST_DOMAIN) metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None), - page_id=change.get("pageid", None)) + page_id=change.get("pageid", None), timestamp=''.join(filter(str.isdigit, change.get("timestamp", "1970101000000")))) logger.debug(change) context = Context(settings["appearance"]["mode"], "recentchanges", settings["webhookURL"], client, formatters_i18n, settings) if ("actionhidden" in change or "suppressed" in change) and "suppressed" not in settings["ignored"]: # if event is hidden using suppression @@ -267,7 +267,10 @@ def rc_processor(change, changed_categories): delete_messages(dict(logid=logid)) elif context.event == "delete/revision" and AUTO_SUPPRESSION_ENABLED: logparams = change.get('logparams', {"ids": []}) - if logparams.get("type", "") in ("revision", "logging", "oldimage"): + if logparams.get("type", "") == "oldimage": + logger.debug("Revdeleting image code reached") + redact_image_files(logparams.get("ids", []), logparams.get("new", {}), change.get("pageid")) + elif logparams.get("type", "") in ("revision", "logging"): if settings["appearance"]["mode"] == "embed": redact_messages(logparams.get("ids", []), 0, logparams.get("new", {})) if "content" in logparams.get("new", {}) and settings.get("appearance", {}).get("embed", {}).get("show_edit_changes", False): # Also redact revisions in the middle and next ones in case of content (diffs leak) diff --git a/test/test_queue.py b/test/test_queue.py index 66b3161..67e4ee8 100644 --- a/test/test_queue.py +++ b/test/test_queue.py @@ -23,7 +23,8 @@ from src.discord.message import DiscordMessage, DiscordMessageMetadata def create_dummy(id: int = 0, **kwargs) -> Tuple[DiscordMessage, DiscordMessageMetadata]: dm = DiscordMessage(event_type="log/{}".format(id), message_type="embed", webhook_url="https://example.com/") dmm = DiscordMessageMetadata("POST", log_id=kwargs.get("log_id", int(id)*10), page_id=kwargs.get("page_id", int(id)), - rev_id=kwargs.get("rev_id", int(id)*10), webhook_url=kwargs.get("webhook_url", "https://example.com/")) + rev_id=kwargs.get("rev_id", int(id)*10), webhook_url=kwargs.get("webhook_url", "https://example.com/"), + timestamp=''.join(filter(str.isdigit, kwargs.get("timestamp", "1970101000000")))) return dm, dmm