From 6858be4e61c414fd6fd94b398de2c82b66ccb947 Mon Sep 17 00:00:00 2001 From: Frisk Date: Fri, 30 Apr 2021 22:11:22 +0200 Subject: [PATCH] Added a few more formatters, added support for aliases argument for formatter decorators --- docs/API spec.md | 7 +++ extensions/base/mediawiki.py | 117 +++++++++++++++++++++++++++++++---- src/api/formatter.py | 13 ++-- src/rc_formatters.py | 78 +---------------------- src/rcgcdw.py | 11 +++- src/wiki.py | 2 +- 6 files changed, 132 insertions(+), 96 deletions(-) diff --git a/docs/API spec.md b/docs/API spec.md index 5d9d332..1f7c02d 100644 --- a/docs/API spec.md +++ b/docs/API spec.md @@ -28,6 +28,13 @@ Directory with extensions should be possible to be changed using settings.json ## API api object exposes various data which allows to extend the usefulness of what can be then sent to Discord. +### Language support + + + +### Formatter event types +Formatters can be added based on their "event type". Event type is determined by `type` property for events in Recent Changes MediaWiki API. However in case of log events this becomes not enough and log events are chosen by "logtype/logaction" combination (for example `upload/overwrite`). +There are also additional made up cases like a single event type of "abuselog" for all abuselog related events and "discussion/discussiontype" for Fandom's Discussion technology integration. ## Example formatter diff --git a/extensions/base/mediawiki.py b/extensions/base/mediawiki.py index 6148f81..405685d 100644 --- a/extensions/base/mediawiki.py +++ b/extensions/base/mediawiki.py @@ -15,11 +15,13 @@ import logging import math +import re +import time from src.discord.message import DiscordMessage from src.api import formatter from src.i18n import rc_formatters from src.api.context import Context -from src.api.util import embed_helper, sanitize_to_url, parse_mediawiki_changes, clean_link, compact_author +from src.api.util import embed_helper, sanitize_to_url, parse_mediawiki_changes, clean_link, compact_author, create_article_path from src.configloader import settings from src.exceptions import * @@ -28,9 +30,9 @@ _ = rc_formatters.gettext logger = logging.getLogger("extensions.base") -# Page edit - event edit +# Page edit - event edit, New - page creation -@formatter.embed(event="edit", mode="embed") +@formatter.embed(event="edit", mode="embed", aliases=["new"]) def embed_edit(ctx: Context, change: dict) -> DiscordMessage: embed = DiscordMessage(ctx.message_type, ctx.event, ctx.webhook_url) embed_helper(ctx, embed, change) @@ -81,7 +83,7 @@ def embed_edit(ctx: Context, change: dict) -> DiscordMessage: return embed -@formatter.compact(event="edit", mode="compact") +@formatter.compact(event="edit", mode="compact", aliases=["new"]) def compact_edit(ctx: Context, change: dict): parsed_comment = "" if ctx.parsedcomment is None else " *(" + ctx.parsedcomment + ")*" author, author_url = compact_author(ctx, change) @@ -110,14 +112,105 @@ def compact_edit(ctx: Context, change: dict): return DiscordMessage(ctx.message_type, ctx.event, ctx.webhook_url, content=content) -# Page creation - event new aliases to embed_edit since they share a lot of their code - -@formatter.embed(event="new", mode="embed") -def embed_new(ctx, change): - return embed_edit(ctx, change) +# Upload - upload/reupload, upload/upload +@formatter.embed(event="upload/upload", mode="embed", aliases=["upload/overwrite", "upload/revert"]) +def embed_upload_upload(ctx, change): + license = None + embed = DiscordMessage(ctx.message_type, ctx.event, ctx.webhook_url) + action = ctx.event + embed_helper(ctx, embed, change) + urls = ctx.client.make_api_request("{wiki}?action=query&format=json&prop=imageinfo&list=&meta=&titles={filename}&iiprop=timestamp%7Curl%7Carchivename&iilimit=5".format( + wiki=ctx.WIKI_API_PATH, filename=sanitize_to_url(change["title"])), "query", "pages") + link = create_article_path(sanitize_to_url(change["title"])) + image_direct_url = None + # Make a request for file revisions so we can get direct URL to the image for embed + if urls is not None: + logger.debug(urls) + if "-1" not in urls: # image still exists and not removed + try: + img_info = next(iter(urls.values()))["imageinfo"] + for num, revision in enumerate(img_info): + if revision["timestamp"] == change["logparams"]["img_timestamp"]: # find the correct revision corresponding for this log entry + image_direct_url = "{rev}?{cache}".format(rev=revision["url"], cache=int(time.time() * 5)) # cachebusting + break + except KeyError: + logger.exception( + "Wiki did not respond with extended information about file. The preview will not be shown.") + else: + logger.warning("Request for additional image information have failed. The preview will not be shown.") + if action in ("upload/overwrite", "upload/revert"): + if image_direct_url: + try: + revision = img_info[num + 1] + except IndexError: + logger.exception( + "Could not analize the information about the image (does it have only one version when expected more in overwrite?) which resulted in no Options field: {}".format( + img_info)) + else: + undolink = "{wiki}index.php?title={filename}&action=revert&oldimage={archiveid}".format( + wiki=ctx.client.WIKI_SCRIPT_PATH, filename=sanitize_to_url(change["title"]), archiveid=revision["archivename"]) + embed.add_field(_("Options"), _("([preview]({link}) | [undo]({undolink}))").format( + link=image_direct_url, undolink=undolink)) + if settings["appearance"]["embed"]["embed_images"]: + embed["image"]["url"] = image_direct_url + if action == "upload/overwrite": + embed["title"] = _("Uploaded a new version of {name}").format(name=change["title"]) + elif action == "upload/revert": + embed["title"] = _("Reverted a version of {name}").format(name=change["title"]) + else: + embed["title"] = _("Uploaded {name}").format(name=change["title"]) + if settings["license_detection"]: + article_content = ctx.client.make_api_request( + "{wiki}?action=query&format=json&prop=revisions&titles={article}&rvprop=content".format( + wiki=ctx.client.WIKI_API_PATH, article=sanitize_to_url(change["title"])), "query", "pages") + if article_content is None: + logger.warning("Something went wrong when getting license for the image") + return 0 + if "-1" not in article_content: + content = list(article_content.values())[0]['revisions'][0]['*'] + try: + matches = re.search(re.compile(settings["license_regex"], re.IGNORECASE), content) + if matches is not None: + license = matches.group("license") + else: + if re.search(re.compile(settings["license_regex_detect"], re.IGNORECASE), content) is None: + license = _("**No license!**") + else: + license = "?" + except IndexError: + logger.error( + "Given regex for the license detection is incorrect. It does not have a capturing group called \"license\" specified. Please fix license_regex value in the config!") + license = "?" + except re.error: + logger.error( + "Given regex for the license detection is incorrect. Please fix license_regex or license_regex_detect values in the config!") + license = "?" + if license is not None: + ctx.parsedcomment += _("\nLicense: {}").format(license) + if image_direct_url: + embed.add_field(_("Options"), _("([preview]({link}))").format(link=image_direct_url)) + if settings["appearance"]["embed"]["embed_images"]: + embed["image"]["url"] = image_direct_url + return embed -@formatter.compact(event="new", mode="compact") -def compact_new(ctx, change): - return compact_edit(ctx, change) +@formatter.compact(event="upload/upload", mode="compact") +def compact_upload_upload(ctx, change): + author, author_url = compact_author(ctx, change) + file_link = clean_link(create_article_path(sanitize_to_url(change["title"]))) + content = _("[{author}]({author_url}) uploaded [{file}]({file_link}){comment}").format(author=author, + author_url=author_url, + file=change["title"], + file_link=file_link, + comment="" if ctx.parsedcomment is None else " *(" + ctx.parsedcomment + ")*") + return DiscordMessage(ctx.message_type, ctx.event, ctx.webhook_url, content=content) +# delete - Page deletion +@formatter.embed(event="delete/delete", mode="embed") +def embed_delete(ctx, change): + embed = DiscordMessage(ctx.message_type, ctx.event, ctx.webhook_url) + embed_helper(ctx, embed, change) + link = create_article_path(sanitize_to_url(change["title"])) + embed["title"] = _("Deleted page {article}").format(article=change["title"]) + if AUTO_SUPPRESSION_ENABLED: + delete_messages(dict(pageid=change.get("pageid"))) \ No newline at end of file diff --git a/src/api/formatter.py b/src/api/formatter.py index 76ea727..b808c79 100644 --- a/src/api/formatter.py +++ b/src/api/formatter.py @@ -36,11 +36,12 @@ def _register_formatter(func: Callable[[dict], DiscordMessage], kwargs: dict[str if action_type is None: raise FormatterBreaksAPISpec("event type") if settings["appearance"]["mode"] == formatter_type: - if action_type in src.api.hooks.formatter_hooks: - logger.warning(f"Action {action_type} is already defined inside of " - f"{src.api.hooks.formatter_hooks[action_type].__module__}! " - f"Overwriting it with one from {func.__module__}") - src.api.hooks.formatter_hooks[action_type] = func + for act in [action_type] + kwargs.get("aliases", []): # Make action_type string a list and merge with aliases + if act in src.api.hooks.formatter_hooks: + logger.warning(f"Action {act} is already defined inside of " + f"{src.api.hooks.formatter_hooks[act].__module__}! " + f"Overwriting it with one from {func.__module__}") + src.api.hooks.formatter_hooks[act] = func def embed(**kwargs): @@ -49,6 +50,7 @@ def embed(**kwargs): :key event: Event string :key mode: Discord Message mode + :key aliases: Allows to register multiple events under same function :return: """ @@ -65,6 +67,7 @@ def compact(**kwargs): :key event: Event string :key mode: Discord Message mode + :key aliases: Allows to register multiple events under same function :return: """ diff --git a/src/rc_formatters.py b/src/rc_formatters.py index a1a1474..4b3b254 100644 --- a/src/rc_formatters.py +++ b/src/rc_formatters.py @@ -715,83 +715,9 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes): if action in ("edit", "new"): # edit or new page elif action in ("upload/overwrite", "upload/upload", "upload/revert"): # sending files - license = None - urls = safe_read(recent_changes._safe_request( - "{wiki}?action=query&format=json&prop=imageinfo&list=&meta=&titles={filename}&iiprop=timestamp%7Curl%7Carchivename&iilimit=5".format( - wiki=WIKI_API_PATH, filename=change["title"])), "query", "pages") - link = create_article_path(change["title"]) - additional_info_retrieved = False - if urls is not None: - logger.debug(urls) - if "-1" not in urls: # image still exists and not removed - try: - img_info = next(iter(urls.values()))["imageinfo"] - for num, revision in enumerate(img_info): - if revision["timestamp"] == change["logparams"]["img_timestamp"]: # find the correct revision corresponding for this log entry - image_direct_url = "{rev}?{cache}".format(rev=revision["url"], cache=int(time.time()*5)) # cachebusting - additional_info_retrieved = True - break - except KeyError: - logger.warning("Wiki did not respond with extended information about file. The preview will not be shown.") - else: - logger.warning("Request for additional image information have failed. The preview will not be shown.") - if action in ("upload/overwrite", "upload/revert"): - if additional_info_retrieved: - article_encoded = change["title"].replace(" ", "_").replace("%", "%25").replace("\\", "%5C").replace("&", "%26").replace(')', '\\)') - try: - revision = img_info[num+1] - except IndexError: - logger.exception("Could not analize the information about the image (does it have only one version when expected more in overwrite?) which resulted in no Options field: {}".format(img_info)) - else: - undolink = "{wiki}index.php?title={filename}&action=revert&oldimage={archiveid}".format( - wiki=WIKI_SCRIPT_PATH, filename=article_encoded, archiveid=revision["archivename"]) - embed.add_field(_("Options"), _("([preview]({link}) | [undo]({undolink}))").format( - link=image_direct_url, undolink=undolink)) - if settings["appearance"]["embed"]["embed_images"]: - embed["image"]["url"] = image_direct_url - if action == "upload/overwrite": - embed["title"] = _("Uploaded a new version of {name}").format(name=change["title"]) - elif action == "upload/revert": - embed["title"] = _("Reverted a version of {name}").format(name=change["title"]) - else: - embed["title"] = _("Uploaded {name}").format(name=change["title"]) - if settings["license_detection"]: - article_content = safe_read(recent_changes._safe_request( - "{wiki}?action=query&format=json&prop=revisions&titles={article}&rvprop=content".format( - wiki=WIKI_API_PATH, article=quote_plus(change["title"], safe=''))), "query", "pages") - if article_content is None: - logger.warning("Something went wrong when getting license for the image") - return 0 - if "-1" not in article_content: - content = list(article_content.values())[0]['revisions'][0]['*'] - try: - matches = re.search(re.compile(settings["license_regex"], re.IGNORECASE), content) - if matches is not None: - license = matches.group("license") - else: - if re.search(re.compile(settings["license_regex_detect"], re.IGNORECASE), content) is None: - license = _("**No license!**") - else: - license = "?" - except IndexError: - logger.error( - "Given regex for the license detection is incorrect. It does not have a capturing group called \"license\" specified. Please fix license_regex value in the config!") - license = "?" - except re.error: - logger.error( - "Given regex for the license detection is incorrect. Please fix license_regex or license_regex_detect values in the config!") - license = "?" - if license is not None: - parsed_comment += _("\nLicense: {}").format(license) - if additional_info_retrieved: - embed.add_field(_("Options"), _("([preview]({link}))").format(link=image_direct_url)) - if settings["appearance"]["embed"]["embed_images"]: - embed["image"]["url"] = image_direct_url + elif action == "delete/delete": - link = create_article_path(change["title"]) - embed["title"] = _("Deleted page {article}").format(article=change["title"]) - if AUTO_SUPPRESSION_ENABLED: - delete_messages(dict(pageid=change.get("pageid"))) + elif action == "delete/delete_redir": link = create_article_path(change["title"]) embed["title"] = _("Deleted redirect {article} by overwriting").format(article=change["title"]) diff --git a/src/rcgcdw.py b/src/rcgcdw.py index 6fe2da6..a436278 100644 --- a/src/rcgcdw.py +++ b/src/rcgcdw.py @@ -266,8 +266,15 @@ def rc_processor(change, changed_categories): discord_message: Optional[DiscordMessage] = default_message(identification_string, formatter_hooks)(context, change) send_to_discord(discord_message, metadata) -def abuselog_processing(entry, recent_changes): - abuselog_appearance_mode(entry, recent_changes) + +def abuselog_processing(entry): + action = "abuselog" + if action in settings["ignored"]: + return + context = Context(settings["appearance"]["mode"], settings["webhookURL"], client) + context.event = action + discord_message: Optional[DiscordMessage] = default_message(action, formatter_hooks)(context, entry) + send_to_discord(discord_message, DiscordMessageMetadata("POST")) load_extensions() diff --git a/src/wiki.py b/src/wiki.py index cf9ce03..2fd8792 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -217,7 +217,7 @@ class Wiki(object): continue if entry["id"] <= recent_id: continue - self.abuse_processor(entry, self) + self.abuse_processor(entry) return entry["id"] def fetch_changes(self, amount):