mirror of
https://gitlab.com/chicken-riders/RcGcDw.git
synced 2025-02-23 00:24:09 +00:00
Chenges in structure, updated API docs
This commit is contained in:
Frisk
parent
a9d6fa1830
commit
2aa8387b72
|
|
@ -6,6 +6,7 @@ A class allowing to change the message content and/or execute additional actions
|
|||
|
||||
### Formatters
|
||||
Formatters allow to specify how does a Discord message look like depending on message mode (embed, compact) and type of the event that had happened on the wiki (new, edit etc).
|
||||
If formatter for given event is not registered, the script will look for formatter for event "generic" and if this is also not found it will throw a wartning.
|
||||
|
||||
### Post-processing hook
|
||||
A class allowing to change the message content and/or execute additional actions after message has been processed by the formatter. This type of hook executes after a formatter.
|
||||
|
|
@ -35,23 +36,30 @@ api object exposes various data which allows to extend the usefulness of what ca
|
|||
import logging
|
||||
from src.discord.message import DiscordMessage
|
||||
from src.api import formatter
|
||||
from src.api.context import Context
|
||||
from src.api.util import create_article_path, link_formatter
|
||||
from src.i18n import rc_formatters
|
||||
|
||||
_ = rc_formatters.gettext
|
||||
|
||||
logger = logging.getLogger("extensions.abusefilter")
|
||||
|
||||
class abusefilter(Formatter):
|
||||
def __init__(self, api):
|
||||
super().__init__(api)
|
||||
class abusefilter():
|
||||
def __init__(self, api):
|
||||
super().__init__(api)
|
||||
|
||||
@formatter.embed(event="abuselog/modify", mode="embed")
|
||||
def embed_modify(self, change: dict) -> DiscordMessage:
|
||||
return DiscordMessage
|
||||
@formatter.embed(event="abuselog/modify", mode="embed")
|
||||
def embed_modify(self, ctx: Context, change: dict) -> DiscordMessage:
|
||||
embed = DiscordMessage(ctx.message_type, ctx.event, webhook_url=ctx.webhook_url)
|
||||
embed.set_link(create_article_path("Special:AbuseFilter/history/{number}/diff/prev/{historyid}".format(number=change["logparams"]['newId'], historyid=change["logparams"]["historyId"])))
|
||||
embed["title"] = _("Edited abuse filter number {number}").format(number=change["logparams"]['newId'])
|
||||
return embed
|
||||
|
||||
@formatter.compact(event="abuselog/modify")
|
||||
def compact_modify(self, change: dict) -> DiscordMessage:
|
||||
return DiscordMessage
|
||||
@formatter.compact(event="abuselog/modify")
|
||||
def embed_modify(self, ctx: Context, change: dict) -> DiscordMessage:
|
||||
link = link_formatter(create_article_path("Special:AbuseFilter/history/{number}/diff/prev/{historyid}".format(number=change["logparams"]['newId'], historyid=change["logparams"]["historyId"])))
|
||||
content = _("[{author}]({author_url}) edited abuse filter [number {number}]({filter_url})").format(author=author, author_url=author_url, number=change["logparams"]['newId'], filter_url=link)
|
||||
return DiscordMessage
|
||||
|
||||
```
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,6 @@
|
|||
# along with RcGcDw. If not, see <http://www.gnu.org/licenses/>.
|
||||
|
||||
import src.rcgcdw
|
||||
import src.rc
|
||||
import src.misc
|
||||
from typing import Union
|
||||
from collections import OrderedDict
|
||||
|
|
@ -25,7 +24,7 @@ class Client:
|
|||
"""
|
||||
def __init__(self):
|
||||
self._formatters = src.rcgcdw.formatter_hooks
|
||||
self.__recent_changes = src.rc.wiki
|
||||
self.__recent_changes = src.rcgcdw.wiki
|
||||
self.WIKI_API_PATH = src.misc.WIKI_API_PATH
|
||||
self.WIKI_ARTICLE_PATH = src.misc.WIKI_ARTICLE_PATH
|
||||
self.WIKI_SCRIPT_PATH = src.misc.WIKI_SCRIPT_PATH
|
||||
|
|
@ -63,4 +62,5 @@ class Client:
|
|||
def get_formatters(self):
|
||||
return self._formatters
|
||||
|
||||
|
||||
client = Client()
|
||||
|
|
|
|||
|
|
@ -21,3 +21,6 @@ class Context:
|
|||
self.webhook_url = webhook_url
|
||||
self.message_type = message_type
|
||||
self.event = None
|
||||
|
||||
def set_categories(self, cats):
|
||||
self.categories = cats
|
||||
|
|
@ -89,6 +89,9 @@ class DiscordMessage:
|
|||
def set_name(self, name):
|
||||
self.webhook_object["username"] = name
|
||||
|
||||
def set_link(self, link):
|
||||
self.embed["link"] = link
|
||||
|
||||
|
||||
class DiscordMessageRaw(DiscordMessage):
|
||||
def __init__(self, content: dict, webhook_url: str):
|
||||
|
|
|
|||
|
|
@ -174,14 +174,14 @@ def send_to_discord(data: Optional[DiscordMessage], meta: DiscordMessageMetadata
|
|||
logger.info("Message \"{}\" has been rejected due to matching filter ({}).".format(
|
||||
to_check, regex))
|
||||
return # discard the message without anything
|
||||
if messagequeue:
|
||||
messagequeue.add_message((data, meta))
|
||||
else:
|
||||
code = send_to_discord_webhook(data, metadata=meta)
|
||||
if code == 3:
|
||||
if messagequeue:
|
||||
messagequeue.add_message((data, meta))
|
||||
elif code == 2:
|
||||
time.sleep(5.0)
|
||||
messagequeue.add_message((data, meta))
|
||||
elif code < 2:
|
||||
pass
|
||||
else:
|
||||
code = send_to_discord_webhook(data, metadata=meta)
|
||||
if code == 3:
|
||||
messagequeue.add_message((data, meta))
|
||||
elif code == 2:
|
||||
time.sleep(5.0)
|
||||
messagequeue.add_message((data, meta))
|
||||
elif code < 2:
|
||||
pass
|
||||
|
|
@ -19,7 +19,7 @@ import gettext
|
|||
from urllib.parse import quote_plus
|
||||
|
||||
from src.configloader import settings
|
||||
from src.misc import link_formatter, create_article_path, escape_formatting
|
||||
from src.api.util import link_formatter, escape_formatting, create_article_path
|
||||
from src.discord.queue import send_to_discord
|
||||
from src.discord.message import DiscordMessage, DiscordMessageMetadata
|
||||
from src.i18n import discussion_formatters
|
||||
|
|
|
|||
21
src/misc.py
21
src/misc.py
|
|
@ -15,10 +15,12 @@
|
|||
# You should have received a copy of the GNU General Public License
|
||||
# along with RcGcDw. If not, see <http://www.gnu.org/licenses/>.
|
||||
import base64
|
||||
import json, logging, sys, re
|
||||
import json, logging, sys
|
||||
from html.parser import HTMLParser
|
||||
from urllib.parse import urlparse, urlunparse, quote
|
||||
from urllib.parse import urlparse, urlunparse
|
||||
import requests
|
||||
|
||||
from src.api.util import escape_formatting
|
||||
from src.configloader import settings
|
||||
from src.discord.message import DiscordMessage, DiscordMessageMetadata
|
||||
from src.discord.queue import messagequeue, send_to_discord
|
||||
|
|
@ -105,16 +107,6 @@ def weighted_average(value, weight, new_value):
|
|||
return round(((value * weight) + new_value) / (weight + 1), 2)
|
||||
|
||||
|
||||
def link_formatter(link):
|
||||
"""Formats a link to not embed it"""
|
||||
return "<" + quote(link.replace(" ", "_"), "/:?=&") + ">"
|
||||
|
||||
|
||||
def escape_formatting(data):
|
||||
"""Escape Discord formatting"""
|
||||
return re.sub(r"([`_*~<>{}@/|\\])", "\\\\\\1", data, 0)
|
||||
|
||||
|
||||
class ContentParser(HTMLParser):
|
||||
"""ContentPerser is an implementation of HTMLParser that parses output of action=compare&prop=diff API request
|
||||
for two MediaWiki revisions. It extracts the following:
|
||||
|
|
@ -276,11 +268,6 @@ def prepare_paths(path, dry=False):
|
|||
prepare_paths(settings["wiki_url"])
|
||||
|
||||
|
||||
def create_article_path(article: str) -> str:
|
||||
"""Takes the string and creates an URL with it as the article name"""
|
||||
return WIKI_ARTICLE_PATH.replace("$1", article)
|
||||
|
||||
|
||||
def send_simple(msgtype, message, name, avatar):
|
||||
discord_msg = DiscordMessage("compact", msgtype, settings["webhookURL"], content=message)
|
||||
discord_msg.set_avatar(avatar)
|
||||
|
|
|
|||
|
|
@ -27,8 +27,9 @@ from urllib.parse import quote_plus, quote
|
|||
from bs4 import BeautifulSoup
|
||||
|
||||
from src.configloader import settings
|
||||
from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, safe_read, \
|
||||
from src.misc import WIKI_SCRIPT_PATH, safe_read, \
|
||||
WIKI_API_PATH, ContentParser, profile_field_name, LinkParser, AUTO_SUPPRESSION_ENABLED
|
||||
from src.api.util import link_formatter, create_article_path
|
||||
from src.discord.queue import send_to_discord
|
||||
from src.discord.message import DiscordMessage, DiscordMessageMetadata
|
||||
|
||||
|
|
|
|||
|
|
@ -19,18 +19,23 @@
|
|||
# WARNING! SHITTY CODE AHEAD. ENTER ONLY IF YOU ARE SURE YOU CAN TAKE IT
|
||||
# You have been warned
|
||||
|
||||
import time, logging.config, requests, datetime, gettext, math, os.path, schedule, sys
|
||||
import time, logging.config, requests, datetime, gettext, math, os.path, schedule, sys, re
|
||||
|
||||
import src.misc
|
||||
from collections import defaultdict, Counter
|
||||
|
||||
import src.api.client
|
||||
from typing import Optional
|
||||
from src.api.context import Context
|
||||
from src.configloader import settings
|
||||
from src.misc import add_to_dict, datafile, \
|
||||
WIKI_API_PATH, create_article_path
|
||||
WIKI_API_PATH, LinkParser
|
||||
from src.api.util import create_article_path, default_message
|
||||
from src.discord.queue import send_to_discord
|
||||
from src.discord.message import DiscordMessage, DiscordMessageMetadata
|
||||
from src.rc import wiki
|
||||
from src.exceptions import MWError
|
||||
from src.i18n import rcgcdw
|
||||
from src.wiki import Wiki
|
||||
|
||||
_ = rcgcdw.gettext
|
||||
ngettext = rcgcdw.ngettext
|
||||
|
|
@ -67,6 +72,12 @@ if settings["limitrefetch"] != -1 and os.path.exists("lastchange.txt") is True:
|
|||
os.remove("lastchange.txt")
|
||||
|
||||
|
||||
def no_formatter(ctx: Context, change: dict) -> None:
|
||||
logger.warning(f"There is no formatter specified for {ctx.event}! Ignoring event.")
|
||||
return
|
||||
|
||||
formatter_hooks["no_formatter"] = no_formatter
|
||||
|
||||
def day_overview_request():
|
||||
logger.info("Fetching daily overview... This may take up to 30 seconds!")
|
||||
timestamp = (datetime.datetime.utcnow() - datetime.timedelta(hours=24)).isoformat(timespec='milliseconds')
|
||||
|
|
@ -216,7 +227,48 @@ def day_overview():
|
|||
logger.debug("function requesting changes for day overview returned with error code")
|
||||
|
||||
|
||||
def rc_processor(change, changed_categories):
|
||||
"""Prepares essential information for both embed and compact message format."""
|
||||
LinkParser = LinkParser()
|
||||
metadata = DiscordMessageMetadata("POST", rev_id=change.get("revid", None), log_id=change.get("logid", None),
|
||||
page_id=change.get("pageid", None))
|
||||
logger.debug(change)
|
||||
context = Context(settings["appearance"]["mode"], settings["webhookURL"], src.api.client.client)
|
||||
if ("actionhidden" in change or "suppressed" in change) and "suppressed" not in settings["ignored"]: # if event is hidden using suppression
|
||||
context.event = "suppressed"
|
||||
discord_message: Optional[DiscordMessage] = default_message("suppressed", formatter_hooks)(context, change)
|
||||
else:
|
||||
if "commenthidden" not in change:
|
||||
LinkParser.feed(change.get("parsedcomment", ""))
|
||||
parsed_comment = LinkParser.new_string
|
||||
parsed_comment = re.sub(r"(`|_|\*|~|{|}|\|\|)", "\\\\\\1", parsed_comment)
|
||||
else:
|
||||
parsed_comment = _("~~hidden~~")
|
||||
if "userhidden" in change:
|
||||
change["user"] = _("hidden")
|
||||
if change.get("ns", -1) in settings.get("ignored_namespaces", ()):
|
||||
return
|
||||
if change["type"] in ["edit", "new"]:
|
||||
logger.debug("List of categories in essential_info: {}".format(changed_categories))
|
||||
identification_string = change["type"]
|
||||
context.set_categories(changed_categories)
|
||||
elif change["type"] == "categorize":
|
||||
return
|
||||
elif change["type"] == "log":
|
||||
identification_string = "{logtype}/{logaction}".format(logtype=change["logtype"], logaction=change["logaction"])
|
||||
else:
|
||||
identification_string = change.get("type", "unknown") # If event doesn't have a type
|
||||
if identification_string in settings["ignored"]:
|
||||
return
|
||||
discord_message: Optional[DiscordMessage] = default_message(identification_string, formatter_hooks)(context, change)
|
||||
send_to_discord(discord_message, metadata)
|
||||
|
||||
def abuselog_processing(entry, recent_changes):
|
||||
abuselog_appearance_mode(entry, recent_changes)
|
||||
|
||||
|
||||
# Log in and download wiki information
|
||||
wiki = Wiki(rc_processor, abuselog_processing)
|
||||
try:
|
||||
if settings["wiki_bot_login"] and settings["wiki_bot_password"]:
|
||||
wiki.log_in()
|
||||
|
|
@ -254,6 +306,7 @@ if 1 == 2: # additional translation strings in unreachable code
|
|||
|
||||
load_extensions()
|
||||
|
||||
|
||||
if TESTING:
|
||||
logger.debug("DEBUGGING ")
|
||||
storage["rcid"] = 1
|
||||
|
|
|
|||
|
|
@ -20,17 +20,15 @@ import sys
|
|||
import time
|
||||
import logging
|
||||
import requests
|
||||
import src.api.client
|
||||
from bs4 import BeautifulSoup
|
||||
|
||||
from src.configloader import settings
|
||||
from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, LinkParser, \
|
||||
from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, safe_read, \
|
||||
AUTO_SUPPRESSION_ENABLED, parse_mw_request_info
|
||||
from src.discord.queue import messagequeue
|
||||
from src.exceptions import MWError, BadRequest, ClientError, ServerError, MediaWikiError
|
||||
from src.session import session
|
||||
from src.api.context import Context
|
||||
from typing import Union
|
||||
from typing import Union, Callable
|
||||
# from src.rc_formatters import compact_formatter, embed_formatter, compact_abuselog_formatter, embed_abuselog_formatter
|
||||
from src.i18n import rc
|
||||
from collections import OrderedDict
|
||||
|
|
@ -41,11 +39,11 @@ storage = datafile
|
|||
|
||||
logger = logging.getLogger("rcgcdw.rc")
|
||||
|
||||
LinkParser = LinkParser()
|
||||
|
||||
class Wiki(object):
|
||||
"""Store verious data and functions related to wiki and fetching of Recent Changes"""
|
||||
def __init__(self):
|
||||
def __init__(self, rc_processor: Callable, abuse_processor: Callable):
|
||||
self.rc_processor = rc_processor
|
||||
self.abuse_processor = abuse_processor
|
||||
self.map_ips = {}
|
||||
self.downtimecredibility = 0
|
||||
self.last_downtime = 0
|
||||
|
|
@ -58,6 +56,7 @@ class Wiki(object):
|
|||
self.logged_in = False
|
||||
self.initial_run_complete = False
|
||||
|
||||
|
||||
@staticmethod
|
||||
def handle_mw_errors(request):
|
||||
if "errors" in request:
|
||||
|
|
@ -166,30 +165,30 @@ class Wiki(object):
|
|||
"There were too many new events, but the limit was high enough we don't care anymore about fetching them all.")
|
||||
if change["type"] == "categorize":
|
||||
if "commenthidden" not in change:
|
||||
if len(wiki.mw_messages.keys()) > 0:
|
||||
if len(self.mw_messages.keys()) > 0:
|
||||
cat_title = change["title"].split(':', 1)[1]
|
||||
# I so much hate this, blame Markus for making me do this
|
||||
if change["revid"] not in categorize_events:
|
||||
categorize_events[change["revid"]] = {"new": set(), "removed": set()}
|
||||
comment_to_match = re.sub(r'<.*?a>', '', change["parsedcomment"])
|
||||
if wiki.mw_messages["recentchanges-page-added-to-category"] in comment_to_match or \
|
||||
wiki.mw_messages[
|
||||
if self.mw_messages["recentchanges-page-added-to-category"] in comment_to_match or \
|
||||
self.mw_messages[
|
||||
"recentchanges-page-added-to-category-bundled"] in comment_to_match:
|
||||
categorize_events[change["revid"]]["new"].add(cat_title)
|
||||
logger.debug("Matched {} to added category for {}".format(cat_title, change["revid"]))
|
||||
elif wiki.mw_messages[
|
||||
elif self.mw_messages[
|
||||
"recentchanges-page-removed-from-category"] in comment_to_match or \
|
||||
wiki.mw_messages[
|
||||
self.mw_messages[
|
||||
"recentchanges-page-removed-from-category-bundled"] in comment_to_match:
|
||||
categorize_events[change["revid"]]["removed"].add(cat_title)
|
||||
logger.debug("Matched {} to removed category for {}".format(cat_title, change["revid"]))
|
||||
else:
|
||||
logger.debug(
|
||||
"Unknown match for category change with messages {}, {}, {}, {} and comment_to_match {}".format(
|
||||
wiki.mw_messages["recentchanges-page-added-to-category"],
|
||||
wiki.mw_messages["recentchanges-page-removed-from-category"],
|
||||
wiki.mw_messages["recentchanges-page-removed-from-category-bundled"],
|
||||
wiki.mw_messages["recentchanges-page-added-to-category-bundled"],
|
||||
self.mw_messages["recentchanges-page-added-to-category"],
|
||||
self.mw_messages["recentchanges-page-removed-from-category"],
|
||||
self.mw_messages["recentchanges-page-removed-from-category-bundled"],
|
||||
self.mw_messages["recentchanges-page-added-to-category-bundled"],
|
||||
comment_to_match))
|
||||
else:
|
||||
logger.warning(
|
||||
|
|
@ -204,7 +203,7 @@ class Wiki(object):
|
|||
logger.debug("Change ({}) is lower or equal to recent_id {}".format(change["rcid"], recent_id))
|
||||
continue
|
||||
logger.debug(recent_id)
|
||||
rc_processor(change, categorize_events.get(change.get("revid"), None))
|
||||
self.rc_processor(change, categorize_events.get(change.get("revid"), None))
|
||||
return highest_id
|
||||
|
||||
def prepare_abuse_log(self, abuse_log: list):
|
||||
|
|
@ -218,7 +217,7 @@ class Wiki(object):
|
|||
continue
|
||||
if entry["id"] <= recent_id:
|
||||
continue
|
||||
abuselog_processing(entry, self)
|
||||
self.abuse_processor(entry, self)
|
||||
return entry["id"]
|
||||
|
||||
def fetch_changes(self, amount):
|
||||
|
|
@ -368,7 +367,7 @@ class Wiki(object):
|
|||
if not looped:
|
||||
while 1: # recursed loop, check for connection (every 10 seconds) as long as three services are down, don't do anything else
|
||||
if self.check_connection(looped=True):
|
||||
wiki.fetch(amount=settings["limitrefetch"])
|
||||
self.fetch(amount=settings["limitrefetch"])
|
||||
break
|
||||
time.sleep(10)
|
||||
return False
|
||||
|
|
@ -451,49 +450,3 @@ class Wiki(object):
|
|||
comment = comment[0:1000] + "…"
|
||||
return comment
|
||||
return ""
|
||||
|
||||
|
||||
wiki = Wiki()
|
||||
|
||||
|
||||
def rc_processor(change, changed_categories):
|
||||
"""Prepares essential information for both embed and compact message format."""
|
||||
formatters = src.api.client.client.get_formatters() # TODO Make it better? Importing might be a hell
|
||||
logger.debug(change)
|
||||
context = Context(settings["appearance"]["mode"], settings["webhookURL"], src.api.client.client)
|
||||
if ("actionhidden" in change or "suppressed" in change) and "suppressed" not in settings["ignored"]: # if event is hidden using suppression
|
||||
context.event = "suppressed"
|
||||
if "commenthidden" not in change:
|
||||
LinkParser.feed(change["parsedcomment"])
|
||||
parsed_comment = LinkParser.new_string
|
||||
LinkParser.new_string = ""
|
||||
parsed_comment = re.sub(r"(`|_|\*|~|{|}|\|\|)", "\\\\\\1", parsed_comment, 0)
|
||||
else:
|
||||
parsed_comment = _("~~hidden~~")
|
||||
if not parsed_comment:
|
||||
parsed_comment = None
|
||||
if "userhidden" in change:
|
||||
change["user"] = _("hidden")
|
||||
if change.get("ns", -1) in settings.get("ignored_namespaces", ()):
|
||||
return
|
||||
if change["type"] in ["edit", "new"]:
|
||||
logger.debug("List of categories in essential_info: {}".format(changed_categories))
|
||||
identification_string = change["type"]
|
||||
elif change["type"] == "log":
|
||||
identification_string = "{logtype}/{logaction}".format(logtype=change["logtype"], logaction=change["logaction"])
|
||||
if identification_string not in supported_logs:
|
||||
logger.warning(
|
||||
"This event is not implemented in the script. Please make an issue on the tracker attaching the following info: wiki url, time, and this information: {}".format(
|
||||
change))
|
||||
return
|
||||
elif change["type"] == "categorize":
|
||||
return
|
||||
else:
|
||||
logger.warning("This event is not implemented in the script. Please make an issue on the tracker attaching the following info: wiki url, time, and this information: {}".format(change))
|
||||
return
|
||||
if identification_string in settings["ignored"]:
|
||||
return
|
||||
appearance_mode(identification_string, change, parsed_comment, changed_categories, wiki)
|
||||
|
||||
def abuselog_processing(entry, recent_changes):
|
||||
abuselog_appearance_mode(entry, recent_changes)
|
||||
Loading…
Reference in a new issue