From d50707ecdda2b8db7fd6d3ffb63f4f4807713ee4 Mon Sep 17 00:00:00 2001 From: Frisk Date: Tue, 28 Jul 2020 14:39:32 +0200 Subject: [PATCH] Fixed some issues, code cleanup --- src/bot.py | 30 +++++++++++++++++------------- src/discord.py | 6 ++---- src/formatters/rc.py | 10 ++++------ src/misc.py | 1 - src/msgqueue.py | 2 +- src/queue_handler.py | 3 ++- src/wiki.py | 9 ++++++--- 7 files changed, 32 insertions(+), 29 deletions(-) diff --git a/src/bot.py b/src/bot.py index 630b740..8bb0219 100644 --- a/src/bot.py +++ b/src/bot.py @@ -1,19 +1,20 @@ +import aiohttp +import asyncio import logging.config -from src.config import settings -import sqlite3 -import sys import signal -from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info -import asyncio, aiohttp -from src.misc import get_paths -from src.exceptions import * -from src.database import db_cursor +import sys from collections import defaultdict -from src.queue_handler import DBHandler -from src.discord import DiscordMessage -from src.msgqueue import messagequeue + import requests +from src.config import settings +from src.database import db_cursor +from src.exceptions import * +from src.misc import get_paths +from src.msgqueue import messagequeue +from src.queue_handler import DBHandler +from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info + logging.config.dictConfig(settings["logging"]) logger = logging.getLogger("rcgcdb.bot") logger.debug("Current settings: {settings}".format(settings=settings)) @@ -58,7 +59,8 @@ async def wiki_scanner(): try: while True: calc_delay = calculate_delay() - fetch_all = db_cursor.execute('SELECT webhook, wiki, lang, display, wikiid, rcid, postid FROM rcgcdw GROUP BY wiki') + fetch_all = db_cursor.execute( + 'SELECT webhook, wiki, lang, display, wikiid, rcid, postid FROM rcgcdw GROUP BY wiki') for db_wiki in fetch_all.fetchall(): logger.debug("Wiki {}".format(db_wiki["wiki"])) extended = False @@ -134,11 +136,13 @@ def shutdown(loop, signal=None): task.cancel() sys.exit(0) + def global_exception_handler(loop, context): """Global exception handler for asyncio, lets us know when something crashes""" msg = context.get("exception", context["message"]) logger.error("Global exception handler:" + msg) - requests.post("https://discord.com/api/webhooks/"+settings["monitoring_webhook"], data={"content": "test"}) + requests.post("https://discord.com/api/webhooks/" + settings["monitoring_webhook"], data={"content": "test"}) + async def main_loop(): loop = asyncio.get_event_loop() diff --git a/src/discord.py b/src/discord.py index 0fb5019..1ce80b8 100644 --- a/src/discord.py +++ b/src/discord.py @@ -1,13 +1,11 @@ import json, random, math, logging from collections import defaultdict -from src.config import settings -from src.database import db_cursor from src.misc import logger from src.config import settings from src.database import db_cursor from src.i18n import langs -import aiohttp, gettext +import aiohttp logger = logging.getLogger("rcgcdb.discord") @@ -21,7 +19,7 @@ async def wiki_removal(wiki_url, status): """Our own translation string to make it compatible with async""" return langs[observer["lang"]].gettext(string) reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"), - 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible"), 410: _("wiki becoming inaccessible")} + 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")} reason = reasons.get(status, _("unknown error")) await send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[], content=_("The webhook for {} has been removed due to {}.".format(wiki_url, reason)), wiki=None), webhook_url=observer["webhook"]) header = settings["header"] diff --git a/src/formatters/rc.py b/src/formatters/rc.py index a1138b4..ea1c531 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -3,20 +3,18 @@ import math import re import time import logging -import base64 from src.config import settings from src.misc import link_formatter, create_article_path, parse_link, profile_field_name, ContentParser from src.discord import DiscordMessage -from urllib.parse import quote_plus from src.msgqueue import send_to_discord # from html.parser import HTMLParser +# noinspection PyPackageRequirements from bs4 import BeautifulSoup #from src.configloader import settings #from src.misc import link_formatter, create_article_path, WIKI_SCRIPT_PATH, send_to_discord, DiscordMessage, safe_read, \ # WIKI_API_PATH, ContentParser, profile_field_name, LinkParser -from src.i18n import langs #from src.rc import recent_changes, pull_comment logger = logging.getLogger("rcgcdw.rc_formatters") @@ -254,7 +252,7 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c content = _("[{author}]({author_url}) created abuse filter [number {number}]({filter_url})").format(author=author, author_url=author_url, number=change["logparams"]['newId'], filter_url=link) elif action == "merge/merge": link = link_formatter(create_article_path(change["title"], WIKI_ARTICLE_PATH)) - link_dest = link_formatter(create_article_path(change["logparams"]["dest_title"])) + link_dest = link_formatter(create_article_path(change["logparams"]["dest_title"], WIKI_ARTICLE_PATH)) content = _("[{author}]({author_url}) merged revision histories of [{article}]({article_url}) into [{dest}]({dest_url}){comment}").format(author=author, author_url=author_url, article=change["title"], article_url=link, dest_url=link_dest, dest=change["logparams"]["dest_title"], comment=parsed_comment) elif action == "interwiki/iw_add": @@ -532,9 +530,9 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha link = create_article_path("UserProfile:{target}".format(target=change["title"].split(':')[1].replace(" ", "_").replace(')', '\)')), WIKI_ARTICLE_PATH) embed["title"] = _("Edited {target}'s profile").format(target=change["title"].split(':')[1]) if change["user"] != change["title"].split(':')[1] else _("Edited their own profile") if not change["parsedcomment"]: # If the field is empty - parsed_comment = _("Cleared the {field} field").format(field=profile_field_name(change["logparams"]['4:section'], True)) + parsed_comment = _("Cleared the {field} field").format(field=profile_field_name(change["logparams"]['4:section'], True, _)) else: - parsed_comment = _("{field} field changed to: {desc}").format(field=profile_field_name(change["logparams"]['4:section'], True), desc=BeautifulSoup(change["parsedcomment"], "lxml").get_text()) + parsed_comment = _("{field} field changed to: {desc}").format(field=profile_field_name(change["logparams"]['4:section'], True, _), desc=BeautifulSoup(change["parsedcomment"], "lxml").get_text()) elif action == "curseprofile/comment-purged": link = create_article_path("Special:CommentPermalink/{commentid}".format(commentid=change["logparams"]["4:comment_id"]), WIKI_ARTICLE_PATH) embed["title"] = _("Purged a comment on {target}'s profile").format(target=change["title"].split(':')[1]) diff --git a/src/misc.py b/src/misc.py index 7e8865c..9ec52a8 100644 --- a/src/misc.py +++ b/src/misc.py @@ -3,7 +3,6 @@ import base64, re import logging from urllib.parse import urlparse, urlunparse -import aiohttp logger = logging.getLogger("rcgcdw.misc") diff --git a/src/msgqueue.py b/src/msgqueue.py index 3ba8794..d16fd5a 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -1,7 +1,7 @@ import asyncio, logging, aiohttp from src.discord import send_to_discord_webhook from src.config import settings -from collections import defaultdict, ItemsView +from collections import defaultdict logger = logging.getLogger("rcgcdw.msgqueue") class MessageQueue: diff --git a/src/queue_handler.py b/src/queue_handler.py index a54f434..8525b63 100644 --- a/src/queue_handler.py +++ b/src/queue_handler.py @@ -3,7 +3,8 @@ from src.database import db_cursor, db_connection logger = logging.getLogger("rcgcdb.queue_handler") -class UpdateDB(): + +class UpdateDB: def __init__(self): self.updated = [] diff --git a/src/wiki.py b/src/wiki.py index 4948cc8..86ed364 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -9,6 +9,7 @@ from src.i18n import langs import src.discord import asyncio from src.config import settings +# noinspection PyPackageRequirements from bs4 import BeautifulSoup logger = logging.getLogger("rcgcdb.wiki") @@ -23,7 +24,8 @@ class Wiki: session: aiohttp.ClientSession = None - async def fetch_wiki(self, extended, script_path, session: aiohttp.ClientSession) -> aiohttp.ClientResponse: + @staticmethod + async def fetch_wiki(extended, script_path, session: aiohttp.ClientSession) -> aiohttp.ClientResponse: url_path = script_path + "api.php" amount = 20 if extended: @@ -84,7 +86,8 @@ class Wiki: logger.warning("Wiki {} responded with HTTP code {}, skipping...".format(wiki_url, status, self.fail_times)) raise WikiServerError - async def remove(self, wiki_url, reason): + @staticmethod + async def remove(wiki_url, reason): await src.discord.wiki_removal(wiki_url, reason) await src.discord.wiki_removal_monitor(wiki_url, reason) db_cursor.execute('DELETE FROM rcgcdw WHERE wiki = ?', (wiki_url,)) @@ -179,7 +182,7 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ ngettext = lang.ngettext # recent_changes = RecentChangesClass() # TODO Look into replacing RecentChangesClass with local_wiki appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter - if ("actionhidden" in change or "suppressed" in change): # if event is hidden using suppression + if "actionhidden" in change or "suppressed" in change: # if event is hidden using suppression await appearance_mode("suppressed", change, "", changed_categories, local_wiki, target, _, ngettext, paths) return if "commenthidden" not in change: