From f3594ac12beebd952df7d0fc14c8212b0cf32266 Mon Sep 17 00:00:00 2001 From: Frisk Date: Wed, 22 Jul 2020 13:43:18 +0200 Subject: [PATCH] Fixed one thing Broke 10 more --- src/bot.py | 99 ++++++++++++++++++++++++-------------------- src/formatters/rc.py | 14 ++++--- src/misc.py | 7 +--- src/msgqueue.py | 1 + src/queue_handler.py | 2 +- src/wiki.py | 7 ++-- 6 files changed, 69 insertions(+), 61 deletions(-) diff --git a/src/bot.py b/src/bot.py index 5020c5a..e72ce4c 100644 --- a/src/bot.py +++ b/src/bot.py @@ -8,6 +8,7 @@ from src.exceptions import * from src.database import db_cursor from collections import defaultdict from src.queue_handler import DBHandler +from src.msgqueue import messagequeue logging.config.dictConfig(settings["logging"]) logger = logging.getLogger("rcgcdb.bot") @@ -37,7 +38,7 @@ def calculate_delay() -> float: def generate_targets(wiki_url: str) -> defaultdict: combinations = defaultdict(list) - for webhook in db_cursor.execute('SELECT ROWID, * FROM rcgcdw WHERE wiki = ?', wiki_url): + for webhook in db_cursor.execute('SELECT ROWID, * FROM rcgcdw WHERE wiki = ?', (wiki_url,)): # rowid, guild, configid, webhook, wiki, lang, display, rcid, wikiid, postid combination = (webhook[5], webhook[6]) # lang, display combinations[combination].append(webhook[3]) @@ -45,60 +46,66 @@ def generate_targets(wiki_url: str) -> defaultdict: async def wiki_scanner(): - calc_delay = calculate_delay() - # db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'): - for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'): - extended = False - if db_wiki[3] not in all_wikis: - logger.debug("New wiki: {}".format(db_wiki[3])) - all_wikis[db_wiki[3]] = Wiki() - local_wiki = all_wikis[db_wiki[3]] # set a reference to a wiki object from memory - if local_wiki.mw_messages is None: - extended = True - logger.debug("test") - try: - wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3]) - await local_wiki.check_status(wiki[3], wiki_response.status) - except (WikiServerError, WikiError): - continue # ignore this wiki if it throws errors - try: - recent_changes_resp = await wiki_response.json(encoding="UTF-8") - recent_changes = recent_changes_resp['query']['recentchanges'].reverse() - except: - logger.exception("On loading json of response.") - continue - if extended: - await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs) - if db_wiki[6] is None: # new wiki, just get the last rc to not spam the channel - if len(recent_changes) > 0: - DBHandler.add(db_wiki[3], recent_changes[-1]["rcid"]) + while True: + calc_delay = calculate_delay() + # db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'): + for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'): + extended = False + if db_wiki[3] not in all_wikis: + logger.debug("New wiki: {}".format(db_wiki[3])) + all_wikis[db_wiki[3]] = Wiki() + local_wiki = all_wikis[db_wiki[3]] # set a reference to a wiki object from memory + if local_wiki.mw_messages is None: + extended = True + logger.debug("test") + try: + wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3]) + await local_wiki.check_status(db_wiki[3], wiki_response.status) + except (WikiServerError, WikiError): + logger.exception("Exeption when fetching the wiki") + continue # ignore this wiki if it throws errors + try: + recent_changes_resp = await wiki_response.json(encoding="UTF-8") + recent_changes = recent_changes_resp['query']['recentchanges'] + recent_changes.reverse() + except: + logger.exception("On loading json of response.") continue - else: - DBHandler.add(db_wiki[3], 0) - continue - categorize_events = {} - targets = generate_targets(db_wiki[3]) - paths = get_paths(db_wiki[3], recent_changes_resp) - for change in recent_changes: - await process_cats(change, local_wiki, mw_msgs, categorize_events) - for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up - if change["rcid"] < db_wiki[6]: - for target in targets.items(): - await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp) - if recent_changes: - DBHandler.add(db_wiki[3], change["rcid"]) - await asyncio.sleep(delay=calc_delay) - - DBHandler.update_db() + if extended: + await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs) + if db_wiki[6] is None: # new wiki, just get the last rc to not spam the channel + if len(recent_changes) > 0: + DBHandler.add(db_wiki[3], recent_changes[-1]["rcid"]) + continue + else: + DBHandler.add(db_wiki[3], 0) + continue + categorize_events = {} + targets = generate_targets(db_wiki[3]) + paths = get_paths(db_wiki[3], recent_changes_resp) + for change in recent_changes: + await process_cats(change, local_wiki, mw_msgs, categorize_events) + for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up + if change["rcid"] < db_wiki[6]: + for target in targets.items(): + await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp) + if recent_changes: + DBHandler.add(db_wiki[3], change["rcid"]) + await asyncio.sleep(delay=calc_delay) + DBHandler.update_db() async def message_sender(): - pass + while True: + await messagequeue.resend_msgs() + async def main_loop(): task1 = asyncio.create_task(wiki_scanner()) task2 = asyncio.create_task(message_sender()) + await task1 + await task2 asyncio.run(main_loop()) diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 6af84bf..970e904 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -23,13 +23,14 @@ logger = logging.getLogger("rcgcdw.rc_formatters") async def compact_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None): + global LinkParser if additional_data is None: additional_data = {"namespaces": {}, "tags": {}} WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] WIKI_JUST_DOMAIN = paths[3] - LinkParser = LinkParser("domain") + LinkParser = LinkParser(paths[3]) if action != "suppressed": author_url = link_formatter(create_article_path("User:{user}".format(user=change["user"]), WIKI_ARTICLE_PATH)) author = change["user"] @@ -318,13 +319,14 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None): + global LinkParser if additional_data is None: additional_data = {"namespaces": {}, "tags": {}} WIKI_API_PATH = paths[0] WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] WIKI_JUST_DOMAIN = paths[3] - LinkParser = LinkParser() + LinkParser = LinkParser(paths[3]) embed = DiscordMessage("embed", action, target[1], wiki=WIKI_SCRIPT_PATH) if parsed_comment is None: parsed_comment = _("No description provided") @@ -694,16 +696,16 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha if "tags" in change and change["tags"]: tag_displayname = [] for tag in change["tags"]: - if tag in additional_data.tags: - if additional_data.tags[tag] is None: + if tag in additional_data["tags"]: + if additional_data["tags"][tag] is None: continue # Ignore hidden tags else: - tag_displayname.append(additional_data.tags[tag]) + tag_displayname.append(additional_data["tags"][tag]) else: tag_displayname.append(tag) embed.add_field(_("Tags"), ", ".join(tag_displayname)) logger.debug("Current params in edit action: {}".format(change)) - if categories is not None and not (len(categories["new"]) == 0 and len(categories["removed"]) == 0): + if categories and not (len(categories["new"]) == 0 and len(categories["removed"]) == 0): new_cat = (_("**Added**: ") + ", ".join(list(categories["new"])[0:16]) + ("\n" if len(categories["new"])<=15 else _(" and {} more\n").format(len(categories["new"])-15))) if categories["new"] else "" del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else "" embed.add_field(_("Changed categories"), new_cat + del_cat) diff --git a/src/misc.py b/src/misc.py index 346ecb5..f46c24e 100644 --- a/src/misc.py +++ b/src/misc.py @@ -1,4 +1,3 @@ -from abc import ABC from html.parser import HTMLParser import base64, re from src.config import settings @@ -14,7 +13,7 @@ logger = logging.getLogger("rcgcdw.misc") class DiscordMessage(): """A class defining a typical Discord JSON representation of webhook payload.""" def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None): - self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, "")) + self.webhook_object = dict(allowed_mentions={"parse": []}) self.webhook_url = webhook_url self.wiki = wiki @@ -123,15 +122,13 @@ def get_paths(wiki: str, request) -> tuple: class LinkParser(HTMLParser): - def error(self, message): - pass new_string = "" recent_href = "" def __init__(self, domain): - super().__init__() self.WIKI_JUST_DOMAIN = domain + super().__init__() def handle_starttag(self, tag, attrs): for attr in attrs: diff --git a/src/msgqueue.py b/src/msgqueue.py index 3f177ea..e8327ee 100644 --- a/src/msgqueue.py +++ b/src/msgqueue.py @@ -52,6 +52,7 @@ class MessageQueue: logger.debug("Queue emptied, all messages delivered") self.cut_messages(num) logger.debug(self._queue) + await asyncio.sleep(4.0) messagequeue = MessageQueue() diff --git a/src/queue_handler.py b/src/queue_handler.py index 41a948e..a54f434 100644 --- a/src/queue_handler.py +++ b/src/queue_handler.py @@ -15,7 +15,7 @@ class UpdateDB(): def update_db(self): for update in self.updated: - db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", update[1], update[0]) + db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", (update[1], update[0],)) db_connection.commit() self.clear_list() diff --git a/src/wiki.py b/src/wiki.py index 5b66e22..6a38706 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -77,7 +77,7 @@ class Wiki: async def remove(self, wiki_id, reason): src.discord.wiki_removal(wiki_id, reason) src.discord.wiki_removal_monitor(wiki_id, reason) - db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", wiki_id) + db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", (wiki_id,)) logger.warning("{} rows affected by DELETE FROM rcgcdw WHERE wiki = {}".format(db_cursor.rowcount, wiki_id)) db_connection.commit() @@ -141,7 +141,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict): :return: """ msgs = [] - for message in wiki_response["allmessages"]: + for message in wiki_response["query"]["allmessages"]: if not "missing" in message: # ignore missing strings msgs.append((message["name"], re.sub(r'\[\[.*?\]\]', '', message["*"]))) else: @@ -157,6 +157,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict): local_wiki.mw_messages = key async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_wiki: tuple, target: tuple, paths: tuple, request: dict): + global LinkParser """Prepares essential information for both embed and compact message format.""" def _(string: str) -> str: """Our own translation string to make it compatible with async""" @@ -165,7 +166,7 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ lang = langs[target[0][0]] ngettext = lang.ngettext # recent_changes = RecentChangesClass() # TODO Look into replacing RecentChangesClass with local_wiki - LinkParser = LinkParser("domain") + LinkParser = LinkParser(paths[3]) logger.debug(change) appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter if ("actionhidden" in change or "suppressed" in change): # if event is hidden using suppression