diff --git a/src/bot.py b/src/bot.py index a9217b5..6993d59 100644 --- a/src/bot.py +++ b/src/bot.py @@ -15,7 +15,8 @@ from src.exceptions import * from src.misc import get_paths from src.msgqueue import messagequeue from src.queue_handler import DBHandler -from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info +from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info, essential_feeds +from src.formatters.discussions import embed_formatter, compact_formatter from src.discord import DiscordMessage, formatter_exception_logger, msg_sender_exception_logger logging.config.dictConfig(settings["logging"]) @@ -72,68 +73,124 @@ async def wiki_scanner(): 'SELECT webhook, wiki, lang, display, wikiid, rcid, postid FROM rcgcdw GROUP BY wiki') for db_wiki in fetch_all.fetchall(): logger.debug("Wiki {}".format(db_wiki["wiki"])) - extended = False if db_wiki["wiki"] not in all_wikis: logger.info("Registering new wiki locally: {}".format(db_wiki["wiki"])) all_wikis[db_wiki["wiki"]] = Wiki() local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory - if local_wiki.mw_messages is None: - extended = True - async with aiohttp.ClientSession(headers=settings["header"], - timeout=aiohttp.ClientTimeout(3.0)) as session: - try: - wiki_response = await local_wiki.fetch_wiki(extended, db_wiki["wiki"], session) - await local_wiki.check_status(db_wiki["wiki"], wiki_response.status) - except (WikiServerError, WikiError): - logger.error("Exeption when fetching the wiki") - continue # ignore this wiki if it throws errors - try: - recent_changes_resp = await wiki_response.json() - if "error" in recent_changes_resp or "errors" in recent_changes_resp: - error = recent_changes_resp.get("error", recent_changes_resp["errors"]) - if error["code"] == "readapidenied": - await local_wiki.fail_add(db_wiki["wiki"], 410) - continue - raise WikiError - recent_changes = recent_changes_resp['query']['recentchanges'] - recent_changes.reverse() - except aiohttp.ContentTypeError: - logger.exception("Wiki seems to be resulting in non-json content.") - await local_wiki.fail_add(db_wiki["wiki"], 410) + if db_wiki["rcid"] != -1: + extended = False + if local_wiki.mw_messages is None: + extended = True + async with aiohttp.ClientSession(headers=settings["header"], + timeout=aiohttp.ClientTimeout(3.0)) as session: + try: + wiki_response = await local_wiki.fetch_wiki(extended, db_wiki["wiki"], session) + await local_wiki.check_status(db_wiki["wiki"], wiki_response.status) + except (WikiServerError, WikiError): + logger.error("Exeption when fetching the wiki") + continue # ignore this wiki if it throws errors + try: + recent_changes_resp = await wiki_response.json() + if "error" in recent_changes_resp or "errors" in recent_changes_resp: + error = recent_changes_resp.get("error", recent_changes_resp["errors"]) + if error["code"] == "readapidenied": + await local_wiki.fail_add(db_wiki["wiki"], 410) + continue + raise WikiError + recent_changes = recent_changes_resp['query']['recentchanges'] + recent_changes.reverse() + except aiohttp.ContentTypeError: + logger.exception("Wiki seems to be resulting in non-json content.") + await local_wiki.fail_add(db_wiki["wiki"], 410) + continue + except: + logger.exception("On loading json of response.") + continue + if extended: + await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs) + if db_wiki["rcid"] is None: # new wiki, just get the last rc to not spam the channel + if len(recent_changes) > 0: + DBHandler.add(db_wiki["wiki"], recent_changes[-1]["rcid"]) + else: + DBHandler.add(db_wiki["wiki"], 0) + DBHandler.update_db() continue - except: - logger.exception("On loading json of response.") - continue - if extended: - await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs) - if db_wiki["rcid"] is None: # new wiki, just get the last rc to not spam the channel - if len(recent_changes) > 0: - DBHandler.add(db_wiki["wiki"], recent_changes[-1]["rcid"]) - else: - DBHandler.add(db_wiki["wiki"], 0) + categorize_events = {} + targets = generate_targets(db_wiki["wiki"]) + paths = get_paths(db_wiki["wiki"], recent_changes_resp) + for change in recent_changes: + await process_cats(change, local_wiki, mw_msgs, categorize_events) + for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up + if change["rcid"] > db_wiki["rcid"]: + for target in targets.items(): + try: + await essential_info(change, categorize_events, local_wiki, db_wiki, + target, paths, recent_changes_resp) + except: + if command_line_args.debug: + raise # reraise the issue + else: + logger.exception("Exception on RC formatter") + await formatter_exception_logger(db_wiki["wiki"], change, traceback.format_exc()) + if recent_changes: + DBHandler.add(db_wiki["wiki"], change["rcid"]) DBHandler.update_db() - continue - categorize_events = {} - targets = generate_targets(db_wiki["wiki"]) - paths = get_paths(db_wiki["wiki"], recent_changes_resp) - for change in recent_changes: - await process_cats(change, local_wiki, mw_msgs, categorize_events) - for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up - if change["rcid"] > db_wiki["rcid"]: - for target in targets.items(): - try: - await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, - recent_changes_resp) - except: - if command_line_args.debug: - raise # reraise the issue - else: - logger.exception("Exception on RC formatter") - await formatter_exception_logger(db_wiki["wiki"], change, traceback.format_exc()) - if recent_changes: - DBHandler.add(db_wiki["wiki"], change["rcid"]) - DBHandler.update_db() - await asyncio.sleep(delay=calc_delay) + await asyncio.sleep(delay=calc_delay) + if db_wiki["wikiid"] is not None: + header = settings["header"] + header["Accept"] = "application/hal+json" + async with aiohttp.ClientSession(headers=header, + timeout=aiohttp.ClientTimeout(3.0)) as session: + try: + feeds_response = await local_wiki.fetch_feeds(db_wiki["wikiid"], session) + # await local_wiki.check_status(db_wiki["wiki"], wiki_response.status) + # NEED A GOAT TO CHECK THIS + except (WikiServerError, WikiError): + logger.error("Exeption when fetching the wiki") + continue # ignore this wiki if it throws errors + try: + discussion_feed_resp = await feeds_response.json() + if "title" in discussion_feed_resp: + error = discussion_feed_resp["error"] + if error == "site doesn't exists": + db_cursor.execute("UPDATE rcgcdw SET wikiid = ? WHERE wiki = ?", + (None, db_wiki["wiki"],)) + DBHandler.update_db() + continue + raise WikiError + discussion_feed = discussion_feed_resp["_embedded"]["doc:posts"] + discussion_feed.reverse() + except aiohttp.ContentTypeError: + logger.exception("Wiki seems to be resulting in non-json content.") + # NEED A GOAT TO CHECK THIS + # await local_wiki.fail_add(db_wiki["wiki"], 410) + continue + except: + logger.exception("On loading json of response.") + continue + if db_wiki["postid"] is None: # new wiki, just get the last post to not spam the channel + if len(discussion_feed) > 0: + DBHandler.add(db_wiki["wiki"], discussion_feed[-1]["id"], True) + else: + DBHandler.add(db_wiki["wiki"], "0", True) + DBHandler.update_db() + continue + targets = generate_targets(db_wiki["wiki"]) + for post in discussion_feed: + if post["id"] > db_wiki["postid"]: + for target in targets.items(): + try: + await essential_feeds(post, db_wiki, target) + except: + if command_line_args.debug: + raise # reraise the issue + else: + logger.exception("Exception on Feeds formatter") + await formatter_exception_logger(db_wiki["wiki"], post, traceback.format_exc()) + if discussion_feed: + DBHandler.add(db_wiki["wiki"], post["id"], True) + DBHandler.update_db() + await asyncio.sleep(delay=calc_delay) except asyncio.CancelledError: raise diff --git a/src/formatters/discussions.py b/src/formatters/discussions.py index b12c9c7..e0dd68e 100644 --- a/src/formatters/discussions.py +++ b/src/formatters/discussions.py @@ -1,19 +1,15 @@ import datetime, logging import json -import gettext from urllib.parse import quote_plus from src.config import settings from src.misc import send_to_discord, escape_formatting from discord import DiscordMessage -from src.i18n import disc - -_ = disc.gettext discussion_logger = logging.getLogger("rcgcdw.discussion_formatter") -def embed_formatter(post, post_type): +def feeds_embed_formatter(post_type, post, message_target, wiki, _): """Embed formatter for Fandom discussions.""" embed = DiscordMessage("embed", "discussion", settings["fandom_discussions"]["webhookURL"]) embed.set_author(post["createdBy"]["name"], "{wikiurl}f/u/{creatorId}".format( @@ -74,13 +70,15 @@ def embed_formatter(post, post_type): embed.add_field(option["text"] if image_type is True else _("Option {}").format(num+1), option["text"] if image_type is False else _("__[View image]({image_url})__").format(image_url=option["image"]["url"]), inline=True) + else: + # UNKNOWN EVENT embed["footer"]["text"] = post["forumName"] embed["timestamp"] = datetime.datetime.fromtimestamp(post["creationDate"]["epochSecond"], tz=datetime.timezone.utc).isoformat() embed.finish_embed() send_to_discord(embed) -def compact_formatter(post, post_type): +def feeds_compact_formatter(post_type, post, message_target, wiki, _): """Compact formatter for Fandom discussions.""" message = None discussion_post_type = post["_embedded"]["thread"][0].get("containerType", @@ -117,12 +115,13 @@ def compact_formatter(post, post_type): "[{author}](<{url}f/u/{creatorId}>) replied to [{title}](<{wikiurl}wiki/Message_Wall:{user_wall}?threadId={threadid}#{replyId}>) on {user}'s Message Wall").format( author=post["createdBy"]["name"], url=settings["fandom_discussions"]["wiki_url"], creatorId=post["creatorId"], title=post["_embedded"]["thread"][0]["title"], user=user_wall, wikiurl=settings["fandom_discussions"]["wiki_url"], user_wall=quote_plus(user_wall.replace(" ", "_")), threadid=post["threadId"], replyId=post["id"]) - elif post_type == "POLL": message = _( "[{author}](<{url}f/u/{creatorId}>) created a poll [{title}](<{url}f/p/{threadId}>) in {forumName}").format( author=post["createdBy"]["name"], url=settings["fandom_discussions"]["wiki_url"], creatorId=post["creatorId"], title=post["title"], threadId=post["threadId"], forumName=post["forumName"]) + else: + # UNKNOWN EVENT send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message)) diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 56e4752..bfcc2dd 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -330,7 +330,7 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c await send_to_discord(DiscordMessage("compact", action, message_target[1], content=content, wiki=WIKI_SCRIPT_PATH)) -async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None): +async def embed_formatter(action, change, parsed_comment, categories, recent_changes, message_target, _, ngettext, paths, additional_data=None): """Recent Changes embed formatter, part of RcGcDw""" if additional_data is None: additional_data = {"namespaces": {}, "tags": {}} @@ -338,7 +338,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha WIKI_SCRIPT_PATH = paths[1] WIKI_ARTICLE_PATH = paths[2] WIKI_JUST_DOMAIN = paths[3] - embed = DiscordMessage("embed", action, target[1], wiki=WIKI_SCRIPT_PATH) + embed = DiscordMessage("embed", action, message_target[1], wiki=WIKI_SCRIPT_PATH) if parsed_comment is None: parsed_comment = _("No description provided") if action != "suppressed": @@ -369,7 +369,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = "{redirect}{article} ({new}{minor}{bot}{space}{editsize})".format(redirect="⤷ " if "redirect" in change else "", article=change["title"], editsize="+" + str( editsize) if editsize > 0 else editsize, new=_("(N!) ") if action == "new" else "", minor=_("m") if action == "edit" and "minor" in change else "", bot=_('b') if "bot" in change else "", space=" " if "bot" in change or (action == "edit" and "minor" in change) or action == "new" else "") - if target[0][1] == 3: + if message_target[0][1] == 3: if action == "new": changed_content = await recent_changes.safe_request( "{wiki}?action=compare&format=json&fromtext=&torev={diff}&topst=1&prop=diff".format( @@ -433,7 +433,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha wiki=WIKI_SCRIPT_PATH, filename=article_encoded, archiveid=revision["archivename"]) embed.add_field(_("Options"), _("([preview]({link}) | [undo]({undolink}))").format( link=image_direct_url, undolink=undolink)) - if target[0][1] > 1: + if message_target[0][1] > 1: embed["image"]["url"] = image_direct_url if action == "upload/overwrite": embed["title"] = _("Uploaded a new version of {name}").format(name=change["title"]) @@ -443,7 +443,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = _("Uploaded {name}").format(name=change["title"]) if additional_info_retrieved: embed.add_field(_("Options"), _("([preview]({link}))").format(link=image_direct_url)) - if target[0][1] > 1: + if message_target[0][1] > 1: embed["image"]["url"] = image_direct_url elif action == "delete/delete": link = create_article_path(change["title"], WIKI_ARTICLE_PATH) diff --git a/src/matrix.py b/src/matrix.py deleted file mode 100644 index e69de29..0000000 diff --git a/src/queue_handler.py b/src/queue_handler.py index 8525b63..f2a3f93 100644 --- a/src/queue_handler.py +++ b/src/queue_handler.py @@ -8,15 +8,18 @@ class UpdateDB: def __init__(self): self.updated = [] - def add(self, wiki, rc_id): - self.updated.append((wiki, rc_id)) + def add(self, wiki, rc_id, feeds: None): + self.updated.append((wiki, rc_id, feeds)) def clear_list(self): self.updated.clear() def update_db(self): for update in self.updated: - db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", (update[1], update[0],)) + id = "rcid" + if update[2] is not None: + id = "postid" + db_cursor.execute("UPDATE rcgcdw SET {} = ? WHERE wiki = ?".format(id),(update[1],update[0],)) db_connection.commit() self.clear_list() diff --git a/src/wiki.py b/src/wiki.py index 3fef827..e97966b 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -4,6 +4,7 @@ import logging, aiohttp from src.exceptions import * from src.database import db_cursor, db_connection from src.formatters.rc import embed_formatter, compact_formatter +from src.formatters.discussions import feeds_embed_formatter, feeds_compact_formatter from src.misc import parse_link from src.i18n import langs import src.discord @@ -49,6 +50,17 @@ class Wiki: raise WikiServerError return response + @staticmethod + async def fetch_feeds(wiki_id, session: aiohttp.ClientSession) -> aiohttp.ClientResponse: + url_path = "https://services.fandom.com/discussion/{wikiid}/posts".format(wikiid=wiki_id) + params = {"sortDirection": "descending", "sortKey": "creation_date", "limit": 20} + try: + response = await session.get(url_path, params=params) + except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError): + logger.exception("A connection error occurred while requesting {}".format(url_path)) + raise WikiServerError + return response + @staticmethod async def safe_request(url, *keys): try: @@ -212,3 +224,14 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_ except KeyError: additional_data["tags"][tag["name"]] = None # Tags with no displ await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, _, ngettext, paths, additional_data=additional_data) + + +async def essential_feeds(change: dict, db_wiki: tuple, target: tuple): + """Prepares essential information for both embed and compact message format.""" + def _(string: str) -> str: + """Our own translation string to make it compatible with async""" + return lang.gettext(string) + + lang = langs[target[0][0]] + appearance_mode = feeds_embed_formatter if target[0][1] > 0 else feeds_compact_formatter + await appearance_mode(change.get("funnel", "TEXT"), change, target, db_wiki["wiki"], _)