diff --git a/requirements.txt b/requirements.txt index 1c45373..c752fe3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ beautifulsoup4 >= 4.6.0; python_version >= '3.6' aiohttp >= 3.6.2 lxml >= 4.2.1 -nest-asyncio >= 1.4.0 \ No newline at end of file +nest-asyncio >= 1.4.0 +irc >= 19.0.1 \ No newline at end of file diff --git a/settings.json.example b/settings.json.example index 3b5ca77..67dd2f2 100644 --- a/settings.json.example +++ b/settings.json.example @@ -7,6 +7,17 @@ "database_path": "rcgcdb.db", "monitoring_webhook": "111111111111111111/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", "support": "https://discord.gg/v77RTk5", + "irc_overtime": 3600, + "irc_servers": { + "your custom name for the farm": { + "domains": ["wikipedia.org", "otherwikipedia.org"], + "irc_host": "randomIRC.domain.com", + "irc_port": "6667", + "irc_nickname": "BotIRCNickname", + "irc_name": "BotIRCName", + "irc_channel_mapping": {"rc": "#rcchannel", "discussion": "#discussionchannel"} + } + }, "logging": { "version": 1, "disable_existing_loggers": false, diff --git a/src/bot.py b/src/bot.py index 68b40d2..a4a70a6 100644 --- a/src/bot.py +++ b/src/bot.py @@ -130,7 +130,7 @@ class RcQueue: shutdown(asyncio.get_event_loop()) else: logger.exception("Group task returned error") - await generic_msg_sender_exception_logger(traceback.format_exc(), "Group task error logger (really bad)", Group=group) + await generic_msg_sender_exception_logger(traceback.format_exc(), "Group task error logger", Group=group) else: self.domain_list[group]["query"].pop(0) @@ -162,11 +162,11 @@ class RcQueue: try: current_domain: dict = self[domain] if current_domain["irc"]: - logger.info('CURRENT STATUS:') - logger.info("DOMAIN LIST FOR IRC: {}".format(current_domain["irc"].updated)) - logger.info("CURRENT DOMAIN INFO: {}".format(domain)) - logger.info("IS WIKI IN A LIST?: {}".format(db_wiki["wiki"] in current_domain["irc"].updated)) - logger.info("LAST CHECK FOR THE WIKI {} IS {}".format(db_wiki["wiki"], all_wikis[db_wiki["wiki"]].last_check)) + logger.debug('CURRENT STATUS:') + logger.debug("DOMAIN LIST FOR IRC: {}".format(current_domain["irc"].updated)) + logger.debug("CURRENT DOMAIN INFO: {}".format(domain)) + logger.debug("IS WIKI IN A LIST?: {}".format(db_wiki["wiki"] in current_domain["irc"].updated)) + logger.debug("LAST CHECK FOR THE WIKI {} IS {}".format(db_wiki["wiki"], all_wikis[db_wiki["wiki"]].last_check)) if db_wiki["wiki"] not in current_domain["irc"].updated and all_wikis[db_wiki["wiki"]].last_check+settings["irc_overtime"] > time.time(): continue # if domain has IRC, has not been updated, and it was updated less than an hour ago else: # otherwise remove it from the list @@ -177,7 +177,6 @@ class RcQueue: if not db_wiki["ROWID"] < current_domain["last_rowid"]: current_domain["query"].append(QueuedWiki(db_wiki["wiki"], 20)) except KeyError: - raise await self.start_group(domain, [QueuedWiki(db_wiki["wiki"], 20)]) logger.info("A new domain group ({}) has been added since last time, adding it to the domain_list and starting a task...".format(domain)) except ListFull: @@ -196,7 +195,7 @@ class RcQueue: shutdown(asyncio.get_event_loop()) else: logger.exception("Exception on queue updater") - await generic_msg_sender_exception_logger(traceback.format_exc(), "Queue updator (ok)") + await generic_msg_sender_exception_logger(traceback.format_exc(), "Queue updator") def __getitem__(self, item): @@ -250,7 +249,8 @@ async def scan_group(group: str): while True: try: async with rcqueue.retrieve_next_queued(group) as queued_wiki: # acquire next wiki in queue - await asyncio.sleep(calculate_delay_for_group(len(rcqueue[group]["query"]))) + if "irc" not in rcqueue[group]: + await asyncio.sleep(calculate_delay_for_group(len(rcqueue[group]["query"]))) logger.debug("Wiki {}".format(queued_wiki.url)) local_wiki = all_wikis[queued_wiki.url] # set a reference to a wiki object from memory extended = False @@ -334,7 +334,7 @@ async def scan_group(group: str): raise else: logger.exception("Exception on RC formatter") - await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in RC formatter (ok)", Wiki=queued_wiki.url, Change=str(change)[0:1000]) + await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in RC formatter", Wiki=queued_wiki.url, Change=str(change)[0:1000]) # Lets stack the messages for messages in message_list.values(): messages = stack_message_list(messages) @@ -347,7 +347,7 @@ async def scan_group(group: str): except asyncio.CancelledError: return except QueueEmpty: - await asyncio.sleep(21.0) + await asyncio.sleep(10.0) continue @@ -387,13 +387,17 @@ async def message_sender(): await generic_msg_sender_exception_logger(traceback.format_exc(), "Message sender exception") async def discussion_handler(): - # Handler for Fandom Discussions, it has the entire look of things from queuing to sending try: while True: fetch_all = db_cursor.execute( "SELECT wiki, rcid, postid FROM rcgcdw WHERE postid != '-1' OR postid IS NULL GROUP BY wiki") for db_wiki in fetch_all.fetchall(): - if db_wiki["wiki"] not in rcqueue.irc_mapping["fandom.com"].updated_discussions and all_wikis[db_wiki["wiki"]].last_discussion_check+settings["irc_overtime"] > time.time(): # I swear if another wiki farm ever starts using Fandom discussions I'm gonna use explosion magic + try: + local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory + except KeyError: + local_wiki = all_wikis[db_wiki["wiki"]] = Wiki() + local_wiki.rc_active = db_wiki["rcid"] + if db_wiki["wiki"] not in rcqueue.irc_mapping["fandom.com"].updated_discussions and local_wiki.last_discussion_check+settings["irc_overtime"] > time.time(): # I swear if another wiki farm ever starts using Fandom discussions I'm gonna use explosion magic continue else: try: @@ -404,20 +408,15 @@ async def discussion_handler(): header["Accept"] = "application/hal+json" async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(6.0)) as session: - try: - local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory - except KeyError: - local_wiki = all_wikis[db_wiki["wiki"]] = Wiki() - local_wiki.rc_active = db_wiki["rcid"] try: feeds_response = await local_wiki.fetch_feeds(db_wiki["wiki"], session) except (WikiServerError, WikiError): continue # ignore this wiki if it throws errors try: discussion_feed_resp = await feeds_response.json(encoding="UTF-8") - if "title" in discussion_feed_resp: + if "error" in discussion_feed_resp: error = discussion_feed_resp["error"] - if error == "site doesn't exists": # Discussions disabled + if error == "NotFoundException": # Discussions disabled if db_wiki["rcid"] != -1: # RC feed is disabled db_cursor.execute("UPDATE rcgcdw SET postid = ? WHERE wiki = ?", ("-1", db_wiki["wiki"],)) @@ -483,7 +482,7 @@ async def discussion_handler(): shutdown(loop=asyncio.get_event_loop()) else: logger.exception("Exception on Feeds formatter") - await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in feed formatter (ok)", Post=str(post)[0:1000], Wiki=db_wiki["wiki"]) + await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in feed formatter", Post=str(post)[0:1000], Wiki=db_wiki["wiki"]) # Lets stack the messages for messages in message_list.values(): messages = stack_message_list(messages) @@ -501,13 +500,11 @@ async def discussion_handler(): raise # reraise the issue else: logger.exception("Exception on Feeds formatter") - await generic_msg_sender_exception_logger(traceback.format_exc(), "Discussion handler task exception (bad)", Wiki=db_wiki["wiki"]) + await generic_msg_sender_exception_logger(traceback.format_exc(), "Discussion handler task exception", Wiki=db_wiki["wiki"]) def shutdown(loop, signal=None): - # This is our best attempt at shutting down gently - we save and close the database, wait for messages to be sent, - # stop all of the tasks and stop the look effectively shutting down all asyncio operations global main_tasks DBHandler.update_db() db_connection.close() diff --git a/src/config.py b/src/config.py index cffd432..2a2ddc2 100644 --- a/src/config.py +++ b/src/config.py @@ -4,7 +4,7 @@ try: # load settings with open("settings.json", encoding="utf8") as sfile: settings = json.load(sfile) if "user-agent" in settings["header"]: - settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.0") # set the version in the useragent + settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.1") # set the version in the useragent except FileNotFoundError: logging.critical("No config file could be found. Please make sure settings.json is in the directory.") sys.exit(1) \ No newline at end of file diff --git a/src/formatters/discussions.py b/src/formatters/discussions.py index b7ea5b5..156bc88 100644 --- a/src/formatters/discussions.py +++ b/src/formatters/discussions.py @@ -45,20 +45,20 @@ async def feeds_compact_formatter(post_type, post, message_target, wiki, article else: logger.warning("No entry for {event} with params: {params}".format(event=thread_funnel, params=post)) event_type = "unknown" - message = msg_text.format(author=author, author_url=author_url, title=post["title"], url=wiki, threadId=post["threadId"], forumName=post["forumName"]) + message = msg_text.format(author=author, author_url=author_url, title=escape_formatting(post["title"]), url=wiki, threadId=post["threadId"], forumName=post["forumName"]) else: event_type = "discussion/forum/reply" - message = _("[{author}]({author_url}) created a [reply](<{url}f/p/{threadId}/r/{postId}>) to [{title}](<{url}f/p/{threadId}>) in {forumName}").format(author=author, author_url=author_url, url=wiki, threadId=post["threadId"], postId=post["id"], title=post["_embedded"]["thread"][0]["title"], forumName=post["forumName"]) + message = _("[{author}]({author_url}) created a [reply](<{url}f/p/{threadId}/r/{postId}>) to [{title}](<{url}f/p/{threadId}>) in {forumName}").format(author=author, author_url=author_url, url=wiki, threadId=post["threadId"], postId=post["id"], title=escape_formatting(post["_embedded"]["thread"][0]["title"]), forumName=post["forumName"]) elif post_type == "WALL": user_wall = _("unknown") # Fail safe if post["forumName"].endswith(' Message Wall'): user_wall = post["forumName"][:-13] if not post["isReply"]: event_type = "discussion/wall/post" - message = _("[{author}]({author_url}) created [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, title=post["title"], url=wiki, user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"]) + message = _("[{author}]({author_url}) created [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, title=escape_formatting(post["title"]), url=wiki, user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"]) else: event_type = "discussion/wall/reply" - message = _("[{author}]({author_url}) created a [reply](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}>) to [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, url=wiki, title=post["_embedded"]["thread"][0]["title"], user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"]) + message = _("[{author}]({author_url}) created a [reply](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}>) to [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, url=wiki, title=escape_formatting(post["_embedded"]["thread"][0]["title"]), user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"]) elif post_type == "ARTICLE_COMMENT": if article_page is None: article_page = {"title": _("unknown"), "fullUrl": wiki} # No page known @@ -112,11 +112,11 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p if post_type == "FORUM": if not post["isReply"]: embed["url"] = "{url}f/p/{threadId}".format(url=wiki, threadId=post["threadId"]) - embed["title"] = _("Created \"{title}\"").format(title=post["title"]) + embed["title"] = _("Created \"{title}\"").format(title=escape_formatting(post["title"])) thread_funnel = post.get("funnel") if thread_funnel == "POLL": embed.event_type = "discussion/forum/poll" - embed["title"] = _("Created a poll \"{title}\"").format(title=post["title"]) + embed["title"] = _("Created a poll \"{title}\"").format(title=escape_formatting(post["title"])) if message_target[0][1] > 1: poll = post["poll"] image_type = False @@ -128,7 +128,7 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p inline=True) elif thread_funnel == "QUIZ": embed.event_type = "discussion/forum/quiz" - embed["title"] = _("Created a quiz \"{title}\"").format(title=post["title"]) + embed["title"] = _("Created a quiz \"{title}\"").format(title=escape_formatting(post["title"])) if message_target[0][1] > 1: quiz = post["_embedded"]["quizzes"][0] embed["description"] = quiz["title"] @@ -149,7 +149,7 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p embed.add_field(_("Tags"), ", ".join(tag_displayname)) else: embed.event_type = "discussion/forum/reply" - embed["title"] = _("Replied to \"{title}\"").format(title=post["_embedded"]["thread"][0]["title"]) + embed["title"] = _("Replied to \"{title}\"").format(title=escape_formatting(post["_embedded"]["thread"][0]["title"])) embed["url"] = "{url}f/p/{threadId}/r/{postId}".format(url=wiki, threadId=post["threadId"], postId=post["id"]) elif post_type == "WALL": user_wall = _("unknown") # Fail safe @@ -158,11 +158,11 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p if not post["isReply"]: embed.event_type = "discussion/wall/post" embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}".format(url=wiki, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"]) - embed["title"] = _("Created \"{title}\" on {user}'s Message Wall").format(title=post["title"], user=user_wall) + embed["title"] = _("Created \"{title}\" on {user}'s Message Wall").format(title=escape_formatting(post["title"]), user=user_wall) else: embed.event_type = "discussion/wall/reply" embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}".format(url=wiki, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"]) - embed["title"] = _("Replied to \"{title}\" on {user}'s Message Wall").format(title=post["_embedded"]["thread"][0]["title"], user=user_wall) + embed["title"] = _("Replied to \"{title}\" on {user}'s Message Wall").format(title=escape_formatting(post["_embedded"]["thread"][0]["title"]), user=user_wall) elif post_type == "ARTICLE_COMMENT": if article_page is None: article_page = {"title": _("unknown"), "fullUrl": wiki} # No page known diff --git a/src/formatters/rc.py b/src/formatters/rc.py index 1f9420a..9ee8a40 100644 --- a/src/formatters/rc.py +++ b/src/formatters/rc.py @@ -388,7 +388,7 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c author=author, author_url=author_url, group_name=group_name, comment=parsed_comment ) elif action == "managewiki/undelete": - content = _("[{author}]({author_url}) restored a wiki *{wiki_name}*{comment}").format( + content = _("[{author}]({author_url}) undeleted a wiki *{wiki_name}*{comment}").format( author=author, author_url=author_url, wiki_name=change["logparams"].get("wiki", _("Unknown")), comment=parsed_comment ) elif action == "managewiki/unlock": @@ -1018,7 +1018,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha embed["title"] = _("Modified \"{usergroup_name}\" usergroup").format(usergroup_name=group_name) link = create_article_path(change["title"], WIKI_ARTICLE_PATH) elif action == "managewiki/undelete": - embed["title"] = _("Restored a \"{wiki}\" wiki").format(wiki=change["logparams"].get("wiki", _("Unknown"))) + embed["title"] = _("Undeleted a \"{wiki}\" wiki").format(wiki=change["logparams"].get("wiki", _("Unknown"))) link = create_article_path(change["title"], WIKI_ARTICLE_PATH) elif action == "managewiki/unlock": embed["title"] = _("Unlocked a \"{wiki}\" wiki").format(wiki=change["logparams"].get("wiki", _("Unknown"))) diff --git a/src/wiki.py b/src/wiki.py index 1bf3a89..4366efa 100644 --- a/src/wiki.py +++ b/src/wiki.py @@ -70,7 +70,7 @@ class Wiki: await ratelimiter.timeout_wait() try: async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(6.0)) as session: - request = await session.get(url, allow_redirects=False) + request = await session.get(url) ratelimiter.timeout_add(1.0) request.raise_for_status() json_request = await request.json(encoding="UTF-8")