mirror of
https://gitlab.com/chicken-riders/RcGcDb.git
synced 2025-02-23 00:54:09 +00:00
Merge remote-tracking branch 'origin/master'
# Conflicts: # src/bot.py
This commit is contained in:
Frisk
commit
359df7e879
|
|
@ -2,3 +2,4 @@ beautifulsoup4 >= 4.6.0; python_version >= '3.6'
|
|||
aiohttp >= 3.6.2
|
||||
lxml >= 4.2.1
|
||||
nest-asyncio >= 1.4.0
|
||||
irc >= 19.0.1
|
||||
|
|
@ -7,6 +7,17 @@
|
|||
"database_path": "rcgcdb.db",
|
||||
"monitoring_webhook": "111111111111111111/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
"support": "https://discord.gg/v77RTk5",
|
||||
"irc_overtime": 3600,
|
||||
"irc_servers": {
|
||||
"your custom name for the farm": {
|
||||
"domains": ["wikipedia.org", "otherwikipedia.org"],
|
||||
"irc_host": "randomIRC.domain.com",
|
||||
"irc_port": "6667",
|
||||
"irc_nickname": "BotIRCNickname",
|
||||
"irc_name": "BotIRCName",
|
||||
"irc_channel_mapping": {"rc": "#rcchannel", "discussion": "#discussionchannel"}
|
||||
}
|
||||
},
|
||||
"logging": {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": false,
|
||||
|
|
|
|||
43
src/bot.py
43
src/bot.py
|
|
@ -130,7 +130,7 @@ class RcQueue:
|
|||
shutdown(asyncio.get_event_loop())
|
||||
else:
|
||||
logger.exception("Group task returned error")
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Group task error logger (really bad)", Group=group)
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Group task error logger", Group=group)
|
||||
else:
|
||||
self.domain_list[group]["query"].pop(0)
|
||||
|
||||
|
|
@ -162,11 +162,11 @@ class RcQueue:
|
|||
try:
|
||||
current_domain: dict = self[domain]
|
||||
if current_domain["irc"]:
|
||||
logger.info('CURRENT STATUS:')
|
||||
logger.info("DOMAIN LIST FOR IRC: {}".format(current_domain["irc"].updated))
|
||||
logger.info("CURRENT DOMAIN INFO: {}".format(domain))
|
||||
logger.info("IS WIKI IN A LIST?: {}".format(db_wiki["wiki"] in current_domain["irc"].updated))
|
||||
logger.info("LAST CHECK FOR THE WIKI {} IS {}".format(db_wiki["wiki"], all_wikis[db_wiki["wiki"]].last_check))
|
||||
logger.debug('CURRENT STATUS:')
|
||||
logger.debug("DOMAIN LIST FOR IRC: {}".format(current_domain["irc"].updated))
|
||||
logger.debug("CURRENT DOMAIN INFO: {}".format(domain))
|
||||
logger.debug("IS WIKI IN A LIST?: {}".format(db_wiki["wiki"] in current_domain["irc"].updated))
|
||||
logger.debug("LAST CHECK FOR THE WIKI {} IS {}".format(db_wiki["wiki"], all_wikis[db_wiki["wiki"]].last_check))
|
||||
if db_wiki["wiki"] not in current_domain["irc"].updated and all_wikis[db_wiki["wiki"]].last_check+settings["irc_overtime"] > time.time():
|
||||
continue # if domain has IRC, has not been updated, and it was updated less than an hour ago
|
||||
else: # otherwise remove it from the list
|
||||
|
|
@ -177,7 +177,6 @@ class RcQueue:
|
|||
if not db_wiki["ROWID"] < current_domain["last_rowid"]:
|
||||
current_domain["query"].append(QueuedWiki(db_wiki["wiki"], 20))
|
||||
except KeyError:
|
||||
raise
|
||||
await self.start_group(domain, [QueuedWiki(db_wiki["wiki"], 20)])
|
||||
logger.info("A new domain group ({}) has been added since last time, adding it to the domain_list and starting a task...".format(domain))
|
||||
except ListFull:
|
||||
|
|
@ -196,7 +195,7 @@ class RcQueue:
|
|||
shutdown(asyncio.get_event_loop())
|
||||
else:
|
||||
logger.exception("Exception on queue updater")
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Queue updator (ok)")
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Queue updator")
|
||||
|
||||
|
||||
def __getitem__(self, item):
|
||||
|
|
@ -250,6 +249,7 @@ async def scan_group(group: str):
|
|||
while True:
|
||||
try:
|
||||
async with rcqueue.retrieve_next_queued(group) as queued_wiki: # acquire next wiki in queue
|
||||
if "irc" not in rcqueue[group]:
|
||||
await asyncio.sleep(calculate_delay_for_group(len(rcqueue[group]["query"])))
|
||||
logger.debug("Wiki {}".format(queued_wiki.url))
|
||||
local_wiki = all_wikis[queued_wiki.url] # set a reference to a wiki object from memory
|
||||
|
|
@ -334,7 +334,7 @@ async def scan_group(group: str):
|
|||
raise
|
||||
else:
|
||||
logger.exception("Exception on RC formatter")
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in RC formatter (ok)", Wiki=queued_wiki.url, Change=str(change)[0:1000])
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in RC formatter", Wiki=queued_wiki.url, Change=str(change)[0:1000])
|
||||
# Lets stack the messages
|
||||
for messages in message_list.values():
|
||||
messages = stack_message_list(messages)
|
||||
|
|
@ -347,7 +347,7 @@ async def scan_group(group: str):
|
|||
except asyncio.CancelledError:
|
||||
return
|
||||
except QueueEmpty:
|
||||
await asyncio.sleep(21.0)
|
||||
await asyncio.sleep(10.0)
|
||||
continue
|
||||
|
||||
|
||||
|
|
@ -387,13 +387,17 @@ async def message_sender():
|
|||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Message sender exception")
|
||||
|
||||
async def discussion_handler():
|
||||
# Handler for Fandom Discussions, it has the entire look of things from queuing to sending
|
||||
try:
|
||||
while True:
|
||||
fetch_all = db_cursor.execute(
|
||||
"SELECT wiki, rcid, postid FROM rcgcdw WHERE postid != '-1' OR postid IS NULL GROUP BY wiki")
|
||||
for db_wiki in fetch_all.fetchall():
|
||||
if db_wiki["wiki"] not in rcqueue.irc_mapping["fandom.com"].updated_discussions and all_wikis[db_wiki["wiki"]].last_discussion_check+settings["irc_overtime"] > time.time(): # I swear if another wiki farm ever starts using Fandom discussions I'm gonna use explosion magic
|
||||
try:
|
||||
local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory
|
||||
except KeyError:
|
||||
local_wiki = all_wikis[db_wiki["wiki"]] = Wiki()
|
||||
local_wiki.rc_active = db_wiki["rcid"]
|
||||
if db_wiki["wiki"] not in rcqueue.irc_mapping["fandom.com"].updated_discussions and local_wiki.last_discussion_check+settings["irc_overtime"] > time.time(): # I swear if another wiki farm ever starts using Fandom discussions I'm gonna use explosion magic
|
||||
continue
|
||||
else:
|
||||
try:
|
||||
|
|
@ -404,20 +408,15 @@ async def discussion_handler():
|
|||
header["Accept"] = "application/hal+json"
|
||||
async with aiohttp.ClientSession(headers=header,
|
||||
timeout=aiohttp.ClientTimeout(6.0)) as session:
|
||||
try:
|
||||
local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory
|
||||
except KeyError:
|
||||
local_wiki = all_wikis[db_wiki["wiki"]] = Wiki()
|
||||
local_wiki.rc_active = db_wiki["rcid"]
|
||||
try:
|
||||
feeds_response = await local_wiki.fetch_feeds(db_wiki["wiki"], session)
|
||||
except (WikiServerError, WikiError):
|
||||
continue # ignore this wiki if it throws errors
|
||||
try:
|
||||
discussion_feed_resp = await feeds_response.json(encoding="UTF-8")
|
||||
if "title" in discussion_feed_resp:
|
||||
if "error" in discussion_feed_resp:
|
||||
error = discussion_feed_resp["error"]
|
||||
if error == "site doesn't exists": # Discussions disabled
|
||||
if error == "NotFoundException": # Discussions disabled
|
||||
if db_wiki["rcid"] != -1: # RC feed is disabled
|
||||
db_cursor.execute("UPDATE rcgcdw SET postid = ? WHERE wiki = ?",
|
||||
("-1", db_wiki["wiki"],))
|
||||
|
|
@ -483,7 +482,7 @@ async def discussion_handler():
|
|||
shutdown(loop=asyncio.get_event_loop())
|
||||
else:
|
||||
logger.exception("Exception on Feeds formatter")
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in feed formatter (ok)", Post=str(post)[0:1000], Wiki=db_wiki["wiki"])
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Exception in feed formatter", Post=str(post)[0:1000], Wiki=db_wiki["wiki"])
|
||||
# Lets stack the messages
|
||||
for messages in message_list.values():
|
||||
messages = stack_message_list(messages)
|
||||
|
|
@ -501,13 +500,11 @@ async def discussion_handler():
|
|||
raise # reraise the issue
|
||||
else:
|
||||
logger.exception("Exception on Feeds formatter")
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Discussion handler task exception (bad)", Wiki=db_wiki["wiki"])
|
||||
await generic_msg_sender_exception_logger(traceback.format_exc(), "Discussion handler task exception", Wiki=db_wiki["wiki"])
|
||||
|
||||
|
||||
|
||||
def shutdown(loop, signal=None):
|
||||
# This is our best attempt at shutting down gently - we save and close the database, wait for messages to be sent,
|
||||
# stop all of the tasks and stop the look effectively shutting down all asyncio operations
|
||||
global main_tasks
|
||||
DBHandler.update_db()
|
||||
db_connection.close()
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ try: # load settings
|
|||
with open("settings.json", encoding="utf8") as sfile:
|
||||
settings = json.load(sfile)
|
||||
if "user-agent" in settings["header"]:
|
||||
settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.0") # set the version in the useragent
|
||||
settings["header"]["user-agent"] = settings["header"]["user-agent"].format(version="1.1") # set the version in the useragent
|
||||
except FileNotFoundError:
|
||||
logging.critical("No config file could be found. Please make sure settings.json is in the directory.")
|
||||
sys.exit(1)
|
||||
|
|
@ -45,20 +45,20 @@ async def feeds_compact_formatter(post_type, post, message_target, wiki, article
|
|||
else:
|
||||
logger.warning("No entry for {event} with params: {params}".format(event=thread_funnel, params=post))
|
||||
event_type = "unknown"
|
||||
message = msg_text.format(author=author, author_url=author_url, title=post["title"], url=wiki, threadId=post["threadId"], forumName=post["forumName"])
|
||||
message = msg_text.format(author=author, author_url=author_url, title=escape_formatting(post["title"]), url=wiki, threadId=post["threadId"], forumName=post["forumName"])
|
||||
else:
|
||||
event_type = "discussion/forum/reply"
|
||||
message = _("[{author}]({author_url}) created a [reply](<{url}f/p/{threadId}/r/{postId}>) to [{title}](<{url}f/p/{threadId}>) in {forumName}").format(author=author, author_url=author_url, url=wiki, threadId=post["threadId"], postId=post["id"], title=post["_embedded"]["thread"][0]["title"], forumName=post["forumName"])
|
||||
message = _("[{author}]({author_url}) created a [reply](<{url}f/p/{threadId}/r/{postId}>) to [{title}](<{url}f/p/{threadId}>) in {forumName}").format(author=author, author_url=author_url, url=wiki, threadId=post["threadId"], postId=post["id"], title=escape_formatting(post["_embedded"]["thread"][0]["title"]), forumName=post["forumName"])
|
||||
elif post_type == "WALL":
|
||||
user_wall = _("unknown") # Fail safe
|
||||
if post["forumName"].endswith(' Message Wall'):
|
||||
user_wall = post["forumName"][:-13]
|
||||
if not post["isReply"]:
|
||||
event_type = "discussion/wall/post"
|
||||
message = _("[{author}]({author_url}) created [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, title=post["title"], url=wiki, user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"])
|
||||
message = _("[{author}]({author_url}) created [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, title=escape_formatting(post["title"]), url=wiki, user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"])
|
||||
else:
|
||||
event_type = "discussion/wall/reply"
|
||||
message = _("[{author}]({author_url}) created a [reply](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}>) to [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, url=wiki, title=post["_embedded"]["thread"][0]["title"], user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"])
|
||||
message = _("[{author}]({author_url}) created a [reply](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}>) to [{title}](<{url}wiki/Message_Wall:{user_wall}?threadId={threadId}>) on [{user}'s Message Wall](<{url}wiki/Message_Wall:{user_wall}>)").format(author=author, author_url=author_url, url=wiki, title=escape_formatting(post["_embedded"]["thread"][0]["title"]), user=user_wall, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"])
|
||||
elif post_type == "ARTICLE_COMMENT":
|
||||
if article_page is None:
|
||||
article_page = {"title": _("unknown"), "fullUrl": wiki} # No page known
|
||||
|
|
@ -112,11 +112,11 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p
|
|||
if post_type == "FORUM":
|
||||
if not post["isReply"]:
|
||||
embed["url"] = "{url}f/p/{threadId}".format(url=wiki, threadId=post["threadId"])
|
||||
embed["title"] = _("Created \"{title}\"").format(title=post["title"])
|
||||
embed["title"] = _("Created \"{title}\"").format(title=escape_formatting(post["title"]))
|
||||
thread_funnel = post.get("funnel")
|
||||
if thread_funnel == "POLL":
|
||||
embed.event_type = "discussion/forum/poll"
|
||||
embed["title"] = _("Created a poll \"{title}\"").format(title=post["title"])
|
||||
embed["title"] = _("Created a poll \"{title}\"").format(title=escape_formatting(post["title"]))
|
||||
if message_target[0][1] > 1:
|
||||
poll = post["poll"]
|
||||
image_type = False
|
||||
|
|
@ -128,7 +128,7 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p
|
|||
inline=True)
|
||||
elif thread_funnel == "QUIZ":
|
||||
embed.event_type = "discussion/forum/quiz"
|
||||
embed["title"] = _("Created a quiz \"{title}\"").format(title=post["title"])
|
||||
embed["title"] = _("Created a quiz \"{title}\"").format(title=escape_formatting(post["title"]))
|
||||
if message_target[0][1] > 1:
|
||||
quiz = post["_embedded"]["quizzes"][0]
|
||||
embed["description"] = quiz["title"]
|
||||
|
|
@ -149,7 +149,7 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p
|
|||
embed.add_field(_("Tags"), ", ".join(tag_displayname))
|
||||
else:
|
||||
embed.event_type = "discussion/forum/reply"
|
||||
embed["title"] = _("Replied to \"{title}\"").format(title=post["_embedded"]["thread"][0]["title"])
|
||||
embed["title"] = _("Replied to \"{title}\"").format(title=escape_formatting(post["_embedded"]["thread"][0]["title"]))
|
||||
embed["url"] = "{url}f/p/{threadId}/r/{postId}".format(url=wiki, threadId=post["threadId"], postId=post["id"])
|
||||
elif post_type == "WALL":
|
||||
user_wall = _("unknown") # Fail safe
|
||||
|
|
@ -158,11 +158,11 @@ async def feeds_embed_formatter(post_type, post, message_target, wiki, article_p
|
|||
if not post["isReply"]:
|
||||
embed.event_type = "discussion/wall/post"
|
||||
embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}".format(url=wiki, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"])
|
||||
embed["title"] = _("Created \"{title}\" on {user}'s Message Wall").format(title=post["title"], user=user_wall)
|
||||
embed["title"] = _("Created \"{title}\" on {user}'s Message Wall").format(title=escape_formatting(post["title"]), user=user_wall)
|
||||
else:
|
||||
embed.event_type = "discussion/wall/reply"
|
||||
embed["url"] = "{url}wiki/Message_Wall:{user_wall}?threadId={threadId}#{replyId}".format(url=wiki, user_wall=quote_plus(user_wall.replace(" ", "_")), threadId=post["threadId"], replyId=post["id"])
|
||||
embed["title"] = _("Replied to \"{title}\" on {user}'s Message Wall").format(title=post["_embedded"]["thread"][0]["title"], user=user_wall)
|
||||
embed["title"] = _("Replied to \"{title}\" on {user}'s Message Wall").format(title=escape_formatting(post["_embedded"]["thread"][0]["title"]), user=user_wall)
|
||||
elif post_type == "ARTICLE_COMMENT":
|
||||
if article_page is None:
|
||||
article_page = {"title": _("unknown"), "fullUrl": wiki} # No page known
|
||||
|
|
|
|||
|
|
@ -388,7 +388,7 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c
|
|||
author=author, author_url=author_url, group_name=group_name, comment=parsed_comment
|
||||
)
|
||||
elif action == "managewiki/undelete":
|
||||
content = _("[{author}]({author_url}) restored a wiki *{wiki_name}*{comment}").format(
|
||||
content = _("[{author}]({author_url}) undeleted a wiki *{wiki_name}*{comment}").format(
|
||||
author=author, author_url=author_url, wiki_name=change["logparams"].get("wiki", _("Unknown")), comment=parsed_comment
|
||||
)
|
||||
elif action == "managewiki/unlock":
|
||||
|
|
@ -1018,7 +1018,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
|
|||
embed["title"] = _("Modified \"{usergroup_name}\" usergroup").format(usergroup_name=group_name)
|
||||
link = create_article_path(change["title"], WIKI_ARTICLE_PATH)
|
||||
elif action == "managewiki/undelete":
|
||||
embed["title"] = _("Restored a \"{wiki}\" wiki").format(wiki=change["logparams"].get("wiki", _("Unknown")))
|
||||
embed["title"] = _("Undeleted a \"{wiki}\" wiki").format(wiki=change["logparams"].get("wiki", _("Unknown")))
|
||||
link = create_article_path(change["title"], WIKI_ARTICLE_PATH)
|
||||
elif action == "managewiki/unlock":
|
||||
embed["title"] = _("Unlocked a \"{wiki}\" wiki").format(wiki=change["logparams"].get("wiki", _("Unknown")))
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ class Wiki:
|
|||
await ratelimiter.timeout_wait()
|
||||
try:
|
||||
async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(6.0)) as session:
|
||||
request = await session.get(url, allow_redirects=False)
|
||||
request = await session.get(url)
|
||||
ratelimiter.timeout_add(1.0)
|
||||
request.raise_for_status()
|
||||
json_request = await request.json(encoding="UTF-8")
|
||||
|
|
|
|||
Loading…
Reference in a new issue