RcGcDb/src/bot.py

import logging.config
from src.config import settings
import sqlite3
from src.wiki import Wiki, process_event, process_mwmsgs
import asyncio, aiohttp
from src.exceptions import *
from src.database import db_cursor

logging.config.dictConfig(settings["logging"])
logger = logging.getLogger("rcgcdb.bot")
logger.debug("Current settings: {settings}".format(settings=settings))

# Log Fail states with structure wiki_id: number of fail states
all_wikis: dict = {}
mw_msgs: dict = {}  # will have the type of id: tuple

# First populate the all_wikis list with every wiki
# Reasons for this: 1. we require amount of wikis to calculate the cooldown between requests
# 2. Easier to code

for wiki in db_cursor.execute('SELECT ROWID, * FROM wikis'):
	all_wikis[wiki[0]] = Wiki()

# Start queueing logic

def calculate_delay() -> float:
	min_delay = 60/settings["max_requests_per_minute"]
	if (len(all_wikis) * min_delay) < settings["minimal_cooldown_per_wiki_in_sec"]:
		return settings["minimal_cooldown_per_wiki_in_sec"]/len(all_wikis)
	else:
		return min_delay

async def main_loop():
	calc_delay = calculate_delay()

	for db_wiki in db_cursor.execute('SELECT ROWID, * FROM wikis'):
		extended = False
		if wiki[0] not in all_wikis:
			logger.debug("New wiki: {}".format(wiki[1]))
			all_wikis[wiki[0]] = Wiki()
		local_wiki = all_wikis[wiki[0]]  # set a reference to a wiki object from memory
		if local_wiki.mw_messages is None:
			extended = True
		try:
			wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3], db_wiki[4])
			await local_wiki.check_status(wiki[0], wiki_response.status, db_wiki[1])
		except (WikiServerError, WikiError):
			continue  # ignore this wikis if it throws errors
		try:
			recent_changes_resp = await wiki_response.json(encoding="UTF-8")
			recent_changes = recent_changes_resp['query']['recentchanges'].reverse()
		except:
			logger.exception("On loading json of response.")
			continue
		if extended:
			await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
		for change in recent_changes:
			if change["rcid"] < db_wiki[6]:
				await process_event(change, local_wiki)
		await asyncio.sleep(delay=calc_delay)