Added code

.gitignore

@@ -0,0 +1 @@
+/src/__pycache__

src/bot.py

@@ -1,6 +1,8 @@
 import logging.config
 from src.config import settings
 import sqlite3
+import sys
+import signal
 from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info
 import asyncio, aiohttp
 from src.misc import get_paths
@@ -49,6 +51,7 @@ def generate_targets(wiki_url: str) -> defaultdict:
 
 
 async def wiki_scanner():
+	try:
 		while True:
 			calc_delay = calculate_delay()
 			fetch_all = db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki')
@@ -75,6 +78,9 @@ async def wiki_scanner():
 						raise WikiError
 					recent_changes = recent_changes_resp['query']['recentchanges']
 					recent_changes.reverse()
+				except asyncio.exceptions.TimeoutError:
+					logger.debug("Timeout on fetching {}.".format(db_wiki[3]))
+					continue
 				except:
 					logger.exception("On loading json of response.")
 					continue
@@ -101,6 +107,8 @@ async def wiki_scanner():
 					DBHandler.add(db_wiki[3], change["rcid"])
 				DBHandler.update_db()
 				await asyncio.sleep(delay=calc_delay)
+	except asyncio.CancelledError:
+		return
 
 
 async def message_sender():
@@ -108,17 +116,26 @@ async def message_sender():
 		await messagequeue.resend_msgs()
 
 
+def shutdown(loop, signal=None):
+	DBHandler.update_db()
+	loop.stop()
+	logger.info("Script has shut down due to signal {}.".format(signal))
+	for task in asyncio.all_tasks(loop):
+		task.cancel()
+	sys.exit(0)
+
 def global_exception_handler(loop, context):
 	"""Global exception handler for asyncio, lets us know when something crashes"""
 	msg = context.get("exception", context["message"])
-	logger.error(msg)
-	#requests.post("https://discord.com/api/webhooks/" + settings["monitoring_webhook"],
-	#              data=DiscordMessage("embed", "exception", None, content=
-	#              "[RcGcDb] Exception detected, function might have shut down! Exception: {}".format(msg), wiki=None))
+	logger.error("Global exception handler:" + msg)
 
 
 async def main_loop():
 	loop = asyncio.get_event_loop()
+	signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)
+	for s in signals:
+		loop.add_signal_handler(
+			s, lambda s=s: shutdown(loop, signal=s))
 	loop.set_exception_handler(global_exception_handler)
 	task1 = asyncio.create_task(wiki_scanner())
 	task2 = asyncio.create_task(message_sender())

src/discord.py

@@ -15,7 +15,7 @@ logger = logging.getLogger("rcgcdb.discord")
 
 
 # User facing webhook functions
-def wiki_removal(wiki_id, status):
+async def wiki_removal(wiki_id, status):
 	for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wiki = ?', (wiki_id,)):
 		def _(string: str) -> str:
 			"""Our own translation string to make it compatible with async"""
@@ -23,7 +23,13 @@ def wiki_removal(wiki_id, status):
 		reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"),
 		           402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")}
 		reason = reasons.get(status, _("unknown error"))
-		send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[observer[2]], content=_("The webhook for {} has been removed due to {}.".format(wiki_id, reason)), wiki=None))
+		await send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[observer[2]], content=_("The webhook for {} has been removed due to {}.".format(wiki_id, reason)), wiki=None))
+		header = settings["header"]
+		header['Content-Type'] = 'application/json'
+		header['X-Audit-Log-Reason'] = "Wiki becoming unavailable"
+		async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(5.0)) as session:
+			await session.delete("https://discord.com/api/webhooks/"+observer[2])
+
 
 async def webhook_removal_monitor(webhook_url: list, reason: int):
 	await send_to_discord_webhook_monitoring(DiscordMessage("compact", "webhook/remove", None, content="The webhook {} has been removed due to {}.".format("https://discord.com/api/webhooks/" + webhook_url[0], reason), wiki=None),
@@ -97,28 +103,28 @@ class DiscordMessage:
 
 
 # Monitoring webhook functions
-def wiki_removal_monitor(wiki_id, status):
-	pass
+async def wiki_removal_monitor(wiki_id, status):
+	await send_to_discord_webhook_monitoring(DiscordMessage("compact", "webhook/remove", content="Removing {} because {}.".format(wiki_id, status), webhook_url=[None], wiki=None))
 
 
-async def send_to_discord_webhook_monitoring(data: DiscordMessage, session: aiohttp.ClientSession):
+async def send_to_discord_webhook_monitoring(data: DiscordMessage):
 	header = settings["header"]
 	header['Content-Type'] = 'application/json'
+	async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(5.0)) as session:
 		try:
-		result = await session.post("https://discord.com/api/webhooks/"+settings["monitoring_webhook"], data=repr(data),
-		                       headers=header)
+			result = await session.post("https://discord.com/api/webhooks/"+settings["monitoring_webhook"], data=repr(data))
 		except (aiohttp.ClientConnectionError, aiohttp.ServerConnectionError):
 			logger.exception("Could not send the message to Discord")
 			return 3
 
 
-async def send_to_discord_webhook(data: DiscordMessage, session: aiohttp.ClientSession):
+async def send_to_discord_webhook(data: DiscordMessage):
 	header = settings["header"]
 	header['Content-Type'] = 'application/json'
+	async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(5.0)) as session:
 		for webhook in data.webhook_url:
 			try:
-			result = await session.post("https://discord.com/api/webhooks/"+webhook, data=repr(data),
-			                       headers=header)
+				result = await session.post("https://discord.com/api/webhooks/"+webhook, data=repr(data))
 			except (aiohttp.ClientConnectionError, aiohttp.ServerConnectionError):
 				logger.exception("Could not send the message to Discord")
 				return 3

src/request_tracking.py

@@ -0,0 +1,23 @@
+import aiohttp
+import logging
+from src.config import settings
+
+logger = logging.getLogger("rcgcdb.request_tracking")
+
+class WikiRequestTracking:
+	def __init__(self):
+		self.current_timeout = 0
+
+	async def add_timeout(self, time: float):
+		self.current_timeout += time
+
+	def is_fandom(self, url):
+		if any(x in url for x in ("fandom.com", "gamepedia.com", "wikia.org")):
+			return True
+		return False
+
+async def on_request_start(session, trace_config_ctx, params):
+	if
+
+trace_config = aiohttp.TraceConfig()
+trace_config.on_request_start.append(on_request_start)

src/wiki.py

@@ -40,7 +40,7 @@ class Wiki:
 			          "rcprop": "title|redirect|timestamp|ids|loginfo|parsedcomment|sizes|flags|tags|user",
 			          "rclimit": amount, "rctype": "edit|new|log|external", "siprop": "namespaces|general"}
 		try:
-			async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0)) as session:
+			async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(6.0)) as session:
 				response = await session.get(url_path, params=params)
 		except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError):
 			logger.exception("A connection error occurred while requesting {}".format(url_path))
@@ -73,8 +73,8 @@ class Wiki:
 			raise WikiServerError
 
 	async def remove(self, wiki_id, reason):
-		src.discord.wiki_removal(wiki_id, reason)
-		src.discord.wiki_removal_monitor(wiki_id, reason)
+		await src.discord.wiki_removal(wiki_id, reason)
+		await src.discord.wiki_removal_monitor(wiki_id, reason)
 		db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", (wiki_id,))
 		logger.warning("{} rows affected by DELETE FROM rcgcdw WHERE wiki = {}".format(db_cursor.rowcount, wiki_id))
 		db_connection.commit()