More fixes

src/bot.py

@@ -64,26 +64,31 @@ async def wiki_scanner():
 				local_wiki = all_wikis[db_wiki[3]]  # set a reference to a wiki object from memory
 				if local_wiki.mw_messages is None:
 					extended = True
-				logger.debug("test")
-				try:
-					wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3])
-					await local_wiki.check_status(db_wiki[3], wiki_response.status)
-				except (WikiServerError, WikiError):
-					logger.exception("Exeption when fetching the wiki")
-					continue  # ignore this wiki if it throws errors
-				try:
-					recent_changes_resp = await wiki_response.json(encoding="UTF-8")
-					if "error" in recent_changes_resp or "errors" in recent_changes_resp:
-						# TODO Remove on some errors (example "code": "readapidenied")
-						raise WikiError
-					recent_changes = recent_changes_resp['query']['recentchanges']
-					recent_changes.reverse()
-				except asyncio.exceptions.TimeoutError:
-					logger.debug("Timeout on fetching {}.".format(db_wiki[3]))
-					continue
-				except:
-					logger.exception("On loading json of response.")
-					continue
+				async with aiohttp.ClientSession(headers=settings["header"],
+				                                 timeout=aiohttp.ClientTimeout(2.0)) as session:
+					try:
+						wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3], session)
+						await local_wiki.check_status(db_wiki[3], wiki_response.status)
+					except (WikiServerError, WikiError):
+						logger.exception("Exeption when fetching the wiki")
+						continue  # ignore this wiki if it throws errors
+					try:
+						recent_changes_resp = await wiki_response.json()
+						if "error" in recent_changes_resp or "errors" in recent_changes_resp:
+							error = recent_changes_resp.get("error", recent_changes_resp["errors"])
+							if error["code"] == "readapidenied":
+								await local_wiki.fail_add(db_wiki[3], 410)
+								continue
+							raise WikiError
+						recent_changes = recent_changes_resp['query']['recentchanges']
+						recent_changes.reverse()
+					except aiohttp.ContentTypeError:
+						logger.exception("Wiki seems to be resulting in non-json content.")
+						await local_wiki.fail_add(db_wiki[3], 410)
+						continue
+					except:
+						logger.exception("On loading json of response.")
+						continue
 				if extended:
 					await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
 				if db_wiki[6] is None:  # new wiki, just get the last rc to not spam the channel
@@ -108,7 +113,7 @@ async def wiki_scanner():
 				DBHandler.update_db()
 				await asyncio.sleep(delay=calc_delay)
 	except asyncio.CancelledError:
-		return
+		raise
 
 
 async def message_sender():
@@ -121,6 +126,7 @@ def shutdown(loop, signal=None):
 	loop.stop()
 	logger.info("Script has shut down due to signal {}.".format(signal))
 	for task in asyncio.all_tasks(loop):
+		logger.debug("Killing task {}".format(task.get_name()))
 		task.cancel()
 	sys.exit(0)
 
@@ -134,17 +140,20 @@ async def main_loop():
 	loop = asyncio.get_event_loop()
 	try:
 		signals = (signal.SIGHUP, signal.SIGTERM, signal.SIGINT)
+		for s in signals:
+			loop.add_signal_handler(
+				s, lambda s=s: shutdown(loop, signal=s))
 	except AttributeError:
 		logger.info("Running on Windows huh? This complicates things")
 		signals = (signal.SIGBREAK, signal.SIGTERM, signal.SIGINT)
-	for s in signals:
-		loop.add_signal_handler(
-			s, lambda s=s: shutdown(loop, signal=s))
 	loop.set_exception_handler(global_exception_handler)
-	task1 = asyncio.create_task(wiki_scanner())
-	task2 = asyncio.create_task(message_sender())
-	await task1
-	await task2
+	try:
+		task1 = asyncio.create_task(wiki_scanner())
+		task2 = asyncio.create_task(message_sender())
+		await task1
+		await task2
+	except KeyboardInterrupt:
+		shutdown(loop)
 
 
 asyncio.run(main_loop())

src/discord.py

@@ -21,7 +21,7 @@ async def wiki_removal(wiki_id, status):
 			"""Our own translation string to make it compatible with async"""
 			return langs[observer[4]].gettext(string)
 		reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"),
-		           402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible")}
+		           402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible"), 410: _("wiki becoming inaccessible")}
 		reason = reasons.get(status, _("unknown error"))
 		await send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[observer[2]], content=_("The webhook for {} has been removed due to {}.".format(wiki_id, reason)), wiki=None))
 		header = settings["header"]

src/formatters/rc.py

@@ -104,10 +104,13 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c
 			english_length_num = re.sub(r"(\D+)", "", change["logparams"]["duration"])
 			try:
 				english_length = english_length.rstrip("s").strip()
-				block_time = "{num} {translated_length}".format(num=english_length_num,
+				try:
+					block_time = "{num} {translated_length}".format(num=english_length_num,
 				                                                translated_length=ngettext(english_length,
 				                                                                           english_length + "s",
 				                                                                           int(english_length_num)))
+				except ValueError:
+					logger.exception("Couldn't properly resolve block expiry.")
 			except AttributeError:
 				logger.error("Could not strip s from the block event, seems like the regex didn't work?")
 				return

src/msgqueue.py

@@ -40,9 +40,9 @@ class MessageQueue:
 				logger.debug(
 					"Trying to send a message to Discord from the queue with id of {} and content {}".format(str(num),
 					                                                                                         str(item)))
-				if await send_to_discord_webhook(item, self.session) < 2:
+				if await send_to_discord_webhook(item) < 2:
 					logger.debug("Sending message succeeded")
-					await asyncio.sleep(1.5)
+					await asyncio.sleep(1.9)
 				else:
 					logger.debug("Sending message failed")
 					break

src/wiki.py

@@ -7,6 +7,7 @@ from src.formatters.rc import embed_formatter, compact_formatter
 from src.misc import parse_link
 from src.i18n import langs
 import src.discord
+import asyncio
 from src.config import settings
 from bs4 import BeautifulSoup
 
@@ -22,7 +23,7 @@ class Wiki:
 	session: aiohttp.ClientSession = None
 
 
-	async def fetch_wiki(self, extended, script_path) -> aiohttp.ClientResponse:
+	async def fetch_wiki(self, extended, script_path, session) -> aiohttp.ClientResponse:
 		url_path = script_path + "api.php"
 		amount = 20
 		if extended:
@@ -30,18 +31,17 @@ class Wiki:
 			          "meta": "allmessages|siteinfo",
 			          "utf8": 1, "tglimit": "max", "tgprop": "displayname",
 			          "rcprop": "title|redirect|timestamp|ids|loginfo|parsedcomment|sizes|flags|tags|user",
-			          "rclimit": amount, "rctype": "edit|new|log|external",
+			          "rclimit": amount, "rcshow": "!bot", "rctype": "edit|new|log|external",
 			          "ammessages": "recentchanges-page-added-to-category|recentchanges-page-removed-from-category|recentchanges-page-added-to-category-bundled|recentchanges-page-removed-from-category-bundled",
 			          "amenableparser": 1, "amincludelocal": 1, "siprop": "namespaces|general"}
 		else:
 			params = {"action": "query", "format": "json", "uselang": "content", "list": "tags|recentchanges",
 			          "meta": "siteinfo", "utf8": 1,
-			          "tglimit": "max", "tgprop": "displayname",
+			          "tglimit": "max", "rcshow": "!bot", "tgprop": "displayname",
 			          "rcprop": "title|redirect|timestamp|ids|loginfo|parsedcomment|sizes|flags|tags|user",
 			          "rclimit": amount, "rctype": "edit|new|log|external", "siprop": "namespaces|general"}
 		try:
-			async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(6.0)) as session:
-				response = await session.get(url_path, params=params)
+			response = await session.get(url_path, params=params)
 		except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError):
 			logger.exception("A connection error occurred while requesting {}".format(url_path))
 			raise WikiServerError
@@ -49,7 +49,7 @@ class Wiki:
 
 	async def safe_request(self, url):
 		try:
-			async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0)) as session:
+			async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(2.0)) as session:
 				request = await session.get(url, timeout=5, allow_redirects=False)
 			request.raise_for_status()
 		except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError):
@@ -58,15 +58,19 @@ class Wiki:
 		else:
 			return request
 
+	async def fail_add(self, wiki_url, status):
+		logger.debug("Increasing fail_times to {}".format(self.fail_times+3))
+		self.fail_times += 3
+		if self.fail_times > 9:
+			await self.remove(wiki_url, status)
+
 	async def check_status(self, wiki_url, status):
 		if 199 < status < 300:
-			self.fail_times = 0
+			self.fail_times -= 1
 			pass
 		elif 400 < status < 500:  # ignore 400 error since this might be our fault
-			self.fail_times += 1
+			await self.fail_add(wiki_url, status)
 			logger.warning("Wiki {} responded with HTTP code {}, increased fail_times to {}, skipping...".format(wiki_url, status, self.fail_times))
-			if self.fail_times > 3:
-				await self.remove(wiki_url, status)
 			raise WikiError
 		elif 499 < status < 600:
 			logger.warning("Wiki {} responded with HTTP code {}, skipping...".format(wiki_url, status, self.fail_times))
@@ -75,8 +79,8 @@ class Wiki:
 	async def remove(self, wiki_id, reason):
 		await src.discord.wiki_removal(wiki_id, reason)
 		await src.discord.wiki_removal_monitor(wiki_id, reason)
-		db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", (wiki_id,))
-		logger.warning("{} rows affected by DELETE FROM rcgcdw WHERE wiki = {}".format(db_cursor.rowcount, wiki_id))
+		db_cursor.execute('DELETE FROM rcgcdw WHERE wiki = "?"', (wiki_id,))
+		logger.warning('{} rows affected by DELETE FROM rcgcdw WHERE wiki = "{}"'.format(db_cursor.rowcount, wiki_id))
 		db_connection.commit()
 
 	async def pull_comment(self, comment_id, WIKI_API_PATH):
@@ -143,7 +147,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict):
 		if not "missing" in message:  # ignore missing strings
 			msgs.append((message["name"], re.sub(r'\[\[.*?\]\]', '', message["*"])))
 		else:
-			logging.warning("Could not fetch the MW message translation for: {}".format(message["name"]))
+			logger.warning("Could not fetch the MW message translation for: {}".format(message["name"]))
 	msgs = tuple(msgs)
 	for key, set in mw_msgs.items():
 		if msgs == set:
@@ -163,7 +167,6 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_
 	lang = langs[target[0][0]]
 	ngettext = lang.ngettext
 	# recent_changes = RecentChangesClass()  # TODO Look into replacing RecentChangesClass with local_wiki
-	logger.debug(change)
 	appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter
 	if ("actionhidden" in change or "suppressed" in change):  # if event is hidden using suppression
 		await appearance_mode("suppressed", change, "", changed_categories, local_wiki, target, _, ngettext, paths)