Fixed one thing

Broke 10 more

src/bot.py

@@ -8,6 +8,7 @@ from src.exceptions import *
 from src.database import db_cursor
 from collections import defaultdict
 from src.queue_handler import DBHandler
+from src.msgqueue import messagequeue
 
 logging.config.dictConfig(settings["logging"])
 logger = logging.getLogger("rcgcdb.bot")
@@ -37,7 +38,7 @@ def calculate_delay() -> float:
 
 def generate_targets(wiki_url: str) -> defaultdict:
 	combinations = defaultdict(list)
-	for webhook in db_cursor.execute('SELECT ROWID, * FROM rcgcdw WHERE wiki = ?', wiki_url):
+	for webhook in db_cursor.execute('SELECT ROWID, * FROM rcgcdw WHERE wiki = ?', (wiki_url,)):
 		# rowid, guild, configid, webhook, wiki, lang, display, rcid, wikiid, postid
 		combination = (webhook[5], webhook[6])  # lang, display
 		combinations[combination].append(webhook[3])
@@ -45,60 +46,66 @@ def generate_targets(wiki_url: str) -> defaultdict:
 
 
 async def wiki_scanner():
-	calc_delay = calculate_delay()
+	while True:
-	# db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
+		calc_delay = calculate_delay()
-	for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'):
+		# db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
-		extended = False
+		for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'):
-		if db_wiki[3] not in all_wikis:
+			extended = False
-			logger.debug("New wiki: {}".format(db_wiki[3]))
+			if db_wiki[3] not in all_wikis:
-			all_wikis[db_wiki[3]] = Wiki()
+				logger.debug("New wiki: {}".format(db_wiki[3]))
-		local_wiki = all_wikis[db_wiki[3]]  # set a reference to a wiki object from memory
+				all_wikis[db_wiki[3]] = Wiki()
-		if local_wiki.mw_messages is None:
+			local_wiki = all_wikis[db_wiki[3]]  # set a reference to a wiki object from memory
-			extended = True
+			if local_wiki.mw_messages is None:
-		logger.debug("test")
+				extended = True
-		try:
+			logger.debug("test")
-			wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3])
+			try:
-			await local_wiki.check_status(wiki[3], wiki_response.status)
+				wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3])
-		except (WikiServerError, WikiError):
+				await local_wiki.check_status(db_wiki[3], wiki_response.status)
-			continue  # ignore this wiki if it throws errors
+			except (WikiServerError, WikiError):
-		try:
+				logger.exception("Exeption when fetching the wiki")
-			recent_changes_resp = await wiki_response.json(encoding="UTF-8")
+				continue  # ignore this wiki if it throws errors
-			recent_changes = recent_changes_resp['query']['recentchanges'].reverse()
+			try:
-		except:
+				recent_changes_resp = await wiki_response.json(encoding="UTF-8")
-			logger.exception("On loading json of response.")
+				recent_changes = recent_changes_resp['query']['recentchanges']
-			continue
+				recent_changes.reverse()
-		if extended:
+			except:
-			await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
+				logger.exception("On loading json of response.")
-		if db_wiki[6] is None:  # new wiki, just get the last rc to not spam the channel
-			if len(recent_changes) > 0:
-				DBHandler.add(db_wiki[3], recent_changes[-1]["rcid"])
 				continue
-			else:
+			if extended:
-				DBHandler.add(db_wiki[3], 0)
+				await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
-				continue
+			if db_wiki[6] is None:  # new wiki, just get the last rc to not spam the channel
-		categorize_events = {}
+				if len(recent_changes) > 0:
-		targets = generate_targets(db_wiki[3])
+					DBHandler.add(db_wiki[3], recent_changes[-1]["rcid"])
-		paths = get_paths(db_wiki[3], recent_changes_resp)
+					continue
-		for change in recent_changes:
+				else:
-			await process_cats(change, local_wiki, mw_msgs, categorize_events)
+					DBHandler.add(db_wiki[3], 0)
-		for change in recent_changes:  # Yeah, second loop since the categories require to be all loaded up
+					continue
-			if change["rcid"] < db_wiki[6]:
+			categorize_events = {}
-				for target in targets.items():
+			targets = generate_targets(db_wiki[3])
-					await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp)
+			paths = get_paths(db_wiki[3], recent_changes_resp)
-		if recent_changes:
+			for change in recent_changes:
-			DBHandler.add(db_wiki[3], change["rcid"])
+				await process_cats(change, local_wiki, mw_msgs, categorize_events)
-		await asyncio.sleep(delay=calc_delay)
+			for change in recent_changes:  # Yeah, second loop since the categories require to be all loaded up
-
+				if change["rcid"] < db_wiki[6]:
-	DBHandler.update_db()
+					for target in targets.items():
+						await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp)
+			if recent_changes:
+				DBHandler.add(db_wiki[3], change["rcid"])
+			await asyncio.sleep(delay=calc_delay)
+		DBHandler.update_db()
 
 
 async def message_sender():
-	pass
+	while True:
+		await messagequeue.resend_msgs()
+
 
 
 async def main_loop():
 	task1 = asyncio.create_task(wiki_scanner())
 	task2 = asyncio.create_task(message_sender())
+	await task1
+	await task2
 
 
 asyncio.run(main_loop())

src/formatters/rc.py

@@ -23,13 +23,14 @@ logger = logging.getLogger("rcgcdw.rc_formatters")
 
 async def compact_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths,
                             additional_data=None):
+	global LinkParser
 	if additional_data is None:
 		additional_data = {"namespaces": {}, "tags": {}}
 	WIKI_API_PATH = paths[0]
 	WIKI_SCRIPT_PATH = paths[1]
 	WIKI_ARTICLE_PATH = paths[2]
 	WIKI_JUST_DOMAIN = paths[3]
-	LinkParser = LinkParser("domain")
+	LinkParser = LinkParser(paths[3])
 	if action != "suppressed":
 		author_url = link_formatter(create_article_path("User:{user}".format(user=change["user"]), WIKI_ARTICLE_PATH))
 		author = change["user"]
@@ -318,13 +319,14 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c
 
 
 async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None):
+	global LinkParser
 	if additional_data is None:
 		additional_data = {"namespaces": {}, "tags": {}}
 	WIKI_API_PATH = paths[0]
 	WIKI_SCRIPT_PATH = paths[1]
 	WIKI_ARTICLE_PATH = paths[2]
 	WIKI_JUST_DOMAIN = paths[3]
-	LinkParser = LinkParser()
+	LinkParser = LinkParser(paths[3])
 	embed = DiscordMessage("embed", action, target[1], wiki=WIKI_SCRIPT_PATH)
 	if parsed_comment is None:
 		parsed_comment = _("No description provided")
@@ -694,16 +696,16 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
 	if "tags" in change and change["tags"]:
 		tag_displayname = []
 		for tag in change["tags"]:
-			if tag in additional_data.tags:
+			if tag in additional_data["tags"]:
-				if additional_data.tags[tag] is None:
+				if additional_data["tags"][tag] is None:
 					continue  # Ignore hidden tags
 				else:
-					tag_displayname.append(additional_data.tags[tag])
+					tag_displayname.append(additional_data["tags"][tag])
 			else:
 				tag_displayname.append(tag)
 		embed.add_field(_("Tags"), ", ".join(tag_displayname))
 	logger.debug("Current params in edit action: {}".format(change))
-	if categories is not None and not (len(categories["new"]) == 0 and len(categories["removed"]) == 0):
+	if categories and not (len(categories["new"]) == 0 and len(categories["removed"]) == 0):
 		new_cat = (_("**Added**: ") + ", ".join(list(categories["new"])[0:16]) + ("\n" if len(categories["new"])<=15 else _(" and {} more\n").format(len(categories["new"])-15))) if categories["new"] else ""
 		del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else ""
 		embed.add_field(_("Changed categories"), new_cat + del_cat)

src/misc.py

@@ -1,4 +1,3 @@
-from abc import ABC
 from html.parser import HTMLParser
 import base64, re
 from src.config import settings
@@ -14,7 +13,7 @@ logger = logging.getLogger("rcgcdw.misc")
 class DiscordMessage():
 	"""A class defining a typical Discord JSON representation of webhook payload."""
 	def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None):
-		self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, ""))
+		self.webhook_object = dict(allowed_mentions={"parse": []})
 		self.webhook_url = webhook_url
 		self.wiki = wiki
 
@@ -123,15 +122,13 @@ def get_paths(wiki: str, request) -> tuple:
 
 
 class LinkParser(HTMLParser):
-	def error(self, message):
-		pass
 
 	new_string = ""
 	recent_href = ""
 
 	def __init__(self, domain):
-		super().__init__()
 		self.WIKI_JUST_DOMAIN = domain
+		super().__init__()
 
 	def handle_starttag(self, tag, attrs):
 		for attr in attrs:

src/msgqueue.py

@@ -52,6 +52,7 @@ class MessageQueue:
 				logger.debug("Queue emptied, all messages delivered")
 			self.cut_messages(num)
 			logger.debug(self._queue)
+		await asyncio.sleep(4.0)
 
 
 messagequeue = MessageQueue()

src/queue_handler.py

@@ -15,7 +15,7 @@ class UpdateDB():
 
 	def update_db(self):
 		for update in self.updated:
-			db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", update[1], update[0])
+			db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", (update[1], update[0],))
 		db_connection.commit()
 		self.clear_list()
 

src/wiki.py

@@ -77,7 +77,7 @@ class Wiki:
 	async def remove(self, wiki_id, reason):
 		src.discord.wiki_removal(wiki_id, reason)
 		src.discord.wiki_removal_monitor(wiki_id, reason)
-		db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", wiki_id)
+		db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", (wiki_id,))
 		logger.warning("{} rows affected by DELETE FROM rcgcdw WHERE wiki = {}".format(db_cursor.rowcount, wiki_id))
 		db_connection.commit()
 
@@ -141,7 +141,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict):
 	:return:
 	"""
 	msgs = []
-	for message in wiki_response["allmessages"]:
+	for message in wiki_response["query"]["allmessages"]:
 		if not "missing" in message:  # ignore missing strings
 			msgs.append((message["name"], re.sub(r'\[\[.*?\]\]', '', message["*"])))
 		else:
@@ -157,6 +157,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict):
 	local_wiki.mw_messages = key
 
 async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_wiki: tuple, target: tuple, paths: tuple, request: dict):
+	global LinkParser
 	"""Prepares essential information for both embed and compact message format."""
 	def _(string: str) -> str:
 		"""Our own translation string to make it compatible with async"""
@@ -165,7 +166,7 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_
 	lang = langs[target[0][0]]
 	ngettext = lang.ngettext
 	# recent_changes = RecentChangesClass()  # TODO Look into replacing RecentChangesClass with local_wiki
-	LinkParser = LinkParser("domain")
+	LinkParser = LinkParser(paths[3])
 	logger.debug(change)
 	appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter
 	if ("actionhidden" in change or "suppressed" in change):  # if event is hidden using suppression