Refactoring - moved a few functions to misc

misc.py

@@ -22,6 +22,7 @@ from configloader import settings
 import gettext
 
 # Initialize translation
+
 t = gettext.translation('misc', localedir='locale', languages=[settings["lang"]])
 _ = t.gettext
 
@@ -144,3 +145,78 @@ class ContentParser(HTMLParser):
 			self.current_tag = "afterdel"
 		else:
 			self.current_tag = ""
+
+
+class LinkParser(HTMLParser):
+	new_string = ""
+	recent_href = ""
+
+	def handle_starttag(self, tag, attrs):
+		for attr in attrs:
+			if attr[0] == 'href':
+				self.recent_href = attr[1]
+				if self.recent_href.startswith("//"):
+					self.recent_href = "https:{rest}".format(rest=self.recent_href)
+				elif not self.recent_href.startswith("http"):
+					self.recent_href = "https://{wiki}.gamepedia.com".format(wiki=settings["wiki"]) + self.recent_href
+				self.recent_href = self.recent_href.replace(")", "\\)")
+
+	def handle_data(self, data):
+		if self.recent_href:
+			self.new_string = self.new_string + "[{}](<{}>)".format(data, self.recent_href)
+			self.recent_href = ""
+		else:
+			self.new_string = self.new_string + data
+
+	def handle_comment(self, data):
+		self.new_string = self.new_string + data
+
+	def handle_endtag(self, tag):
+		misc_logger.debug(self.new_string)
+
+
+def safe_read(request, *keys):
+	if request is None:
+		return None
+	try:
+		request = request.json()
+		for item in keys:
+			request = request[item]
+	except KeyError:
+		misc_logger.warning(
+			"Failure while extracting data from request on key {key} in {change}".format(key=item, change=request))
+		return None
+	except ValueError:
+		misc_logger.warning("Failure while extracting data from request in {change}".format(change=request))
+		return None
+	return request
+
+
+def handle_discord_http(code, formatted_embed, result):
+	if 300 > code > 199:  # message went through
+		return 0
+	elif code == 400:  # HTTP BAD REQUEST result.status_code, data, result, header
+		misc_logger.error(
+			"Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:")
+		misc_logger.error(formatted_embed)
+		misc_logger.error(result.text)
+		return 1
+	elif code == 401 or code == 404:  # HTTP UNAUTHORIZED AND NOT FOUND
+		misc_logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.")
+		sys.exit(1)
+	elif code == 429:
+		misc_logger.error("We are sending too many requests to the Discord, slowing down...")
+		return 2
+	elif 499 < code < 600:
+		misc_logger.error(
+			"Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format(
+				code))
+		return 3
+
+
+def add_to_dict(dictionary, key):
+	if key in dictionary:
+		dictionary[key] += 1
+	else:
+		dictionary[key] = 1
+	return dictionary

rcgcdw.py

@@ -25,10 +25,8 @@ import misc
 from bs4 import BeautifulSoup
 from collections import defaultdict, Counter
 from urllib.parse import quote_plus
-from html.parser import HTMLParser
 from configloader import settings
-from misc import link_formatter
+from misc import link_formatter, LinkParser, ContentParser, safe_read, handle_discord_http, add_to_dict
-from misc import ContentParser
 
 if __name__ != "__main__":  # return if called as a module
 	logging.critical("The file is being executed as a module. Please execute the script using the console.")
@@ -68,41 +66,11 @@ if settings["limitrefetch"] != -1 and os.path.exists("lastchange.txt") is True:
 
 logged_in = False
 supported_logs = ["protect/protect", "protect/modify", "protect/unprotect", "upload/overwrite", "upload/upload", "delete/delete", "delete/delete_redir", "delete/restore", "delete/revision", "delete/event", "import/upload", "import/interwiki", "merge/merge", "move/move", "move/move_redir", "protect/move_prot", "block/block", "block/unblock", "block/reblock", "rights/rights", "rights/autopromote", "abusefilter/modify", "abusefilter/create", "interwiki/iw_add", "interwiki/iw_edit", "interwiki/iw_delete", "curseprofile/comment-created", "curseprofile/comment-edited", "curseprofile/comment-deleted", "curseprofile/profile-edited", "curseprofile/comment-replied", "contentmodel/change", "sprite/sprite", "sprite/sheet", "sprite/slice", "managetags/create", "managetags/delete", "managetags/activate", "managetags/deactivate", "tag/update"]
+LinkParser = LinkParser()
 
 class MWError(Exception):
 	pass
 
-class LinkParser(HTMLParser):
-	new_string = ""
-	recent_href = ""
-
-	def handle_starttag(self, tag, attrs):
-		for attr in attrs:
-			if attr[0] == 'href':
-				self.recent_href = attr[1]
-				if self.recent_href.startswith("//"):
-					self.recent_href = "https:{rest}".format(rest=self.recent_href)
-				elif not self.recent_href.startswith("http"):
-					self.recent_href = "https://{wiki}.gamepedia.com".format(wiki=settings["wiki"]) + self.recent_href
-				self.recent_href = self.recent_href.replace(")", "\\)")
-
-	def handle_data(self, data):
-		if self.recent_href:
-			self.new_string = self.new_string + "[{}](<{}>)".format(data, self.recent_href)
-			self.recent_href = ""
-		else:
-			self.new_string = self.new_string + data
-
-	def handle_comment(self, data):
-		self.new_string = self.new_string + data
-
-	def handle_endtag(self, tag):
-		logger.debug(self.new_string)
-
-
-LinkParser = LinkParser()
-
-
 def send(message, name, avatar):
 	dictionary_creator = {"content": message}
 	if name:
@@ -112,23 +80,6 @@ def send(message, name, avatar):
 	send_to_discord(dictionary_creator)
 
 
-def safe_read(request, *keys):
-	if request is None:
-		return None
-	try:
-		request = request.json()
-		for item in keys:
-			request = request[item]
-	except KeyError:
-		logger.warning(
-			"Failure while extracting data from request on key {key} in {change}".format(key=item, change=request))
-		return None
-	except ValueError:
-		logger.warning("Failure while extracting data from request in {change}".format(change=request))
-		return None
-	return request
-
-
 def send_to_discord_webhook(data):
 	header = settings["header"]
 	if "content" not in data:
@@ -881,28 +832,6 @@ def embed_formatter(action, change, parsed_comment, categories):
 	send_to_discord(formatted_embed)
 
 
-def handle_discord_http(code, formatted_embed, result):
-	if 300 > code > 199:  # message went through
-		return 0
-	elif code == 400:  # HTTP BAD REQUEST result.status_code, data, result, header
-		logger.error(
-			"Following message has been rejected by Discord, please submit a bug on our bugtracker adding it:")
-		logger.error(formatted_embed)
-		logger.error(result.text)
-		return 1
-	elif code == 401 or code == 404:  # HTTP UNAUTHORIZED AND NOT FOUND
-		logger.error("Webhook URL is invalid or no longer in use, please replace it with proper one.")
-		sys.exit(1)
-	elif code == 429:
-		logger.error("We are sending too many requests to the Discord, slowing down...")
-		return 2
-	elif 499 < code < 600:
-		logger.error(
-			"Discord have trouble processing the event, and because the HTTP code returned is {} it means we blame them.".format(
-				code))
-		return 3
-
-
 def essential_info(change, changed_categories):
 	"""Prepares essential information for both embed and compact message format."""
 	logger.debug(change)
@@ -981,13 +910,6 @@ def day_overview_request():
 	return result, complete
 
 
-def add_to_dict(dictionary, key):
-	if key in dictionary:
-		dictionary[key] += 1
-	else:
-		dictionary[key] = 1
-	return dictionary
-
 def daily_overview_sync(edits, files, admin, changed_bytes, new_articles, unique_contributors, day_score):
 	weight = storage["daily_overview"]["days_tracked"]
 	if weight == 0: