Some restructuring done

src/rc.py

@@ -110,6 +110,7 @@ class Recent_Changes_Class(object):
 
 	def fetch(self, amount=settings["limit"]):
 		messagequeue.resend_msgs()
+		rcrequest = self.fetch_recentchanges_request(amount)
 		last_check = self.fetch_changes(amount=amount)
 		# If the request succeeds the last_check will be the last rcid from recentchanges query
 		if last_check is not None:
@@ -122,7 +123,24 @@ class Recent_Changes_Class(object):
 		logger.debug("Most recent rcid is: {}".format(self.recent_id))
 		return self.recent_id
 
+	def fetch_recentchanges_request(self, amount):
+		"""Make a typical MW request for rc/abuselog
+
+		If succeeds return the .json() of request and if not raises ConnectionError"""
+		request = self.safe_request(WIKI_API_PATH, params=self.construct_params(amount))
+		if request is not None:
+			try:
+				request = request.json()
+			except ValueError:
+				logger.warning("ValueError in fetching changes")
+				logger.warning("Changes URL:" + request.url)
+				self.downtime_controller(True)
+				raise ConnectionError
+			return request
+		raise ConnectionError
+
 	def construct_params(self, amount):
+		"""Constructs GET parameters for recentchanges/abuselog fetching feature"""
 		params = OrderedDict(action="query", format="json")
 		params["list"] = "recentchanges|abuselog" if settings.get("show_abuselog", False) else "recentchanges"
 		params["rcshow"] = "" if settings.get("show_bots", False) else "!bot"
@@ -134,47 +152,11 @@ class Recent_Changes_Class(object):
 			params["aflprop"] = "ids|user|title|action|result|timestamp|hidden|revid|filter"
 		return params
 
-	def fetch_changes(self, amount, clean=False):
+	def prepare_rc(self, changes: list, clean: bool, amount: int):
-		"""Fetches the :amount: of changes from the wiki.
+		"""Processes recent changes messages"""
-		Returns None on error and int of rcid of latest change if succeeded"""
-		global logged_in
-		if len(self.ids) == 0:
-			logger.debug("ids is empty, triggering clean fetch")
-			clean = True
-		raw_changes = self.safe_request(WIKI_API_PATH, params=self.construct_params(amount))
-		# action=query&format=json&list=recentchanges%7Cabuselog&rcprop=title%7Credirect%7Ctimestamp%7Cids%7Cloginfo%7Cparsedcomment%7Csizes%7Cflags%7Ctags%7Cuser&rcshow=!bot&rclimit=20&rctype=edit%7Cnew%7Clog%7Cexternal&afllimit=10&aflprop=ids%7Cuser%7Ctitle%7Caction%7Cresult%7Ctimestamp%7Chidden%7Crevid%7Cfilter
-		if raw_changes:
-			try:
-				raw_changes = raw_changes.json()
-				changes = raw_changes['query']['recentchanges']
-				# {"batchcomplete":"","warnings":{"query":{"*":"Unrecognized value for parameter \"list\": abuselog."}}}
-				changes.reverse()
-				if "warnings" in raw_changes:
-					warnings = raw_changes.get("warnings", {"query": {"*": ""}})
-					if warnings["query"]["*"] == "Unrecognized value for parameter \"list\": abuselog.":
-						settings["show_abuselog"] = False
-						logger.warning("AbuseLog extension is not enabled on the wiki. Disabling the function...")
-			except ValueError:
-				logger.warning("ValueError in fetching changes")
-				logger.warning("Changes URL:" + raw_changes.url)
-				self.downtime_controller()
-				return None
-			except KeyError:
-				logger.warning("Wiki returned %s" % (raw_changes))
-				return None
-			else:
-				if self.downtimecredibility > 0:
-					self.downtimecredibility -= 1
-					if self.streak > -1:
-						self.streak += 1
-					if self.streak > 8:
-						self.streak = -1
-						send_simple("down_detector", _("Connection to {wiki} seems to be stable now.").format(wiki=settings["wikiname"]),
-						     _("Connection status"), settings["avatars"]["connection_restored"])
-				# In the first for loop we analize the categorize events and figure if we will need more changes to fetch
-				# in order to cover all of the edits
 		categorize_events = {}
 		new_events = 0
+		changes.reverse()
 		for change in changes:
 			if not (change["rcid"] in self.ids or change["rcid"] < self.recent_id) and not clean:
 				new_events += 1
@@ -196,23 +178,30 @@ class Recent_Changes_Class(object):
 						if change["revid"] not in categorize_events:
 							categorize_events[change["revid"]] = {"new": set(), "removed": set()}
 						comment_to_match = re.sub(r'<.*?a>', '', change["parsedcomment"])
-								if recent_changes.mw_messages["recentchanges-page-added-to-category"] in comment_to_match or recent_changes.mw_messages["recentchanges-page-added-to-category-bundled"] in comment_to_match:
+						if recent_changes.mw_messages["recentchanges-page-added-to-category"] in comment_to_match or \
+								recent_changes.mw_messages[
+									"recentchanges-page-added-to-category-bundled"] in comment_to_match:
 							categorize_events[change["revid"]]["new"].add(cat_title)
 							logger.debug("Matched {} to added category for {}".format(cat_title, change["revid"]))
-								elif recent_changes.mw_messages["recentchanges-page-removed-from-category"] in comment_to_match or recent_changes.mw_messages["recentchanges-page-removed-from-category-bundled"] in comment_to_match:
+						elif recent_changes.mw_messages[
+							"recentchanges-page-removed-from-category"] in comment_to_match or \
+								recent_changes.mw_messages[
+									"recentchanges-page-removed-from-category-bundled"] in comment_to_match:
 							categorize_events[change["revid"]]["removed"].add(cat_title)
 							logger.debug("Matched {} to removed category for {}".format(cat_title, change["revid"]))
 						else:
-									logger.debug("Unknown match for category change with messages {}, {}, {}, {} and comment_to_match {}".format(recent_changes.mw_messages["recentchanges-page-added-to-category"], recent_changes.mw_messages["recentchanges-page-removed-from-category"], recent_changes.mw_messages["recentchanges-page-removed-from-category-bundled"], recent_changes.mw_messages["recentchanges-page-added-to-category-bundled"], comment_to_match))
+							logger.debug(
+								"Unknown match for category change with messages {}, {}, {}, {} and comment_to_match {}".format(
+									recent_changes.mw_messages["recentchanges-page-added-to-category"],
+									recent_changes.mw_messages["recentchanges-page-removed-from-category"],
+									recent_changes.mw_messages["recentchanges-page-removed-from-category-bundled"],
+									recent_changes.mw_messages["recentchanges-page-added-to-category-bundled"],
+									comment_to_match))
 					else:
-								logger.warning("Init information not available, could not read category information. Please restart the bot.")
+						logger.warning(
+							"Init information not available, could not read category information. Please restart the bot.")
 				else:
 					logger.debug("Log entry got suppressed, ignoring entry.")
-				# if change["revid"] in categorize_events:
-						# 	categorize_events[change["revid"]].append(cat_title)
-						# else:
-						# 	logger.debug("New category '{}' for {}".format(cat_title, change["revid"]))
-						# 	categorize_events[change["revid"]] = {cat_title: }
 		for change in changes:
 			if change["rcid"] in self.ids or change["rcid"] < self.recent_id:
 				logger.debug("Change ({}) is in ids or is lower than recent_id {}".format(change["rcid"],
@@ -225,13 +214,43 @@ class Recent_Changes_Class(object):
 				logger.debug("Rejected {val}".format(val=change["rcid"]))
 				continue
 			essential_info(change, categorize_events.get(change.get("revid"), None))
-				if "abuselog" in raw_changes["query"]:
+
-					abuse_log = raw_changes['query']['recentchanges']
+	def prepare_abuse_log(self, abuse_log: list):
 		abuse_log.reverse()
 		for entry in abuse_log:
 			abuselog_processing(entry, self)
 		return change["rcid"]
 
+	def fetch_changes(self, amount, clean=False):
+		"""Fetches the :amount: of changes from the wiki.
+		Returns None on error and int of rcid of latest change if succeeded"""
+		global logged_in
+		if len(self.ids) == 0:
+			logger.debug("ids is empty, triggering clean fetch")
+			clean = True
+		try:
+			request_json = self.fetch_recentchanges_request(amount)
+		except ConnectionError:
+			return
+		try:
+			rc = request_json["query"]['recentchanges']
+		except KeyError:
+			logger.warning("Path query.recentchanges not found inside request body. Skipping...")
+			return
+		else:
+			self.prepare_rc(rc, clean, amount)
+		if settings["show_abuselog"]:
+			try:
+				abuselog = request_json["query"]["abuselog"]  # While LYBL approach would be more performant when abuselog is not in request body, I prefer this approach for its clarity
+			except KeyError:
+				if "warnings" in request_json:
+					warnings = request_json.get("warnings", {"query": {"*": ""}})
+					if "Unrecognized value for parameter \"list\": abuselog." in warnings["query"]["*"]:
+						settings["show_abuselog"] = False
+						logger.warning("AbuseLog extension is not enabled on the wiki. Disabling the function...")
+			else:
+				self.prepare_abuse_log(abuselog)
+
 	def safe_request(self, url, params=None):
 		try:
 			if params:
@@ -240,19 +259,19 @@ class Recent_Changes_Class(object):
 				request = self.session.get(url, timeout=10, allow_redirects=False)
 		except requests.exceptions.Timeout:
 			logger.warning("Reached timeout error for request on link {url}".format(url=url))
-			self.downtime_controller()
+			self.downtime_controller(True)
 			return None
 		except requests.exceptions.ConnectionError:
 			logger.warning("Reached connection error for request on link {url}".format(url=url))
-			self.downtime_controller()
+			self.downtime_controller(True)
 			return None
 		except requests.exceptions.ChunkedEncodingError:
 			logger.warning("Detected faulty response from the web server for request on link {url}".format(url=url))
-			self.downtime_controller()
+			self.downtime_controller(True)
 			return None
 		else:
 			if 499 < request.status_code < 600:
-				self.downtime_controller()
+				self.downtime_controller(True)
 				return None
 			elif request.status_code == 302:
 				logger.critical("Redirect detected! Either the wiki given in the script settings (wiki field) is incorrect/the wiki got removed or Gamepedia is giving us the false value. Please provide the real URL to the wiki, current URL redirects to {}".format(request.next.url))
@@ -282,9 +301,10 @@ class Recent_Changes_Class(object):
 			return False
 		return True
 
-	def downtime_controller(self):
+	def downtime_controller(self, down):
 		if not settings["show_updown_messages"]:
 			return
+		if down:
 			if self.streak > -1:  # reset the streak of successful connections when bad one happens
 				self.streak = 0
 			if self.downtimecredibility < 60:
@@ -296,6 +316,16 @@ class Recent_Changes_Class(object):
 					     _("Connection status"), settings["avatars"]["connection_failed"])
 					self.last_downtime = time.time()
 					self.streak = 0
+		else:
+			if self.downtimecredibility > 0:
+				self.downtimecredibility -= 1
+				if self.streak > -1:
+					self.streak += 1
+				if self.streak > 8:
+					self.streak = -1
+					send_simple("down_detector", _("Connection to {wiki} seems to be stable now.").format(
+						wiki=settings["wikiname"]),
+					            _("Connection status"), settings["avatars"]["connection_restored"])
 
 	def clear_cache(self):
 		self.map_ips = {}

src/rcgcdw.py

@@ -78,7 +78,7 @@ def day_overview_request():
 				continuearg = request["continue"]["rccontinue"] if "continue" in request else None
 			except ValueError:
 				logger.warning("ValueError in fetching changes")
-				recent_changes.downtime_controller()
+				recent_changes.downtime_controller(True)
 				complete = 2
 			except KeyError:
 				logger.warning("Wiki returned %s" % (request))