Updated with new CurseProfile extension changes

(cherry picked from commit dc865539f4)

rcgcdw.py

@@ -30,23 +30,21 @@ with open("settings.json") as sfile:
 	settings = json.load(sfile)
 	if settings["limitrefetch"] < settings["limit"] and settings["limitrefetch"] != -1:
 		settings["limitrefetch"] = settings["limit"]
-logged_in = False
 logging.basicConfig(level=settings["verbose_level"])
 if settings["limitrefetch"] != -1 and os.path.exists("lastchange.txt") == False:
 	with open("lastchange.txt", 'w') as sfile:
 		sfile.write("99999999999")
-logging.debug("Current settings: {settings}".format(settings=settings))
+logging.info("Current settings: {settings}".format(settings=settings))
 lang = gettext.translation('rcgcdw', localedir='locale', languages=[settings["lang"]])
 lang.install()
 ngettext = lang.ngettext
-version = "1.5"
+
-print("RcGcDw version {} is starting...".format(version))
 
 class MWError(Exception):
 	pass
 
 
-class LinkParser(HTMLParser):
+class MyHTMLParser(HTMLParser):
 	new_string = ""
 	recent_href = ""
 
@@ -71,9 +69,10 @@ class LinkParser(HTMLParser):
 		self.new_string = self.new_string + data
 
 	def handle_endtag(self, tag):
-		logging.debug(self.new_string)
+		print(self.new_string)
 
-LinkParser = LinkParser()
+
+HTMLParse = MyHTMLParser()
 
 
 def send(message, name, avatar):
@@ -98,12 +97,9 @@ def safe_read(request, *keys):
 
 
 def send_to_discord_webhook(data):
-	header = settings["header"]
-	if "content" not in data:
-		header['Content-Type'] = 'application/json'
 	try:
 		result = requests.post(settings["webhookURL"], data=data,
-		                       headers=header, timeout=10)
+		                       headers={**{'Content-Type': 'application/json'}, **settings["header"]}, timeout=10)
 	except requests.exceptions.Timeout:
 		logging.warning("Timeouted while sending data to the webhook.")
 		return 3
@@ -134,25 +130,23 @@ def webhook_formatter(action, STATIC, **params):
 	colornumber = None if isinstance(STATIC["color"], str) else STATIC["color"]
 	data = {"embeds": []}
 	embed = defaultdict(dict)
-	if STATIC["ipaction"]:
+	if "title" in params:
+		article_encoded = params["title"].replace(" ", "_").replace(')', '\)')
+	if re.match(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", params["user"]) is not None:
 		author_url = "https://{wiki}.gamepedia.com/Special:Contributions/{user}".format(wiki=settings["wiki"],
 		                                                                                user=params["user"])
-		logging.debug("current user: {} with cache of IPs: {}".format(params["user"], recent_changes.map_ips.keys()))
 		if params["user"] not in list(recent_changes.map_ips.keys()):
 			contibs = safe_read(recent_changes.safe_request(
-				"https://{wiki}.gamepedia.com/api.php?action=query&format=json&list=usercontribs&uclimit=max&ucuser={user}&ucstart={timestamp}&ucprop=".format(
+				"https://{wiki}.gamepedia.com/api.php?action=query&format=json&list=usercontribs&uclimit=max&ucuser={user}&ucprop=".format(
-					wiki=settings["wiki"], user=params["user"], timestamp=STATIC["timestamp"])), "query", "usercontribs")
+					wiki=settings["wiki"], user=params["user"])), "query", "usercontribs")
 			if contibs is None:
 				logging.warning(
 					"WARNING: Something went wrong when checking amount of contributions for given IP address")
 				params["user"] = params["user"] + "(?)"
 			else:
-				recent_changes.map_ips[params["user"]] = len(contibs)
-				logging.debug("1Current params user {} and state of map_ips {}".format(params["user"], recent_changes.map_ips))
 				params["user"] = "{author} ({contribs})".format(author=params["user"], contribs=len(contibs))
+				recent_changes.map_ips[params["user"]] = len(contibs)
 		else:
-			logging.debug(
-				"2Current params user {} and state of map_ips {}".format(params["user"], recent_changes.map_ips))
 			recent_changes.map_ips[params["user"]] += 1
 			params["user"] = "{author} ({amount})".format(author=params["user"],
 			                                              amount=recent_changes.map_ips[params["user"]])
@@ -161,6 +155,7 @@ def webhook_formatter(action, STATIC, **params):
 		                                                               user=params["user"].replace(" ", "_"))
 	if action in ("edit", "new"):  # edit or new page
 		editsize = params["size"]
+		print(editsize)
 		if editsize > 0:
 			if editsize > 6032:
 				colornumber = 65280
@@ -176,7 +171,7 @@ def webhook_formatter(action, STATIC, **params):
 		link = "https://{wiki}.gamepedia.com/index.php?title={article}&curid={pageid}&diff={diff}&oldid={oldrev}".format(
 			wiki=settings["wiki"], pageid=params["pageid"], diff=params["diff"], oldrev=params["oldrev"],
 			article=params["title"].replace(" ", "_"))
-		embed["title"] = "{redirect}{article} ({new}{minor}{editsize})".format(redirect="⤷ " if STATIC["redirect"] else "",article=params["title"], editsize="+" + str(
+		embed["title"] = "{article} ({new}{minor}{editsize})".format(article=params["title"], editsize="+" + str(
 			editsize) if editsize > 0 else editsize, new=_("(N!) ") if action == "new" else "",
 		                                                             minor=_("m ") if action == "edit" and params[
 			                                                             "minor"] else "")
@@ -198,7 +193,6 @@ def webhook_formatter(action, STATIC, **params):
 			pass
 		if params["overwrite"]:
 			if additional_info_retrieved:
-				article_encoded = params["title"].replace(" ", "_").replace(')', '\)')
 				img_timestamp = [x for x in img_info[1]["timestamp"] if x.isdigit()]
 				undolink = "https://{wiki}.gamepedia.com/index.php?title={filename}&action=revert&oldimage={timestamp}%21{filenamewon}".format(
 					wiki=settings["wiki"], filename=article_encoded, timestamp="".join(img_timestamp),
@@ -253,16 +247,16 @@ def webhook_formatter(action, STATIC, **params):
 		                                            supress=_("No redirect has been made") if params[
 			                                                                                      "supress"] == True else _(
 			                                            "A redirect has been made"))
-		embed["title"] = _("Moved {redirect}{article} to {target}").format(redirect="⤷ " if STATIC["redirect"] else "", article=params["title"], target=params["target"])
+		embed["title"] = _("Moved {article} to {target}").format(article=params["title"], target=params["target"])
 	elif action == "move/move_redir":
 		link = "https://{wiki}.gamepedia.com/{article}".format(wiki=settings["wiki"],
 		                                                       article=params["target"].replace(" ", "_"))
-		embed["title"] = _("Moved {redirect}{article} to {title} over redirect").format(redirect="⤷ " if STATIC["redirect"] else "", article=params["title"],
+		embed["title"] = _("Moved {article} to {title} over redirect").format(article=params["title"],
 		                                                                      title=params["target"])
 	elif action == "protect/move_prot":
 		link = "https://{wiki}.gamepedia.com/{article}".format(wiki=settings["wiki"],
 		                                                       article=params["title"].replace(" ", "_"))
-		embed["title"] = _("Moved protection settings from {redirect}{article} to {title}").format(redirect="⤷ " if STATIC["redirect"] else "", article=params["title"],
+		embed["title"] = _("Moved protection settings from {article} to {title}").format(article=params["title"],
 		                                                                                 title=params["target"])
 	elif action == "block/block":
 		link = "https://{wiki}.gamepedia.com/{user}".format(wiki=settings["wiki"],
@@ -454,11 +448,11 @@ def webhook_formatter(action, STATIC, **params):
 	elif action == "managetags/create":
 		link = "https://{wiki}.gamepedia.com/Special:Tags".format(wiki=settings["wiki"])
 		embed["title"] = _("Created a tag \"{tag}\"").format(tag=params["additional"]["tag"])
-		recent_changes.init_info()
+		recent_changes.update_tags()
 	elif action == "managetags/delete":
 		link = "https://{wiki}.gamepedia.com/Special:Tags".format(wiki=settings["wiki"])
 		embed["title"] = _("Deleted a tag \"{tag}\"").format(tag=params["additional"]["tag"])
-		recent_changes.init_info()
+		recent_changes.update_tags()
 	elif action == "managetags/activate":
 		link = "https://{wiki}.gamepedia.com/Special:Tags".format(wiki=settings["wiki"])
 		embed["title"] = _("Activated a tag \"{tag}\"").format(tag=params["additional"]["tag"])
@@ -489,14 +483,6 @@ def webhook_formatter(action, STATIC, **params):
 			else:
 				tag_displayname.append(tag)
 		embed["fields"].append({"name": _("Tags"), "value": ", ".join(tag_displayname)})
-	logging.debug("Current params in edit action: {}".format(params))
-	if "changed_categories" in STATIC and STATIC["changed_categories"] is not None:
-		if "fields" not in embed:
-			embed["fields"] = []
-		# embed["fields"].append({"name": _("Changed categories"), "value": ", ".join(params["new_categories"][0:15]) + ("" if (len(params["new_categories"]) < 15) else _(" and {} more").format(len(params["new_categories"])-14))})
-		new_cat = (_("**Added**: ") + ", ".join(STATIC["changed_categories"]["new"][0:16]) + ("\n" if len(STATIC["changed_categories"]["new"])<=15 else _(" and {} more\n").format(len(STATIC["changed_categories"]["new"])-15) ) ) if STATIC["changed_categories"]["new"] else ""
-		del_cat = (_("**Removed**: ") + ", ".join(STATIC["changed_categories"]["removed"][0:16]) + ("" if len(STATIC["changed_categories"]["removed"])<=15 else _(" and {} more").format(len(STATIC["changed_categories"]["removed"])-15) ) ) if STATIC["changed_categories"]["removed"] else ""
-		embed["fields"].append({"name": _("Changed categories"), "value": new_cat + del_cat})
 	data["embeds"].append(dict(embed))
 	data['avatar_url'] = settings["avatars"]["embed"]
 	formatted_embed = json.dumps(data, indent=4)
@@ -525,22 +511,21 @@ def handle_discord_http(code, formatted_embed):
 
 
 def first_pass(
-		change, changed_categories):  # I've decided to split the embed formatter and change handler, maybe it's more messy this way, I don't know
+		change):  # I've decided to split the embed formatter and change handler, maybe it's more messy this way, I don't know
 	if "actionhidden" in change or "suppressed" in change and "suppressed" not in settings["ignored"]:
 		webhook_formatter("suppressed",
 		                  {"timestamp": change["timestamp"], "color": settings["appearance"]["suppressed"]["color"],
 		                   "icon": settings["appearance"]["suppressed"]["icon"]}, user=change["user"])
 		return
-	LinkParser.feed(change["parsedcomment"])
+	HTMLParse.feed(change["parsedcomment"])
 	# parsedcomment = (BeautifulSoup(change["parsedcomment"], "lxml")).get_text()
-	parsedcomment = LinkParser.new_string
+	parsedcomment = HTMLParse.new_string
-	LinkParser.new_string = ""
+	HTMLParse.new_string = ""
 	logging.debug(change)
-	STATIC_VARS = {"timestamp": change["timestamp"], "tags": change["tags"], "redirect": (True if "redirect" in change else False), "ipaction": (True if "anon" in change else False), "changed_categories": changed_categories}
+	STATIC_VARS = {"timestamp": change["timestamp"], "tags": change["tags"]}
 	if not parsedcomment:
 		parsedcomment = _("No description provided")
 	if change["type"] == "edit" and "edit" not in settings["ignored"]:
-		logging.debug("List of categories in first_pass: {}".format(changed_categories))
 		STATIC_VARS = {**STATIC_VARS, **{"color": settings["appearance"]["edit"]["color"],
 		                                 "icon": settings["appearance"]["edit"]["icon"]}}
 		webhook_formatter("edit", STATIC_VARS, user=change["user"], title=change["title"], desc=parsedcomment,
@@ -598,8 +583,8 @@ def first_pass(
 			webhook_formatter(combination, STATIC_VARS, user=change["user"], title=change["title"], desc=parsedcomment,
 			                  target=change["logparams"]["target_title"])
 		elif combination == "protect/move_prot":
-			webhook_formatter(combination, STATIC_VARS, user=change["user"], title=change["logparams"]["oldtitle_title"], desc=parsedcomment,
+			webhook_formatter(combination, STATIC_VARS, user=change["user"], title=change["title"], desc=parsedcomment,
-			                  target=change["title"])
+			                  target=change["logparams"]["oldtitle_title"])
 		elif combination == "block/block":
 			webhook_formatter(combination, STATIC_VARS, user=change["user"], blocked_user=change["title"],
 			                  desc=parsedcomment, duration=change["logparams"]["duration"])
@@ -641,7 +626,7 @@ def first_pass(
 			                  field=change["logparams"]['4:section'], desc=change["parsedcomment"])
 		elif combination == "curseprofile/comment-replied":
 			webhook_formatter(combination, STATIC_VARS, user=change["user"], target=change["title"].split(':')[1],
-			                  commentid=change["logparams"]["4:comment_id"])
+			                  commentid=change["logparams"]["4:section"])
 		elif combination == "contentmodel/change":
 			webhook_formatter(combination, STATIC_VARS, user=change["user"], title=change["title"], desc=parsedcomment,
 			                  oldmodel=change["logparams"]["oldmodel"], newmodel=change["logparams"]["newmodel"])
@@ -667,25 +652,19 @@ def first_pass(
 			webhook_formatter(combination, STATIC_VARS, user=change["user"], title=change["title"], desc=parsedcomment)
 		else:
 			logging.warning("No entry matches given change!")
-			logging.warning("Entry: {}".format(change))
+			print(change)
 			send(_("Unable to process the event"), _("error"), settings["avatars"]["no_event"])
 			return
-	# elif change["type"] == "external":  # not sure what happens then, but it's listed as possible type
+	if change["type"] == "external":  # not sure what happens then, but it's listed as possible type
-	# 	logging.warning("External event happened, ignoring.")
+		logging.warning("External event happened, ignoring.")
-	# 	print(change)
+		print(change)
-	# 	return
+		return
 	elif change["type"] == "new" and "new" not in settings["ignored"]:  # new page
 		STATIC_VARS = {**STATIC_VARS, **{"color": settings["appearance"]["new"]["color"],
 		                                 "icon": settings["appearance"]["new"]["icon"]}}
 		webhook_formatter("new", STATIC_VARS, user=change["user"], title=change["title"], desc=parsedcomment,
 		                  oldrev=change["old_revid"], pageid=change["pageid"], diff=change["revid"],
 		                  size=change["newlen"])
-	elif change["type"] == "categorize":
-		return
-	else:
-		logging.warning("This event is not implemented in the bot.")
-		logging.debug("Cannot process event {}".format(change))
-		return
 
 
 def day_overview_request():
@@ -778,13 +757,14 @@ def day_overview():  # time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime(time.
 		if activity:
 			v = activity.values()
 			active_users = []
-			for user, numberu in Counter(activity).most_common(3):  # find most active users
+			for user, numberu in Counter(activity).most_common(list(v).count(max(v))):  # find most active users
-				active_users.append(user + ngettext(" ({} action)", " ({} actions)", numberu).format(numberu))
+				active_users.append(user)
 			# the_one = random.choice(active_users)
 			v = hours.values()
 			active_hours = []
-			for hour, numberh in Counter(hours).most_common(list(v).count(max(v))):  # find most active hours
+			for hour, numberh in Counter(hours).most_common(list(v).count(max(v))):  # find most active users
 				active_hours.append(str(hour))
+			usramount = ngettext(" ({} action)", " ({} actions)", numberu).format(numberu)
 			houramount = ngettext(" UTC ({} action)", " UTC ({} actions)", numberh).format(numberh)
 		else:
 			active_users = [_("But nobody came")]  # a reference to my favorite game of all the time, sorry ^_^
@@ -793,7 +773,7 @@ def day_overview():  # time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime(time.
 			houramount = ""
 		embed["fields"] = []
 		fields = (
-		(ngettext("Most active user", "Most active users", len(active_users)), ', '.join(active_users)),
+		(ngettext("Most active user", "Most active users", len(active_users)), ', '.join(active_users) + usramount),
 		(_("Edits made"), edits), (_("New files"), files), (_("Admin actions"), admin),
 		(_("Bytes changed"), changed_bytes), (_("New articles"), new_articles),
 		(_("Unique contributors"), str(len(activity))),
@@ -820,7 +800,6 @@ class recent_changes_class(object):
 	groups = {}
 	unsent_messages = []
 	streak = -1
-	mw_messages = {}
 	session = requests.Session()
 	session.headers.update(settings["header"])
 	if settings["limitrefetch"] != -1:
@@ -837,12 +816,11 @@ class recent_changes_class(object):
 
 	def handle_mw_errors(self, request):
 		if "errors" in request:
-			logging.error(request["errors"])
+			print(request["errors"])
 			raise MWError
 		return request
 
 	def log_in(self):
-		global logged_in
 		# session.cookies.clear()
 		if '@' not in settings["wiki_bot_login"]:
 			logging.error(
@@ -875,7 +853,6 @@ class recent_changes_class(object):
 		try:
 			if response.json()['login']['result'] == "Success":
 				logging.info("Logging to the wiki succeeded")
-				logged_in = True
 			else:
 				logging.error("Logging in have not succeeded")
 		except:
@@ -914,16 +891,14 @@ class recent_changes_class(object):
 			with open("lastchange.txt", "w") as record:
 				record.write(str(self.file_id))
 		logging.debug("Most recent rcid is: {}".format(self.recent_id))
-		return self.recent_id
 
 	def fetch_changes(self, amount, clean=False):
-		global logged_in
 		if len(self.ids) == 0:
 			logging.debug("ids is empty, triggering clean fetch")
 			clean = True
 		changes = self.safe_request(
-			"https://{wiki}.gamepedia.com/api.php?action=query&format=json&list=recentchanges&rcshow=!bot&rcprop=title%7Credirect%7Ctimestamp%7Cids%7Cloginfo%7Cparsedcomment%7Csizes%7Cflags%7Ctags%7Cuser&rclimit={amount}&rctype=edit%7Cnew%7Clog%7Cexternal{categorize}".format(
+			"https://{wiki}.gamepedia.com/api.php?action=query&format=json&list=recentchanges&rcshow=!bot&rcprop=title%7Ctimestamp%7Cids%7Cloginfo%7Cparsedcomment%7Csizes%7Cflags%7Ctags%7Cuser&rclimit={amount}&rctype=edit%7Cnew%7Clog%7Cexternal".format(
-				wiki=settings["wiki"], amount=amount, categorize="%7Ccategorize" if settings["show_added_categories"] else ""))
+				wiki=settings["wiki"], amount=amount))
 		if changes:
 			try:
 				changes = changes.json()['query']['recentchanges']
@@ -948,42 +923,6 @@ class recent_changes_class(object):
 						self.streak = -1
 						send(_("Connection to {wiki} seems to be stable now.").format(wiki=settings["wikiname"]),
 						     _("Connection status"), settings["avatars"]["connection_restored"])
-				# In the first for loop we analize the categorize events and figure if we will need more changes to fetch
-				# in order to cover all of the edits
-				categorize_events = {}
-				new_events = 0
-				for change in changes:
-					if not (change["rcid"] in self.ids or change["rcid"] < self.recent_id) and not clean:
-						new_events += 1
-						logging.debug(
-							"New event: {}".format(change["rcid"]))
-						if new_events == settings["limit"]:
-							if amount < 500:
-							# call the function again with max limit for more results, ignore the ones in this request
-								logging.debug("There were too many new events, requesting max amount of events from the wiki.")
-								return self.fetch(amount=5000 if logged_in else 500)
-							else:
-								logging.debug(
-									"There were too many new events, but the limit was high enough we don't care anymore about fetching them all.")
-					if change["type"] == "categorize":
-						cat_title = change["title"].split(':', 1)[1]
-						# I so much hate this, blame Markus for making me do this
-						if change["revid"] not in categorize_events:
-							categorize_events[change["revid"]] = {"new": [], "removed": []}
-						comment_to_match = re.sub('<.*?a>', '', change["parsedcomment"])
-						if recent_changes.mw_messages["recentchanges-page-added-to-category"].replace("[[:$1]]", "") in comment_to_match:
-							categorize_events[change["revid"]]["new"].append(cat_title)
-							logging.debug("Matched {} to added category for {}".format(cat_title, change["revid"]))
-						elif recent_changes.mw_messages["recentchanges-page-removed-from-category"].replace("[[:$1]]", "") in comment_to_match:
-							categorize_events[change["revid"]]["removed"].append(cat_title)
-							logging.debug("Matched {} to removed category for {}".format(cat_title, change["revid"]))
-						else:
-							logging.debug("Unknown match for category change with messages {} and {} and comment_to_match {}".format(recent_changes.mw_messages["recentchanges-page-added-to-category"].replace("[[:$1]]", ""), recent_changes.mw_messages["recentchanges-page-removed-from-category"].replace("[[:$1]]", ""), comment_to_match))
-				# if change["revid"] in categorize_events:
-						# 	categorize_events[change["revid"]].append(cat_title)
-						# else:
-						# 	logging.debug("New category '{}' for {}".format(cat_title, change["revid"]))
-						# 	categorize_events[change["revid"]] = {cat_title: }
 				for change in changes:
 					if change["rcid"] in self.ids or change["rcid"] < self.recent_id:
 						logging.debug("Change ({}) is in ids or is lower than recent_id {}".format(change["rcid"],
@@ -995,7 +934,7 @@ class recent_changes_class(object):
 					if clean and not (self.recent_id == 0 and change["rcid"] > self.file_id):
 						logging.debug("Rejected {val}".format(val=change["rcid"]))
 						continue
-					first_pass(change, categorize_events.get(change.get("revid"), None))
+					first_pass(change)
 				return change["rcid"]
 
 	def safe_request(self, url):
@@ -1053,28 +992,21 @@ class recent_changes_class(object):
 	def clear_cache(self):
 		self.map_ips = {}
 
-	def init_info(self):
+	def update_tags(self):
-		startup_info = safe_read(self.safe_request(
+		tags_read = safe_read(self.safe_request(
-			"https://{wiki}.gamepedia.com/api.php?action=query&format=json&uselang=content&list=tags|recentchanges&meta=allmessages&utf8=1&tglimit=max&tgprop=name|displayname&ammessages=recentchanges-page-added-to-category|recentchanges-page-removed-from-category&amenableparser=1&amincludelocal=1".format(
+			"https://{wiki}.gamepedia.com/api.php?action=query&format=json&list=tags&tglimit=max&tgprop=name|displayname".format(
-				wiki=settings["wiki"])), "query")
+				wiki=settings["wiki"])), "query", "tags")
-		if "tags" in startup_info and "allmessages" in startup_info:
+		if tags_read:
-			for tag in startup_info["tags"]:
+			for tag in tags_read:
 				self.tags[tag["name"]] = (BeautifulSoup(tag["displayname"], "lxml")).get_text()
-			for message in startup_info["allmessages"]:
-				self.mw_messages[message["name"]] = message["*"]
 		else:
-			logging.warning("Could not retrieve initial wiki information. Some features may not work correctly!")
+			logging.warning("Could not retrive tags. Internal names will be used!")
-			logging.debug(startup_info)
 
 
 recent_changes = recent_changes_class()
-try:
 if settings["wiki_bot_login"] and settings["wiki_bot_password"]:
 	recent_changes.log_in()
-	recent_changes.init_info()
+recent_changes.update_tags()
-except requests.exceptions.ConnectionError:
-	logging.critical("A connection can't be established with the wiki. Exiting...")
-	sys.exit(1)
 time.sleep(1.0)
 recent_changes.fetch(amount=settings["limitrefetch"] if settings["limitrefetch"] != -1 else settings["limit"])