Merge branch 'testing' into 'master'

Testing

See merge request piotrex43/RcGcDw!54

configbuilder.py

@@ -48,7 +48,7 @@ def yes_no(answer):
 	else:
 		raise ValueError
 
-print("Welcome in RcGcDw config builder! You can accept the default value if provided in the question by using Enter key without providing any other input.\nWARNING! Your current settings.json will be overwritten if you continue!")
+print("Welcome in RcGcDw config builder! This script is still work in progress so beware! You can accept the default value if provided in the question by using Enter key without providing any other input.\nWARNING! Your current settings.json will be overwritten if you continue!")
 
 try:  # load settings
 	with open("settings.json.example") as sfile:
@@ -78,6 +78,49 @@ def basic():
 			break
 
 def advanced():
+	while True:
+		if set_limit():
+			break
+	while True:
+		if set_refetch_limit():
+			break
+	while True:
+		if set_updown_messages():
+			break
+	if settings["show_updown_messages"]:
+		while True:
+			if set_downup_avatars():
+				break
+	while True:
+		if set_ignored_events():
+			break
+	while True:
+		if set_overview():
+			break
+	if settings["overview"]:
+		while True:
+			if set_overview_time():
+				break
+		while True:
+			if set_empty_overview():
+				break
+	while True:
+		if set_license_detection():
+			break
+	if settings["license_detection"]:
+		while True:
+			if set_license_detection_regex():
+				break
+		while True:
+			if set_license_classification_regex():
+				break
+	while True:
+		if set_login():
+			break
+	if settings["wiki_bot_login"]:
+		while True:
+			if set_password():
+				break
 
 
 def set_cooldown():
@@ -100,7 +143,8 @@ def set_wiki():
 		regex = re.search(r"http(?:s|):\/\/(.*?)\.gamepedia.com", option)
 		if regex.group(1):
 			option = regex.group(1)
-	wiki_request = requests.get("https://{}.gamepedia.com".format(option), timeout=10, allow_redirects=False)
+	print("Checking the wiki...")
+	wiki_request = requests.get("https://{}.gamepedia.com".format(option), timeout=10, allow_redirects=False)  # TODO Actually check the API endpoint
 	if wiki_request.status_code == 404 or wiki_request.status_code == 302:
 		print("Wiki at https://{}.gamepedia.com does not exist, are you sure you have entered the wiki correctly?".format(option))
 		return False
@@ -175,7 +219,7 @@ def set_refetch_limit():
 
 def set_updown_messages():
 	try:
-		option = yes_no(default_or_custom(input("Should the script send messages when the wiki becomes unavailable? (Y/n)"), "y"))
+		option = yes_no(default_or_custom(input("Should the script send messages when the wiki becomes unavailable? (Y/n)\n"), "y"))
 		settings["show_updown_messages"] = option
 		return True
 	except ValueError:
@@ -183,25 +227,26 @@ def set_updown_messages():
 		return False
 
 def set_downup_avatars():
-	option = default_or_custom(input("Provide a link for a custom webhook avatar when the wiki goes DOWN. (default: no avatar)"), "")  #TODO Add a check for the image
+	option = default_or_custom(input("Provide a link for a custom webhook avatar when the wiki goes DOWN. (default: no avatar)\n"), "")  #TODO Add a check for the image
 	settings["avatars"]["connection_failed"] = option
 	option = default_or_custom(
-		input("Provide a link for a custom webhook avatar when the connection to the wiki is RESTORED. (default: no avatar)"),
+		input("Provide a link for a custom webhook avatar when the connection to the wiki is RESTORED. (default: no avatar)\n"),
 		"")  # TODO Add a check for the image
 	settings["avatars"]["connection_failed"] = option
 	return True
 
 def set_ignored_events():
 	option = default_or_custom(
-		input("Provide a list of entry types that are supposed to be ignored. Separate them using commas. Example: external, edit, upload/overwrite. (default: external)"), "external")  # TODO Add a check for the image
+		input("Provide a list of entry types that are supposed to be ignored. Separate them using commas. Example: external, edit, upload/overwrite. (default: external)\n"), "external")  # TODO Add a check for the image
 	entry_types = []
 	for etype in option.split(","):
 		entry_types.append(etype.strip())
 	settings["ignored"] = entry_types
+	return True
 
 def set_overview():
 	try:
-		option = yes_no(default_or_custom(input("Should the script send daily overviews of the actions done on the wiki for past 24 hours? (y/N)"), "n"))
+		option = yes_no(default_or_custom(input("Should the script send daily overviews of the actions done on the wiki for past 24 hours? (y/N)\n"), "n"))
 		settings["overview"] = option
 		return True
 	except ValueError:
@@ -209,15 +254,64 @@ def set_overview():
 		return False
 
 def set_overview_time():
+	option = default_or_custom(input("At what time should the daily overviews be sent? (script uses local machine time, the format of the time should be HH:MM, default is 00:00)\n"), "00:00")
+	check = re.match(r"^\d{2}:\d{2}$", option)
+	if check is not None:
+		settings["overview_time"] = option
+		return True
+	else:
+		print("Response not valid, please enter a time in format HH:MM like for example 00:00 or 15:21!")
+		return False
+
+def set_empty_overview():
 	try:
-		option = default_or_custom(input("At what time should the daily overviews be sent? (script uses local machine time, the format of the time should be HH:MM, default is 00:00)"), "00:00")
+		option = yes_no(default_or_custom(input("Should the script send empty overviews in case nothing happens during the day? (y/N)\n"), "n"))
-		re.match(r"$\d{2}:\d{2}^")
+		settings["send_empty_overview"] = option
-		settings["overview"] = option
 		return True
 	except ValueError:
 		print("Response not valid, please use y (yes) or n (no)")
 		return False
 
+def set_license_detection():
+	try:
+		option = yes_no(default_or_custom(input("Should the script detect licenses in the newly uploaded images? (Y/n)\n"), "y"))
+		settings["license_detection"] = option
+		return True
+	except ValueError:
+		print("Response not valid, please use y (yes) or n (no)")
+		return False
+
+def set_license_detection_regex():
+	try:
+		option = default_or_custom(input("Please provide regex for license detection (only to find it, the next step will be a regex to determine the type of the license). Default: \{\{(license|lizenz|licence|copyright)\n"), "\{\{(license|lizenz|licence|copyright)")
+		re.compile(option)
+		settings["license_regex_detect"] = option
+		return True
+	except re.error:
+		print("Given regex expression could not be compiled. Please provide a proper regex expression in Python.")
+		return False
+
+def set_license_classification_regex():
+	try:
+		option = default_or_custom(input("Please provide regex for license classification where named capture group \"license\" is used as a license type for the image. Default: \{\{(license|lizenz|licence|copyright)(\ |\|)(?P<license>.*?)\}\}\n"), "\{\{(license|lizenz|licence|copyright)(\ |\|)(?P<license>.*?)\}\}")
+		re.compile(option)
+		settings["license_regex"] = option
+		return True
+	except re.error:
+		print("Given regex expression could not be compiled. Please provide a proper regex expression in Python.")
+		return False
+
+def set_login():
+	option = default_or_custom(input("You can provide bot credentials if you want the script to use higher limits than usual. If that's the case, please provide the login. If not, just skip this option \n"), "")
+	settings["wiki_bot_login"] = option
+	return True
+
+def set_password():
+	option = default_or_custom(input("Please give bot password now.\n"), "")
+	settings["wiki_bot_password"] = option
+	return True
+
+
 try:
 	basic()
 	with open("settings.json", "w") as settings_file:

rcgcdw.py

@@ -66,6 +66,8 @@ if settings["limitrefetch"] != -1 and os.path.exists("lastchange.txt") is True:
 
 logged_in = False
 supported_logs = ["protect/protect", "protect/modify", "protect/unprotect", "upload/overwrite", "upload/upload", "delete/delete", "delete/delete_redir", "delete/restore", "delete/revision", "delete/event", "import/upload", "import/interwiki", "merge/merge", "move/move", "move/move_redir", "protect/move_prot", "block/block", "block/unblock", "block/reblock", "rights/rights", "rights/autopromote", "abusefilter/modify", "abusefilter/create", "interwiki/iw_add", "interwiki/iw_edit", "interwiki/iw_delete", "curseprofile/comment-created", "curseprofile/comment-edited", "curseprofile/comment-deleted", "curseprofile/profile-edited", "curseprofile/comment-replied", "contentmodel/change", "sprite/sprite", "sprite/sheet", "sprite/slice", "managetags/create", "managetags/delete", "managetags/activate", "managetags/deactivate", "tag/update"]
+profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XVL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")}
+
 LinkParser = LinkParser()
 
 class MWError(Exception):
@@ -80,6 +82,15 @@ def send(message, name, avatar):
 	send_to_discord(dictionary_creator)
 
 
+def profile_field_name(name, embed):
+	try:
+		return profile_fields[name]
+	except KeyError:
+		if embed:
+			return _("Unknown")
+		else:
+			return _("unknown")
+
 def send_to_discord_webhook(data):
 	header = settings["header"]
 	if "content" not in data:
@@ -113,6 +124,24 @@ def send_to_discord(data):
 			time.sleep(2.0)
 			pass
 
+
+def pull_comment(comment_id):
+	try:
+		comment = recent_changes.handle_mw_errors(recent_changes.safe_request("https://{}.gamepedia.com/api.php?action=comment&do=getRaw&comment_id={}&format=json".format(settings["wiki"], comment_id)).json())["text"]
+		logger.debug("Got the following comment from the API: {}".format(comment))
+	except MWError:
+		pass
+	except TypeError:
+		logger.exception("Could not resolve the comment text.")
+	except KeyError:
+		logger.exception("CurseProfile extension API did not respond with a valid comment content.")
+	else:
+		if len(comment) > 1000:
+			comment = comment[0:1000] + "…"
+		return comment
+	return ""
+
+
 def compact_formatter(action, change, parsed_comment, categories):
 	if action != "suppressed":
 		author_url = link_formatter("https://{wiki}.gamepedia.com/User:{user}".format(wiki=settings["wiki"], user=change["user"]))
@@ -238,39 +267,11 @@ def compact_formatter(action, change, parsed_comment, categories):
 	elif action == "curseprofile/profile-edited":
 		link = link_formatter("https://{wiki}.gamepedia.com/UserProfile:{target}".format(wiki=settings["wiki"],
 		                                                                                 target=change["title"].split(':')[1]))
-		if change["logparams"]['4:section'] == "profile-location":
-			field = _("Location")
-		elif change["logparams"]['4:section'] == "profile-aboutme":
-			field = _("About me")
-		elif change["logparams"]['4:section'] == "profile-link-google":
-			field = _("Google link")
-		elif change["logparams"]['4:section'] == "profile-link-facebook":
-			field = _("Facebook link")
-		elif change["logparams"]['4:section'] == "profile-link-twitter":
-			field = _("Twitter link")
-		elif change["logparams"]['4:section'] == "profile-link-reddit":
-			field = _("Reddit link")
-		elif change["logparams"]['4:section'] == "profile-link-twitch":
-			field = _("Twitch link")
-		elif change["logparams"]['4:section'] == "profile-link-psn":
-			field = _("PSN link")
-		elif change["logparams"]['4:section'] == "profile-link-vk":
-			field = _("VK link")
-		elif change["logparams"]['4:section'] == "profile-link-xbl":
-			field = _("XVL link")
-		elif change["logparams"]['4:section'] == "profile-link-steam":
-			field = _("Steam link")
-		elif change["logparams"]['4:section'] == "profile-link-discord":
-			field = _("Discord handle")
-		elif change["logparams"]['4:section'] == "profile-link-battlenet":
-			field = _("Battle.net handle")
-		else:
-			field = _("unknown")
 		target = _("[{target}]({target_url})'s").format(target=change["title"].split(':')[1], target_url=link) if change["title"].split(':')[1] != author else _("[their own]({target_url})").format(target_url=link)
 		content = _("[{author}]({author_url}) edited the {field} on {target} profile. *({desc})*").format(author=author,
 		                                                                        author_url=author_url,
 		                                                                        target=target,
-		                                                                        field=field,
+		                                                                        field=profile_field_name(change["logparams"]['4:section'], False),
 		                                                                        desc=BeautifulSoup(change["parsedcomment"], "lxml").get_text())
 	elif action in ("rights/rights", "rights/autopromote"):
 		link = link_formatter("https://{wiki}.gamepedia.com/User:{user}".format(wiki=settings["wiki"], user=change["title"].split(":")[1]))
@@ -508,7 +509,11 @@ def embed_formatter(action, change, parsed_comment, categories):
 		if action == "upload/overwrite":
 			if additional_info_retrieved:
 				article_encoded = change["title"].replace(" ", "_").replace(')', '\)')
+				try:
 					img_timestamp = [x for x in img_info[1]["timestamp"] if x.isdigit()]
+				except IndexError:
+					logger.exception("Could not analize the information about the image (does it have only one version when expected more in overwrite?) which resulted in no Options field: {}".format(img_info))
+				else:
 					undolink = "https://{wiki}.gamepedia.com/index.php?title={filename}&action=revert&oldimage={timestamp}%21{filenamewon}".format(
 						wiki=settings["wiki"], filename=article_encoded, timestamp="".join(img_timestamp),
 						filenamewon=article_encoded.split(":", 1)[1])
@@ -609,18 +614,24 @@ def embed_formatter(action, change, parsed_comment, categories):
 		user = change["title"].split(':')[1]
 		embed["title"] = _("Unblocked {blocked_user}").format(blocked_user=user)
 	elif action == "curseprofile/comment-created":
+		if settings["appearance"]["embed"]["show_edit_changes"]:
+			parsed_comment = pull_comment(change["logparams"]["4:comment_id"])
 		link = "https://{wiki}.gamepedia.com/Special:CommentPermalink/{commentid}".format(wiki=settings["wiki"],
 		                                                                                  commentid=change["logparams"]["4:comment_id"])
 		embed["title"] = _("Left a comment on {target}'s profile").format(target=change["title"].split(':')[1]) if change["title"].split(':')[1] != \
 		                                                                                              change["user"] else _(
 			"Left a comment on their own profile")
 	elif action == "curseprofile/comment-replied":
+		if settings["appearance"]["embed"]["show_edit_changes"]:
+			parsed_comment = pull_comment(change["logparams"]["4:comment_id"])
 		link = "https://{wiki}.gamepedia.com/Special:CommentPermalink/{commentid}".format(wiki=settings["wiki"],
 		                                                                                  commentid=change["logparams"]["4:comment_id"])
 		embed["title"] = _("Replied to a comment on {target}'s profile").format(target=change["title"].split(':')[1]) if change["title"].split(':')[1] != \
 		                                                                                                    change["user"] else _(
 			"Replied to a comment on their own profile")
 	elif action == "curseprofile/comment-edited":
+		if settings["appearance"]["embed"]["show_edit_changes"]:
+			parsed_comment = pull_comment(change["logparams"]["4:comment_id"])
 		link = "https://{wiki}.gamepedia.com/Special:CommentPermalink/{commentid}".format(wiki=settings["wiki"],
 		                                                                                  commentid=change["logparams"]["4:comment_id"])
 		embed["title"] = _("Edited a comment on {target}'s profile").format(target=change["title"].split(':')[1]) if change["title"].split(':')[1] != \
@@ -631,39 +642,11 @@ def embed_formatter(action, change, parsed_comment, categories):
 		                                                                  target=change["title"].split(':')[1].replace(" ",
 		                                                                                                  "_").replace(
 			                                                                  ')', '\)'))
-		if change["logparams"]['4:section'] == "profile-location":
-			field = _("Location")
-		elif change["logparams"]['4:section'] == "profile-aboutme":
-			field = _("About me")
-		elif change["logparams"]['4:section'] == "profile-link-google":
-			field = _("Google link")
-		elif change["logparams"]['4:section'] == "profile-link-facebook":
-			field = _("Facebook link")
-		elif change["logparams"]['4:section'] == "profile-link-twitter":
-			field = _("Twitter link")
-		elif change["logparams"]['4:section'] == "profile-link-reddit":
-			field = _("Reddit link")
-		elif change["logparams"]['4:section'] == "profile-link-twitch":
-			field = _("Twitch link")
-		elif change["logparams"]['4:section'] == "profile-link-psn":
-			field = _("PSN link")
-		elif change["logparams"]['4:section'] == "profile-link-vk":
-			field = _("VK link")
-		elif change["logparams"]['4:section'] == "profile-link-xbl":
-			field = _("XVL link")
-		elif change["logparams"]['4:section'] == "profile-link-steam":
-			field = _("Steam link")
-		elif change["logparams"]['4:section'] == "profile-link-discord":
-			field = _("Discord handle")
-		elif change["logparams"]['4:section'] == "profile-link-battlenet":
-			field = _("Battle.net handle")
-		else:
-			field = _("Unknown")
 		embed["title"] = _("Edited {target}'s profile").format(target=change["title"].split(':')[1]) if change["user"] != change["title"].split(':')[1] else _("Edited their own profile")
 		if not change["parsedcomment"]:  # If the field is empty
-			parsed_comment = _("Cleared the {field} field").format(field=field)
+			parsed_comment = _("Cleared the {field} field").format(field=profile_field_name(change["logparams"]['4:section'], True))
 		else:
-			parsed_comment = _("{field} field changed to: {desc}").format(field=field, desc=BeautifulSoup(change["parsedcomment"], "lxml").get_text())
+			parsed_comment = _("{field} field changed to: {desc}").format(field=profile_field_name(change["logparams"]['4:section'], True), desc=BeautifulSoup(change["parsedcomment"], "lxml").get_text())
 	elif action == "curseprofile/comment-deleted":
 		if "4:comment_id" in change["logparams"]:
 			link = "https://{wiki}.gamepedia.com/Special:CommentPermalink/{commentid}".format(wiki=settings["wiki"],