Initial commit, discussions work in progress, split off most important functions from rcgcdw.py since we need them for modularization

configloader.py

@@ -10,3 +10,4 @@ try:  # load settings
 except FileNotFoundError:
 	logging.critical("No config file could be found. Please make sure settings.json is in the directory.")
 	sys.exit(1)
+

discussions.py

@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+
+# Recent changes Goat compatible Discord webhook is a project for using a webhook as recent changes page from MediaWiki.
+# Copyright (C) 2020 Frisk
+
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+
+# You should have received a copy of the GNU General Public License
+# along with this program.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging, gettext
+from configloader import settings
+from misc import datafile, WIKI_SCRIPT_PATH
+
+# Initialize translation
+
+t = gettext.translation('discussions', localedir='locale', languages=[settings["lang"]])
+_ = t.gettext
+
+# Create a custom logger
+
+discussion_logger = logging.getLogger("rcgcdw.disc")
+
+fetch_url = "https://services.fandom.com/discussion/{wikiid}/posts?sortDirection=descending&sortKey=creation_date&limit={limit}".format(wikiid=settings["fandom_discussions"]["wiki_id"], limit=settings["fandom_discussions"]["limit"])
+
+def fetch_discussions():
+	pass

misc.py

@@ -18,6 +18,9 @@
 
 import json, logging, sys, re
 from html.parser import HTMLParser
+from urllib.parse import urlparse, urlunparse
+import requests
+
 from configloader import settings
 import gettext
 
@@ -30,43 +33,53 @@ _ = t.gettext
 
 misc_logger = logging.getLogger("rcgcdw.misc")
 
-data_template = {"rcid": 99999999999,
+data_template = {"rcid": 99999999999, "discussion_id": 0,
                  "daily_overview": {"edits": None, "new_files": None, "admin_actions": None, "bytes_changed": None,
                                     "new_articles": None, "unique_editors": None, "day_score": None, "days_tracked": 0}}
 
+WIKI_API_PATH: str = ""
+WIKI_ARTICLE_PATH: str = ""
+WIKI_SCRIPT_PATH: str = ""
+WIKI_JUST_DOMAIN: str = ""
 
-def generate_datafile():
+class DataFile:
-	"""Generate a data.json file from a template."""
+	"""Data class which instance of is shared by multiple modules to remain consistent and do not cause too many IO operations."""
-	try:
+	def __init__(self):
-		with open("data.json", 'w') as data:
+		self.data = self.load_datafile()
-			data.write(json.dumps(data_template, indent=4))
+
-	except PermissionError:
+	@staticmethod
-		misc_logger.critical("Could not create a data file (no permissions). No way to store last edit.")
+	def generate_datafile():
-		sys.exit(1)
+		"""Generate a data.json file from a template."""
+		try:
+			with open("data.json", 'w') as data:
+				data.write(json.dumps(data_template, indent=4))
+		except PermissionError:
+			misc_logger.critical("Could not create a data file (no permissions). No way to store last edit.")
+			sys.exit(1)
+
+	def load_datafile(self) -> dict:
+		"""Read a data.json file and return a dictionary with contents
+		:rtype: dict
+		"""
+		try:
+			with open("data.json") as data:
+				return json.loads(data.read())
+		except FileNotFoundError:
+			self.generate_datafile()
+			misc_logger.info("The data file could not be found. Generating a new one...")
+			return data_template
+
+	def save_datafile(self):
+		"""Overwrites the data.json file with given dictionary"""
+		try:
+			with open("data.json", "w") as data_file:
+				data_file.write(json.dumps(self.data, indent=4))
+		except PermissionError:
+			misc_logger.critical("Could not modify a data file (no permissions). No way to store last edit.")
+			sys.exit(1)
 
 
-def load_datafile() -> dict:
+datafile = DataFile()
-	"""Read a data.json file and return a dictionary with contents
-	:rtype: dict
-	"""
-	try:
-		with open("data.json") as data:
-			return json.loads(data.read())
-	except FileNotFoundError:
-		generate_datafile()
-		misc_logger.info("The data file could not be found. Generating a new one...")
-		return data_template
-
-
-def save_datafile(data):
-	"""Overwrites the data.json file with given dictionary"""
-	try:
-		with open("data.json", "w") as data_file:
-			data_file.write(json.dumps(data, indent=4))
-	except PermissionError:
-		misc_logger.critical("Could not modify a data file (no permissions). No way to store last edit.")
-		sys.exit(1)
-
 
 def weighted_average(value, weight, new_value):
 	"""Calculates weighted average of value number with weight weight and new_value with weight 1"""
@@ -192,3 +205,44 @@ def add_to_dict(dictionary, key):
 	else:
 		dictionary[key] = 1
 	return dictionary
+
+def prepare_paths():
+	global WIKI_API_PATH
+	global WIKI_ARTICLE_PATH
+	global WIKI_SCRIPT_PATH
+	global WIKI_JUST_DOMAIN
+	"""Set the URL paths for article namespace and script namespace
+	WIKI_API_PATH will be: WIKI_DOMAIN/api.php
+	WIKI_ARTICLE_PATH will be: WIKI_DOMAIN/articlepath/$1 where $1 is the replaced string
+	WIKI_SCRIPT_PATH will be: WIKI_DOMAIN/
+	WIKI_JUST_DOMAIN will be: WIKI_DOMAIN"""
+	def quick_try_url(url):
+		"""Quickly test if URL is the proper script path,
+		False if it appears invalid
+		dictionary when it appears valid"""
+		try:
+			request = requests.get(url, timeout=5)
+			if request.status_code == requests.codes.ok:
+				if request.json()["query"]["general"] is not None:
+					return request
+			return False
+		except (KeyError, requests.exceptions.ConnectionError):
+			return False
+	try:
+		parsed_url = urlparse(settings["wiki_url"])
+	except KeyError:
+		misc_logger.critical("wiki_url is not specified in the settings. Please provide the wiki url in the settings and start the script again.")
+		sys.exit(1)
+	for url_scheme in (settings["wiki_url"], settings["wiki_url"].split("wiki")[0], urlunparse((*parsed_url[0:2], "", "", "", ""))):  # check different combinations, it's supposed to be idiot-proof
+		tested = quick_try_url(url_scheme + "/api.php?action=query&format=json&meta=siteinfo")
+		if tested:
+			WIKI_API_PATH = urlunparse((*parsed_url[0:2], "", "", "", "")) + tested.json()["query"]["general"]["scriptpath"] + "/api.php"
+			WIKI_SCRIPT_PATH = urlunparse((*parsed_url[0:2], "", "", "", "")) + tested.json()["query"]["general"]["scriptpath"] + "/"
+			WIKI_ARTICLE_PATH = urlunparse((*parsed_url[0:2], "", "", "", "")) + tested.json()["query"]["general"]["articlepath"]
+			WIKI_JUST_DOMAIN = urlunparse((*parsed_url[0:2], "", "", "", ""))
+			break
+	else:
+		misc_logger.critical("Could not verify wikis paths. Please make sure you have given the proper wiki URL in settings.json and your Internet connection is working.")
+		sys.exit(1)
+
+prepare_paths()

rcgcdw.py

@@ -26,9 +26,13 @@ from html.parser import HTMLParser
 import misc
 from bs4 import BeautifulSoup
 from collections import defaultdict, Counter
-from urllib.parse import quote_plus, urlparse, urlunparse
+from urllib.parse import quote_plus
 from configloader import settings
-from misc import link_formatter, ContentParser, safe_read, handle_discord_http, add_to_dict, misc_logger
+from misc import link_formatter, ContentParser, safe_read, handle_discord_http, add_to_dict, datafile, \
+	WIKI_API_PATH, WIKI_ARTICLE_PATH, WIKI_SCRIPT_PATH, WIKI_JUST_DOMAIN
+
+if settings["fandom_discussions"]["enabled"]:
+	pass
 
 if __name__ != "__main__":  # return if called as a module
 	logging.critical("The file is being executed as a module. Please execute the script using the console.")
@@ -53,7 +57,7 @@ except FileNotFoundError:
 lang.install()
 ngettext = lang.ngettext
 
-storage = misc.load_datafile()
+storage = datafile.data
 
 # Remove previous data holding file if exists and limitfetch allows
 
@@ -61,7 +65,7 @@ if settings["limitrefetch"] != -1 and os.path.exists("lastchange.txt") is True:
 	with open("lastchange.txt", 'r') as sfile:
 		logger.info("Converting old lastchange.txt file into new data storage data.json...")
 		storage["rcid"] = int(sfile.read().strip())
-		misc.save_datafile(storage)
+		datafile.save_datafile()
 		os.remove("lastchange.txt")
 
 # A few initial vars
@@ -69,10 +73,6 @@ if settings["limitrefetch"] != -1 and os.path.exists("lastchange.txt") is True:
 logged_in = False
 supported_logs = ["protect/protect", "protect/modify", "protect/unprotect", "upload/overwrite", "upload/upload", "delete/delete", "delete/delete_redir", "delete/restore", "delete/revision", "delete/event", "import/upload", "import/interwiki", "merge/merge", "move/move", "move/move_redir", "protect/move_prot", "block/block", "block/unblock", "block/reblock", "rights/rights", "rights/autopromote", "abusefilter/modify", "abusefilter/create", "interwiki/iw_add", "interwiki/iw_edit", "interwiki/iw_delete", "curseprofile/comment-created", "curseprofile/comment-edited", "curseprofile/comment-deleted", "curseprofile/comment-purged", "curseprofile/profile-edited", "curseprofile/comment-replied", "contentmodel/change", "sprite/sprite", "sprite/sheet", "sprite/slice", "managetags/create", "managetags/delete", "managetags/activate", "managetags/deactivate", "tag/update", "cargo/createtable", "cargo/deletetable", "cargo/recreatetable", "cargo/replacetable", "upload/revert"]
 profile_fields = {"profile-location": _("Location"), "profile-aboutme": _("About me"), "profile-link-google": _("Google link"), "profile-link-facebook":_("Facebook link"), "profile-link-twitter": _("Twitter link"), "profile-link-reddit": _("Reddit link"), "profile-link-twitch": _("Twitch link"), "profile-link-psn": _("PSN link"), "profile-link-vk": _("VK link"), "profile-link-xbl": _("XBL link"), "profile-link-steam": _("Steam link"), "profile-link-discord": _("Discord handle"), "profile-link-battlenet": _("Battle.net handle")}
-WIKI_API_PATH: str = ""
-WIKI_ARTICLE_PATH: str = ""
-WIKI_SCRIPT_PATH: str = ""
-WIKI_JUST_DOMAIN: str = ""
 
 
 class LinkParser(HTMLParser):
@@ -112,44 +112,6 @@ LinkParser = LinkParser()
 class MWError(Exception):
 	pass
 
-def prepare_paths():
-	global WIKI_API_PATH
-	global WIKI_ARTICLE_PATH
-	global WIKI_SCRIPT_PATH
-	global WIKI_JUST_DOMAIN
-	"""Set the URL paths for article namespace and script namespace
-	WIKI_API_PATH will be: WIKI_DOMAIN/api.php
-	WIKI_ARTICLE_PATH will be: WIKI_DOMAIN/articlepath/$1 where $1 is the replaced string
-	WIKI_SCRIPT_PATH will be: WIKI_DOMAIN/
-	WIKI_JUST_DOMAIN will be: WIKI_DOMAIN"""
-	def quick_try_url(url):
-		"""Quickly test if URL is the proper script path,
-		False if it appears invalid
-		dictionary when it appears valid"""
-		try:
-			request = requests.get(url, timeout=5)
-			if request.status_code == requests.codes.ok:
-				if request.json()["query"]["general"] is not None:
-					return request
-			return False
-		except (KeyError, requests.exceptions.ConnectionError):
-			return False
-	try:
-		parsed_url = urlparse(settings["wiki_url"])
-	except KeyError:
-		logger.critical("wiki_url is not specified in the settings. Please provide the wiki url in the settings and start the script again.")
-		sys.exit(1)
-	for url_scheme in (settings["wiki_url"], settings["wiki_url"].split("wiki")[0], urlunparse((*parsed_url[0:2], "", "", "", ""))):  # check different combinations, it's supposed to be idiot-proof
-		tested = quick_try_url(url_scheme + "/api.php?action=query&format=json&meta=siteinfo")
-		if tested:
-			WIKI_API_PATH = urlunparse((*parsed_url[0:2], "", "", "", "")) + tested.json()["query"]["general"]["scriptpath"] + "/api.php"
-			WIKI_SCRIPT_PATH = urlunparse((*parsed_url[0:2], "", "", "", "")) + tested.json()["query"]["general"]["scriptpath"] + "/"
-			WIKI_ARTICLE_PATH = urlunparse((*parsed_url[0:2], "", "", "", "")) + tested.json()["query"]["general"]["articlepath"]
-			WIKI_JUST_DOMAIN = urlunparse((*parsed_url[0:2], "", "", "", ""))
-			break
-	else:
-		logger.critical("Could not verify wikis paths. Please make sure you have given the proper wiki URL in settings.json and your Internet connection is working.")
-		sys.exit(1)
 
 def create_article_path(article: str) -> str:
 	"""Takes the string and creates an URL with it as the article name"""
@@ -1069,7 +1031,7 @@ def daily_overview_sync(edits, files, admin, changed_bytes, new_articles, unique
 		storage["daily_overview"].update({"edits": edits_avg, "new_files": files_avg, "admin_actions": admin_avg, "bytes_changed": changed_bytes_avg,
 		             "new_articles": new_articles_avg, "unique_editors": unique_contributors_avg, "day_score": day_score_avg})
 	storage["daily_overview"]["days_tracked"] += 1
-	misc.save_datafile(storage)
+	datafile.save_datafile()
 	return edits, files, admin, changed_bytes, new_articles, unique_contributors, day_score
 
 def day_overview():
@@ -1261,7 +1223,7 @@ class Recent_Changes_Class(object):
 		if settings["limitrefetch"] != -1 and self.recent_id != self.file_id and self.recent_id != 0:  # if saving to database is disabled, don't save the recent_id
 			self.file_id = self.recent_id
 			storage["rcid"] = self.recent_id
-			misc.save_datafile(storage)
+			datafile.save_datafile()
 		logger.debug("Most recent rcid is: {}".format(self.recent_id))
 		return self.recent_id
 
@@ -1453,7 +1415,6 @@ else:
 	sys.exit(1)
 
 # Log in and download wiki information
-prepare_paths()
 try:
 	if settings["wiki_bot_login"] and settings["wiki_bot_password"]:
 		recent_changes.log_in()
@@ -1468,6 +1429,7 @@ recent_changes.fetch(amount=settings["limitrefetch"] if settings["limitrefetch"]
 
 schedule.every(settings["cooldown"]).seconds.do(recent_changes.fetch)
 if 1 == 2: # additional translation strings in unreachable code
+	# noinspection PyUnreachableCode
 	print(_("director"), _("bot"), _("editor"), _("directors"), _("sysop"), _("bureaucrat"), _("reviewer"),
 	      _("autoreview"), _("autopatrol"), _("wiki_guardian"), ngettext("second", "seconds", 1), ngettext("minute", "minutes", 1), ngettext("hour", "hours", 1), ngettext("day", "days", 1), ngettext("week", "weeks", 1), ngettext("month", "months",1), ngettext("year", "years", 1), ngettext("millennium", "millennia", 1), ngettext("decade", "decades", 1), ngettext("century", "centuries", 1))