Further work towards an API

docs/API spec.md

@@ -12,17 +12,17 @@ A class allowing to change the message content and/or execute additional actions
 
 ## File structure
 Directory with extensions should be possible to be changed using settings.json
-/
+/    
- /src
+ /src   
- /extensions
+ /extensions   
- /extensions/base
+ /extensions/base   
- /extensions/abusefilter
+ /extensions/abusefilter    
-  /extensions/abusefilter/abusefilter.py
+  /extensions/abusefilter/abusefilter.py   
- /extensions/managewiki
+ /extensions/managewiki   
-  /extensions/managewiki/managewiki.py
+  /extensions/managewiki/managewiki.py   
- /extensions/prehooks/
+ /extensions/prehooks/   
-  /extensions/prehooks/friskyhooks.py
+  /extensions/prehooks/friskyhooks.py   
- /extensions/posthooks/
+ /extensions/posthooks/   
 
 ## API
 api object exposes various data which allows to extend the usefulness of what can be then sent to Discord.
@@ -46,11 +46,11 @@ class abusefilter(Formatter):
 		super().__init__(api)
 		
     @formatter.embed(event="abuselog/modify", mode="embed")
-	def embed_modify(self, change: dict) -> DiscordMessage:
+    def embed_modify(self, change: dict) -> DiscordMessage:
 		return DiscordMessage
 		
     @formatter.compact(event="abuselog/modify")
-	def compact_modify(self, change: dict) -> DiscordMessage:
+    def compact_modify(self, change: dict) -> DiscordMessage:
 		return DiscordMessage
 
 ```

extensions/__init__.py

@@ -13,3 +13,4 @@
 #  You should have received a copy of the GNU General Public License
 #  along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.
 
+import base

extensions/base/__init__.py

@@ -13,3 +13,4 @@
 #  You should have received a copy of the GNU General Public License
 #  along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.
 
+import mediawiki

extensions/base/base.py

@@ -1,36 +0,0 @@
-#  This file is part of Recent changes Goat compatible Discord webhook (RcGcDw).
-#
-#  RcGcDw is free software: you can redistribute it and/or modify
-#  it under the terms of the GNU General Public License as published by
-#  the Free Software Foundation, either version 3 of the License, or
-#  (at your option) any later version.
-#
-#  RcGcDw is distributed in the hope that it will be useful,
-#  but WITHOUT ANY WARRANTY; without even the implied warranty of
-#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-#  GNU General Public License for more details.
-#
-#  You should have received a copy of the GNU General Public License
-#  along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.
-
-import logging
-from src.discord.message import DiscordMessage
-from src.api import formatter
-from src.i18n import rc_formatters
-
-_ = rc_formatters.gettext
-
-logger = logging.getLogger("extensions.base")
-
-
-class abusefilter():
-	def __init__(self, api):
-		super().__init__(api)
-
-	@formatter.embed(event="edit", mode="embed")
-	def embed_edit(self, change: dict) -> DiscordMessage:
-		return DiscordMessage()
-
-	@formatter.compact(event="edit", mode="embed")
-	def compact_edit(self, change: dict):
-		return DiscordMessage()

extensions/base/mediawiki.py

@@ -0,0 +1,95 @@
+#  This file is part of Recent changes Goat compatible Discord webhook (RcGcDw).
+#
+#  RcGcDw is free software: you can redistribute it and/or modify
+#  it under the terms of the GNU General Public License as published by
+#  the Free Software Foundation, either version 3 of the License, or
+#  (at your option) any later version.
+#
+#  RcGcDw is distributed in the hope that it will be useful,
+#  but WITHOUT ANY WARRANTY; without even the implied warranty of
+#  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#  GNU General Public License for more details.
+#
+#  You should have received a copy of the GNU General Public License
+#  along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.
+
+import logging
+import math
+from src.discord.message import DiscordMessage
+from src.api import formatter
+from src.i18n import rc_formatters
+from src.api.client import Client, client
+from src.configloader import settings
+
+
+_ = rc_formatters.gettext
+
+logger = logging.getLogger("extensions.base")
+
+
+class base():
+	def __init__(self, api):
+		super().__init__(api)
+
+	@formatter.embed(event="edit", mode="embed")
+	def embed_edit(self, ctx: Client, change: dict) -> DiscordMessage:
+		embed = DiscordMessage(ctx.message_type, ctx.event, ctx.webhook_url)
+		action = ctx.event
+		editsize = change["newlen"] - change["oldlen"]
+		if editsize > 0:
+			if editsize > 6032:
+				embed["color"] = 65280
+			else:
+				embed["color"] = 35840 + (math.floor(editsize / 52)) * 256
+		elif editsize < 0:
+			if editsize < -6032:
+				embed["color"] = 16711680
+			else:
+				embed["color"] = 9175040 + (math.floor((editsize * -1) / 52)) * 65536
+		elif editsize == 0:
+			embed["color"] = 8750469
+		if change["title"].startswith("MediaWiki:Tag-"):  # Refresh tag list when tag display name is edited
+			ctx.client.refresh_internal_data()
+		link = "{wiki}index.php?title={article}&curid={pageid}&diff={diff}&oldid={oldrev}".format(
+			wiki=ctx.client.WIKI_SCRIPT_PATH, pageid=change["pageid"], diff=change["revid"], oldrev=change["old_revid"],
+			article=change["title"].replace(" ", "_").replace("%", "%25").replace("\\", "%5C").replace("&", "%26"))
+		embed["title"] = "{redirect}{article} ({new}{minor}{bot}{space}{editsize})".format(
+			redirect="⤷ " if "redirect" in change else "", article=change["title"], editsize="+" + str(
+				editsize) if editsize > 0 else editsize, new=_("(N!) ") if action == "new" else "",
+			minor=_("m") if action == "edit" and "minor" in change else "", bot=_('b') if "bot" in change else "",
+			space=" " if "bot" in change or (action == "edit" and "minor" in change) or action == "new" else "")
+		if settings["appearance"]["embed"]["show_edit_changes"]:
+			if action == "new":
+				changed_content = safe_read(recent_changes.safe_request(
+					"{wiki}?action=compare&format=json&fromtext=&torev={diff}&topst=1&prop=diff".format(
+						wiki=ctx.client.WIKI_API_PATH, diff=change["revid"]
+					)), "compare", "*")
+			else:
+				changed_content = safe_read(recent_changes.safe_request(
+					"{wiki}?action=compare&format=json&fromrev={oldrev}&torev={diff}&topst=1&prop=diff".format(
+						wiki=ctx.client.WIKI_API_PATH, diff=change["revid"], oldrev=change["old_revid"]
+					)), "compare", "*")
+			if changed_content:
+				EditDiff = ctx.client.content_parser()
+				EditDiff.feed(changed_content)
+				if EditDiff.small_prev_del:
+					if EditDiff.small_prev_del.replace("~~", "").isspace():
+						EditDiff.small_prev_del = _('__Only whitespace__')
+					else:
+						EditDiff.small_prev_del = EditDiff.small_prev_del.replace("~~~~", "")
+				if EditDiff.small_prev_ins:
+					if EditDiff.small_prev_ins.replace("**", "").isspace():
+						EditDiff.small_prev_ins = _('__Only whitespace__')
+					else:
+						EditDiff.small_prev_ins = EditDiff.small_prev_ins.replace("****", "")
+				logger.debug("Changed content: {}".format(EditDiff.small_prev_ins))
+				if EditDiff.small_prev_del and not action == "new":
+					embed.add_field(_("Removed"), "{data}".format(data=EditDiff.small_prev_del), inline=True)
+				if EditDiff.small_prev_ins:
+					embed.add_field(_("Added"), "{data}".format(data=EditDiff.small_prev_ins), inline=True)
+			else:
+				logger.warning("Unable to download data on the edit content!")
+
+	@formatter.compact(event="edit", mode="embed")
+	def compact_edit(self, change: dict):
+		return DiscordMessage()

src/api/__init__.py

@@ -1 +1 @@
-from .formatter import *
+from .formatter import *

src/api/client.py

@@ -13,12 +13,29 @@
 #  You should have received a copy of the GNU General Public License
 #  along with RcGcDw.  If not, see <http://www.gnu.org/licenses/>.
 
-from src.rcgcdw import formatter_hooks
+import src.rcgcdw
+import src.rc
+import src.misc
+
 
 class Client:
+	"""
+		A client for interacting with RcGcDw when creating formatters or hooks.
+	"""
 	def __init__(self):
-		self._formatters = formatter_hooks
+		self._formatters = src.rcgcdw.formatter_hooks
-		self.
+		self.__recent_changes = src.rc.recent_changes
+		self.WIKI_API_PATH = src.misc.WIKI_API_PATH
+		self.WIKI_ARTICLE_PATH = src.misc.WIKI_ARTICLE_PATH
+		self.WIKI_SCRIPT_PATH = src.misc.WIKI_SCRIPT_PATH
+		self.WIKI_JUST_DOMAIN = src.misc.WIKI_JUST_DOMAIN
+		self.content_parser = src.misc.ContentParser
+
+	def refresh_internal_data(self):
+		"""Refreshes internal storage data for wiki tags and MediaWiki messages."""
+		self.__recent_changes.init_info()
+
+
 
 
 

src/fileio/database.py

@@ -22,6 +22,7 @@ logger = logging.getLogger("rcgcdw.fileio.database")
 
 
 def create_schema():
+	"""Creates a SQLite database schema"""
 	logger.info("Creating database schema...")
 	db_cursor.executescript(
 	"""BEGIN TRANSACTION;
@@ -43,6 +44,7 @@ def create_schema():
 
 
 def create_connection() -> (sqlite3.Connection, sqlite3.Cursor):
+	"""Creates a connection to the database"""
 	_db_connection = sqlite3.connect(settings['auto_suppression'].get("db_location", ':memory:'))
 	_db_connection.row_factory = sqlite3.Row
 	_db_cursor = _db_connection.cursor()
@@ -67,6 +69,7 @@ def add_entry(pageid: int, revid: int, logid: int, message, message_id: str):
 	logger.debug("Adding an entry to the database (pageid: {}, revid: {}, logid: {}, message: {})".format(pageid, revid, logid, message))
 	db_connection.commit()
 
+
 def clean_entries():
 	"""Cleans entries that are 50+"""
 	cleanup = db_cursor.execute(

src/misc.py

@@ -115,6 +115,13 @@ def escape_formatting(data):
 
 
 class ContentParser(HTMLParser):
+	"""ContentPerser is an implementation of HTMLParser that parses output of action=compare&prop=diff API request
+	for two MediaWiki revisions. It extracts the following:
+	small_prev_ins - storing up to 1000 characters of added text
+	small_prev_del - storing up to 1000 chracters of removed text
+	ins_length - storing length of inserted text
+	del_length - storing length of deleted text
+	"""
 	more = _("\n__And more__")
 	current_tag = ""
 	last_ins = None

src/rc.py

@@ -27,7 +27,7 @@ from src.misc import WIKI_SCRIPT_PATH, WIKI_API_PATH, datafile, send_simple, saf
 from src.discord.queue import messagequeue
 from src.exceptions import MWError
 from src.session import session
-from src.rc_formatters import compact_formatter, embed_formatter, compact_abuselog_formatter, embed_abuselog_formatter
+# from src.rc_formatters import compact_formatter, embed_formatter, compact_abuselog_formatter, embed_abuselog_formatter
 from src.i18n import rc
 from collections import OrderedDict
 
@@ -52,17 +52,6 @@ supported_logs = {"protect/protect", "protect/modify", "protect/unprotect", "upl
                   "managewiki/settings", "managewiki/delete", "managewiki/lock", "managewiki/unlock",
                   "managewiki/namespaces", "managewiki/namespaces-delete", "managewiki/rights", "managewiki/undelete"}
 
-# Set the proper formatter
-if settings["appearance"]["mode"] == "embed":
-	appearance_mode = embed_formatter
-	abuselog_appearance_mode = embed_abuselog_formatter
-elif settings["appearance"]["mode"] == "compact":
-	appearance_mode = compact_formatter
-	abuselog_appearance_mode = compact_abuselog_formatter
-else:
-	logger.critical("Unknown formatter!")
-	sys.exit(1)
-
 
 LinkParser = LinkParser()
 
@@ -298,7 +287,7 @@ class Recent_Changes_Class(object):
 				self.downtime_controller(True)
 				return None
 			elif request.status_code == 302:
-				logger.critical("Redirect detected! Either the wiki given in the script settings (wiki field) is incorrect/the wiki got removed or Gamepedia is giving us the false value. Please provide the real URL to the wiki, current URL redirects to {}".format(request.next.url))
+				logger.critical("Redirect detected! Either the wiki given in the script settings (wiki field) is incorrect/the wiki got removed or is giving us the false value. Please provide the real URL to the wiki, current URL redirects to {}".format(request.next.url))
 				sys.exit(0)
 			return request
 

src/rc_formatters.py

@@ -712,58 +712,7 @@ def embed_formatter(action, change, parsed_comment, categories, recent_changes):
 		change["user"], author_url = format_user(change, recent_changes, action)
 		embed.set_author(change["user"], author_url)
 	if action in ("edit", "new"):  # edit or new page
-		editsize = change["newlen"] - change["oldlen"]
+
-		if editsize > 0:
-			if editsize > 6032:
-				embed["color"] = 65280
-			else:
-				embed["color"] = 35840 + (math.floor(editsize / 52)) * 256
-		elif editsize < 0:
-			if editsize < -6032:
-				embed["color"] = 16711680
-			else:
-				embed["color"] = 9175040 + (math.floor((editsize * -1) / 52)) * 65536
-		elif editsize == 0:
-			embed["color"] = 8750469
-		if change["title"].startswith("MediaWiki:Tag-"):  # Refresh tag list when tag display name is edited
-			recent_changes.init_info()
-		link = "{wiki}index.php?title={article}&curid={pageid}&diff={diff}&oldid={oldrev}".format(
-			wiki=WIKI_SCRIPT_PATH, pageid=change["pageid"], diff=change["revid"], oldrev=change["old_revid"],
-			article=change["title"].replace(" ", "_").replace("%", "%25").replace("\\", "%5C").replace("&", "%26"))
-		embed["title"] = "{redirect}{article} ({new}{minor}{bot}{space}{editsize})".format(redirect="⤷ " if "redirect" in change else "", article=change["title"], editsize="+" + str(
-			editsize) if editsize > 0 else editsize, new=_("(N!) ") if action == "new" else "",
-		                                                             minor=_("m") if action == "edit" and "minor" in change else "", bot=_('b') if "bot" in change else "", space=" " if "bot" in change or (action == "edit" and "minor" in change) or action == "new" else "")
-		if settings["appearance"]["embed"]["show_edit_changes"]:
-			if action == "new":
-				changed_content = safe_read(recent_changes.safe_request(
-				"{wiki}?action=compare&format=json&fromtext=&torev={diff}&topst=1&prop=diff".format(
-					wiki=WIKI_API_PATH, diff=change["revid"]
-				)), "compare", "*")
-			else:
-				changed_content = safe_read(recent_changes.safe_request(
-					"{wiki}?action=compare&format=json&fromrev={oldrev}&torev={diff}&topst=1&prop=diff".format(
-						wiki=WIKI_API_PATH, diff=change["revid"],oldrev=change["old_revid"]
-					)), "compare", "*")
-			if changed_content:
-				EditDiff = ContentParser()
-				EditDiff.feed(changed_content)
-				if EditDiff.small_prev_del:
-					if EditDiff.small_prev_del.replace("~~", "").isspace():
-						EditDiff.small_prev_del = _('__Only whitespace__')
-					else:
-						EditDiff.small_prev_del = EditDiff.small_prev_del.replace("~~~~", "")
-				if EditDiff.small_prev_ins:
-					if EditDiff.small_prev_ins.replace("**", "").isspace():
-						EditDiff.small_prev_ins = _('__Only whitespace__')
-					else:
-						EditDiff.small_prev_ins = EditDiff.small_prev_ins.replace("****", "")
-				logger.debug("Changed content: {}".format(EditDiff.small_prev_ins))
-				if EditDiff.small_prev_del and not action == "new":
-					embed.add_field(_("Removed"), "{data}".format(data=EditDiff.small_prev_del), inline=True)
-				if EditDiff.small_prev_ins:
-					embed.add_field(_("Added"), "{data}".format(data=EditDiff.small_prev_ins), inline=True)
-			else:
-				logger.warning("Unable to download data on the edit content!")
 	elif action in ("upload/overwrite", "upload/upload", "upload/revert"):  # sending files
 		license = None
 		urls = safe_read(recent_changes.safe_request(

src/rcgcdw.py

@@ -46,6 +46,11 @@ logging.config.dictConfig(settings["logging"])
 logger = logging.getLogger("rcgcdw")
 logger.debug("Current settings: {settings}".format(settings=settings))
 from src.migrations import *  # migrations after logging
+try:
+	import exceptions
+except ImportError:
+	logger.critical("No extensions module found. What's going on?")
+	sys.exit(1)
 storage = datafile
 
 # Remove previous data holding file if exists and limitfetch allows

start.py

@@ -1,4 +1,4 @@
 import src.rcgcdw, sys
 
 if __name__ != "__main__":  # return if called as a module
-	sys.exit(1)
+	sys.exit(1)