diff --git a/README.md b/README.md new file mode 100644 index 0000000..da82073 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ +## Introduction +RcGcDb is a backend for handling webhooks to which recent changes of MediaWiki wikis are being pushed to. + +### Dependencies ### +* **Python 3.6>** +* requests 2.18.4> +* beautifulsoup 4.6.0> +* aiohttp 3.6.2> +* lxml 4.2.1> + +#### Installation +``` +$ git clone git@gitlab.com:chicken-riders/rcgcdb.git #clone repo +$ cd RcGcDw +$ python3 -m venv . #(optional, if you want to contain everything (you should!)) +$ source bin/activate #(optional, see above) +$ pip3 install -r requirements.txt #install requirements (lxml may require additional distro packages, more on that here https://lxml.de/build.html) +$ nano settings.json.example #edit the configuration file +$ mv settings.json.example settings.json +$ python3 start.py +``` \ No newline at end of file diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..c92d56c --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +beautifulsoup4 >= 4.6.0; python_version >= '3.6' +requests >= 2.18.4 +aiohttp >= 3.6.2 +lxml >= 4.2.1 \ No newline at end of file diff --git a/settings.json.example b/settings.json.example new file mode 100644 index 0000000..2ff6e72 --- /dev/null +++ b/settings.json.example @@ -0,0 +1,271 @@ +{ + "header": { + "user-agent": "RcGcDb/{version}" + }, + "max_requests_per_minute": 30, + "minimal_cooldown_per_wiki_in_sec": 60, + "monitoring_webhook": "111111111111111111/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA", + "logging": { + "version": 1, + "disable_existing_loggers": false, + "formatters": { + "standard": { + "format": "%(name)s - %(levelname)s: %(message)s" + } + }, + "handlers": { + "default": { + "formatter": "standard", + "class": "logging.StreamHandler", + "stream": "ext://sys.stdout" + }, + "file": { + "formatter": "standard", + "class": "logging.handlers.TimedRotatingFileHandler", + "filename": "error.log", + "interval": 7, + "when": "D" + } + }, + "loggers": { + "": { + "level": 0, + "handlers": [ + "default" + ] + }, + "rcgcdb.bot": {}, + "rcgcdb.config": {}, + "rcgcdb.discord": {}, + "rcgcdb.wiki": {} + } + }, + "appearance": { + "mode": "embed", + "embed": { + "show_footer": true, + "show_edit_changes": true, + "embed_images": true, + "daily_overview": { + "color": 16312092, + "icon": "" + }, + "new": { + "icon": "https://i.imgur.com/6HIbEq8.png", + "color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE" + }, + "edit": { + "icon": "", + "color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE" + }, + "upload/overwrite": { + "icon": "https://i.imgur.com/egJpa81.png", + "color": 12390624 + }, + "upload/upload": { + "icon": "https://i.imgur.com/egJpa81.png", + "color": null + }, + "upload/revert": { + "icon": "https://i.imgur.com/egJpa81.png", + "color": null + }, + "delete/delete": { + "icon": "https://i.imgur.com/BU77GD3.png", + "color": 1 + }, + "delete/delete_redir": { + "icon": "https://i.imgur.com/BU77GD3.png", + "color": 1 + }, + "delete/restore": { + "icon": "https://i.imgur.com/9MnROIU.png", + "color": null + }, + "delete/revision": { + "icon": "https://i.imgur.com/1gps6EZ.png", + "color": null + }, + "delete/event": { + "icon": "https://i.imgur.com/1gps6EZ.png", + "color": null + }, + "merge/merge": { + "icon": "https://i.imgur.com/uQMK9XK.png", + "color": null + }, + "move/move": { + "icon": "https://i.imgur.com/eXz9dog.png", + "color": null + }, + "move/move_redir": { + "icon": "https://i.imgur.com/UtC3YX2.png", + "color": null + }, + "block/block": { + "icon": "https://i.imgur.com/g7KgZHf.png", + "color": 1 + }, + "block/unblock": { + "icon": "https://i.imgur.com/bvtBJ8o.png", + "color": 1 + }, + "block/reblock": { + "icon": "https://i.imgur.com/g7KgZHf.png", + "color": 1 + }, + "protect/protect": { + "icon": "https://i.imgur.com/bzPt89Z.png", + "color": null + }, + "protect/modify": { + "icon": "https://i.imgur.com/bzPt89Z.png", + "color": null + }, + "protect/move_prot": { + "icon": "https://i.imgur.com/bzPt89Z.png", + "color": null + }, + "protect/unprotect": { + "icon": "https://i.imgur.com/2wN3Qcq.png", + "color": null + }, + "import/upload": { + "icon": "", + "color": null + }, + "import/interwiki": { + "icon": "https://i.imgur.com/sFkhghb.png", + "color": null + }, + "rights/rights": { + "icon": "", + "color": null + }, + "abusefilter/abusefilter": { + "icon": "https://i.imgur.com/Sn2NzRJ.png", + "color": null + }, + "abusefilter/modify": { + "icon": "https://i.imgur.com/Sn2NzRJ.png", + "color": null + }, + "abusefilter/create": { + "icon": "https://i.imgur.com/Sn2NzRJ.png", + "color": null + }, + "interwiki/iw_add": { + "icon": "https://i.imgur.com/sFkhghb.png", + "color": null + }, + "interwiki/iw_edit": { + "icon": "https://i.imgur.com/sFkhghb.png", + "color": null + }, + "interwiki/iw_delete": { + "icon": "https://i.imgur.com/sFkhghb.png", + "color": null + }, + "curseprofile/comment-created": { + "icon": "https://i.imgur.com/Lvy5E32.png", + "color": null + }, + "curseprofile/comment-edited": { + "icon": "https://i.imgur.com/Lvy5E32.png", + "color": null + }, + "curseprofile/comment-deleted": { + "icon": "", + "color": null + }, + "curseprofile/comment-purged":{ + "icon":"", + "color":null + }, + "curseprofile/comment-replied": { + "icon": "https://i.imgur.com/hkyYsI1.png", + "color": null + }, + "curseprofile/profile-edited": { + "icon": "", + "color": null + }, + "contentmodel/change": { + "icon": "", + "color": null + }, + "cargo/deletetable": { + "icon": "", + "color": null + }, + "cargo/createtable": { + "icon": "", + "color": null + }, + "cargo/replacetable": { + "icon": "", + "color": null + }, + "cargo/recreatetable": { + "icon": "", + "color": null + }, + "sprite/sprite": { + "icon": "", + "color": null + }, + "sprite/sheet": { + "icon": "", + "color": null + }, + "sprite/slice": { + "icon": "", + "color": null + }, + "managetags/create": { + "icon": "", + "color": null + }, + "managetags/delete": { + "icon": "", + "color": null + }, + "managetags/activate": { + "icon": "", + "color": null + }, + "managetags/deactivate": { + "icon": "", + "color": null + }, + "tag/update": { + "icon": "", + "color": null + }, + "suppressed": { + "icon": "https://i.imgur.com/1gps6EZ.png", + "color": 8092539 + }, + "discussion/forum/post": { + "icon": "", + "color":null + }, + "discussion/forum/reply": { + "icon": "", + "color":null + }, + "discussion/forum/poll": { + "icon": "", + "color":null + }, + "discussion/wall/post": { + "icon": "", + "color":null + }, + "discussion/wall/reply": { + "icon": "", + "color":null + } + } + } +} \ No newline at end of file diff --git a/src/bot.py b/src/bot.py index 48e2fde..a73aeaf 100644 --- a/src/bot.py +++ b/src/bot.py @@ -8,7 +8,9 @@ from src.exceptions import * from src.database import db_cursor from collections import defaultdict from src.queue_handler import DBHandler +from src.discord import DiscordMessage from src.msgqueue import messagequeue +import requests logging.config.dictConfig(settings["logging"]) logger = logging.getLogger("rcgcdb.bot") @@ -25,13 +27,14 @@ mw_msgs: dict = {} # will have the type of id: tuple for wiki in db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'): all_wikis[wiki] = Wiki() + # Start queueing logic def calculate_delay() -> float: - min_delay = 60/settings["max_requests_per_minute"] + min_delay = 60 / settings["max_requests_per_minute"] if (len(all_wikis) * min_delay) < settings["minimal_cooldown_per_wiki_in_sec"]: - return settings["minimal_cooldown_per_wiki_in_sec"]/len(all_wikis) + return settings["minimal_cooldown_per_wiki_in_sec"] / len(all_wikis) else: return min_delay @@ -67,6 +70,9 @@ async def wiki_scanner(): continue # ignore this wiki if it throws errors try: recent_changes_resp = await wiki_response.json(encoding="UTF-8") + if "error" in recent_changes_resp or "errors" in recent_changes_resp: + # TODO Remove on some errors (example "code": "readapidenied") + raise WikiError recent_changes = recent_changes_resp['query']['recentchanges'] recent_changes.reverse() except: @@ -89,7 +95,8 @@ async def wiki_scanner(): for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up if change["rcid"] > db_wiki[6]: for target in targets.items(): - await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp) + await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, + recent_changes_resp) if recent_changes: DBHandler.add(db_wiki[3], change["rcid"]) DBHandler.update_db() @@ -101,8 +108,18 @@ async def message_sender(): await messagequeue.resend_msgs() +def global_exception_handler(loop, context): + """Global exception handler for asyncio, lets us know when something crashes""" + msg = context.get("exception", context["message"]) + logger.error(msg) + requests.post("https://discord.com/api/webhooks/" + settings["monitoring_webhook"], + data=DiscordMessage("embed", "exception", None, content= + "[RcGcDb] Exception detected, function might have shut down! Exception: {}".format(msg), wiki=None)) + async def main_loop(): + loop = asyncio.get_event_loop() + loop.set_exception_handler(global_exception_handler) task1 = asyncio.create_task(wiki_scanner()) task2 = asyncio.create_task(message_sender()) await task1 diff --git a/src/discord.py b/src/discord.py index c1ac431..ed5fed9 100644 --- a/src/discord.py +++ b/src/discord.py @@ -16,7 +16,7 @@ logger = logging.getLogger("rcgcdb.discord") # User facing webhook functions def wiki_removal(wiki_id, status): - for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wikiid = ?', (wiki_id,)): + for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wiki = ?', (wiki_id,)): def _(string: str) -> str: """Our own translation string to make it compatible with async""" return langs[observer[4]].gettext(string) @@ -30,7 +30,7 @@ async def webhook_removal_monitor(webhook_url: list, reason: int): aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(4.0))) -class DiscordMessage(): +class DiscordMessage: """A class defining a typical Discord JSON representation of webhook payload.""" def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None): self.webhook_object = dict(allowed_mentions={"parse": []})