Frisk/RcGcDb
1
0
Fork 0
mirror of https://gitlab.com/chicken-riders/RcGcDb.git synced 2025-02-23 00:54:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Made it easier to setup

Browse source
This commit is contained in:
Frisk 2020-07-23 21:12:07 +02:00
parent e4ae528e70
commit d7b93d55bf
No known key found for this signature in database
GPG key ID: 213F7C15068AF8AC
5 changed files with 318 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

21
README.md Normal file
View file

@ -0,0 +1,21 @@
## Introduction
RcGcDb is a backend for handling webhooks to which recent changes of MediaWiki wikis are being pushed to.
### Dependencies ###
* **Python 3.6>**
* requests 2.18.4>
* beautifulsoup 4.6.0>
* aiohttp 3.6.2>
* lxml 4.2.1>
#### Installation
```
$ git clone git@gitlab.com:chicken-riders/rcgcdb.git #clone repo
$ cd RcGcDw
$ python3 -m venv . #(optional, if you want to contain everything (you should!))
$ source bin/activate #(optional, see above)
$ pip3 install -r requirements.txt #install requirements (lxml may require additional distro packages, more on that here https://lxml.de/build.html)
$ nano settings.json.example #edit the configuration file
$ mv settings.json.example settings.json
$ python3 start.py
```

4
requirements.txt Normal file
View file

@ -0,0 +1,4 @@
beautifulsoup4 >= 4.6.0; python_version >= '3.6'
requests >= 2.18.4
aiohttp >= 3.6.2
lxml >= 4.2.1

271
settings.json.example Normal file
View file

@ -0,0 +1,271 @@
{
"header": {
"user-agent": "RcGcDb/{version}"
},
"max_requests_per_minute": 30,
"minimal_cooldown_per_wiki_in_sec": 60,
"monitoring_webhook": "111111111111111111/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
"logging": {
"version": 1,
"disable_existing_loggers": false,
"formatters": {
"standard": {
"format": "%(name)s - %(levelname)s: %(message)s"
}
},
"handlers": {
"default": {
"formatter": "standard",
"class": "logging.StreamHandler",
"stream": "ext://sys.stdout"
},
"file": {
"formatter": "standard",
"class": "logging.handlers.TimedRotatingFileHandler",
"filename": "error.log",
"interval": 7,
"when": "D"
}
},
"loggers": {
"": {
"level": 0,
"handlers": [
"default"
]
},
"rcgcdb.bot": {},
"rcgcdb.config": {},
"rcgcdb.discord": {},
"rcgcdb.wiki": {}
}
},
"appearance": {
"mode": "embed",
"embed": {
"show_footer": true,
"show_edit_changes": true,
"embed_images": true,
"daily_overview": {
"color": 16312092,
"icon": ""
},
"new": {
"icon": "https://i.imgur.com/6HIbEq8.png",
"color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE"
},
"edit": {
"icon": "",
"color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE"
},
"upload/overwrite": {
"icon": "https://i.imgur.com/egJpa81.png",
"color": 12390624
},
"upload/upload": {
"icon": "https://i.imgur.com/egJpa81.png",
"color": null
},
"upload/revert": {
"icon": "https://i.imgur.com/egJpa81.png",
"color": null
},
"delete/delete": {
"icon": "https://i.imgur.com/BU77GD3.png",
"color": 1
},
"delete/delete_redir": {
"icon": "https://i.imgur.com/BU77GD3.png",
"color": 1
},
"delete/restore": {
"icon": "https://i.imgur.com/9MnROIU.png",
"color": null
},
"delete/revision": {
"icon": "https://i.imgur.com/1gps6EZ.png",
"color": null
},
"delete/event": {
"icon": "https://i.imgur.com/1gps6EZ.png",
"color": null
},
"merge/merge": {
"icon": "https://i.imgur.com/uQMK9XK.png",
"color": null
},
"move/move": {
"icon": "https://i.imgur.com/eXz9dog.png",
"color": null
},
"move/move_redir": {
"icon": "https://i.imgur.com/UtC3YX2.png",
"color": null
},
"block/block": {
"icon": "https://i.imgur.com/g7KgZHf.png",
"color": 1
},
"block/unblock": {
"icon": "https://i.imgur.com/bvtBJ8o.png",
"color": 1
},
"block/reblock": {
"icon": "https://i.imgur.com/g7KgZHf.png",
"color": 1
},
"protect/protect": {
"icon": "https://i.imgur.com/bzPt89Z.png",
"color": null
},
"protect/modify": {
"icon": "https://i.imgur.com/bzPt89Z.png",
"color": null
},
"protect/move_prot": {
"icon": "https://i.imgur.com/bzPt89Z.png",
"color": null
},
"protect/unprotect": {
"icon": "https://i.imgur.com/2wN3Qcq.png",
"color": null
},
"import/upload": {
"icon": "",
"color": null
},
"import/interwiki": {
"icon": "https://i.imgur.com/sFkhghb.png",
"color": null
},
"rights/rights": {
"icon": "",
"color": null
},
"abusefilter/abusefilter": {
"icon": "https://i.imgur.com/Sn2NzRJ.png",
"color": null
},
"abusefilter/modify": {
"icon": "https://i.imgur.com/Sn2NzRJ.png",
"color": null
},
"abusefilter/create": {
"icon": "https://i.imgur.com/Sn2NzRJ.png",
"color": null
},
"interwiki/iw_add": {
"icon": "https://i.imgur.com/sFkhghb.png",
"color": null
},
"interwiki/iw_edit": {
"icon": "https://i.imgur.com/sFkhghb.png",
"color": null
},
"interwiki/iw_delete": {
"icon": "https://i.imgur.com/sFkhghb.png",
"color": null
},
"curseprofile/comment-created": {
"icon": "https://i.imgur.com/Lvy5E32.png",
"color": null
},
"curseprofile/comment-edited": {
"icon": "https://i.imgur.com/Lvy5E32.png",
"color": null
},
"curseprofile/comment-deleted": {
"icon": "",
"color": null
},
"curseprofile/comment-purged":{
"icon":"",
"color":null
},
"curseprofile/comment-replied": {
"icon": "https://i.imgur.com/hkyYsI1.png",
"color": null
},
"curseprofile/profile-edited": {
"icon": "",
"color": null
},
"contentmodel/change": {
"icon": "",
"color": null
},
"cargo/deletetable": {
"icon": "",
"color": null
},
"cargo/createtable": {
"icon": "",
"color": null
},
"cargo/replacetable": {
"icon": "",
"color": null
},
"cargo/recreatetable": {
"icon": "",
"color": null
},
"sprite/sprite": {
"icon": "",
"color": null
},
"sprite/sheet": {
"icon": "",
"color": null
},
"sprite/slice": {
"icon": "",
"color": null
},
"managetags/create": {
"icon": "",
"color": null
},
"managetags/delete": {
"icon": "",
"color": null
},
"managetags/activate": {
"icon": "",
"color": null
},
"managetags/deactivate": {
"icon": "",
"color": null
},
"tag/update": {
"icon": "",
"color": null
},
"suppressed": {
"icon": "https://i.imgur.com/1gps6EZ.png",
"color": 8092539
},
"discussion/forum/post": {
"icon": "",
"color":null
},
"discussion/forum/reply": {
"icon": "",
"color":null
},
"discussion/forum/poll": {
"icon": "",
"color":null
},
"discussion/wall/post": {
"icon": "",
"color":null
},
"discussion/wall/reply": {
"icon": "",
"color":null
}
}
}
}

23
src/bot.py
View file

@ -8,7 +8,9 @@ from src.exceptions import *
from src.database import db_cursor from src.database import db_cursor
from collections import defaultdict from collections import defaultdict
from src.queue_handler import DBHandler from src.queue_handler import DBHandler
from src.discord import DiscordMessage
from src.msgqueue import messagequeue from src.msgqueue import messagequeue
import requests
logging.config.dictConfig(settings["logging"]) logging.config.dictConfig(settings["logging"])
logger = logging.getLogger("rcgcdb.bot") logger = logging.getLogger("rcgcdb.bot")
@ -25,13 +27,14 @@ mw_msgs: dict = {} # will have the type of id: tuple
for wiki in db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'): for wiki in db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
all_wikis[wiki] = Wiki() all_wikis[wiki] = Wiki()
# Start queueing logic # Start queueing logic
def calculate_delay() -> float: def calculate_delay() -> float:
min_delay = 60/settings["max_requests_per_minute"] min_delay = 60 / settings["max_requests_per_minute"]
if (len(all_wikis) * min_delay) < settings["minimal_cooldown_per_wiki_in_sec"]: if (len(all_wikis) * min_delay) < settings["minimal_cooldown_per_wiki_in_sec"]:
return settings["minimal_cooldown_per_wiki_in_sec"]/len(all_wikis) return settings["minimal_cooldown_per_wiki_in_sec"] / len(all_wikis)
else: else:
return min_delay return min_delay
@ -67,6 +70,9 @@ async def wiki_scanner():
continue # ignore this wiki if it throws errors continue # ignore this wiki if it throws errors
try: try:
recent_changes_resp = await wiki_response.json(encoding="UTF-8") recent_changes_resp = await wiki_response.json(encoding="UTF-8")
if "error" in recent_changes_resp or "errors" in recent_changes_resp:
# TODO Remove on some errors (example "code": "readapidenied")
raise WikiError
recent_changes = recent_changes_resp['query']['recentchanges'] recent_changes = recent_changes_resp['query']['recentchanges']
recent_changes.reverse() recent_changes.reverse()
except: except:
@ -89,7 +95,8 @@ async def wiki_scanner():
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
if change["rcid"] > db_wiki[6]: if change["rcid"] > db_wiki[6]:
for target in targets.items(): for target in targets.items():
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp) await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths,
recent_changes_resp)
if recent_changes: if recent_changes:
DBHandler.add(db_wiki[3], change["rcid"]) DBHandler.add(db_wiki[3], change["rcid"])
DBHandler.update_db() DBHandler.update_db()
@ -101,8 +108,18 @@ async def message_sender():
await messagequeue.resend_msgs() await messagequeue.resend_msgs()
def global_exception_handler(loop, context):
"""Global exception handler for asyncio, lets us know when something crashes"""
msg = context.get("exception", context["message"])
logger.error(msg)
requests.post("https://discord.com/api/webhooks/" + settings["monitoring_webhook"],
data=DiscordMessage("embed", "exception", None, content=
"[RcGcDb] Exception detected, function might have shut down! Exception: {}".format(msg), wiki=None))
async def main_loop(): async def main_loop():
loop = asyncio.get_event_loop()
loop.set_exception_handler(global_exception_handler)
task1 = asyncio.create_task(wiki_scanner()) task1 = asyncio.create_task(wiki_scanner())
task2 = asyncio.create_task(message_sender()) task2 = asyncio.create_task(message_sender())
await task1 await task1

4
src/discord.py
View file

@ -16,7 +16,7 @@ logger = logging.getLogger("rcgcdb.discord")
# User facing webhook functions # User facing webhook functions
def wiki_removal(wiki_id, status): def wiki_removal(wiki_id, status):
for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wikiid = ?', (wiki_id,)): for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wiki = ?', (wiki_id,)):
def _(string: str) -> str: def _(string: str) -> str:
"""Our own translation string to make it compatible with async""" """Our own translation string to make it compatible with async"""
return langs[observer[4]].gettext(string) return langs[observer[4]].gettext(string)
@ -30,7 +30,7 @@ async def webhook_removal_monitor(webhook_url: list, reason: int):
aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(4.0))) aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(4.0)))
class DiscordMessage(): class DiscordMessage:
"""A class defining a typical Discord JSON representation of webhook payload.""" """A class defining a typical Discord JSON representation of webhook payload."""
def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None): def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None):
self.webhook_object = dict(allowed_mentions={"parse": []}) self.webhook_object = dict(allowed_mentions={"parse": []})