mirror of
https://gitlab.com/chicken-riders/RcGcDb.git
synced 2025-02-23 00:54:09 +00:00
Made it easier to setup
This commit is contained in:
Frisk
parent
e4ae528e70
commit
d7b93d55bf
21
README.md
Normal file
21
README.md
Normal file
|
|
@ -0,0 +1,21 @@
|
|||
## Introduction
|
||||
RcGcDb is a backend for handling webhooks to which recent changes of MediaWiki wikis are being pushed to.
|
||||
|
||||
### Dependencies ###
|
||||
* **Python 3.6>**
|
||||
* requests 2.18.4>
|
||||
* beautifulsoup 4.6.0>
|
||||
* aiohttp 3.6.2>
|
||||
* lxml 4.2.1>
|
||||
|
||||
#### Installation
|
||||
```
|
||||
$ git clone git@gitlab.com:chicken-riders/rcgcdb.git #clone repo
|
||||
$ cd RcGcDw
|
||||
$ python3 -m venv . #(optional, if you want to contain everything (you should!))
|
||||
$ source bin/activate #(optional, see above)
|
||||
$ pip3 install -r requirements.txt #install requirements (lxml may require additional distro packages, more on that here https://lxml.de/build.html)
|
||||
$ nano settings.json.example #edit the configuration file
|
||||
$ mv settings.json.example settings.json
|
||||
$ python3 start.py
|
||||
```
|
||||
4
requirements.txt
Normal file
4
requirements.txt
Normal file
|
|
@ -0,0 +1,4 @@
|
|||
beautifulsoup4 >= 4.6.0; python_version >= '3.6'
|
||||
requests >= 2.18.4
|
||||
aiohttp >= 3.6.2
|
||||
lxml >= 4.2.1
|
||||
271
settings.json.example
Normal file
271
settings.json.example
Normal file
|
|
@ -0,0 +1,271 @@
|
|||
{
|
||||
"header": {
|
||||
"user-agent": "RcGcDb/{version}"
|
||||
},
|
||||
"max_requests_per_minute": 30,
|
||||
"minimal_cooldown_per_wiki_in_sec": 60,
|
||||
"monitoring_webhook": "111111111111111111/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||
"logging": {
|
||||
"version": 1,
|
||||
"disable_existing_loggers": false,
|
||||
"formatters": {
|
||||
"standard": {
|
||||
"format": "%(name)s - %(levelname)s: %(message)s"
|
||||
}
|
||||
},
|
||||
"handlers": {
|
||||
"default": {
|
||||
"formatter": "standard",
|
||||
"class": "logging.StreamHandler",
|
||||
"stream": "ext://sys.stdout"
|
||||
},
|
||||
"file": {
|
||||
"formatter": "standard",
|
||||
"class": "logging.handlers.TimedRotatingFileHandler",
|
||||
"filename": "error.log",
|
||||
"interval": 7,
|
||||
"when": "D"
|
||||
}
|
||||
},
|
||||
"loggers": {
|
||||
"": {
|
||||
"level": 0,
|
||||
"handlers": [
|
||||
"default"
|
||||
]
|
||||
},
|
||||
"rcgcdb.bot": {},
|
||||
"rcgcdb.config": {},
|
||||
"rcgcdb.discord": {},
|
||||
"rcgcdb.wiki": {}
|
||||
}
|
||||
},
|
||||
"appearance": {
|
||||
"mode": "embed",
|
||||
"embed": {
|
||||
"show_footer": true,
|
||||
"show_edit_changes": true,
|
||||
"embed_images": true,
|
||||
"daily_overview": {
|
||||
"color": 16312092,
|
||||
"icon": ""
|
||||
},
|
||||
"new": {
|
||||
"icon": "https://i.imgur.com/6HIbEq8.png",
|
||||
"color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE"
|
||||
},
|
||||
"edit": {
|
||||
"icon": "",
|
||||
"color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE"
|
||||
},
|
||||
"upload/overwrite": {
|
||||
"icon": "https://i.imgur.com/egJpa81.png",
|
||||
"color": 12390624
|
||||
},
|
||||
"upload/upload": {
|
||||
"icon": "https://i.imgur.com/egJpa81.png",
|
||||
"color": null
|
||||
},
|
||||
"upload/revert": {
|
||||
"icon": "https://i.imgur.com/egJpa81.png",
|
||||
"color": null
|
||||
},
|
||||
"delete/delete": {
|
||||
"icon": "https://i.imgur.com/BU77GD3.png",
|
||||
"color": 1
|
||||
},
|
||||
"delete/delete_redir": {
|
||||
"icon": "https://i.imgur.com/BU77GD3.png",
|
||||
"color": 1
|
||||
},
|
||||
"delete/restore": {
|
||||
"icon": "https://i.imgur.com/9MnROIU.png",
|
||||
"color": null
|
||||
},
|
||||
"delete/revision": {
|
||||
"icon": "https://i.imgur.com/1gps6EZ.png",
|
||||
"color": null
|
||||
},
|
||||
"delete/event": {
|
||||
"icon": "https://i.imgur.com/1gps6EZ.png",
|
||||
"color": null
|
||||
},
|
||||
"merge/merge": {
|
||||
"icon": "https://i.imgur.com/uQMK9XK.png",
|
||||
"color": null
|
||||
},
|
||||
"move/move": {
|
||||
"icon": "https://i.imgur.com/eXz9dog.png",
|
||||
"color": null
|
||||
},
|
||||
"move/move_redir": {
|
||||
"icon": "https://i.imgur.com/UtC3YX2.png",
|
||||
"color": null
|
||||
},
|
||||
"block/block": {
|
||||
"icon": "https://i.imgur.com/g7KgZHf.png",
|
||||
"color": 1
|
||||
},
|
||||
"block/unblock": {
|
||||
"icon": "https://i.imgur.com/bvtBJ8o.png",
|
||||
"color": 1
|
||||
},
|
||||
"block/reblock": {
|
||||
"icon": "https://i.imgur.com/g7KgZHf.png",
|
||||
"color": 1
|
||||
},
|
||||
"protect/protect": {
|
||||
"icon": "https://i.imgur.com/bzPt89Z.png",
|
||||
"color": null
|
||||
},
|
||||
"protect/modify": {
|
||||
"icon": "https://i.imgur.com/bzPt89Z.png",
|
||||
"color": null
|
||||
},
|
||||
"protect/move_prot": {
|
||||
"icon": "https://i.imgur.com/bzPt89Z.png",
|
||||
"color": null
|
||||
},
|
||||
"protect/unprotect": {
|
||||
"icon": "https://i.imgur.com/2wN3Qcq.png",
|
||||
"color": null
|
||||
},
|
||||
"import/upload": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"import/interwiki": {
|
||||
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||
"color": null
|
||||
},
|
||||
"rights/rights": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"abusefilter/abusefilter": {
|
||||
"icon": "https://i.imgur.com/Sn2NzRJ.png",
|
||||
"color": null
|
||||
},
|
||||
"abusefilter/modify": {
|
||||
"icon": "https://i.imgur.com/Sn2NzRJ.png",
|
||||
"color": null
|
||||
},
|
||||
"abusefilter/create": {
|
||||
"icon": "https://i.imgur.com/Sn2NzRJ.png",
|
||||
"color": null
|
||||
},
|
||||
"interwiki/iw_add": {
|
||||
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||
"color": null
|
||||
},
|
||||
"interwiki/iw_edit": {
|
||||
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||
"color": null
|
||||
},
|
||||
"interwiki/iw_delete": {
|
||||
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||
"color": null
|
||||
},
|
||||
"curseprofile/comment-created": {
|
||||
"icon": "https://i.imgur.com/Lvy5E32.png",
|
||||
"color": null
|
||||
},
|
||||
"curseprofile/comment-edited": {
|
||||
"icon": "https://i.imgur.com/Lvy5E32.png",
|
||||
"color": null
|
||||
},
|
||||
"curseprofile/comment-deleted": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"curseprofile/comment-purged":{
|
||||
"icon":"",
|
||||
"color":null
|
||||
},
|
||||
"curseprofile/comment-replied": {
|
||||
"icon": "https://i.imgur.com/hkyYsI1.png",
|
||||
"color": null
|
||||
},
|
||||
"curseprofile/profile-edited": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"contentmodel/change": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"cargo/deletetable": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"cargo/createtable": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"cargo/replacetable": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"cargo/recreatetable": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"sprite/sprite": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"sprite/sheet": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"sprite/slice": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"managetags/create": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"managetags/delete": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"managetags/activate": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"managetags/deactivate": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"tag/update": {
|
||||
"icon": "",
|
||||
"color": null
|
||||
},
|
||||
"suppressed": {
|
||||
"icon": "https://i.imgur.com/1gps6EZ.png",
|
||||
"color": 8092539
|
||||
},
|
||||
"discussion/forum/post": {
|
||||
"icon": "",
|
||||
"color":null
|
||||
},
|
||||
"discussion/forum/reply": {
|
||||
"icon": "",
|
||||
"color":null
|
||||
},
|
||||
"discussion/forum/poll": {
|
||||
"icon": "",
|
||||
"color":null
|
||||
},
|
||||
"discussion/wall/post": {
|
||||
"icon": "",
|
||||
"color":null
|
||||
},
|
||||
"discussion/wall/reply": {
|
||||
"icon": "",
|
||||
"color":null
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
23
src/bot.py
23
src/bot.py
|
|
@ -8,7 +8,9 @@ from src.exceptions import *
|
|||
from src.database import db_cursor
|
||||
from collections import defaultdict
|
||||
from src.queue_handler import DBHandler
|
||||
from src.discord import DiscordMessage
|
||||
from src.msgqueue import messagequeue
|
||||
import requests
|
||||
|
||||
logging.config.dictConfig(settings["logging"])
|
||||
logger = logging.getLogger("rcgcdb.bot")
|
||||
|
|
@ -25,13 +27,14 @@ mw_msgs: dict = {} # will have the type of id: tuple
|
|||
for wiki in db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
|
||||
all_wikis[wiki] = Wiki()
|
||||
|
||||
|
||||
# Start queueing logic
|
||||
|
||||
|
||||
def calculate_delay() -> float:
|
||||
min_delay = 60/settings["max_requests_per_minute"]
|
||||
min_delay = 60 / settings["max_requests_per_minute"]
|
||||
if (len(all_wikis) * min_delay) < settings["minimal_cooldown_per_wiki_in_sec"]:
|
||||
return settings["minimal_cooldown_per_wiki_in_sec"]/len(all_wikis)
|
||||
return settings["minimal_cooldown_per_wiki_in_sec"] / len(all_wikis)
|
||||
else:
|
||||
return min_delay
|
||||
|
||||
|
|
@ -67,6 +70,9 @@ async def wiki_scanner():
|
|||
continue # ignore this wiki if it throws errors
|
||||
try:
|
||||
recent_changes_resp = await wiki_response.json(encoding="UTF-8")
|
||||
if "error" in recent_changes_resp or "errors" in recent_changes_resp:
|
||||
# TODO Remove on some errors (example "code": "readapidenied")
|
||||
raise WikiError
|
||||
recent_changes = recent_changes_resp['query']['recentchanges']
|
||||
recent_changes.reverse()
|
||||
except:
|
||||
|
|
@ -89,7 +95,8 @@ async def wiki_scanner():
|
|||
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
|
||||
if change["rcid"] > db_wiki[6]:
|
||||
for target in targets.items():
|
||||
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp)
|
||||
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths,
|
||||
recent_changes_resp)
|
||||
if recent_changes:
|
||||
DBHandler.add(db_wiki[3], change["rcid"])
|
||||
DBHandler.update_db()
|
||||
|
|
@ -101,8 +108,18 @@ async def message_sender():
|
|||
await messagequeue.resend_msgs()
|
||||
|
||||
|
||||
def global_exception_handler(loop, context):
|
||||
"""Global exception handler for asyncio, lets us know when something crashes"""
|
||||
msg = context.get("exception", context["message"])
|
||||
logger.error(msg)
|
||||
requests.post("https://discord.com/api/webhooks/" + settings["monitoring_webhook"],
|
||||
data=DiscordMessage("embed", "exception", None, content=
|
||||
"[RcGcDb] Exception detected, function might have shut down! Exception: {}".format(msg), wiki=None))
|
||||
|
||||
|
||||
async def main_loop():
|
||||
loop = asyncio.get_event_loop()
|
||||
loop.set_exception_handler(global_exception_handler)
|
||||
task1 = asyncio.create_task(wiki_scanner())
|
||||
task2 = asyncio.create_task(message_sender())
|
||||
await task1
|
||||
|
|
|
|||
|
|
@ -16,7 +16,7 @@ logger = logging.getLogger("rcgcdb.discord")
|
|||
|
||||
# User facing webhook functions
|
||||
def wiki_removal(wiki_id, status):
|
||||
for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wikiid = ?', (wiki_id,)):
|
||||
for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wiki = ?', (wiki_id,)):
|
||||
def _(string: str) -> str:
|
||||
"""Our own translation string to make it compatible with async"""
|
||||
return langs[observer[4]].gettext(string)
|
||||
|
|
@ -30,7 +30,7 @@ async def webhook_removal_monitor(webhook_url: list, reason: int):
|
|||
aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(4.0)))
|
||||
|
||||
|
||||
class DiscordMessage():
|
||||
class DiscordMessage:
|
||||
"""A class defining a typical Discord JSON representation of webhook payload."""
|
||||
def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None):
|
||||
self.webhook_object = dict(allowed_mentions={"parse": []})
|
||||
|
|
|
|||
Loading…
Reference in a new issue