mirror of
https://gitlab.com/chicken-riders/RcGcDb.git
synced 2025-02-23 00:54:09 +00:00
Made it easier to setup
This commit is contained in:
Frisk
parent
e4ae528e70
commit
d7b93d55bf
21
README.md
Normal file
21
README.md
Normal file
|
|
@ -0,0 +1,21 @@
|
||||||
|
## Introduction
|
||||||
|
RcGcDb is a backend for handling webhooks to which recent changes of MediaWiki wikis are being pushed to.
|
||||||
|
|
||||||
|
### Dependencies ###
|
||||||
|
* **Python 3.6>**
|
||||||
|
* requests 2.18.4>
|
||||||
|
* beautifulsoup 4.6.0>
|
||||||
|
* aiohttp 3.6.2>
|
||||||
|
* lxml 4.2.1>
|
||||||
|
|
||||||
|
#### Installation
|
||||||
|
```
|
||||||
|
$ git clone git@gitlab.com:chicken-riders/rcgcdb.git #clone repo
|
||||||
|
$ cd RcGcDw
|
||||||
|
$ python3 -m venv . #(optional, if you want to contain everything (you should!))
|
||||||
|
$ source bin/activate #(optional, see above)
|
||||||
|
$ pip3 install -r requirements.txt #install requirements (lxml may require additional distro packages, more on that here https://lxml.de/build.html)
|
||||||
|
$ nano settings.json.example #edit the configuration file
|
||||||
|
$ mv settings.json.example settings.json
|
||||||
|
$ python3 start.py
|
||||||
|
```
|
||||||
4
requirements.txt
Normal file
4
requirements.txt
Normal file
|
|
@ -0,0 +1,4 @@
|
||||||
|
beautifulsoup4 >= 4.6.0; python_version >= '3.6'
|
||||||
|
requests >= 2.18.4
|
||||||
|
aiohttp >= 3.6.2
|
||||||
|
lxml >= 4.2.1
|
||||||
271
settings.json.example
Normal file
271
settings.json.example
Normal file
|
|
@ -0,0 +1,271 @@
|
||||||
|
{
|
||||||
|
"header": {
|
||||||
|
"user-agent": "RcGcDb/{version}"
|
||||||
|
},
|
||||||
|
"max_requests_per_minute": 30,
|
||||||
|
"minimal_cooldown_per_wiki_in_sec": 60,
|
||||||
|
"monitoring_webhook": "111111111111111111/AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA",
|
||||||
|
"logging": {
|
||||||
|
"version": 1,
|
||||||
|
"disable_existing_loggers": false,
|
||||||
|
"formatters": {
|
||||||
|
"standard": {
|
||||||
|
"format": "%(name)s - %(levelname)s: %(message)s"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"handlers": {
|
||||||
|
"default": {
|
||||||
|
"formatter": "standard",
|
||||||
|
"class": "logging.StreamHandler",
|
||||||
|
"stream": "ext://sys.stdout"
|
||||||
|
},
|
||||||
|
"file": {
|
||||||
|
"formatter": "standard",
|
||||||
|
"class": "logging.handlers.TimedRotatingFileHandler",
|
||||||
|
"filename": "error.log",
|
||||||
|
"interval": 7,
|
||||||
|
"when": "D"
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"loggers": {
|
||||||
|
"": {
|
||||||
|
"level": 0,
|
||||||
|
"handlers": [
|
||||||
|
"default"
|
||||||
|
]
|
||||||
|
},
|
||||||
|
"rcgcdb.bot": {},
|
||||||
|
"rcgcdb.config": {},
|
||||||
|
"rcgcdb.discord": {},
|
||||||
|
"rcgcdb.wiki": {}
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"appearance": {
|
||||||
|
"mode": "embed",
|
||||||
|
"embed": {
|
||||||
|
"show_footer": true,
|
||||||
|
"show_edit_changes": true,
|
||||||
|
"embed_images": true,
|
||||||
|
"daily_overview": {
|
||||||
|
"color": 16312092,
|
||||||
|
"icon": ""
|
||||||
|
},
|
||||||
|
"new": {
|
||||||
|
"icon": "https://i.imgur.com/6HIbEq8.png",
|
||||||
|
"color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE"
|
||||||
|
},
|
||||||
|
"edit": {
|
||||||
|
"icon": "",
|
||||||
|
"color": "THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE"
|
||||||
|
},
|
||||||
|
"upload/overwrite": {
|
||||||
|
"icon": "https://i.imgur.com/egJpa81.png",
|
||||||
|
"color": 12390624
|
||||||
|
},
|
||||||
|
"upload/upload": {
|
||||||
|
"icon": "https://i.imgur.com/egJpa81.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"upload/revert": {
|
||||||
|
"icon": "https://i.imgur.com/egJpa81.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"delete/delete": {
|
||||||
|
"icon": "https://i.imgur.com/BU77GD3.png",
|
||||||
|
"color": 1
|
||||||
|
},
|
||||||
|
"delete/delete_redir": {
|
||||||
|
"icon": "https://i.imgur.com/BU77GD3.png",
|
||||||
|
"color": 1
|
||||||
|
},
|
||||||
|
"delete/restore": {
|
||||||
|
"icon": "https://i.imgur.com/9MnROIU.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"delete/revision": {
|
||||||
|
"icon": "https://i.imgur.com/1gps6EZ.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"delete/event": {
|
||||||
|
"icon": "https://i.imgur.com/1gps6EZ.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"merge/merge": {
|
||||||
|
"icon": "https://i.imgur.com/uQMK9XK.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"move/move": {
|
||||||
|
"icon": "https://i.imgur.com/eXz9dog.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"move/move_redir": {
|
||||||
|
"icon": "https://i.imgur.com/UtC3YX2.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"block/block": {
|
||||||
|
"icon": "https://i.imgur.com/g7KgZHf.png",
|
||||||
|
"color": 1
|
||||||
|
},
|
||||||
|
"block/unblock": {
|
||||||
|
"icon": "https://i.imgur.com/bvtBJ8o.png",
|
||||||
|
"color": 1
|
||||||
|
},
|
||||||
|
"block/reblock": {
|
||||||
|
"icon": "https://i.imgur.com/g7KgZHf.png",
|
||||||
|
"color": 1
|
||||||
|
},
|
||||||
|
"protect/protect": {
|
||||||
|
"icon": "https://i.imgur.com/bzPt89Z.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"protect/modify": {
|
||||||
|
"icon": "https://i.imgur.com/bzPt89Z.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"protect/move_prot": {
|
||||||
|
"icon": "https://i.imgur.com/bzPt89Z.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"protect/unprotect": {
|
||||||
|
"icon": "https://i.imgur.com/2wN3Qcq.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"import/upload": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"import/interwiki": {
|
||||||
|
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"rights/rights": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"abusefilter/abusefilter": {
|
||||||
|
"icon": "https://i.imgur.com/Sn2NzRJ.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"abusefilter/modify": {
|
||||||
|
"icon": "https://i.imgur.com/Sn2NzRJ.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"abusefilter/create": {
|
||||||
|
"icon": "https://i.imgur.com/Sn2NzRJ.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"interwiki/iw_add": {
|
||||||
|
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"interwiki/iw_edit": {
|
||||||
|
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"interwiki/iw_delete": {
|
||||||
|
"icon": "https://i.imgur.com/sFkhghb.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"curseprofile/comment-created": {
|
||||||
|
"icon": "https://i.imgur.com/Lvy5E32.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"curseprofile/comment-edited": {
|
||||||
|
"icon": "https://i.imgur.com/Lvy5E32.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"curseprofile/comment-deleted": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"curseprofile/comment-purged":{
|
||||||
|
"icon":"",
|
||||||
|
"color":null
|
||||||
|
},
|
||||||
|
"curseprofile/comment-replied": {
|
||||||
|
"icon": "https://i.imgur.com/hkyYsI1.png",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"curseprofile/profile-edited": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"contentmodel/change": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"cargo/deletetable": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"cargo/createtable": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"cargo/replacetable": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"cargo/recreatetable": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"sprite/sprite": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"sprite/sheet": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"sprite/slice": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"managetags/create": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"managetags/delete": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"managetags/activate": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"managetags/deactivate": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"tag/update": {
|
||||||
|
"icon": "",
|
||||||
|
"color": null
|
||||||
|
},
|
||||||
|
"suppressed": {
|
||||||
|
"icon": "https://i.imgur.com/1gps6EZ.png",
|
||||||
|
"color": 8092539
|
||||||
|
},
|
||||||
|
"discussion/forum/post": {
|
||||||
|
"icon": "",
|
||||||
|
"color":null
|
||||||
|
},
|
||||||
|
"discussion/forum/reply": {
|
||||||
|
"icon": "",
|
||||||
|
"color":null
|
||||||
|
},
|
||||||
|
"discussion/forum/poll": {
|
||||||
|
"icon": "",
|
||||||
|
"color":null
|
||||||
|
},
|
||||||
|
"discussion/wall/post": {
|
||||||
|
"icon": "",
|
||||||
|
"color":null
|
||||||
|
},
|
||||||
|
"discussion/wall/reply": {
|
||||||
|
"icon": "",
|
||||||
|
"color":null
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
19
src/bot.py
19
src/bot.py
|
|
@ -8,7 +8,9 @@ from src.exceptions import *
|
||||||
from src.database import db_cursor
|
from src.database import db_cursor
|
||||||
from collections import defaultdict
|
from collections import defaultdict
|
||||||
from src.queue_handler import DBHandler
|
from src.queue_handler import DBHandler
|
||||||
|
from src.discord import DiscordMessage
|
||||||
from src.msgqueue import messagequeue
|
from src.msgqueue import messagequeue
|
||||||
|
import requests
|
||||||
|
|
||||||
logging.config.dictConfig(settings["logging"])
|
logging.config.dictConfig(settings["logging"])
|
||||||
logger = logging.getLogger("rcgcdb.bot")
|
logger = logging.getLogger("rcgcdb.bot")
|
||||||
|
|
@ -25,6 +27,7 @@ mw_msgs: dict = {} # will have the type of id: tuple
|
||||||
for wiki in db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
|
for wiki in db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
|
||||||
all_wikis[wiki] = Wiki()
|
all_wikis[wiki] = Wiki()
|
||||||
|
|
||||||
|
|
||||||
# Start queueing logic
|
# Start queueing logic
|
||||||
|
|
||||||
|
|
||||||
|
|
@ -67,6 +70,9 @@ async def wiki_scanner():
|
||||||
continue # ignore this wiki if it throws errors
|
continue # ignore this wiki if it throws errors
|
||||||
try:
|
try:
|
||||||
recent_changes_resp = await wiki_response.json(encoding="UTF-8")
|
recent_changes_resp = await wiki_response.json(encoding="UTF-8")
|
||||||
|
if "error" in recent_changes_resp or "errors" in recent_changes_resp:
|
||||||
|
# TODO Remove on some errors (example "code": "readapidenied")
|
||||||
|
raise WikiError
|
||||||
recent_changes = recent_changes_resp['query']['recentchanges']
|
recent_changes = recent_changes_resp['query']['recentchanges']
|
||||||
recent_changes.reverse()
|
recent_changes.reverse()
|
||||||
except:
|
except:
|
||||||
|
|
@ -89,7 +95,8 @@ async def wiki_scanner():
|
||||||
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
|
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
|
||||||
if change["rcid"] > db_wiki[6]:
|
if change["rcid"] > db_wiki[6]:
|
||||||
for target in targets.items():
|
for target in targets.items():
|
||||||
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp)
|
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths,
|
||||||
|
recent_changes_resp)
|
||||||
if recent_changes:
|
if recent_changes:
|
||||||
DBHandler.add(db_wiki[3], change["rcid"])
|
DBHandler.add(db_wiki[3], change["rcid"])
|
||||||
DBHandler.update_db()
|
DBHandler.update_db()
|
||||||
|
|
@ -101,8 +108,18 @@ async def message_sender():
|
||||||
await messagequeue.resend_msgs()
|
await messagequeue.resend_msgs()
|
||||||
|
|
||||||
|
|
||||||
|
def global_exception_handler(loop, context):
|
||||||
|
"""Global exception handler for asyncio, lets us know when something crashes"""
|
||||||
|
msg = context.get("exception", context["message"])
|
||||||
|
logger.error(msg)
|
||||||
|
requests.post("https://discord.com/api/webhooks/" + settings["monitoring_webhook"],
|
||||||
|
data=DiscordMessage("embed", "exception", None, content=
|
||||||
|
"[RcGcDb] Exception detected, function might have shut down! Exception: {}".format(msg), wiki=None))
|
||||||
|
|
||||||
|
|
||||||
async def main_loop():
|
async def main_loop():
|
||||||
|
loop = asyncio.get_event_loop()
|
||||||
|
loop.set_exception_handler(global_exception_handler)
|
||||||
task1 = asyncio.create_task(wiki_scanner())
|
task1 = asyncio.create_task(wiki_scanner())
|
||||||
task2 = asyncio.create_task(message_sender())
|
task2 = asyncio.create_task(message_sender())
|
||||||
await task1
|
await task1
|
||||||
|
|
|
||||||
|
|
@ -16,7 +16,7 @@ logger = logging.getLogger("rcgcdb.discord")
|
||||||
|
|
||||||
# User facing webhook functions
|
# User facing webhook functions
|
||||||
def wiki_removal(wiki_id, status):
|
def wiki_removal(wiki_id, status):
|
||||||
for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wikiid = ?', (wiki_id,)):
|
for observer in db_cursor.execute('SELECT * FROM rcgcdw WHERE wiki = ?', (wiki_id,)):
|
||||||
def _(string: str) -> str:
|
def _(string: str) -> str:
|
||||||
"""Our own translation string to make it compatible with async"""
|
"""Our own translation string to make it compatible with async"""
|
||||||
return langs[observer[4]].gettext(string)
|
return langs[observer[4]].gettext(string)
|
||||||
|
|
@ -30,7 +30,7 @@ async def webhook_removal_monitor(webhook_url: list, reason: int):
|
||||||
aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(4.0)))
|
aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(4.0)))
|
||||||
|
|
||||||
|
|
||||||
class DiscordMessage():
|
class DiscordMessage:
|
||||||
"""A class defining a typical Discord JSON representation of webhook payload."""
|
"""A class defining a typical Discord JSON representation of webhook payload."""
|
||||||
def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None):
|
def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None):
|
||||||
self.webhook_object = dict(allowed_mentions={"parse": []})
|
self.webhook_object = dict(allowed_mentions={"parse": []})
|
||||||
|
|
|
||||||
Loading…
Reference in a new issue