mirror of
https://gitlab.com/chicken-riders/RcGcDb.git
synced 2025-02-23 00:54:09 +00:00
Fixed one thing
Broke 10 more
This commit is contained in:
Frisk
parent
76ef334843
commit
f3594ac12b
99
src/bot.py
99
src/bot.py
|
|
@ -8,6 +8,7 @@ from src.exceptions import *
|
|||
from src.database import db_cursor
|
||||
from collections import defaultdict
|
||||
from src.queue_handler import DBHandler
|
||||
from src.msgqueue import messagequeue
|
||||
|
||||
logging.config.dictConfig(settings["logging"])
|
||||
logger = logging.getLogger("rcgcdb.bot")
|
||||
|
|
@ -37,7 +38,7 @@ def calculate_delay() -> float:
|
|||
|
||||
def generate_targets(wiki_url: str) -> defaultdict:
|
||||
combinations = defaultdict(list)
|
||||
for webhook in db_cursor.execute('SELECT ROWID, * FROM rcgcdw WHERE wiki = ?', wiki_url):
|
||||
for webhook in db_cursor.execute('SELECT ROWID, * FROM rcgcdw WHERE wiki = ?', (wiki_url,)):
|
||||
# rowid, guild, configid, webhook, wiki, lang, display, rcid, wikiid, postid
|
||||
combination = (webhook[5], webhook[6]) # lang, display
|
||||
combinations[combination].append(webhook[3])
|
||||
|
|
@ -45,60 +46,66 @@ def generate_targets(wiki_url: str) -> defaultdict:
|
|||
|
||||
|
||||
async def wiki_scanner():
|
||||
calc_delay = calculate_delay()
|
||||
# db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
|
||||
for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'):
|
||||
extended = False
|
||||
if db_wiki[3] not in all_wikis:
|
||||
logger.debug("New wiki: {}".format(db_wiki[3]))
|
||||
all_wikis[db_wiki[3]] = Wiki()
|
||||
local_wiki = all_wikis[db_wiki[3]] # set a reference to a wiki object from memory
|
||||
if local_wiki.mw_messages is None:
|
||||
extended = True
|
||||
logger.debug("test")
|
||||
try:
|
||||
wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3])
|
||||
await local_wiki.check_status(wiki[3], wiki_response.status)
|
||||
except (WikiServerError, WikiError):
|
||||
continue # ignore this wiki if it throws errors
|
||||
try:
|
||||
recent_changes_resp = await wiki_response.json(encoding="UTF-8")
|
||||
recent_changes = recent_changes_resp['query']['recentchanges'].reverse()
|
||||
except:
|
||||
logger.exception("On loading json of response.")
|
||||
continue
|
||||
if extended:
|
||||
await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
|
||||
if db_wiki[6] is None: # new wiki, just get the last rc to not spam the channel
|
||||
if len(recent_changes) > 0:
|
||||
DBHandler.add(db_wiki[3], recent_changes[-1]["rcid"])
|
||||
while True:
|
||||
calc_delay = calculate_delay()
|
||||
# db_cursor.execute('SELECT DISTINCT wiki FROM rcgcdw'):
|
||||
for db_wiki in db_cursor.execute('SELECT * FROM rcgcdw GROUP BY wiki'):
|
||||
extended = False
|
||||
if db_wiki[3] not in all_wikis:
|
||||
logger.debug("New wiki: {}".format(db_wiki[3]))
|
||||
all_wikis[db_wiki[3]] = Wiki()
|
||||
local_wiki = all_wikis[db_wiki[3]] # set a reference to a wiki object from memory
|
||||
if local_wiki.mw_messages is None:
|
||||
extended = True
|
||||
logger.debug("test")
|
||||
try:
|
||||
wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[3])
|
||||
await local_wiki.check_status(db_wiki[3], wiki_response.status)
|
||||
except (WikiServerError, WikiError):
|
||||
logger.exception("Exeption when fetching the wiki")
|
||||
continue # ignore this wiki if it throws errors
|
||||
try:
|
||||
recent_changes_resp = await wiki_response.json(encoding="UTF-8")
|
||||
recent_changes = recent_changes_resp['query']['recentchanges']
|
||||
recent_changes.reverse()
|
||||
except:
|
||||
logger.exception("On loading json of response.")
|
||||
continue
|
||||
else:
|
||||
DBHandler.add(db_wiki[3], 0)
|
||||
continue
|
||||
categorize_events = {}
|
||||
targets = generate_targets(db_wiki[3])
|
||||
paths = get_paths(db_wiki[3], recent_changes_resp)
|
||||
for change in recent_changes:
|
||||
await process_cats(change, local_wiki, mw_msgs, categorize_events)
|
||||
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
|
||||
if change["rcid"] < db_wiki[6]:
|
||||
for target in targets.items():
|
||||
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp)
|
||||
if recent_changes:
|
||||
DBHandler.add(db_wiki[3], change["rcid"])
|
||||
await asyncio.sleep(delay=calc_delay)
|
||||
|
||||
DBHandler.update_db()
|
||||
if extended:
|
||||
await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
|
||||
if db_wiki[6] is None: # new wiki, just get the last rc to not spam the channel
|
||||
if len(recent_changes) > 0:
|
||||
DBHandler.add(db_wiki[3], recent_changes[-1]["rcid"])
|
||||
continue
|
||||
else:
|
||||
DBHandler.add(db_wiki[3], 0)
|
||||
continue
|
||||
categorize_events = {}
|
||||
targets = generate_targets(db_wiki[3])
|
||||
paths = get_paths(db_wiki[3], recent_changes_resp)
|
||||
for change in recent_changes:
|
||||
await process_cats(change, local_wiki, mw_msgs, categorize_events)
|
||||
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
|
||||
if change["rcid"] < db_wiki[6]:
|
||||
for target in targets.items():
|
||||
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, recent_changes_resp)
|
||||
if recent_changes:
|
||||
DBHandler.add(db_wiki[3], change["rcid"])
|
||||
await asyncio.sleep(delay=calc_delay)
|
||||
DBHandler.update_db()
|
||||
|
||||
|
||||
async def message_sender():
|
||||
pass
|
||||
while True:
|
||||
await messagequeue.resend_msgs()
|
||||
|
||||
|
||||
|
||||
async def main_loop():
|
||||
task1 = asyncio.create_task(wiki_scanner())
|
||||
task2 = asyncio.create_task(message_sender())
|
||||
await task1
|
||||
await task2
|
||||
|
||||
|
||||
asyncio.run(main_loop())
|
||||
|
|
|
|||
|
|
@ -23,13 +23,14 @@ logger = logging.getLogger("rcgcdw.rc_formatters")
|
|||
|
||||
async def compact_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths,
|
||||
additional_data=None):
|
||||
global LinkParser
|
||||
if additional_data is None:
|
||||
additional_data = {"namespaces": {}, "tags": {}}
|
||||
WIKI_API_PATH = paths[0]
|
||||
WIKI_SCRIPT_PATH = paths[1]
|
||||
WIKI_ARTICLE_PATH = paths[2]
|
||||
WIKI_JUST_DOMAIN = paths[3]
|
||||
LinkParser = LinkParser("domain")
|
||||
LinkParser = LinkParser(paths[3])
|
||||
if action != "suppressed":
|
||||
author_url = link_formatter(create_article_path("User:{user}".format(user=change["user"]), WIKI_ARTICLE_PATH))
|
||||
author = change["user"]
|
||||
|
|
@ -318,13 +319,14 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c
|
|||
|
||||
|
||||
async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None):
|
||||
global LinkParser
|
||||
if additional_data is None:
|
||||
additional_data = {"namespaces": {}, "tags": {}}
|
||||
WIKI_API_PATH = paths[0]
|
||||
WIKI_SCRIPT_PATH = paths[1]
|
||||
WIKI_ARTICLE_PATH = paths[2]
|
||||
WIKI_JUST_DOMAIN = paths[3]
|
||||
LinkParser = LinkParser()
|
||||
LinkParser = LinkParser(paths[3])
|
||||
embed = DiscordMessage("embed", action, target[1], wiki=WIKI_SCRIPT_PATH)
|
||||
if parsed_comment is None:
|
||||
parsed_comment = _("No description provided")
|
||||
|
|
@ -694,16 +696,16 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
|
|||
if "tags" in change and change["tags"]:
|
||||
tag_displayname = []
|
||||
for tag in change["tags"]:
|
||||
if tag in additional_data.tags:
|
||||
if additional_data.tags[tag] is None:
|
||||
if tag in additional_data["tags"]:
|
||||
if additional_data["tags"][tag] is None:
|
||||
continue # Ignore hidden tags
|
||||
else:
|
||||
tag_displayname.append(additional_data.tags[tag])
|
||||
tag_displayname.append(additional_data["tags"][tag])
|
||||
else:
|
||||
tag_displayname.append(tag)
|
||||
embed.add_field(_("Tags"), ", ".join(tag_displayname))
|
||||
logger.debug("Current params in edit action: {}".format(change))
|
||||
if categories is not None and not (len(categories["new"]) == 0 and len(categories["removed"]) == 0):
|
||||
if categories and not (len(categories["new"]) == 0 and len(categories["removed"]) == 0):
|
||||
new_cat = (_("**Added**: ") + ", ".join(list(categories["new"])[0:16]) + ("\n" if len(categories["new"])<=15 else _(" and {} more\n").format(len(categories["new"])-15))) if categories["new"] else ""
|
||||
del_cat = (_("**Removed**: ") + ", ".join(list(categories["removed"])[0:16]) + ("" if len(categories["removed"])<=15 else _(" and {} more").format(len(categories["removed"])-15))) if categories["removed"] else ""
|
||||
embed.add_field(_("Changed categories"), new_cat + del_cat)
|
||||
|
|
|
|||
|
|
@ -1,4 +1,3 @@
|
|||
from abc import ABC
|
||||
from html.parser import HTMLParser
|
||||
import base64, re
|
||||
from src.config import settings
|
||||
|
|
@ -14,7 +13,7 @@ logger = logging.getLogger("rcgcdw.misc")
|
|||
class DiscordMessage():
|
||||
"""A class defining a typical Discord JSON representation of webhook payload."""
|
||||
def __init__(self, message_type: str, event_type: str, webhook_url: list, wiki, content=None):
|
||||
self.webhook_object = dict(allowed_mentions={"parse": []}, avatar_url=settings["avatars"].get(message_type, ""))
|
||||
self.webhook_object = dict(allowed_mentions={"parse": []})
|
||||
self.webhook_url = webhook_url
|
||||
self.wiki = wiki
|
||||
|
||||
|
|
@ -123,15 +122,13 @@ def get_paths(wiki: str, request) -> tuple:
|
|||
|
||||
|
||||
class LinkParser(HTMLParser):
|
||||
def error(self, message):
|
||||
pass
|
||||
|
||||
new_string = ""
|
||||
recent_href = ""
|
||||
|
||||
def __init__(self, domain):
|
||||
super().__init__()
|
||||
self.WIKI_JUST_DOMAIN = domain
|
||||
super().__init__()
|
||||
|
||||
def handle_starttag(self, tag, attrs):
|
||||
for attr in attrs:
|
||||
|
|
|
|||
|
|
@ -52,6 +52,7 @@ class MessageQueue:
|
|||
logger.debug("Queue emptied, all messages delivered")
|
||||
self.cut_messages(num)
|
||||
logger.debug(self._queue)
|
||||
await asyncio.sleep(4.0)
|
||||
|
||||
|
||||
messagequeue = MessageQueue()
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ class UpdateDB():
|
|||
|
||||
def update_db(self):
|
||||
for update in self.updated:
|
||||
db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", update[1], update[0])
|
||||
db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", (update[1], update[0],))
|
||||
db_connection.commit()
|
||||
self.clear_list()
|
||||
|
||||
|
|
|
|||
|
|
@ -77,7 +77,7 @@ class Wiki:
|
|||
async def remove(self, wiki_id, reason):
|
||||
src.discord.wiki_removal(wiki_id, reason)
|
||||
src.discord.wiki_removal_monitor(wiki_id, reason)
|
||||
db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", wiki_id)
|
||||
db_cursor.execute("DELETE FROM rcgcdw WHERE wiki = ?", (wiki_id,))
|
||||
logger.warning("{} rows affected by DELETE FROM rcgcdw WHERE wiki = {}".format(db_cursor.rowcount, wiki_id))
|
||||
db_connection.commit()
|
||||
|
||||
|
|
@ -141,7 +141,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict):
|
|||
:return:
|
||||
"""
|
||||
msgs = []
|
||||
for message in wiki_response["allmessages"]:
|
||||
for message in wiki_response["query"]["allmessages"]:
|
||||
if not "missing" in message: # ignore missing strings
|
||||
msgs.append((message["name"], re.sub(r'\[\[.*?\]\]', '', message["*"])))
|
||||
else:
|
||||
|
|
@ -157,6 +157,7 @@ async def process_mwmsgs(wiki_response: dict, local_wiki: Wiki, mw_msgs: dict):
|
|||
local_wiki.mw_messages = key
|
||||
|
||||
async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_wiki: tuple, target: tuple, paths: tuple, request: dict):
|
||||
global LinkParser
|
||||
"""Prepares essential information for both embed and compact message format."""
|
||||
def _(string: str) -> str:
|
||||
"""Our own translation string to make it compatible with async"""
|
||||
|
|
@ -165,7 +166,7 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_
|
|||
lang = langs[target[0][0]]
|
||||
ngettext = lang.ngettext
|
||||
# recent_changes = RecentChangesClass() # TODO Look into replacing RecentChangesClass with local_wiki
|
||||
LinkParser = LinkParser("domain")
|
||||
LinkParser = LinkParser(paths[3])
|
||||
logger.debug(change)
|
||||
appearance_mode = embed_formatter if target[0][1] > 0 else compact_formatter
|
||||
if ("actionhidden" in change or "suppressed" in change): # if event is hidden using suppression
|
||||
|
|
|
|||
Loading…
Reference in a new issue