Frisk/RcGcDb
1
0
Fork 0
mirror of https://gitlab.com/chicken-riders/RcGcDb.git synced 2025-02-23 00:54:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

use db field names

Browse source
This commit is contained in:
Markus-Rost 2020-07-27 14:13:36 +02:00
parent 49b9e8de20
commit d4f1d89862
2 changed files with 20 additions and 20 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

34
src/bot.py
View file

@ -44,8 +44,8 @@ def calculate_delay() -> float:
def generate_targets(wiki_url: str) -> defaultdict: def generate_targets(wiki_url: str) -> defaultdict:
combinations = defaultdict(list) combinations = defaultdict(list)
for webhook in db_cursor.execute('SELECT webhook, lang, display FROM rcgcdw WHERE wiki = ?', (wiki_url,)): for webhook in db_cursor.execute('SELECT webhook, lang, display FROM rcgcdw WHERE wiki = ?', (wiki_url,)):
combination = (webhook[1], webhook[2]) # lang, display combination = (webhook["lang"], webhook["display"])
combinations[combination].append(webhook[0]) combinations[combination].append(webhook["webhook"])
return combinations return combinations
@ -56,19 +56,19 @@ async def wiki_scanner():
fetch_all = db_cursor.execute('SELECT webhook, wiki, lang, display, wikiid, rcid, postid FROM rcgcdw GROUP BY wiki') fetch_all = db_cursor.execute('SELECT webhook, wiki, lang, display, wikiid, rcid, postid FROM rcgcdw GROUP BY wiki')
# webhook, wiki, lang, display, wikiid, rcid, postid # webhook, wiki, lang, display, wikiid, rcid, postid
for db_wiki in fetch_all.fetchall(): for db_wiki in fetch_all.fetchall():
logger.debug("Wiki {}".format(db_wiki[1])) logger.debug("Wiki {}".format(db_wiki["wiki"]))
extended = False extended = False
if db_wiki[1] not in all_wikis: if db_wiki["wiki"] not in all_wikis:
logger.debug("New wiki: {}".format(db_wiki[1])) logger.debug("New wiki: {}".format(db_wiki["wiki"]))
all_wikis[db_wiki["wiki"]] = Wiki() all_wikis[db_wiki["wiki"]] = Wiki()
local_wiki = all_wikis[db_wiki[1]] # set a reference to a wiki object from memory local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory
if local_wiki.mw_messages is None: if local_wiki.mw_messages is None:
extended = True extended = True
async with aiohttp.ClientSession(headers=settings["header"], async with aiohttp.ClientSession(headers=settings["header"],
timeout=aiohttp.ClientTimeout(2.0)) as session: timeout=aiohttp.ClientTimeout(2.0)) as session:
try: try:
wiki_response = await local_wiki.fetch_wiki(extended, db_wiki[1], session) wiki_response = await local_wiki.fetch_wiki(extended, db_wiki["wiki"], session)
await local_wiki.check_status(db_wiki[1], wiki_response.status) await local_wiki.check_status(db_wiki["wiki"], wiki_response.status)
except (WikiServerError, WikiError): except (WikiServerError, WikiError):
logger.exception("Exeption when fetching the wiki") logger.exception("Exeption when fetching the wiki")
continue # ignore this wiki if it throws errors continue # ignore this wiki if it throws errors
@ -77,39 +77,39 @@ async def wiki_scanner():
if "error" in recent_changes_resp or "errors" in recent_changes_resp: if "error" in recent_changes_resp or "errors" in recent_changes_resp:
error = recent_changes_resp.get("error", recent_changes_resp["errors"]) error = recent_changes_resp.get("error", recent_changes_resp["errors"])
if error["code"] == "readapidenied": if error["code"] == "readapidenied":
await local_wiki.fail_add(db_wiki[1], 410) await local_wiki.fail_add(db_wiki["wiki"], 410)
continue continue
raise WikiError raise WikiError
recent_changes = recent_changes_resp['query']['recentchanges'] recent_changes = recent_changes_resp['query']['recentchanges']
recent_changes.reverse() recent_changes.reverse()
except aiohttp.ContentTypeError: except aiohttp.ContentTypeError:
logger.exception("Wiki seems to be resulting in non-json content.") logger.exception("Wiki seems to be resulting in non-json content.")
await local_wiki.fail_add(db_wiki[1], 410) await local_wiki.fail_add(db_wiki["wiki"], 410)
continue continue
except: except:
logger.exception("On loading json of response.") logger.exception("On loading json of response.")
continue continue
if extended: if extended:
await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs) await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
if db_wiki[5] is None: # new wiki, just get the last rc to not spam the channel if db_wiki["rcid"] is None: # new wiki, just get the last rc to not spam the channel
if len(recent_changes) > 0: if len(recent_changes) > 0:
DBHandler.add(db_wiki[1], recent_changes[-1]["rcid"]) DBHandler.add(db_wiki["wiki"], recent_changes[-1]["rcid"])
else: else:
DBHandler.add(db_wiki[1], 0) DBHandler.add(db_wiki["wiki"], 0)
DBHandler.update_db() DBHandler.update_db()
continue continue
categorize_events = {} categorize_events = {}
targets = generate_targets(db_wiki[1]) targets = generate_targets(db_wiki["wiki"])
paths = get_paths(db_wiki[1], recent_changes_resp) paths = get_paths(db_wiki["wiki"], recent_changes_resp)
for change in recent_changes: for change in recent_changes:
await process_cats(change, local_wiki, mw_msgs, categorize_events) await process_cats(change, local_wiki, mw_msgs, categorize_events)
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
if change["rcid"] > db_wiki[5]: if change["rcid"] > db_wiki["rcid"]:
for target in targets.items(): for target in targets.items():
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths, await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths,
recent_changes_resp) recent_changes_resp)
if recent_changes: if recent_changes:
DBHandler.add(db_wiki[1], change["rcid"]) DBHandler.add(db_wiki["wiki"], change["rcid"])
DBHandler.update_db() DBHandler.update_db()
await asyncio.sleep(delay=calc_delay) await asyncio.sleep(delay=calc_delay)
except asyncio.CancelledError: except asyncio.CancelledError:

6
src/discord.py
View file

@ -19,16 +19,16 @@ async def wiki_removal(wiki_url, status):
for observer in db_cursor.execute('SELECT webhook, lang FROM rcgcdw WHERE wiki = ?', (wiki_url,)): for observer in db_cursor.execute('SELECT webhook, lang FROM rcgcdw WHERE wiki = ?', (wiki_url,)):
def _(string: str) -> str: def _(string: str) -> str:
"""Our own translation string to make it compatible with async""" """Our own translation string to make it compatible with async"""
return langs[observer[1]].gettext(string) return langs[observer["lang"]].gettext(string)
reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"), reasons = {410: _("wiki deletion"), 404: _("wiki deletion"), 401: _("wiki becoming inaccessible"),
402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible"), 410: _("wiki becoming inaccessible")} 402: _("wiki becoming inaccessible"), 403: _("wiki becoming inaccessible"), 410: _("wiki becoming inaccessible")}
reason = reasons.get(status, _("unknown error")) reason = reasons.get(status, _("unknown error"))
await send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[observer[0]], content=_("The webhook for {} has been removed due to {}.".format(wiki_url, reason)), wiki=None)) await send_to_discord_webhook(DiscordMessage("compact", "webhook/remove", webhook_url=[observer["webhook"]], content=_("The webhook for {} has been removed due to {}.".format(wiki_url, reason)), wiki=None))
header = settings["header"] header = settings["header"]
header['Content-Type'] = 'application/json' header['Content-Type'] = 'application/json'
header['X-Audit-Log-Reason'] = "Wiki becoming unavailable" header['X-Audit-Log-Reason'] = "Wiki becoming unavailable"
async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(5.0)) as session: async with aiohttp.ClientSession(headers=header, timeout=aiohttp.ClientTimeout(5.0)) as session:
await session.delete("https://discord.com/api/webhooks/"+observer[0]) await session.delete("https://discord.com/api/webhooks/"+observer["webhook"])
async def webhook_removal_monitor(webhook_url: list, reason: int): async def webhook_removal_monitor(webhook_url: list, reason: int):