Frisk/RcGcDb
1
0
Fork 0
mirror of https://gitlab.com/chicken-riders/RcGcDb.git synced 2025-02-23 00:54:09 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

Small improvement

Browse source
This commit is contained in:
Frisk 2020-08-06 03:07:13 +02:00
parent 493a1c8194
commit 71a3bdd91d
No known key found for this signature in database
GPG key ID: 213F7C15068AF8AC
1 changed files with 56 additions and 57 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

113
src/bot.py
View file

@ -168,64 +168,63 @@ async def scan_group(group: str):
async with rcqueue.retrieve_next_queued(group) as db_wiki: # acquire next wiki in queue async with rcqueue.retrieve_next_queued(group) as db_wiki: # acquire next wiki in queue
logger.debug("Wiki {}".format(db_wiki["wiki"])) logger.debug("Wiki {}".format(db_wiki["wiki"]))
local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory
if db_wiki["rcid"] != -1: extended = False
extended = False if local_wiki.mw_messages is None:
if local_wiki.mw_messages is None: extended = True
extended = True async with aiohttp.ClientSession(headers=settings["header"],
async with aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(3.0)) as session:
timeout=aiohttp.ClientTimeout(3.0)) as session: try:
try: wiki_response = await local_wiki.fetch_wiki(extended, db_wiki["wiki"], session, rate_limiter)
wiki_response = await local_wiki.fetch_wiki(extended, db_wiki["wiki"], session, rate_limiter) await local_wiki.check_status(db_wiki["wiki"], wiki_response.status)
await local_wiki.check_status(db_wiki["wiki"], wiki_response.status) except (WikiServerError, WikiError):
except (WikiServerError, WikiError): logger.error("Exeption when fetching the wiki")
logger.error("Exeption when fetching the wiki") continue # ignore this wiki if it throws errors
continue # ignore this wiki if it throws errors try:
try: recent_changes_resp = await wiki_response.json()
recent_changes_resp = await wiki_response.json() if "error" in recent_changes_resp or "errors" in recent_changes_resp:
if "error" in recent_changes_resp or "errors" in recent_changes_resp: error = recent_changes_resp.get("error", recent_changes_resp["errors"])
error = recent_changes_resp.get("error", recent_changes_resp["errors"]) if error["code"] == "readapidenied":
if error["code"] == "readapidenied": await local_wiki.fail_add(db_wiki["wiki"], 410)
await local_wiki.fail_add(db_wiki["wiki"], 410) continue
continue raise WikiError
raise WikiError recent_changes = recent_changes_resp['query']['recentchanges']
recent_changes = recent_changes_resp['query']['recentchanges'] recent_changes.reverse()
recent_changes.reverse() except aiohttp.ContentTypeError:
except aiohttp.ContentTypeError: logger.exception("Wiki seems to be resulting in non-json content.")
logger.exception("Wiki seems to be resulting in non-json content.") await local_wiki.fail_add(db_wiki["wiki"], 410)
await local_wiki.fail_add(db_wiki["wiki"], 410)
continue
except:
logger.exception("On loading json of response.")
continue
if extended:
await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
if db_wiki["rcid"] is None: # new wiki, just get the last rc to not spam the channel
if len(recent_changes) > 0:
DBHandler.add(db_wiki["wiki"], recent_changes[-1]["rcid"])
else:
DBHandler.add(db_wiki["wiki"], 0)
DBHandler.update_db()
continue continue
categorize_events = {} except:
targets = generate_targets(db_wiki["wiki"]) logger.exception("On loading json of response.")
paths = get_paths(db_wiki["wiki"], recent_changes_resp) continue
for change in recent_changes: if extended:
await process_cats(change, local_wiki, mw_msgs, categorize_events) await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up if db_wiki["rcid"] is None: # new wiki, just get the last rc to not spam the channel
if change["rcid"] > db_wiki["rcid"]: if len(recent_changes) > 0:
for target in targets.items(): DBHandler.add(db_wiki["wiki"], recent_changes[-1]["rcid"])
try: else:
await essential_info(change, categorize_events, local_wiki, db_wiki, DBHandler.add(db_wiki["wiki"], 0)
target, paths, recent_changes_resp, rate_limiter) DBHandler.update_db()
except: continue
if command_line_args.debug: categorize_events = {}
logger.exception("Exception on RC formatter") targets = generate_targets(db_wiki["wiki"])
raise paths = get_paths(db_wiki["wiki"], recent_changes_resp)
else: for change in recent_changes:
logger.exception("Exception on RC formatter") await process_cats(change, local_wiki, mw_msgs, categorize_events)
await formatter_exception_logger(db_wiki["wiki"], change, traceback.format_exc()) for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
if recent_changes: if change["rcid"] > db_wiki["rcid"]:
DBHandler.add(db_wiki["wiki"], change["rcid"]) for target in targets.items():
try:
await essential_info(change, categorize_events, local_wiki, db_wiki,
target, paths, recent_changes_resp, rate_limiter)
except:
if command_line_args.debug:
logger.exception("Exception on RC formatter")
raise
else:
logger.exception("Exception on RC formatter")
await formatter_exception_logger(db_wiki["wiki"], change, traceback.format_exc())
if recent_changes:
DBHandler.add(db_wiki["wiki"], change["rcid"])
delay_between_wikis = calculate_delay_for_group(len(rcqueue[group]["query"])) delay_between_wikis = calculate_delay_for_group(len(rcqueue[group]["query"]))
await asyncio.sleep(delay_between_wikis) await asyncio.sleep(delay_between_wikis)
DBHandler.update_db() DBHandler.update_db()