if I ever complete this project this planet will be free from all evil

This commit is contained in:
Frisk 2020-07-10 16:11:45 +02:00
parent 415c278778
commit 2e7f12a7bc
No known key found for this signature in database
GPG key ID: 213F7C15068AF8AC
4 changed files with 46 additions and 4 deletions

View file

@ -33,4 +33,7 @@ async def main_loop():
local_wiki = all_wikis[wiki[0]] # set a reference to a wiki object from memory
if all_wikis[wiki[0]].mw_messages is None:
extended = True
wiki_response = await all_wikis[wiki[0]].fetch_wiki(extended)
wiki_response = await local_wiki.fetch_wiki(extended)
try:
await local_wiki.check_status(wiki[0], wiki_response.status, db_wiki[1])
except:

17
src/exceptions.py Normal file
View file

@ -0,0 +1,17 @@
class WikiError(Exception):
pass
class WikiServerError(Exception):
pass
class WikiNotFoundError(Exception):
pass
class WikiRemovedError(Exception):
pass
class WikiUnauthorizedError(Exception):
pass
class OtherWikiError(Exception):
pass

View file

@ -1,4 +1,4 @@
import aiohttp
from src.config import settings
session = aiohttp.ClientSession(headers=settings["header"])
session = aiohttp.ClientSession(headers=settings["header"], timeout=aiohttp.ClientTimeout(5.0))

View file

@ -1,13 +1,16 @@
from dataclasses import dataclass
from src.session import session
import logging, aiohttp
from src.exceptions import *
logger = logging.getLogger("rcgcdb.wiki")
@dataclass
class Wiki:
mw_messages: int = None
fail_times: int = 0 # corresponding to amount of times connection with wiki failed for client reasons (400-499)
async def fetch_wiki(self, extended, api_path):
async def fetch_wiki(self, extended, api_path) -> aiohttp.ClientResponse:
url_path = api_path
amount = 20
if extended:
@ -25,4 +28,23 @@ class Wiki:
"rcprop": "title|redirect|timestamp|ids|loginfo|parsedcomment|sizes|flags|tags|user",
"rclimit": amount, "rctype": "edit|new|log|external", "siprop": "namespaces"}
try:
await session.get(url_path, params=params)
response = await session.get(url_path, params=params)
except:
raise NotImplemented
return response
async def check_status(self, wiki_id, status, name):
if 199 < status < 300:
self.fail_times = 0
pass
elif 400 < status < 500: # ignore 400 error since this might be our fault
self.fail_times += 1
logger.warning("Wiki {} responded with HTTP code {}, increased fail_times to {}, skipping...".format(name, status, self.fail_times))
if self.fail_times > 3:
await self.remove(wiki_id)
raise WikiError
elif 499 < status < 600:
logger.warning("Wiki {} responded with HTTP code {}, skipping...".format(name, status, self.fail_times))
raise WikiServerError
async def remove(self, wiki_id):