mirror of
https://gitlab.com/chicken-riders/RcGcDb.git
synced 2025-02-23 00:54:09 +00:00
add discussions check
This commit is contained in:
Markus-Rost
parent
ca3be79ef1
commit
4d618f4268
171
src/bot.py
171
src/bot.py
|
|
@ -15,7 +15,8 @@ from src.exceptions import *
|
|||
from src.misc import get_paths
|
||||
from src.msgqueue import messagequeue
|
||||
from src.queue_handler import DBHandler
|
||||
from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info
|
||||
from src.wiki import Wiki, process_cats, process_mwmsgs, essential_info, essential_feeds
|
||||
from src.formatters.discussions import embed_formatter, compact_formatter
|
||||
from src.discord import DiscordMessage, formatter_exception_logger, msg_sender_exception_logger
|
||||
|
||||
logging.config.dictConfig(settings["logging"])
|
||||
|
|
@ -72,68 +73,124 @@ async def wiki_scanner():
|
|||
'SELECT webhook, wiki, lang, display, wikiid, rcid, postid FROM rcgcdw GROUP BY wiki')
|
||||
for db_wiki in fetch_all.fetchall():
|
||||
logger.debug("Wiki {}".format(db_wiki["wiki"]))
|
||||
extended = False
|
||||
if db_wiki["wiki"] not in all_wikis:
|
||||
logger.info("Registering new wiki locally: {}".format(db_wiki["wiki"]))
|
||||
all_wikis[db_wiki["wiki"]] = Wiki()
|
||||
local_wiki = all_wikis[db_wiki["wiki"]] # set a reference to a wiki object from memory
|
||||
if local_wiki.mw_messages is None:
|
||||
extended = True
|
||||
async with aiohttp.ClientSession(headers=settings["header"],
|
||||
timeout=aiohttp.ClientTimeout(3.0)) as session:
|
||||
try:
|
||||
wiki_response = await local_wiki.fetch_wiki(extended, db_wiki["wiki"], session)
|
||||
await local_wiki.check_status(db_wiki["wiki"], wiki_response.status)
|
||||
except (WikiServerError, WikiError):
|
||||
logger.error("Exeption when fetching the wiki")
|
||||
continue # ignore this wiki if it throws errors
|
||||
try:
|
||||
recent_changes_resp = await wiki_response.json()
|
||||
if "error" in recent_changes_resp or "errors" in recent_changes_resp:
|
||||
error = recent_changes_resp.get("error", recent_changes_resp["errors"])
|
||||
if error["code"] == "readapidenied":
|
||||
await local_wiki.fail_add(db_wiki["wiki"], 410)
|
||||
continue
|
||||
raise WikiError
|
||||
recent_changes = recent_changes_resp['query']['recentchanges']
|
||||
recent_changes.reverse()
|
||||
except aiohttp.ContentTypeError:
|
||||
logger.exception("Wiki seems to be resulting in non-json content.")
|
||||
await local_wiki.fail_add(db_wiki["wiki"], 410)
|
||||
if db_wiki["rcid"] != -1:
|
||||
extended = False
|
||||
if local_wiki.mw_messages is None:
|
||||
extended = True
|
||||
async with aiohttp.ClientSession(headers=settings["header"],
|
||||
timeout=aiohttp.ClientTimeout(3.0)) as session:
|
||||
try:
|
||||
wiki_response = await local_wiki.fetch_wiki(extended, db_wiki["wiki"], session)
|
||||
await local_wiki.check_status(db_wiki["wiki"], wiki_response.status)
|
||||
except (WikiServerError, WikiError):
|
||||
logger.error("Exeption when fetching the wiki")
|
||||
continue # ignore this wiki if it throws errors
|
||||
try:
|
||||
recent_changes_resp = await wiki_response.json()
|
||||
if "error" in recent_changes_resp or "errors" in recent_changes_resp:
|
||||
error = recent_changes_resp.get("error", recent_changes_resp["errors"])
|
||||
if error["code"] == "readapidenied":
|
||||
await local_wiki.fail_add(db_wiki["wiki"], 410)
|
||||
continue
|
||||
raise WikiError
|
||||
recent_changes = recent_changes_resp['query']['recentchanges']
|
||||
recent_changes.reverse()
|
||||
except aiohttp.ContentTypeError:
|
||||
logger.exception("Wiki seems to be resulting in non-json content.")
|
||||
await local_wiki.fail_add(db_wiki["wiki"], 410)
|
||||
continue
|
||||
except:
|
||||
logger.exception("On loading json of response.")
|
||||
continue
|
||||
if extended:
|
||||
await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
|
||||
if db_wiki["rcid"] is None: # new wiki, just get the last rc to not spam the channel
|
||||
if len(recent_changes) > 0:
|
||||
DBHandler.add(db_wiki["wiki"], recent_changes[-1]["rcid"])
|
||||
else:
|
||||
DBHandler.add(db_wiki["wiki"], 0)
|
||||
DBHandler.update_db()
|
||||
continue
|
||||
except:
|
||||
logger.exception("On loading json of response.")
|
||||
continue
|
||||
if extended:
|
||||
await process_mwmsgs(recent_changes_resp, local_wiki, mw_msgs)
|
||||
if db_wiki["rcid"] is None: # new wiki, just get the last rc to not spam the channel
|
||||
if len(recent_changes) > 0:
|
||||
DBHandler.add(db_wiki["wiki"], recent_changes[-1]["rcid"])
|
||||
else:
|
||||
DBHandler.add(db_wiki["wiki"], 0)
|
||||
categorize_events = {}
|
||||
targets = generate_targets(db_wiki["wiki"])
|
||||
paths = get_paths(db_wiki["wiki"], recent_changes_resp)
|
||||
for change in recent_changes:
|
||||
await process_cats(change, local_wiki, mw_msgs, categorize_events)
|
||||
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
|
||||
if change["rcid"] > db_wiki["rcid"]:
|
||||
for target in targets.items():
|
||||
try:
|
||||
await essential_info(change, categorize_events, local_wiki, db_wiki,
|
||||
target, paths, recent_changes_resp)
|
||||
except:
|
||||
if command_line_args.debug:
|
||||
raise # reraise the issue
|
||||
else:
|
||||
logger.exception("Exception on RC formatter")
|
||||
await formatter_exception_logger(db_wiki["wiki"], change, traceback.format_exc())
|
||||
if recent_changes:
|
||||
DBHandler.add(db_wiki["wiki"], change["rcid"])
|
||||
DBHandler.update_db()
|
||||
continue
|
||||
categorize_events = {}
|
||||
targets = generate_targets(db_wiki["wiki"])
|
||||
paths = get_paths(db_wiki["wiki"], recent_changes_resp)
|
||||
for change in recent_changes:
|
||||
await process_cats(change, local_wiki, mw_msgs, categorize_events)
|
||||
for change in recent_changes: # Yeah, second loop since the categories require to be all loaded up
|
||||
if change["rcid"] > db_wiki["rcid"]:
|
||||
for target in targets.items():
|
||||
try:
|
||||
await essential_info(change, categorize_events, local_wiki, db_wiki, target, paths,
|
||||
recent_changes_resp)
|
||||
except:
|
||||
if command_line_args.debug:
|
||||
raise # reraise the issue
|
||||
else:
|
||||
logger.exception("Exception on RC formatter")
|
||||
await formatter_exception_logger(db_wiki["wiki"], change, traceback.format_exc())
|
||||
if recent_changes:
|
||||
DBHandler.add(db_wiki["wiki"], change["rcid"])
|
||||
DBHandler.update_db()
|
||||
await asyncio.sleep(delay=calc_delay)
|
||||
await asyncio.sleep(delay=calc_delay)
|
||||
if db_wiki["wikiid"] is not None:
|
||||
header = settings["header"]
|
||||
header["Accept"] = "application/hal+json"
|
||||
async with aiohttp.ClientSession(headers=header,
|
||||
timeout=aiohttp.ClientTimeout(3.0)) as session:
|
||||
try:
|
||||
feeds_response = await local_wiki.fetch_feeds(db_wiki["wikiid"], session)
|
||||
# await local_wiki.check_status(db_wiki["wiki"], wiki_response.status)
|
||||
# NEED A GOAT TO CHECK THIS
|
||||
except (WikiServerError, WikiError):
|
||||
logger.error("Exeption when fetching the wiki")
|
||||
continue # ignore this wiki if it throws errors
|
||||
try:
|
||||
discussion_feed_resp = await feeds_response.json()
|
||||
if "title" in discussion_feed_resp:
|
||||
error = discussion_feed_resp["error"]
|
||||
if error == "site doesn't exists":
|
||||
db_cursor.execute("UPDATE rcgcdw SET wikiid = ? WHERE wiki = ?",
|
||||
(None, db_wiki["wiki"],))
|
||||
DBHandler.update_db()
|
||||
continue
|
||||
raise WikiError
|
||||
discussion_feed = discussion_feed_resp["_embedded"]["doc:posts"]
|
||||
discussion_feed.reverse()
|
||||
except aiohttp.ContentTypeError:
|
||||
logger.exception("Wiki seems to be resulting in non-json content.")
|
||||
# NEED A GOAT TO CHECK THIS
|
||||
# await local_wiki.fail_add(db_wiki["wiki"], 410)
|
||||
continue
|
||||
except:
|
||||
logger.exception("On loading json of response.")
|
||||
continue
|
||||
if db_wiki["postid"] is None: # new wiki, just get the last post to not spam the channel
|
||||
if len(discussion_feed) > 0:
|
||||
DBHandler.add(db_wiki["wiki"], discussion_feed[-1]["id"], True)
|
||||
else:
|
||||
DBHandler.add(db_wiki["wiki"], "0", True)
|
||||
DBHandler.update_db()
|
||||
continue
|
||||
targets = generate_targets(db_wiki["wiki"])
|
||||
for post in discussion_feed:
|
||||
if post["id"] > db_wiki["postid"]:
|
||||
for target in targets.items():
|
||||
try:
|
||||
await essential_feeds(post, db_wiki, target)
|
||||
except:
|
||||
if command_line_args.debug:
|
||||
raise # reraise the issue
|
||||
else:
|
||||
logger.exception("Exception on Feeds formatter")
|
||||
await formatter_exception_logger(db_wiki["wiki"], post, traceback.format_exc())
|
||||
if discussion_feed:
|
||||
DBHandler.add(db_wiki["wiki"], post["id"], True)
|
||||
DBHandler.update_db()
|
||||
await asyncio.sleep(delay=calc_delay)
|
||||
except asyncio.CancelledError:
|
||||
raise
|
||||
|
||||
|
|
|
|||
|
|
@ -1,19 +1,15 @@
|
|||
import datetime, logging
|
||||
import json
|
||||
import gettext
|
||||
from urllib.parse import quote_plus
|
||||
|
||||
from src.config import settings
|
||||
from src.misc import send_to_discord, escape_formatting
|
||||
from discord import DiscordMessage
|
||||
from src.i18n import disc
|
||||
|
||||
_ = disc.gettext
|
||||
|
||||
|
||||
discussion_logger = logging.getLogger("rcgcdw.discussion_formatter")
|
||||
|
||||
def embed_formatter(post, post_type):
|
||||
def feeds_embed_formatter(post_type, post, message_target, wiki, _):
|
||||
"""Embed formatter for Fandom discussions."""
|
||||
embed = DiscordMessage("embed", "discussion", settings["fandom_discussions"]["webhookURL"])
|
||||
embed.set_author(post["createdBy"]["name"], "{wikiurl}f/u/{creatorId}".format(
|
||||
|
|
@ -74,13 +70,15 @@ def embed_formatter(post, post_type):
|
|||
embed.add_field(option["text"] if image_type is True else _("Option {}").format(num+1),
|
||||
option["text"] if image_type is False else _("__[View image]({image_url})__").format(image_url=option["image"]["url"]),
|
||||
inline=True)
|
||||
else:
|
||||
# UNKNOWN EVENT
|
||||
embed["footer"]["text"] = post["forumName"]
|
||||
embed["timestamp"] = datetime.datetime.fromtimestamp(post["creationDate"]["epochSecond"], tz=datetime.timezone.utc).isoformat()
|
||||
embed.finish_embed()
|
||||
send_to_discord(embed)
|
||||
|
||||
|
||||
def compact_formatter(post, post_type):
|
||||
def feeds_compact_formatter(post_type, post, message_target, wiki, _):
|
||||
"""Compact formatter for Fandom discussions."""
|
||||
message = None
|
||||
discussion_post_type = post["_embedded"]["thread"][0].get("containerType",
|
||||
|
|
@ -117,12 +115,13 @@ def compact_formatter(post, post_type):
|
|||
"[{author}](<{url}f/u/{creatorId}>) replied to [{title}](<{wikiurl}wiki/Message_Wall:{user_wall}?threadId={threadid}#{replyId}>) on {user}'s Message Wall").format(
|
||||
author=post["createdBy"]["name"], url=settings["fandom_discussions"]["wiki_url"], creatorId=post["creatorId"], title=post["_embedded"]["thread"][0]["title"], user=user_wall,
|
||||
wikiurl=settings["fandom_discussions"]["wiki_url"], user_wall=quote_plus(user_wall.replace(" ", "_")), threadid=post["threadId"], replyId=post["id"])
|
||||
|
||||
elif post_type == "POLL":
|
||||
message = _(
|
||||
"[{author}](<{url}f/u/{creatorId}>) created a poll [{title}](<{url}f/p/{threadId}>) in {forumName}").format(
|
||||
author=post["createdBy"]["name"], url=settings["fandom_discussions"]["wiki_url"],
|
||||
creatorId=post["creatorId"], title=post["title"], threadId=post["threadId"], forumName=post["forumName"])
|
||||
else:
|
||||
# UNKNOWN EVENT
|
||||
send_to_discord(DiscordMessage("compact", "discussion", settings["fandom_discussions"]["webhookURL"], content=message))
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -330,7 +330,7 @@ async def compact_formatter(action, change, parsed_comment, categories, recent_c
|
|||
await send_to_discord(DiscordMessage("compact", action, message_target[1], content=content, wiki=WIKI_SCRIPT_PATH))
|
||||
|
||||
|
||||
async def embed_formatter(action, change, parsed_comment, categories, recent_changes, target, _, ngettext, paths, additional_data=None):
|
||||
async def embed_formatter(action, change, parsed_comment, categories, recent_changes, message_target, _, ngettext, paths, additional_data=None):
|
||||
"""Recent Changes embed formatter, part of RcGcDw"""
|
||||
if additional_data is None:
|
||||
additional_data = {"namespaces": {}, "tags": {}}
|
||||
|
|
@ -338,7 +338,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
|
|||
WIKI_SCRIPT_PATH = paths[1]
|
||||
WIKI_ARTICLE_PATH = paths[2]
|
||||
WIKI_JUST_DOMAIN = paths[3]
|
||||
embed = DiscordMessage("embed", action, target[1], wiki=WIKI_SCRIPT_PATH)
|
||||
embed = DiscordMessage("embed", action, message_target[1], wiki=WIKI_SCRIPT_PATH)
|
||||
if parsed_comment is None:
|
||||
parsed_comment = _("No description provided")
|
||||
if action != "suppressed":
|
||||
|
|
@ -369,7 +369,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
|
|||
embed["title"] = "{redirect}{article} ({new}{minor}{bot}{space}{editsize})".format(redirect="⤷ " if "redirect" in change else "", article=change["title"], editsize="+" + str(
|
||||
editsize) if editsize > 0 else editsize, new=_("(N!) ") if action == "new" else "",
|
||||
minor=_("m") if action == "edit" and "minor" in change else "", bot=_('b') if "bot" in change else "", space=" " if "bot" in change or (action == "edit" and "minor" in change) or action == "new" else "")
|
||||
if target[0][1] == 3:
|
||||
if message_target[0][1] == 3:
|
||||
if action == "new":
|
||||
changed_content = await recent_changes.safe_request(
|
||||
"{wiki}?action=compare&format=json&fromtext=&torev={diff}&topst=1&prop=diff".format(
|
||||
|
|
@ -433,7 +433,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
|
|||
wiki=WIKI_SCRIPT_PATH, filename=article_encoded, archiveid=revision["archivename"])
|
||||
embed.add_field(_("Options"), _("([preview]({link}) | [undo]({undolink}))").format(
|
||||
link=image_direct_url, undolink=undolink))
|
||||
if target[0][1] > 1:
|
||||
if message_target[0][1] > 1:
|
||||
embed["image"]["url"] = image_direct_url
|
||||
if action == "upload/overwrite":
|
||||
embed["title"] = _("Uploaded a new version of {name}").format(name=change["title"])
|
||||
|
|
@ -443,7 +443,7 @@ async def embed_formatter(action, change, parsed_comment, categories, recent_cha
|
|||
embed["title"] = _("Uploaded {name}").format(name=change["title"])
|
||||
if additional_info_retrieved:
|
||||
embed.add_field(_("Options"), _("([preview]({link}))").format(link=image_direct_url))
|
||||
if target[0][1] > 1:
|
||||
if message_target[0][1] > 1:
|
||||
embed["image"]["url"] = image_direct_url
|
||||
elif action == "delete/delete":
|
||||
link = create_article_path(change["title"], WIKI_ARTICLE_PATH)
|
||||
|
|
|
|||
|
|
@ -8,15 +8,18 @@ class UpdateDB:
|
|||
def __init__(self):
|
||||
self.updated = []
|
||||
|
||||
def add(self, wiki, rc_id):
|
||||
self.updated.append((wiki, rc_id))
|
||||
def add(self, wiki, rc_id, feeds: None):
|
||||
self.updated.append((wiki, rc_id, feeds))
|
||||
|
||||
def clear_list(self):
|
||||
self.updated.clear()
|
||||
|
||||
def update_db(self):
|
||||
for update in self.updated:
|
||||
db_cursor.execute("UPDATE rcgcdw SET rcid = ? WHERE wiki = ?", (update[1], update[0],))
|
||||
id = "rcid"
|
||||
if update[2] is not None:
|
||||
id = "postid"
|
||||
db_cursor.execute("UPDATE rcgcdw SET {} = ? WHERE wiki = ?".format(id),(update[1],update[0],))
|
||||
db_connection.commit()
|
||||
self.clear_list()
|
||||
|
||||
|
|
|
|||
23
src/wiki.py
23
src/wiki.py
|
|
@ -4,6 +4,7 @@ import logging, aiohttp
|
|||
from src.exceptions import *
|
||||
from src.database import db_cursor, db_connection
|
||||
from src.formatters.rc import embed_formatter, compact_formatter
|
||||
from src.formatters.discussions import feeds_embed_formatter, feeds_compact_formatter
|
||||
from src.misc import parse_link
|
||||
from src.i18n import langs
|
||||
import src.discord
|
||||
|
|
@ -49,6 +50,17 @@ class Wiki:
|
|||
raise WikiServerError
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
async def fetch_feeds(wiki_id, session: aiohttp.ClientSession) -> aiohttp.ClientResponse:
|
||||
url_path = "https://services.fandom.com/discussion/{wikiid}/posts".format(wikiid=wiki_id)
|
||||
params = {"sortDirection": "descending", "sortKey": "creation_date", "limit": 20}
|
||||
try:
|
||||
response = await session.get(url_path, params=params)
|
||||
except (aiohttp.ClientConnectionError, aiohttp.ServerTimeoutError, asyncio.TimeoutError):
|
||||
logger.exception("A connection error occurred while requesting {}".format(url_path))
|
||||
raise WikiServerError
|
||||
return response
|
||||
|
||||
@staticmethod
|
||||
async def safe_request(url, *keys):
|
||||
try:
|
||||
|
|
@ -212,3 +224,14 @@ async def essential_info(change: dict, changed_categories, local_wiki: Wiki, db_
|
|||
except KeyError:
|
||||
additional_data["tags"][tag["name"]] = None # Tags with no displ
|
||||
await appearance_mode(identification_string, change, parsed_comment, changed_categories, local_wiki, target, _, ngettext, paths, additional_data=additional_data)
|
||||
|
||||
|
||||
async def essential_feeds(change: dict, db_wiki: tuple, target: tuple):
|
||||
"""Prepares essential information for both embed and compact message format."""
|
||||
def _(string: str) -> str:
|
||||
"""Our own translation string to make it compatible with async"""
|
||||
return lang.gettext(string)
|
||||
|
||||
lang = langs[target[0][0]]
|
||||
appearance_mode = feeds_embed_formatter if target[0][1] > 0 else feeds_compact_formatter
|
||||
await appearance_mode(change.get("funnel", "TEXT"), change, target, db_wiki["wiki"], _)
|
||||
|
|
|
|||
Loading…
Reference in a new issue