I have no productivity today so I'm just commiting some more work I'll have to rework anyways

This commit is contained in:
Frisk 2021-01-27 18:48:46 +01:00
parent 49c36c9688
commit 6b8a2e217b
No known key found for this signature in database
GPG key ID: 213F7C15068AF8AC
3 changed files with 25 additions and 7 deletions

View file

@ -62,13 +62,19 @@ class RcQueue:
def __init__(self): def __init__(self):
self.domain_list = {} self.domain_list = {}
self.to_remove = [] self.to_remove = []
self.irc_mapping = {}
async def start_group(self, group, initial_wikis): async def start_group(self, group, initial_wikis):
"""Starts a task for given domain group""" """Starts a task for given domain group"""
if group not in self.domain_list: if group not in self.domain_list:
if group in self.irc_mapping: # Hopefully there are no race conditions....
irc_connection = self.irc_mapping[group]
else:
for irc_server in settings["irc_servers"].keys(): for irc_server in settings["irc_servers"].keys():
if group in settings["irc_servers"]["irc_server"]["domains"]: if group in settings["irc_servers"][irc_server]["domains"]:
irc_connection = AioIRCCat(settings["irc_servers"]["group"]["irc_channel_mapping"], all_wikis) irc_connection = AioIRCCat(settings["irc_servers"]["group"]["irc_channel_mapping"], all_wikis)
for domain in settings["irc_servers"][irc_server]["domains"]:
self.irc_mapping[domain] = irc_connection
irc_connection.connect(settings["irc_servers"][irc_server]["irc_host"], settings["irc_servers"][irc_server]["irc_port"], settings["irc_servers"][irc_server]["irc_name"]) irc_connection.connect(settings["irc_servers"][irc_server]["irc_host"], settings["irc_servers"][irc_server]["irc_port"], settings["irc_servers"][irc_server]["irc_name"])
break break
else: else:
@ -285,6 +291,7 @@ async def scan_group(group: str):
targets = generate_targets(queued_wiki.url, "AND (rcid != -1 OR rcid IS NULL)") targets = generate_targets(queued_wiki.url, "AND (rcid != -1 OR rcid IS NULL)")
paths = get_paths(queued_wiki.url, recent_changes_resp) paths = get_paths(queued_wiki.url, recent_changes_resp)
new_events = 0 new_events = 0
local_wiki.last_check = time.time() # on successful check, save new last check time
for change in recent_changes: for change in recent_changes:
if change["rcid"] > local_wiki.rc_active and queued_wiki.amount != 450: if change["rcid"] > local_wiki.rc_active and queued_wiki.amount != 450:
new_events += 1 new_events += 1

View file

@ -1,4 +1,5 @@
import irc.client_aio import irc.client_aio
import json
from urllib.parse import urlparse, quote from urllib.parse import urlparse, quote
class AioIRCCat(irc.client_aio.AioSimpleIRCClient): class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
@ -6,6 +7,7 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
irc.client.SimpleIRCClient.__init__(self) irc.client.SimpleIRCClient.__init__(self)
self.targets = targets self.targets = targets
self.updated = [] # Storage for edited wikis self.updated = [] # Storage for edited wikis
self.updated_discussions = []
self.wikis = all_wikis self.wikis = all_wikis
def on_welcome(self, connection, event): # Join IRC channels def on_welcome(self, connection, event): # Join IRC channels
@ -21,7 +23,7 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
def on_nicknameinuse(self, c, e): def on_nicknameinuse(self, c, e):
c.nick(c.get_nickname() + "_") c.nick(c.get_nickname() + "_")
async def parse_fandom_message(self, message): def parse_fandom_message(self, message):
message = message.split("\x035*\x03") message = message.split("\x035*\x03")
# print(asyncio.all_tasks()) # print(asyncio.all_tasks())
half = message[0].find("\x0302http") half = message[0].find("\x0302http")
@ -34,6 +36,14 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
if full_url in self.wikis: if full_url in self.wikis:
self.updated.append(full_url) self.updated.append(full_url)
def parse_discussions_message(self, message):
post = json.loads(message)
if post.get('action', 'unknown') != "deleted": # ignore deletion events
url = urlparse(post.get('url'))
full_url = url.netloc + recognize_langs(url.path)
self.updated_discussions.append(full_url)
def recognize_langs(path): def recognize_langs(path):
lang = "" lang = ""
new_path = path.split("/") new_path = path.split("/")

View file

@ -25,6 +25,7 @@ class Wiki:
session: aiohttp.ClientSession = None session: aiohttp.ClientSession = None
rc_active: int = 0 rc_active: int = 0
last_check: float = 0.0 last_check: float = 0.0
last_discussion_check: float = 0.0
@staticmethod @staticmethod
async def fetch_wiki(extended, script_path, session: aiohttp.ClientSession, ratelimiter: RateLimiter, amount=20) -> aiohttp.ClientResponse: async def fetch_wiki(extended, script_path, session: aiohttp.ClientSession, ratelimiter: RateLimiter, amount=20) -> aiohttp.ClientResponse: