mirror of
https://gitlab.com/chicken-riders/RcGcDb.git
synced 2025-02-23 00:54:09 +00:00
I have no productivity today so I'm just commiting some more work I'll have to rework anyways
This commit is contained in:
parent
49c36c9688
commit
6b8a2e217b
|
@ -62,13 +62,19 @@ class RcQueue:
|
|||
def __init__(self):
|
||||
self.domain_list = {}
|
||||
self.to_remove = []
|
||||
self.irc_mapping = {}
|
||||
|
||||
async def start_group(self, group, initial_wikis):
|
||||
"""Starts a task for given domain group"""
|
||||
if group not in self.domain_list:
|
||||
if group in self.irc_mapping: # Hopefully there are no race conditions....
|
||||
irc_connection = self.irc_mapping[group]
|
||||
else:
|
||||
for irc_server in settings["irc_servers"].keys():
|
||||
if group in settings["irc_servers"]["irc_server"]["domains"]:
|
||||
if group in settings["irc_servers"][irc_server]["domains"]:
|
||||
irc_connection = AioIRCCat(settings["irc_servers"]["group"]["irc_channel_mapping"], all_wikis)
|
||||
for domain in settings["irc_servers"][irc_server]["domains"]:
|
||||
self.irc_mapping[domain] = irc_connection
|
||||
irc_connection.connect(settings["irc_servers"][irc_server]["irc_host"], settings["irc_servers"][irc_server]["irc_port"], settings["irc_servers"][irc_server]["irc_name"])
|
||||
break
|
||||
else:
|
||||
|
@ -285,6 +291,7 @@ async def scan_group(group: str):
|
|||
targets = generate_targets(queued_wiki.url, "AND (rcid != -1 OR rcid IS NULL)")
|
||||
paths = get_paths(queued_wiki.url, recent_changes_resp)
|
||||
new_events = 0
|
||||
local_wiki.last_check = time.time() # on successful check, save new last check time
|
||||
for change in recent_changes:
|
||||
if change["rcid"] > local_wiki.rc_active and queued_wiki.amount != 450:
|
||||
new_events += 1
|
||||
|
|
|
@ -1,4 +1,5 @@
|
|||
import irc.client_aio
|
||||
import json
|
||||
from urllib.parse import urlparse, quote
|
||||
|
||||
class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
|
||||
|
@ -6,6 +7,7 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
|
|||
irc.client.SimpleIRCClient.__init__(self)
|
||||
self.targets = targets
|
||||
self.updated = [] # Storage for edited wikis
|
||||
self.updated_discussions = []
|
||||
self.wikis = all_wikis
|
||||
|
||||
def on_welcome(self, connection, event): # Join IRC channels
|
||||
|
@ -21,7 +23,7 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
|
|||
def on_nicknameinuse(self, c, e):
|
||||
c.nick(c.get_nickname() + "_")
|
||||
|
||||
async def parse_fandom_message(self, message):
|
||||
def parse_fandom_message(self, message):
|
||||
message = message.split("\x035*\x03")
|
||||
# print(asyncio.all_tasks())
|
||||
half = message[0].find("\x0302http")
|
||||
|
@ -34,6 +36,14 @@ class AioIRCCat(irc.client_aio.AioSimpleIRCClient):
|
|||
if full_url in self.wikis:
|
||||
self.updated.append(full_url)
|
||||
|
||||
def parse_discussions_message(self, message):
|
||||
post = json.loads(message)
|
||||
if post.get('action', 'unknown') != "deleted": # ignore deletion events
|
||||
url = urlparse(post.get('url'))
|
||||
full_url = url.netloc + recognize_langs(url.path)
|
||||
self.updated_discussions.append(full_url)
|
||||
|
||||
|
||||
def recognize_langs(path):
|
||||
lang = ""
|
||||
new_path = path.split("/")
|
||||
|
|
|
@ -25,6 +25,7 @@ class Wiki:
|
|||
session: aiohttp.ClientSession = None
|
||||
rc_active: int = 0
|
||||
last_check: float = 0.0
|
||||
last_discussion_check: float = 0.0
|
||||
|
||||
@staticmethod
|
||||
async def fetch_wiki(extended, script_path, session: aiohttp.ClientSession, ratelimiter: RateLimiter, amount=20) -> aiohttp.ClientResponse:
|
||||
|
|
Loading…
Reference in a new issue