Don't add the same wiki to queue multiple times

This commit is contained in:
MarkusRost 2024-07-22 22:30:07 +00:00
parent cfa01a2613
commit 60d635ccfd

View file

@ -153,7 +153,7 @@ class RcQueue:
full = set()
async with db.pool().acquire() as connection:
async with connection.transaction():
async for db_wiki in connection.cursor('SELECT DISTINCT wiki, row_number() OVER (ORDER BY webhook) AS rowid, webhook, lang, display, rcid FROM rcgcdw WHERE rcid != -1 OR rcid IS NULL ORDER BY webhook'):
async for db_wiki in connection.cursor('SELECT DISTINCT wiki, row_number() OVER (ORDER BY wiki) AS rowid, rcid FROM rcgcdw WHERE rcid != -1 OR rcid IS NULL GROUP BY wiki, rcid ORDER BY rowid'):
domain = get_domain(db_wiki["wiki"])
try:
if db_wiki["wiki"] not in all_wikis:
@ -253,7 +253,7 @@ async def generate_domain_groups():
domain_wikis = defaultdict(list)
async with db.pool().acquire() as connection:
async with connection.transaction():
async for db_wiki in connection.cursor('SELECT DISTINCT wiki, webhook, lang, display, rcid FROM rcgcdw WHERE rcid != -1 OR rcid IS NULL'):
async for db_wiki in connection.cursor('SELECT DISTINCT wiki FROM rcgcdw WHERE rcid != -1 OR rcid IS NULL'):
domain_wikis[get_domain(db_wiki["wiki"])].append(QueuedWiki(db_wiki["wiki"], 20))
for group, db_wikis in domain_wikis.items():
yield group, db_wikis