This commit is contained in:
Frisk 2018-06-22 01:35:24 +02:00
parent 1801d02b2a
commit a47a093392
No known key found for this signature in database
GPG key ID: 0E9A7D3C0A01586C
10 changed files with 461 additions and 42 deletions

View file

@ -6,6 +6,7 @@ Recent changes Gamepedia compatible Discord webhook is a project made from earli
* requests 2.18.4>
* beautifulsoup 4.6.0>
* lxml 4.2.1>
* schedule 0.5.0>
### Features ###
* Fetch recent changes from Gamepedia wiki and send them to Discord channel using a webhook

Binary file not shown.

View file

@ -5,9 +5,9 @@
msgid ""
msgstr ""
"Project-Id-Version: \n"
"POT-Creation-Date: 2018-06-18 16:13+0200\n"
"PO-Revision-Date: 2018-06-18 16:27+0200\n"
"Last-Translator: MarkusRost\n"
"POT-Creation-Date: 2018-06-21 23:24+0200\n"
"PO-Revision-Date: 2018-06-21 23:39+0200\n"
"Last-Translator: Frisk <piotrex43@protonmail.ch>\n"
"Language-Team: \n"
"Language: de\n"
"MIME-Version: 1.0\n"
@ -224,11 +224,11 @@ msgstr "Ereignis kann nicht verabreitet werden"
msgid "error"
msgstr "Fehler"
msgid "Connection status"
msgstr "Verbindungsstatus"
msgid "Daily overview"
msgstr ""
msgid "{wiki} seems to be down or unreachable."
msgstr "Das {wiki} scheint unerreichbar zu sein."
msgid "But nobody came"
msgstr ""
msgid "Admin actions"
msgstr "Adminaktionen"
@ -242,7 +242,14 @@ msgstr "Tageswert"
msgid "Edits made"
msgstr "Bearbeitungen"
msgid "Most active user"
#, fuzzy
#| msgid "Most active user"
msgid "Most active hours"
msgstr "Aktivster Benutzer"
#, fuzzy
#| msgid "Most active user"
msgid "Most active users"
msgstr "Aktivster Benutzer"
#, fuzzy
@ -256,5 +263,11 @@ msgstr "Neue Dateien"
msgid "Unique contributors"
msgstr "Einzelne Autoren"
msgid "Connection status"
msgstr "Verbindungsstatus"
msgid "{wiki} seems to be down or unreachable."
msgstr "Das {wiki} scheint unerreichbar zu sein."
msgid "{wiki} is back up!"
msgstr "Das {wiki} ist wieder erreichbar."

Binary file not shown.

View file

@ -0,0 +1,269 @@
# UN TITRE MERVEILLEUSEMENT DESCRIPTIF
# COPYRIGHT HAHAHAHA I WISH J'AI PAS DE COPYRIGHT
# AUTEUR #1 C'EST MOI MWAHAHAHAHAHA
#
msgid ""
msgstr ""
"Project-Id-Version: \n"
"POT-Creation-Date: 2018-06-21 23:24+0200\n"
"PO-Revision-Date: 2018-06-21 23:38+0200\n"
"Last-Translator: Frisk <piotrex43@protonmail.ch>\n"
"Language-Team: \n"
"Language: pl\n"
"MIME-Version: 1.0\n"
"Content-Type: text/plain; charset=UTF-8\n"
"Content-Transfer-Encoding: 8bit\n"
"Generated-By: pygettext.py 1.5\n"
"X-Generator: Poedit 2.0.8\n"
"X-Poedit-Basepath: ../../..\n"
"X-Poedit-SearchPath-0: rcgcdw.pot\n"
msgid "(N!) "
msgstr "(N!) "
msgid "m "
msgstr "m "
msgid "Uploaded a new version of {name}"
msgstr "Téléversement d'une nouvelle version de {name}"
msgid "([preview]({link}) | [undo]({undolink}))"
msgstr "([Aperçu]({link}) | [Annuler]({undolink}))"
msgid "Options"
msgstr "Options"
msgid "Uploaded {name}"
msgstr "Téléversement de {name}"
msgid "**No license!**"
msgstr "**Aucune license!**"
msgid "([preview]({link}))"
msgstr "([Aperçu]({link}))"
msgid ""
"{desc}\n"
"License: {license}"
msgstr ""
"{desc}\n"
"License: {license}"
msgid "Deleted page {article}"
msgstr "Suppresion de la page {article}"
msgid "Deleted redirect {article} by overwriting"
msgstr "Suppresion par écrasement de la redirection {article}"
msgid "A redirect has been made"
msgstr "Une redirection a été créée"
msgid "No redirect has been made"
msgstr "Aucune redirection créée"
msgid "Moved {article} to {target}"
msgstr "Déplacement de {article} vers {target}"
msgid "Moved {article} to {title} over redirect"
msgstr "Déplacement de {article} vers {title} par redirection"
msgid "Moved protection settings from {article} to {title}"
msgstr "Transfert des paramètres de protection de {article} vers {title}"
msgid "infinity and beyond"
msgstr "toujours"
msgid "Blocked {blocked_user} for {time}"
msgstr "{blocked user} a été bloqué pour {time}"
msgid "Changed block settings for {blocked_user}"
msgstr "Modification des paramètres de blocage pour {blocked_user}"
msgid "Unblocked {blocked_user}"
msgstr "{blocked_user} a été débloqué"
msgid "Left a comment on {target}'s profile"
msgstr "Ajout d'un commentaire sur le profil de {target}"
msgid "Replied to a comment on {target}'s profile"
msgstr "Réponse à un commentaire sur le profil de {target}"
msgid "Edited a comment on {target}'s profile"
msgstr "Édition d'un commentaire sur le profil de {target}"
msgid "Location"
msgstr "Emplacement"
msgid "About me"
msgstr "À propos de moi"
msgid "Google link"
msgstr "Lien Google"
msgid "Facebook link"
msgstr "Lien Facebook"
msgid "Twitter link"
msgstr "Lien Twitter"
msgid "Reddit link"
msgstr "Lien Reddit"
msgid "Twitch link"
msgstr "Lien Twitch"
msgid "PSN link"
msgstr "Lien PSN"
msgid "VK link"
msgstr "Lien VK"
msgid "XVL link"
msgstr "Lien XVL"
msgid "Steam link"
msgstr "Lien Steam"
msgid "Unknown"
msgstr "Inconnu"
msgid "Edited {target}'s profile"
msgstr "Modification du profil de {target}"
msgid "{field} field changed to: {desc}"
msgstr "{field} modifié pour: {desc}"
msgid "Deleted a comment on {target}'s profile"
msgstr "Retrait d'un commentaire sur le profil de {target}"
msgid "Changed group membership for {target}"
msgstr "Modification des groupes pour {target}"
msgid "none"
msgstr "aucun"
msgid "No description provided"
msgstr "Aucune description"
msgid "Groups changed from {old_groups} to {new_groups} {reason}"
msgstr "Groupe modifié de {old_groups} vers {new_groups}: {reason}"
msgid "Protected {target}"
msgstr "Protection de {target}"
msgid "Changed protection level for {article}"
msgstr "Modification du niveau de protection de {article}"
msgid "Removed protection from {article}"
msgstr "Retrait de la protection de {article}"
msgid "Changed visibility of revision(s) on page {article} "
msgstr "Modification de la visibilité de révision(s) sur la page {article} "
msgid "Imported {article} with {count} revision(s)"
msgstr "Article {article} importé avec {count} révision(s)"
msgid "Restored {article}"
msgstr "Restauration de {article}"
msgid "Changed visibility of log events"
msgstr "Modification de la visibilité d'évènements des journaux"
msgid "Imported interwiki"
msgstr "Importation d'interwiki"
msgid "Edited abuse filter number {number}"
msgstr "Édition de la règle {number} du filtre anti-abus"
msgid "Merged revision histories of {article} into {dest}"
msgstr "Fusion de l'historique de {article} vers {dest}"
msgid "Added an entry to the interwiki table"
msgstr "Ajout d'une entrée à la table interwiki"
msgid "Prefix: {prefix}, website: {website} | {desc}"
msgstr "Préfixe: {prefix}, site: {website} | {desc}"
msgid "Edited an entry in interwiki table"
msgstr "Modification d'une entrée de la table interwiki"
msgid "Deleted an entry in interwiki table"
msgstr "Retrait d'une entrée de la table interwiki"
msgid "Prefix: {prefix} | {desc}"
msgstr "Préfixe: {prefix} | {desc}"
msgid "Changed the content model of the page {article}"
msgstr "Modification du modèle de contenu de l'article {article}"
msgid "Model changed from {old} to {new}: {reason}"
msgstr "Modèle changé de {old} à {new}: {reason}"
msgid "Edited the sprite for {article}"
msgstr "Édition du sprite de {article}"
msgid "Created the sprite sheet for {article}"
msgstr "Création d'une feuille de sprite pour {article}"
msgid "Edited the slice for {article}"
msgstr ""
msgid "Created a tag \"{tag}\""
msgstr "Création du tag « {tag} »"
msgid "Deleted a tag \"{tag}\""
msgstr "Suppression du tag « {tag} »"
msgid "Activated a tag \"{tag}\""
msgstr "Activation du tag « {tag} »"
msgid "Deactivated a tag \"{tag}\""
msgstr "Désactivation du tag « {tag} »"
msgid "Unable to process the event"
msgstr "Impossible d'analyser l'évènement"
msgid "error"
msgstr "erreur"
msgid "Daily overview"
msgstr ""
msgid "But nobody came"
msgstr ""
msgid "Admin actions"
msgstr "Actions d'administrateur"
msgid "Bytes changed"
msgstr "Octet modifié"
msgid "Day score"
msgstr "Score du jour"
msgid "Edits made"
msgstr "Modifications effectuées"
msgid "Most active hours"
msgstr "-"
msgid "Most active users"
msgstr "-"
msgid "New articles"
msgstr "Nouveaux articles"
msgid "New files"
msgstr "Nouveaux fichiers"
msgid "Unique contributors"
msgstr "Contributeurs uniques"
msgid "Connection status"
msgstr "Statut de connexion"
msgid "{wiki} seems to be down or unreachable."
msgstr "{wiki} semble être down ou inatteignable."
msgid "{wiki} is back up!"
msgstr "{wiki} est de retour!"

Binary file not shown.

View file

@ -5,8 +5,8 @@
msgid ""
msgstr ""
"Project-Id-Version: RcGcDw\n"
"POT-Creation-Date: 2018-06-18 16:13+0200\n"
"PO-Revision-Date: 2018-06-19 14:46+0200\n"
"POT-Creation-Date: 2018-06-21 23:24+0200\n"
"PO-Revision-Date: 2018-06-21 23:40+0200\n"
"Last-Translator: Frisk <piotrex43@protonmail.ch>\n"
"Language-Team: \n"
"Language: pl\n"
@ -228,11 +228,11 @@ msgstr "Nie udało się odczytać wydarzenia"
msgid "error"
msgstr "błąd"
msgid "Connection status"
msgstr "Problem z połączeniem"
msgid "Daily overview"
msgstr "Podsumowanie dnia"
msgid "{wiki} seems to be down or unreachable."
msgstr "{wiki} nie działa lub jest nieosiągalna."
msgid "But nobody came"
msgstr "Ale nikt nie przyszedł"
msgid "Admin actions"
msgstr "Akcji administratorskich"
@ -246,8 +246,11 @@ msgstr "Wynik dnia"
msgid "Edits made"
msgstr "Zrobionych edycji"
msgid "Most active user"
msgstr "Najbardziej aktywny użytkownik"
msgid "Most active hours"
msgstr "Najbardziej aktywne godziny"
msgid "Most active users"
msgstr "Najbardziej aktywni użytkownicy"
msgid "New articles"
msgstr "Nowych artykułów"
@ -258,6 +261,12 @@ msgstr "Nowych plików"
msgid "Unique contributors"
msgstr "Unikalnych edytujących"
msgid "Connection status"
msgstr "Problem z połączeniem"
msgid "{wiki} seems to be down or unreachable."
msgstr "{wiki} nie działa lub jest nieosiągalna."
msgid "{wiki} is back up!"
msgstr "{wiki} jest ponownie osiągalna!"

View file

@ -5,7 +5,7 @@
msgid ""
msgstr ""
"Project-Id-Version: PACKAGE VERSION\n"
"POT-Creation-Date: 2018-06-18 16:13+0200\n"
"POT-Creation-Date: 2018-06-21 23:24+0200\n"
"PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n"
"Last-Translator: FULL NAME <EMAIL@ADDRESS>\n"
"Language-Team: LANGUAGE <LL@li.org>\n"
@ -221,10 +221,10 @@ msgstr ""
msgid "error"
msgstr ""
msgid "Connection status"
msgid "Daily overview"
msgstr ""
msgid "{wiki} seems to be down or unreachable."
msgid "But nobody came"
msgstr ""
msgid "Admin actions"
@ -239,7 +239,10 @@ msgstr ""
msgid "Edits made"
msgstr ""
msgid "Most active user"
msgid "Most active hours"
msgstr ""
msgid "Most active users"
msgstr ""
msgid "New articles"
@ -251,6 +254,12 @@ msgstr ""
msgid "Unique contributors"
msgstr ""
msgid "Connection status"
msgstr ""
msgid "{wiki} seems to be down or unreachable."
msgstr ""
msgid "{wiki} is back up!"
msgstr ""

158
rcgcdw.py
View file

@ -1,9 +1,12 @@
#!/usr/bin/python
# -*- coding: utf-8 -*-
import time, logging, json, requests, datetime, re, gettext, math, random, os.path
#WARNING! SHITTY CODE AHEAD. ENTER ONLY IF YOU ARE SURE YOU CAN TAKE IT
#You have been warned
import time, logging, json, requests, datetime, re, gettext, math, random, os.path, schedule, sys
from bs4 import BeautifulSoup
from collections import defaultdict
from collections import defaultdict, Counter
from urllib.parse import quote_plus
#logging.warning('Watch out!')
#DEBUG, INFO, WARNING, ERROR, CRITICAL
@ -22,7 +25,6 @@ if settings["lang"] != "en" or settings["lang"] == "":
else:
_ = lambda s: s
def send(message, name, avatar):
try:
req = requests.post(settings["webhookURL"], data={"content": message, "avatar_url": avatar, "username": name}, timeout=10)
@ -105,10 +107,10 @@ def webhook_formatter(action, STATIC, **params):
logging.warning("Something went wrong when getting license for the image")
return 0
content = list(article_content.values())[0]['revisions'][0]['*'].lower()
if "{{license" not in content:
if "{{license" not in content and "{{lizenz" not in content: #de-mcw
license = _("**No license!**")
else:
matches = re.search(r"\{\{license\ (.*?)\}\}", content)
matches = re.search(r"\{\{(license|lizenz)(\ |\|)(.*?)\}\}", content)
if matches is not None:
license = matches.group(1)
else:
@ -268,7 +270,7 @@ def webhook_formatter(action, STATIC, **params):
logging.warning("No entry for {event} with params: {params}".format(event=action, params=params))
embed["author"]["name"] = params["user"]
embed["author"]["url"] = author_url
embed["author"]["icon"] = STATIC["icon"]
embed["author"]["icon_url"] = STATIC["icon"]
embed["url"] = link
if "desc" not in params:
params["desc"] = ""
@ -390,15 +392,122 @@ def first_pass(change): #I've decided to split the embed formatter and change ha
elif change["type"] == "new": #new page
STATIC_VARS = {**STATIC_VARS ,**{"color": settings["appearance"]["new"]["color"], "icon": settings["appearance"]["new"]["icon"]}}
webhook_formatter(37, STATIC_VARS, user=change["user"], title=change["title"], desc=parsedcomment, oldrev=change["old_revid"], pageid=change["pageid"], diff=change["revid"], size=change["newlen"])
def day_overview_request():
logging.info("Fetching daily overview... This may take up to 30 seconds!")
timestamp = (datetime.datetime.utcnow()-datetime.timedelta(hours=24)).isoformat(timespec='milliseconds')
logging.debug("timestamp is {}".format(timestamp))
complete = False
result = []
passes = 0
continuearg = ""
while not complete and passes < 10:
request = recent_changes.safe_request("https://{wiki}.gamepedia.com/api.php?action=query&format=json&list=recentchanges&rcend={timestamp}Z&rcprop=title%7Ctimestamp%7Csizes%7Cloginfo%7Cuser&rcshow=!bot&rclimit=500&rctype=edit%7Cnew%7Clog%7Ccategorize{continuearg}".format(wiki=settings["wiki"], timestamp=timestamp, continuearg=continuearg))
if request:
try:
request = request.json()
rc = request['query']['recentchanges']
continuearg = request["continue"]["rccontinue"] if "continue" in request else None
except ValueError:
logging.warning("ValueError in fetching changes")
self.downtime_controller()
complete = 2
except KeyError:
logging.warning("Wiki returned %s" % (request.json()))
complete = 2
else:
result+= rc
if continuearg:
continuearg = "&rccontinue={}".format(continuearg)
passes+=1
logging.debug("continuing requesting next pages of recent changes with {} passes and continuearg being {}".format(passes, continuearg))
time.sleep(3.0)
else:
complete = 1
else:
complete = 2
if passes == 10:
logging.debug("quit the loop because there been too many passes")
return (result, complete)
class recent_changes(object):
def add_to_dict(dictionary, key):
if key in dictionary:
dictionary[key]+=1
else:
dictionary[key]=1
return dictionary
def day_overview(): #time.strftime('%Y-%m-%dT%H:%M:%S.000Z', time.gmtime(time.time()))
#(datetime.datetime.utcnow()+datetime.timedelta(hours=0)).isoformat(timespec='milliseconds')+'Z'
result = day_overview_request()
if result[1] == 1:
activity = defaultdict(dict)
hours = defaultdict(dict)
edits = 0
files = 0
admin = 0
changed_bytes = 0
new_articles = 0
for item in result[0]:
activity = add_to_dict(activity, item["user"])
hours = add_to_dict(hours, datetime.datetime.strptime(item["timestamp"], "%Y-%m-%dT%H:%M:%SZ").hour)
if item["type"]=="edit":
edits += 1
changed_bytes += item["newlen"]-item["oldlen"]
if item["type"] == "new":
if item["ns"] == 0:
new_articles+=1
changed_bytes += item["newlen"]
if item["type"] == "log":
files = files+1 if item["logtype"] == item["logaction"] == "upload" else files
admin = admin+1 if item["logtype"] in ["delete", "merge", "block", "protect", "import", "rights", "abusefilter", "interwiki", "managetags"] else admin
overall = new_articles+edits*0.1+files*0.3+admin*0.1+changed_bytes*0.01
embed = defaultdict(dict)
embed["title"] = _("Daily overview")
embed["url"] = "https://{wiki}.gamepedia.com/Special:Statistics".format(wiki=settings["wiki"])
embed["color"] = settings["appearance"]["daily_overview"]["color"]
if activity:
v = activity.values()
active_users = []
for user, numberu in Counter(activity).most_common(list(v).count(max(v))): #find most active users
active_users.append(user)
the_one = random.choice(active_users)
embed["author"]["icon_url"] = settings["appearance"]["daily_overview"]["icon"]
embed["author"]["name"] = the_one
if re.match(r"\b\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}\b", the_one) is not None:
author_url = "https://{wiki}.gamepedia.com/Special:Contributions/{user}".format(wiki=settings["wiki"], user=the_one)
else:
author_url = "https://{wiki}.gamepedia.com/User:{user}".format(wiki=settings["wiki"], user=the_one)
embed["author"]["url"] = author_url
v = hours.values()
active_hours = []
for hour, numberh in Counter(hours).most_common(list(v).count(max(v))): #find most active users
active_hours.append(str(hour))
else:
active_users = [_("But nobody came")] #a reference to my favorite game of all the time, sorry ^_^
active_hours = [_("But nobody came")]
numberu, numberh = (0, 0)
embed["fields"] = []
fields = ((_("Most active users"), ', '.join(active_users) + "({})".format(numberu)), (_("Edits made"), edits), (_("New files"), files), (_("Admin actions"), admin), (_("Bytes changed"), changed_bytes), (_("New articles"), new_articles), (_("Unique contributors"), str(len(activity))), (_("Most active hours"), ', '.join(active_hours) + "({})".format(numberh)), (_("Day score"), str(overall)))
for name, value in fields:
embed["fields"].append({"name": name, "value": value})
data = {}
data["embeds"] = [dict(embed)]
formatted_embed = json.dumps(data, indent=4)
headers = {'Content-Type': 'application/json'}
logging.debug(formatted_embed)
result = requests.post(settings["webhookURL"], data=formatted_embed, headers=headers)
else:
logging.debug("function requesting changes for day overview returned with error code")
class recent_changes_class(object):
starttime = time.time()
day = datetime.date.fromtimestamp(time.time()).day
ids = []
map_ips = {}
recent_id = 0
downtimecredibility = 0
last_downtime = 0
clock = 0
if settings["limitrefetch"] != -1:
with open("lastchange.txt", "r") as record:
file_id = int(record.read().strip())
@ -432,7 +541,7 @@ class recent_changes(object):
self.downtime_controller()
return None
except KeyError:
logging.warning("Wiki returned %s" % (request.json()))
logging.warning("Wiki returned %s" % (changes.json()))
return None
else:
if self.downtimecredibility > 0:
@ -489,19 +598,24 @@ class recent_changes(object):
if(time.time() - self.last_downtime)>1800 and self.check_connection(): #check if last downtime happened within 30 minutes, if yes, don't send a message
send(_("{wiki} seems to be down or unreachable.").format(wiki=settings["wikiname"]), _("Connection status"), settings["avatars"]["connection_failed"])
self.last_downtime = time.time()
recent_changes = recent_changes()
recent_changes.fetch(amount=settings["limitrefetch" ] if settings["limitrefetch"] != -1 else settings["limit"])
if 1 == 2:
#some translations for later use in different places
print ([_("{wiki} is back up!"), _("Most active user"), _("Edits made"), _("New files"), _("Admin actions"), _("Unique contributors"), _("Bytes changed"), _("Day score"), _("New articles")])
recent_changes = recent_changes_class()
recent_changes.fetch(amount=settings["limitrefetch" ] if settings["limitrefetch"] != -1 else settings["limit"])
schedule.every(settings["cooldown"]).seconds.do(recent_changes.fetch)
if 1==2: #dummy for future translations
print (_("{wiki} is back up!"))
if settings["overview"]:
ovUTC_time = (time.strptime(settings["overview_UTC_time"], '%H:%M').tm_hour, time.strptime(settings["overview_UTC_time"], '%H:%M').tm_min) #i don't know what I did there
diff = (datetime.datetime.now().hour - datetime.datetime.utcnow().hour, datetime.datetime.now().minute - datetime.datetime.utcnow().minute)
tim = (ovUTC_time[0]+diff[0], ovUTC_time[1]+diff[1])
schedule.every().day.at("{}:{}".format(int(math.fabs(tim[0])), int(math.fabs(tim[1])))).do(day_overview)
while 1:
time.sleep(float(settings["cooldown"]))
recent_changes.fetch()
if (recent_changes.day != datetime.date.fromtimestamp(time.time()).day):
logging.info("A brand new day! Printing the summary and clearing the cache")
#recent_changes.summary()
#recent_changes.clear_cache()
recent_changes.day = datetime.date.fromtimestamp(time.time()).day
time.sleep(1.0)
schedule.run_pending()
#if (recent_changes.day != ):
#logging.info("A brand new day! Printing the summary and clearing the cache")
##recent_changes.clear_cache()
#recent_changes.day = datetime.datetime.now().day

View file

@ -17,6 +17,10 @@
"verbose_level": 0,
"show_updown_messages": true,
"appearance":{
"daily_overview": {
"color": 16312092,
"icon":""
},
"new":{
"icon":"https://framapic.org/VBVcOznftNsV/4a0fbBL7wkUo.png",
"color":"THIS COLOR DEPENDS ON EDIT SIZE, PLEASE DON'T CHANGE"