Cancellable tasks
All checks were successful
Gitea/kapitanbooru-uploader/pipeline/head This commit looks good

This commit is contained in:
Michał Leśniak 2025-02-28 18:57:55 +01:00
parent 5571e18102
commit 9f187afc22
5 changed files with 283 additions and 114 deletions

View File

@ -6,7 +6,7 @@ import queue
import threading import threading
import tkinter as tk import tkinter as tk
from tkinter import filedialog, messagebox, ttk from tkinter import filedialog, messagebox, ttk
from typing import Tuple from typing import Dict, Tuple, Optional
import networkx as nx import networkx as nx
import requests import requests
@ -41,6 +41,7 @@ class ProcessingDialog:
# Setup communication queue and periodic checker # Setup communication queue and periodic checker
self.queue = queue.Queue() self.queue = queue.Queue()
self.running = True self.running = True
self.cancel_event = threading.Event() # Cancellation flag
self.thread = threading.Thread( self.thread = threading.Thread(
target=self.run_task, args=(target_function, *args) target=self.run_task, args=(target_function, *args)
) )
@ -75,10 +76,12 @@ class ProcessingDialog:
"""Execute target function with progress queue if supported""" """Execute target function with progress queue if supported"""
try: try:
sig = inspect.signature(target_function) sig = inspect.signature(target_function)
kwargs = {}
if "progress_queue" in sig.parameters: if "progress_queue" in sig.parameters:
target_function(*args, progress_queue=self.queue) kwargs["progress_queue"] = self.queue
else: if "cancel_event" in sig.parameters:
target_function(*args) kwargs["cancel_event"] = self.cancel_event
target_function(*args, **kwargs)
finally: finally:
self.close_dialog() self.close_dialog()
@ -88,10 +91,12 @@ class ProcessingDialog:
self.running = False self.running = False
self.progress.stop() self.progress.stop()
self.top.after(0, self.top.destroy) self.top.after(0, self.top.destroy)
self.thread.join()
def on_close(self): def on_close(self):
"""Handle manual window closure""" """Handle manual window closure"""
self.running = False self.running = False
self.cancel_event.set() # Notify target function that cancellation is requested
self.top.destroy() self.top.destroy()
@ -121,6 +126,14 @@ class ImageBrowser(tk.Tk):
self.image_files_md5 = [] self.image_files_md5 = []
self.current_index = None self.current_index = None
self.image_cache = None self.image_cache = None
self.tagger_thread_idx = 0
self.tagger = wdt.Tagger()
self.check_uploaded_files_stop_event = threading.Event()
self.check_uploaded_files_thread: Optional[threading.Thread] = None
self.process_tagger_queue_stop_event = threading.Event()
self.process_tagger_queue_thread: Optional[threading.Thread] = None
self.run_tagger_threads: Dict[str, threading.Thread] = {}
# Liczniki statusu # Liczniki statusu
self.total_files = 0 self.total_files = 0
@ -273,9 +286,8 @@ class ImageBrowser(tk.Tk):
self.tagger_processed.add(md5) self.tagger_processed.add(md5)
return cached["result"] return cached["result"]
try: try:
tagger = wdt.Tagger()
with Image.open(file_path) as img: with Image.open(file_path) as img:
result = tagger.tag(img) result = self.tagger.tag(img)
self.tagger_cache[md5] = result self.tagger_cache[md5] = result
self.tagger_processed.add(md5) self.tagger_processed.add(md5)
self.after(0, self.update_status_bar) self.after(0, self.update_status_bar)
@ -407,18 +419,6 @@ class ImageBrowser(tk.Tk):
btn_save.pack(pady=10) btn_save.pack(pady=10)
def create_widgets(self): def create_widgets(self):
# Górna ramka
top_frame = tk.Frame(self)
top_frame.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5)
btn_select_folder = tk.Button(
top_frame, text="Wybierz folder", command=self.select_folder
)
btn_select_folder.pack(side=tk.LEFT, padx=5)
btn_prev = tk.Button(top_frame, text="<<", command=self.show_prev)
btn_prev.pack(side=tk.LEFT, padx=5)
btn_next = tk.Button(top_frame, text=">>", command=self.show_next)
btn_next.pack(side=tk.LEFT, padx=5)
# Główna ramka trzy kolumny # Główna ramka trzy kolumny
main_frame = tk.Frame(self) main_frame = tk.Frame(self)
main_frame.pack(fill=tk.BOTH, expand=True, padx=5, pady=5) main_frame.pack(fill=tk.BOTH, expand=True, padx=5, pady=5)
@ -486,15 +486,9 @@ class ImageBrowser(tk.Tk):
manual_frame = tk.LabelFrame(right_frame, text="Manual Tags") manual_frame = tk.LabelFrame(right_frame, text="Manual Tags")
manual_frame.grid(row=2, column=0, sticky=tk.NSEW, padx=5, pady=5) manual_frame.grid(row=2, column=0, sticky=tk.NSEW, padx=5, pady=5)
self.manual_tags_manager = TagManager( self.manual_tags_manager = TagManager(
manual_frame, self.settings, self.tags_repo manual_frame, self.settings, self.tags_repo, self.update_final_tags
) )
self.manual_tags_manager.pack(fill=tk.BOTH, expand=True) self.manual_tags_manager.pack(fill=tk.BOTH, expand=True)
self.manual_tags_manager.bind(
"<KeyRelease>", lambda e: self.update_final_tags(), add="+"
)
self.manual_tags_manager.bind(
"<Button-1>", lambda e: self.update_final_tags(), add="+"
)
# Final Tags widget Text z scrollbar, który rozszerza się # Final Tags widget Text z scrollbar, który rozszerza się
final_frame = tk.LabelFrame(right_frame, text="Final Tags") final_frame = tk.LabelFrame(right_frame, text="Final Tags")
@ -526,6 +520,10 @@ class ImageBrowser(tk.Tk):
) )
self.view_post_button.pack(side=tk.LEFT, padx=5) self.view_post_button.pack(side=tk.LEFT, padx=5)
self.view_post_button.config(state=tk.DISABLED) self.view_post_button.config(state=tk.DISABLED)
btn_prev = tk.Button(upload_frame, text="<<", command=self.show_prev)
btn_prev.pack(side=tk.LEFT, padx=5)
btn_next = tk.Button(upload_frame, text=">>", command=self.show_next)
btn_next.pack(side=tk.LEFT, padx=5)
# Na końcu okna głównego dodaj status bar: # Na końcu okna głównego dodaj status bar:
self.status_label = tk.Label( self.status_label = tk.Label(
@ -601,8 +599,30 @@ class ImageBrowser(tk.Tk):
""" """
Po załadowaniu plików, sprawdza czy jakieś pliki do uploadu oraz przetwarza Taggerem pliki. Po załadowaniu plików, sprawdza czy jakieś pliki do uploadu oraz przetwarza Taggerem pliki.
""" """
threading.Thread(target=self.check_uploaded_files, daemon=True).start() self.join_check_uploaded_files_thread()
threading.Thread(target=self.process_tagger_queue, daemon=True).start() self.check_uploaded_files_thread = threading.Thread(
target=self.check_uploaded_files
)
self.check_uploaded_files_thread.start()
self.join_process_tagger_queue_thread()
self.process_tagger_queue_thread = threading.Thread(
target=self.process_tagger_queue
)
self.process_tagger_queue_thread.start()
def join_check_uploaded_files_thread(self):
if self.check_uploaded_files_thread is not None:
self.check_uploaded_files_stop_event.set()
self.check_uploaded_files_thread.join()
self.check_uploaded_files_thread = None
self.check_uploaded_files_stop_event = threading.Event()
def join_process_tagger_queue_thread(self):
if self.process_tagger_queue_thread is not None:
self.process_tagger_queue_stop_event.set()
self.process_tagger_queue_thread.join()
self.process_tagger_queue_thread = None
self.process_tagger_queue_stop_event = threading.Event()
def check_uploaded_files(self): def check_uploaded_files(self):
""" """
@ -617,6 +637,8 @@ class ImageBrowser(tk.Tk):
batch_size = 100 batch_size = 100
for i in range(0, len(file_md5_list), batch_size): for i in range(0, len(file_md5_list), batch_size):
if self.check_uploaded_files_stop_event.is_set():
break
batch = file_md5_list[i : i + batch_size] batch = file_md5_list[i : i + batch_size]
batch_md5 = [item[2] for item in batch] batch_md5 = [item[2] for item in batch]
md5_param = ",".join(batch_md5) md5_param = ",".join(batch_md5)
@ -627,11 +649,15 @@ class ImageBrowser(tk.Tk):
root = response.json() root = response.json()
found = {} found = {}
for elem in root: for elem in root:
if self.check_uploaded_files_stop_event.is_set():
break
post_md5 = elem.get("md5", "").lower() post_md5 = elem.get("md5", "").lower()
post_id = elem.get("id") post_id = elem.get("id")
if post_md5 and post_id: if post_md5 and post_id:
found[post_md5] = post_id found[post_md5] = post_id
for idx, file_path, md5 in batch: for idx, file_path, md5 in batch:
if self.check_uploaded_files_stop_event.is_set():
break
self.upload_verified += 1 # Każdy plik w batchu jest zweryfikowany self.upload_verified += 1 # Każdy plik w batchu jest zweryfikowany
if md5.lower() in found: if md5.lower() in found:
self.uploaded[file_path] = found[md5.lower()] self.uploaded[file_path] = found[md5.lower()]
@ -650,6 +676,7 @@ class ImageBrowser(tk.Tk):
self.after(0, self.update_status_bar) self.after(0, self.update_status_bar)
except Exception as e: except Exception as e:
print("Błąd podczas sprawdzania paczki uploadu:", e) print("Błąd podczas sprawdzania paczki uploadu:", e)
self.after(100, self.join_check_uploaded_files_thread)
def update_button_states(self): def update_button_states(self):
""" """
@ -774,7 +801,11 @@ class ImageBrowser(tk.Tk):
# Uaktualnij widget PNG Tags # Uaktualnij widget PNG Tags
self.update_png_tags_widget(parsed_parameters.split()) self.update_png_tags_widget(parsed_parameters.split())
# Uruchom Taggera w osobnym wątku # Uruchom Taggera w osobnym wątku
threading.Thread(target=self.run_tagger, daemon=True).start() thread_name = f"tagger{self.tagger_thread_idx}"
thread = threading.Thread(target=self.run_tagger, args=(thread_name,))
self.run_tagger_threads[thread_name] = thread
self.tagger_thread_idx += 1
thread.start()
except Exception as e: except Exception as e:
messagebox.showerror("Błąd", f"Nie można załadować obrazka:\n{e}") messagebox.showerror("Błąd", f"Nie można załadować obrazka:\n{e}")
@ -1196,6 +1227,8 @@ class ImageBrowser(tk.Tk):
def process_tagger_queue(self): def process_tagger_queue(self):
"""Przetwarza wszystkie obrazki w tle (pomijając aktualnie wybrany).""" """Przetwarza wszystkie obrazki w tle (pomijając aktualnie wybrany)."""
for file_path in self.image_files: for file_path in self.image_files:
if self.process_tagger_queue_stop_event.is_set():
break
# Jeśli obrazek jest aktualnie wybrany, pomijamy on będzie przetwarzany w foreground # Jeśli obrazek jest aktualnie wybrany, pomijamy on będzie przetwarzany w foreground
if ( if (
self.current_index is not None self.current_index is not None
@ -1203,8 +1236,9 @@ class ImageBrowser(tk.Tk):
): ):
continue continue
self.process_tagger_for_image(file_path) self.process_tagger_for_image(file_path)
self.after(100, self.join_process_tagger_queue_thread)
def run_tagger(self): def run_tagger(self, thread_name):
""" """
Przetwarza aktualnie wybrany obrazek. Przetwarza aktualnie wybrany obrazek.
Jeśli wynik jest już w cache, wykorzystuje go; w przeciwnym razie uruchamia Taggera Jeśli wynik jest już w cache, wykorzystuje go; w przeciwnym razie uruchamia Taggera
@ -1223,9 +1257,15 @@ class ImageBrowser(tk.Tk):
self.rating_var.set(new_rating) self.rating_var.set(new_rating)
self.after(0, lambda: self.update_tagger_tags_widget(result)) self.after(0, lambda: self.update_tagger_tags_widget(result))
self.update_listbox_item_color_by_rating(file_path, result.rating) self.update_listbox_item_color_by_rating(file_path, result.rating)
self.after(100, lambda: self.run_tagger_threads[thread_name].join())
def upload_file( def upload_file(
self, file_path, final_tags=None, final_rating=None, progress_queue=None self,
file_path,
final_tags=None,
final_rating=None,
progress_queue=None,
cancel_event=None,
): ):
base_file_name = os.path.basename(file_path) base_file_name = os.path.basename(file_path)
if progress_queue: if progress_queue:
@ -1258,7 +1298,9 @@ class ImageBrowser(tk.Tk):
progress_queue.put(("progress", percentage)) progress_queue.put(("progress", percentage))
with open(file_path, "rb") as f: with open(file_path, "rb") as f:
wrapped_file = ProgressFile(f, progress_callback, total_size) wrapped_file = ProgressFile(
f, progress_callback, total_size, cancel_event
)
files = {"file": (base_file_name, wrapped_file, "image/png")} files = {"file": (base_file_name, wrapped_file, "image/png")}
response = requests.post(url, data=fields, files=files) response = requests.post(url, data=fields, files=files)
if progress_queue: if progress_queue:
@ -1293,25 +1335,21 @@ class ImageBrowser(tk.Tk):
self.uploaded[file_path] = post_id self.uploaded[file_path] = post_id
self.uploaded_count += 1 self.uploaded_count += 1
self.after(0, self.update_status_bar) self.after(0, self.update_status_bar)
self.after(0, self.update_button_states)
except Exception as e: except Exception as e:
self.upload_button.after(0, self.update_button_states)
messagebox.showerror("Błąd uploadu", str(e)) messagebox.showerror("Błąd uploadu", str(e))
finally:
self.upload_button.after(0, self.update_button_states)
def edit_file( def edit_file(
self, file_path, final_tags=None, final_rating=None, progress_queue=None self,
file_path,
final_tags=None,
final_rating=None,
progress_queue=None,
cancel_event=None,
): ):
""" """
Update tags and rating for an existing post without uploading the file. Update tags and rating for an existing post without uploading the file.
Args:
file_path: Path to the image file associated with the post
final_tags: Optional override tags (space-separated string)
final_rating: Optional override rating
progress_queue: Progress communication queue
Requires:
- The file must have an existing post ID in self.uploaded
""" """
base_file_name = os.path.basename(file_path) base_file_name = os.path.basename(file_path)
post_id = self.uploaded.get(file_path) post_id = self.uploaded.get(file_path)
@ -1328,17 +1366,28 @@ class ImageBrowser(tk.Tk):
progress_queue.put(("label", f"Aktualizuję tagi dla {base_file_name}...")) progress_queue.put(("label", f"Aktualizuję tagi dla {base_file_name}..."))
try: try:
# Check for cancellation before starting the operation.
if cancel_event is not None and cancel_event.is_set():
if progress_queue:
progress_queue.put(("label", "Operacja anulowana"))
return
# Get authentication session and token # Get authentication session and token
session = login(self.settings) session = login(self.settings)
auth_token = get_auth_token(session, self.settings) auth_token = get_auth_token(session, self.settings)
# Check cancellation after login if needed.
if cancel_event is not None and cancel_event.is_set():
if progress_queue:
progress_queue.put(("label", "Operacja anulowana"))
return
# Prepare tags and rating # Prepare tags and rating
tags = ( tags = (
self.final_tags_text.get("1.0", tk.END).strip() self.final_tags_text.get("1.0", tk.END).strip()
if final_tags is None if final_tags is None
else final_tags else final_tags
) )
rating_value = self.rating_map.get( rating_value = self.rating_map.get(
self.rating_var.get() if final_rating is None else final_rating, "?" self.rating_var.get() if final_rating is None else final_rating, "?"
) )
@ -1358,6 +1407,12 @@ class ImageBrowser(tk.Tk):
if progress_queue: if progress_queue:
progress_queue.put(("progress", 50)) progress_queue.put(("progress", 50))
# Check for cancellation before sending the update request.
if cancel_event is not None and cancel_event.is_set():
if progress_queue:
progress_queue.put(("label", "Operacja anulowana"))
return
# Send update request # Send update request
response = session.post(url, data=payload, allow_redirects=False) response = session.post(url, data=payload, allow_redirects=False)
@ -1365,11 +1420,9 @@ class ImageBrowser(tk.Tk):
if response.status_code == 302: if response.status_code == 302:
if progress_queue: if progress_queue:
progress_queue.put(("progress", 100)) progress_queue.put(("progress", 100))
message = "Tagi zostały zaktualizowane!" message = "Tagi zostały zaktualizowane!"
if not final_tags: # Only show success if not bulk operation if not final_tags: # Only show success if not bulk operation
messagebox.showinfo("Sukces edycji", message) messagebox.showinfo("Sukces edycji", message)
# Update UI state # Update UI state
self.after(0, self.update_button_states) self.after(0, self.update_button_states)
return return
@ -1400,8 +1453,22 @@ class ImageBrowser(tk.Tk):
): ):
return return
def worker(): def worker(progress_queue: queue = None, cancel_event: threading.Event = None):
files_count = len(self.image_files)
if progress_queue:
progress_queue.put(("mode", "determinate"))
progress_queue.put(("max", 100))
file_idx = 0
for file_path in self.image_files: for file_path in self.image_files:
if progress_queue:
progress_queue.put(("progress", file_idx * 1.0 / files_count))
progress_queue.put(
("label", f"Wysyłam plik {file_idx+1}/{files_count}...")
)
if cancel_event is not None and cancel_event.is_set():
if progress_queue:
progress_queue.put(("label", f"Anulowano operację!"))
return
if not self.uploaded.get(file_path, False): if not self.uploaded.get(file_path, False):
final_tags, final_rating = ( final_tags, final_rating = (
self.compute_final_tags_and_rating_for_file(file_path) self.compute_final_tags_and_rating_for_file(file_path)
@ -1412,5 +1479,8 @@ class ImageBrowser(tk.Tk):
self.upload_file( self.upload_file(
file_path, final_tags=final_tags, final_rating=final_rating file_path, final_tags=final_tags, final_rating=final_rating
) )
if progress_queue:
progress_queue.put(("label", f"Przesłano pliki!"))
progress_queue.put(("progress", 100))
threading.Thread(target=worker, daemon=True).start() self.processing_dialog = ProcessingDialog(self, worker)

View File

@ -1,12 +1,17 @@
# Klasa pomocnicza do monitorowania postępu uploadu # Klasa pomocnicza do monitorowania postępu uploadu
class ProgressFile: class ProgressFile:
def __init__(self, f, callback, total_size): def __init__(self, f, callback, total_size, cancel_event=None):
self.f = f self.f = f
self.callback = callback self.callback = callback
self.cancel_event = cancel_event
self.total_size = total_size self.total_size = total_size
self.read_bytes = 0 self.read_bytes = 0
def read(self, size=-1): def read(self, size=-1):
# Check for cancellation before reading more data
if self.cancel_event is not None and self.cancel_event.is_set():
raise Exception("Upload cancelled by user.")
data = self.f.read(size) data = self.f.read(size)
self.read_bytes += len(data) self.read_bytes += len(data)
self.callback(self.read_bytes, self.total_size) self.callback(self.read_bytes, self.total_size)

View File

@ -134,40 +134,67 @@ class TagsRepo:
except Exception as e: except Exception as e:
print("Błąd przy inicjalizacji bazy tagów:", e) print("Błąd przy inicjalizacji bazy tagów:", e)
def regenerate_db(self): def regenerate_db(self, progress_queue=None, cancel_event=None):
# Połączenie z bazą SQLite i pobranie tagów """
Regenerate the database of tags, aliases, and tag implications.
Optionally updates a progress_queue with status messages and checks
cancel_event (a threading.Event) to cancel the operation.
"""
# Connect to SQLite and clear old data
conn = sqlite3.connect(self.db_path) conn = sqlite3.connect(self.db_path)
cursor = conn.cursor() cursor = conn.cursor()
if progress_queue:
progress_queue.put(("label", "Czyszczenie bazy danych..."))
cursor.execute("DELETE FROM tags") cursor.execute("DELETE FROM tags")
cursor.execute("DELETE FROM tag_aliases") cursor.execute("DELETE FROM tag_aliases")
conn.commit() conn.commit()
rate_limit = 10 # requests per second rate_limit = 10 # requests per second
min_interval = 1.0 / rate_limit # minimum seconds between requests (0.1 sec) min_interval = 1.0 / rate_limit # minimum seconds between requests
# -----------------------
# Fetch Tags
# -----------------------
data_list = [] data_list = []
last_id = 0
page = 0
while True: while True:
print(f"Tagi - Pobieranie od id {page}...") if cancel_event and cancel_event.is_set():
if progress_queue:
progress_queue.put(("label", "Anulowano pobieranie tagów."))
conn.close()
return
if progress_queue:
progress_queue.put(("label", f"Pobieranie tagów (od ID {last_id})..."))
start_time = time.monotonic() start_time = time.monotonic()
url = f"https://danbooru.donmai.us/tags.json?limit=1000&page=a{page}" url = f"https://danbooru.donmai.us/tags.json?limit=1000&page=a{last_id}"
response = requests.get(url) response = requests.get(url)
if response.status_code != 200: if response.status_code != 200:
print( if progress_queue:
f"Błąd przy pobieraniu strony {page}: HTTP {response.status_code}" progress_queue.put(
(
"label",
f"Błąd przy pobieraniu tagów od ID {last_id}: HTTP {response.status_code}",
)
) )
break break
data = response.json() data = response.json()
if not data: if not data:
break break
page = None
last_id = None
for item in data: for item in data:
id = item.get("id") if cancel_event and cancel_event.is_set():
if not page: if progress_queue:
page = id progress_queue.put(("label", "Anulowano przetwarzanie tagów."))
conn.close()
return
tag_id = item.get("id")
if not last_id:
last_id = tag_id
name = item.get("name") name = item.get("name")
post_count = item.get("post_count") post_count = item.get("post_count")
category = item.get("category") category = item.get("category")
@ -177,7 +204,7 @@ class TagsRepo:
words = json.dumps(item.get("words")) words = json.dumps(item.get("words"))
data_list.append( data_list.append(
( (
id, tag_id,
name, name,
post_count, post_count,
category, category,
@ -191,15 +218,15 @@ class TagsRepo:
if len(data) < 1000: if len(data) < 1000:
break break
# Calculate elapsed time and sleep if necessary to enforce the rate limit
elapsed = time.monotonic() - start_time elapsed = time.monotonic() - start_time
if elapsed < min_interval: if elapsed < min_interval:
time.sleep(min_interval - elapsed) time.sleep(min_interval - elapsed)
print(f"Tagi - Pobrano {len(data_list)} tagów...") if progress_queue:
progress_queue.put(("label", f"Pobrano {len(data_list)} tagów..."))
data_list = sorted(data_list, key=lambda x: x[0]) data_list = sorted(data_list, key=lambda x: x[0])
data_list = [(idx,) + row for idx, row in enumerate(data_list)] data_list = [(idx,) + row for idx, row in enumerate(data_list)]
cursor.executemany( cursor.executemany(
""" """
INSERT INTO tags ("index", id, name, post_count, category, created_at, updated_at, is_deprecated, words) INSERT INTO tags ("index", id, name, post_count, category, created_at, updated_at, is_deprecated, words)
@ -210,26 +237,53 @@ class TagsRepo:
conn.commit() conn.commit()
data_list = [] data_list = []
page = 0 # -----------------------
# Fetch Tag Aliases
# -----------------------
last_id = 0
while True: while True:
print(f"Aliasy tagów - Pobieranie od id {page}...") if cancel_event and cancel_event.is_set():
if progress_queue:
progress_queue.put(("label", "Anulowano pobieranie aliasów tagów."))
conn.close()
return
if progress_queue:
progress_queue.put(
("label", f"Pobieranie aliasów tagów (od ID {last_id})...")
)
start_time = time.monotonic() start_time = time.monotonic()
url = f"https://danbooru.donmai.us/tag_aliases.json?limit=1000&only=id,antecedent_name,consequent_name&search[status]=active&page=a{page}" url = (
f"https://danbooru.donmai.us/tag_aliases.json?limit=1000"
f"&only=id,antecedent_name,consequent_name&search[status]=active&page=a{last_id}"
)
response = requests.get(url) response = requests.get(url)
if response.status_code != 200: if response.status_code != 200:
print( if progress_queue:
f"Błąd przy pobieraniu strony {page}: HTTP {response.status_code}" progress_queue.put(
(
"label",
f"Błąd przy pobieraniu aliasów tagów od ID {last_id}: HTTP {response.status_code}",
)
) )
break break
data = response.json() data = response.json()
if not data: if not data:
break break
page = None
last_id = None
for item in data: for item in data:
id = item.get("id") if cancel_event and cancel_event.is_set():
if not page: if progress_queue:
page = id progress_queue.put(
("label", "Anulowano przetwarzanie aliasów tagów.")
)
conn.close()
return
tag_id = item.get("id")
if not last_id:
last_id = tag_id
antecedent = item.get("antecedent_name") antecedent = item.get("antecedent_name")
consequent = item.get("consequent_name") consequent = item.get("consequent_name")
data_list.append((antecedent, consequent)) data_list.append((antecedent, consequent))
@ -237,15 +291,15 @@ class TagsRepo:
if len(data) < 1000: if len(data) < 1000:
break break
# Calculate elapsed time and sleep if necessary to enforce the rate limit
elapsed = time.monotonic() - start_time elapsed = time.monotonic() - start_time
if elapsed < min_interval: if elapsed < min_interval:
time.sleep(min_interval - elapsed) time.sleep(min_interval - elapsed)
print(f"Aliasy tagów - Pobrano {len(data_list)} aliasów tagów...") if progress_queue:
progress_queue.put(("label", f"Pobrano {len(data_list)} aliasów tagów..."))
data_list = sorted(data_list, key=lambda x: x[0]) data_list = sorted(data_list, key=lambda x: x[0])
data_list = [(idx,) + row for idx, row in enumerate(data_list)] data_list = [(idx,) + row for idx, row in enumerate(data_list)]
cursor.executemany( cursor.executemany(
""" """
INSERT INTO tag_aliases ("index", alias, tag) INSERT INTO tag_aliases ("index", alias, tag)
@ -256,19 +310,15 @@ class TagsRepo:
conn.commit() conn.commit()
data_list = [] data_list = []
# Pobranie tagów kategorii "character" (category = 4) # -----------------------
# Fetch Tag Categories for Implications
# -----------------------
cursor.execute("SELECT name FROM tags WHERE category = 4") cursor.execute("SELECT name FROM tags WHERE category = 4")
character_tags = {row[0] for row in cursor.fetchall()} character_tags = {row[0] for row in cursor.fetchall()}
# Pobranie tagów kategorii "copyright" (category = 3)
cursor.execute("SELECT name FROM tags WHERE category = 3") cursor.execute("SELECT name FROM tags WHERE category = 3")
copyright_tags = {row[0] for row in cursor.fetchall()} copyright_tags = {row[0] for row in cursor.fetchall()}
# Pobranie tagów kategorii "meta" (category = 5)
cursor.execute("SELECT name FROM tags WHERE category = 5") cursor.execute("SELECT name FROM tags WHERE category = 5")
meta_tags = {row[0] for row in cursor.fetchall()} meta_tags = {row[0] for row in cursor.fetchall()}
# Pobranie tagów kategorii "artist" (category = 1)
cursor.execute("SELECT name FROM tags WHERE category = 1") cursor.execute("SELECT name FROM tags WHERE category = 1")
artist_tags = {row[0] for row in cursor.fetchall()} artist_tags = {row[0] for row in cursor.fetchall()}
@ -276,36 +326,60 @@ class TagsRepo:
cursor.execute("DELETE FROM tag_closure") # Optional: reset table cursor.execute("DELETE FROM tag_closure") # Optional: reset table
conn.commit() conn.commit()
# Budujemy strukturę implikacji: słownik, gdzie # -----------------------
# kluczem jest antecedent_name, a wartością zbiór consequent_name. # Fetch Tag Implications
# -----------------------
tag_dict = {} tag_dict = {}
last_id = 0
page = 0
while True: while True:
print(f"Implikacje tagów - Pobieranie od id {page}...") if cancel_event and cancel_event.is_set():
url = f"https://danbooru.donmai.us/tag_implications.json?limit=1000&page=a{page}" if progress_queue:
progress_queue.put(
("label", "Anulowano pobieranie implikacji tagów.")
)
conn.close()
return
if progress_queue:
progress_queue.put(
("label", f"Pobieranie implikacji tagów (od ID {last_id})...")
)
start_time = time.monotonic()
url = f"https://danbooru.donmai.us/tag_implications.json?limit=1000&page=a{last_id}"
response = requests.get(url) response = requests.get(url)
if response.status_code != 200: if response.status_code != 200:
print( if progress_queue:
f"Błąd przy pobieraniu strony {page}: HTTP {response.status_code}" progress_queue.put(
(
"label",
f"Błąd przy pobieraniu implikacji tagów od ID {last_id}: HTTP {response.status_code}",
)
) )
break break
data = response.json() data = response.json()
if not data: if not data:
break break
page = None
last_id = None
for item in data: for item in data:
id = item.get("id") if cancel_event and cancel_event.is_set():
if not page: if progress_queue:
page = id progress_queue.put(
("label", "Anulowano przetwarzanie implikacji tagów.")
)
conn.close()
return
tag_id = item.get("id")
if not last_id:
last_id = tag_id
if item.get("status") != "active": if item.get("status") != "active":
continue continue
antecedent = item.get("antecedent_name") antecedent = item.get("antecedent_name")
consequent = item.get("consequent_name") consequent = item.get("consequent_name")
# Dodanie prefiksu, jeżeli tag należy do jednej z kategorii # Prefix tags based on their category
if antecedent in character_tags: if antecedent in character_tags:
antecedent = f"character:{antecedent}" antecedent = f"character:{antecedent}"
elif antecedent in copyright_tags: elif antecedent in copyright_tags:
@ -330,13 +404,19 @@ class TagsRepo:
if len(data) < 1000: if len(data) < 1000:
break break
# Calculate elapsed time and sleep if necessary to enforce the rate limit
elapsed = time.monotonic() - start_time elapsed = time.monotonic() - start_time
if elapsed < min_interval: if elapsed < min_interval:
time.sleep(min_interval - elapsed) time.sleep(min_interval - elapsed)
print(f"Implikacje tagów - Pobrano {len(tag_dict)} implikacji tagów...") if progress_queue:
# Batch insert all unique pairs progress_queue.put(
("label", f"Pobrano implikacje dla {len(tag_dict)} tagów...")
)
# -----------------------
# Insert Tag Implications and Build Transitive Closure
# -----------------------
for antecedent, consequents in tag_dict.items(): for antecedent, consequents in tag_dict.items():
for consequent in consequents: for consequent in consequents:
if antecedent != consequent: if antecedent != consequent:
@ -345,13 +425,18 @@ class TagsRepo:
(antecedent, consequent), (antecedent, consequent),
) )
conn.commit() conn.commit()
closure_data = self.build_transitive_closure(tag_dict)
cursor.executemany( cursor.executemany(
"INSERT INTO tag_closure VALUES (?, ?, ?)", "INSERT INTO tag_closure VALUES (?, ?, ?)",
self.build_transitive_closure(tag_dict), closure_data,
) )
conn.commit() conn.commit()
conn.close() conn.close()
if progress_queue:
progress_queue.put(("label", "Regeneracja bazy zakończona."))
def build_transitive_closure(self, tag_dict): def build_transitive_closure(self, tag_dict):
closure = set() closure = set()
for antecedent in tag_dict: for antecedent in tag_dict:

View File

@ -180,9 +180,16 @@ class TagManager(tk.Frame):
""" """
def __init__( def __init__(
self, master, settings: Settings, tags_repo: TagsRepo, *args, **kwargs self,
master,
settings: Settings,
tags_repo: TagsRepo,
tag_change_callback=None,
*args,
**kwargs,
): ):
super().__init__(master, *args, **kwargs) super().__init__(master, *args, **kwargs)
self.tag_change_callback = tag_change_callback
self.tags_repo = tags_repo self.tags_repo = tags_repo
self.settings = settings self.settings = settings
self.manual_tags = [] # List to hold manually entered tags self.manual_tags = [] # List to hold manually entered tags
@ -234,6 +241,8 @@ class TagManager(tk.Frame):
self.tags_display.tag_bind(tag_name, "<Button-3>", self.open_tag_wiki_url) self.tags_display.tag_bind(tag_name, "<Button-3>", self.open_tag_wiki_url)
self.tags_display.insert(tk.INSERT, " ") self.tags_display.insert(tk.INSERT, " ")
self.tags_display.config(state=tk.DISABLED) self.tags_display.config(state=tk.DISABLED)
if self.tag_change_callback:
self.tag_change_callback()
def remove_tag(self, event): def remove_tag(self, event):
"""Remove the clicked tag from the list and update the display.""" """Remove the clicked tag from the list and update the display."""

View File

@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
[project] [project]
name = "kapitanbooru-uploader" name = "kapitanbooru-uploader"
version = "0.2.0" version = "0.3.0"
description = "A GUI application for uploading images to KapitanBooru" description = "A GUI application for uploading images to KapitanBooru"
authors = [ authors = [
{name = "Michał Leśniak", email = "kapitan@mlesniak.pl"} {name = "Michał Leśniak", email = "kapitan@mlesniak.pl"}