Cancellable tasks
	
		
			
	
		
	
	
		
	
		
			All checks were successful
		
		
	
	
		
			
				
	
				Gitea/kapitanbooru-uploader/pipeline/head This commit looks good
				
			
		
		
	
	
				
					
				
			
		
			All checks were successful
		
		
	
	Gitea/kapitanbooru-uploader/pipeline/head This commit looks good
				
			This commit is contained in:
		| @@ -6,7 +6,7 @@ import queue | ||||
| import threading | ||||
| import tkinter as tk | ||||
| from tkinter import filedialog, messagebox, ttk | ||||
| from typing import Tuple | ||||
| from typing import Dict, Tuple, Optional | ||||
|  | ||||
| import networkx as nx | ||||
| import requests | ||||
| @@ -41,6 +41,7 @@ class ProcessingDialog: | ||||
|         # Setup communication queue and periodic checker | ||||
|         self.queue = queue.Queue() | ||||
|         self.running = True | ||||
|         self.cancel_event = threading.Event()  # Cancellation flag | ||||
|         self.thread = threading.Thread( | ||||
|             target=self.run_task, args=(target_function, *args) | ||||
|         ) | ||||
| @@ -75,10 +76,12 @@ class ProcessingDialog: | ||||
|         """Execute target function with progress queue if supported""" | ||||
|         try: | ||||
|             sig = inspect.signature(target_function) | ||||
|             kwargs = {} | ||||
|             if "progress_queue" in sig.parameters: | ||||
|                 target_function(*args, progress_queue=self.queue) | ||||
|             else: | ||||
|                 target_function(*args) | ||||
|                 kwargs["progress_queue"] = self.queue | ||||
|             if "cancel_event" in sig.parameters: | ||||
|                 kwargs["cancel_event"] = self.cancel_event | ||||
|             target_function(*args, **kwargs) | ||||
|         finally: | ||||
|             self.close_dialog() | ||||
|  | ||||
| @@ -88,10 +91,12 @@ class ProcessingDialog: | ||||
|             self.running = False | ||||
|             self.progress.stop() | ||||
|             self.top.after(0, self.top.destroy) | ||||
|         self.thread.join() | ||||
|  | ||||
|     def on_close(self): | ||||
|         """Handle manual window closure""" | ||||
|         self.running = False | ||||
|         self.cancel_event.set()  # Notify target function that cancellation is requested | ||||
|         self.top.destroy() | ||||
|  | ||||
|  | ||||
| @@ -121,6 +126,14 @@ class ImageBrowser(tk.Tk): | ||||
|         self.image_files_md5 = [] | ||||
|         self.current_index = None | ||||
|         self.image_cache = None | ||||
|         self.tagger_thread_idx = 0 | ||||
|         self.tagger = wdt.Tagger() | ||||
|  | ||||
|         self.check_uploaded_files_stop_event = threading.Event() | ||||
|         self.check_uploaded_files_thread: Optional[threading.Thread] = None | ||||
|         self.process_tagger_queue_stop_event = threading.Event() | ||||
|         self.process_tagger_queue_thread: Optional[threading.Thread] = None | ||||
|         self.run_tagger_threads: Dict[str, threading.Thread] = {} | ||||
|  | ||||
|         # Liczniki statusu | ||||
|         self.total_files = 0 | ||||
| @@ -273,9 +286,8 @@ class ImageBrowser(tk.Tk): | ||||
|             self.tagger_processed.add(md5) | ||||
|             return cached["result"] | ||||
|         try: | ||||
|             tagger = wdt.Tagger() | ||||
|             with Image.open(file_path) as img: | ||||
|                 result = tagger.tag(img) | ||||
|                 result = self.tagger.tag(img) | ||||
|             self.tagger_cache[md5] = result | ||||
|             self.tagger_processed.add(md5) | ||||
|             self.after(0, self.update_status_bar) | ||||
| @@ -407,18 +419,6 @@ class ImageBrowser(tk.Tk): | ||||
|         btn_save.pack(pady=10) | ||||
|  | ||||
|     def create_widgets(self): | ||||
|         # Górna ramka | ||||
|         top_frame = tk.Frame(self) | ||||
|         top_frame.pack(side=tk.TOP, fill=tk.X, padx=5, pady=5) | ||||
|         btn_select_folder = tk.Button( | ||||
|             top_frame, text="Wybierz folder", command=self.select_folder | ||||
|         ) | ||||
|         btn_select_folder.pack(side=tk.LEFT, padx=5) | ||||
|         btn_prev = tk.Button(top_frame, text="<<", command=self.show_prev) | ||||
|         btn_prev.pack(side=tk.LEFT, padx=5) | ||||
|         btn_next = tk.Button(top_frame, text=">>", command=self.show_next) | ||||
|         btn_next.pack(side=tk.LEFT, padx=5) | ||||
|  | ||||
|         # Główna ramka – trzy kolumny | ||||
|         main_frame = tk.Frame(self) | ||||
|         main_frame.pack(fill=tk.BOTH, expand=True, padx=5, pady=5) | ||||
| @@ -486,15 +486,9 @@ class ImageBrowser(tk.Tk): | ||||
|         manual_frame = tk.LabelFrame(right_frame, text="Manual Tags") | ||||
|         manual_frame.grid(row=2, column=0, sticky=tk.NSEW, padx=5, pady=5) | ||||
|         self.manual_tags_manager = TagManager( | ||||
|             manual_frame, self.settings, self.tags_repo | ||||
|             manual_frame, self.settings, self.tags_repo, self.update_final_tags | ||||
|         ) | ||||
|         self.manual_tags_manager.pack(fill=tk.BOTH, expand=True) | ||||
|         self.manual_tags_manager.bind( | ||||
|             "<KeyRelease>", lambda e: self.update_final_tags(), add="+" | ||||
|         ) | ||||
|         self.manual_tags_manager.bind( | ||||
|             "<Button-1>", lambda e: self.update_final_tags(), add="+" | ||||
|         ) | ||||
|  | ||||
|         # Final Tags – widget Text z scrollbar, który rozszerza się | ||||
|         final_frame = tk.LabelFrame(right_frame, text="Final Tags") | ||||
| @@ -526,6 +520,10 @@ class ImageBrowser(tk.Tk): | ||||
|         ) | ||||
|         self.view_post_button.pack(side=tk.LEFT, padx=5) | ||||
|         self.view_post_button.config(state=tk.DISABLED) | ||||
|         btn_prev = tk.Button(upload_frame, text="<<", command=self.show_prev) | ||||
|         btn_prev.pack(side=tk.LEFT, padx=5) | ||||
|         btn_next = tk.Button(upload_frame, text=">>", command=self.show_next) | ||||
|         btn_next.pack(side=tk.LEFT, padx=5) | ||||
|  | ||||
|         # Na końcu okna głównego dodaj status bar: | ||||
|         self.status_label = tk.Label( | ||||
| @@ -601,8 +599,30 @@ class ImageBrowser(tk.Tk): | ||||
|         """ | ||||
|         Po załadowaniu plików, sprawdza czy są jakieś pliki do uploadu oraz przetwarza Taggerem pliki. | ||||
|         """ | ||||
|         threading.Thread(target=self.check_uploaded_files, daemon=True).start() | ||||
|         threading.Thread(target=self.process_tagger_queue, daemon=True).start() | ||||
|         self.join_check_uploaded_files_thread() | ||||
|         self.check_uploaded_files_thread = threading.Thread( | ||||
|             target=self.check_uploaded_files | ||||
|         ) | ||||
|         self.check_uploaded_files_thread.start() | ||||
|         self.join_process_tagger_queue_thread() | ||||
|         self.process_tagger_queue_thread = threading.Thread( | ||||
|             target=self.process_tagger_queue | ||||
|         ) | ||||
|         self.process_tagger_queue_thread.start() | ||||
|  | ||||
|     def join_check_uploaded_files_thread(self): | ||||
|         if self.check_uploaded_files_thread is not None: | ||||
|             self.check_uploaded_files_stop_event.set() | ||||
|             self.check_uploaded_files_thread.join() | ||||
|             self.check_uploaded_files_thread = None | ||||
|             self.check_uploaded_files_stop_event = threading.Event() | ||||
|  | ||||
|     def join_process_tagger_queue_thread(self): | ||||
|         if self.process_tagger_queue_thread is not None: | ||||
|             self.process_tagger_queue_stop_event.set() | ||||
|             self.process_tagger_queue_thread.join() | ||||
|             self.process_tagger_queue_thread = None | ||||
|             self.process_tagger_queue_stop_event = threading.Event() | ||||
|  | ||||
|     def check_uploaded_files(self): | ||||
|         """ | ||||
| @@ -617,6 +637,8 @@ class ImageBrowser(tk.Tk): | ||||
|  | ||||
|         batch_size = 100 | ||||
|         for i in range(0, len(file_md5_list), batch_size): | ||||
|             if self.check_uploaded_files_stop_event.is_set(): | ||||
|                 break | ||||
|             batch = file_md5_list[i : i + batch_size] | ||||
|             batch_md5 = [item[2] for item in batch] | ||||
|             md5_param = ",".join(batch_md5) | ||||
| @@ -627,11 +649,15 @@ class ImageBrowser(tk.Tk): | ||||
|                 root = response.json() | ||||
|                 found = {} | ||||
|                 for elem in root: | ||||
|                     if self.check_uploaded_files_stop_event.is_set(): | ||||
|                         break | ||||
|                     post_md5 = elem.get("md5", "").lower() | ||||
|                     post_id = elem.get("id") | ||||
|                     if post_md5 and post_id: | ||||
|                         found[post_md5] = post_id | ||||
|                 for idx, file_path, md5 in batch: | ||||
|                     if self.check_uploaded_files_stop_event.is_set(): | ||||
|                         break | ||||
|                     self.upload_verified += 1  # Każdy plik w batchu jest zweryfikowany | ||||
|                     if md5.lower() in found: | ||||
|                         self.uploaded[file_path] = found[md5.lower()] | ||||
| @@ -650,6 +676,7 @@ class ImageBrowser(tk.Tk): | ||||
|                     self.after(0, self.update_status_bar) | ||||
|             except Exception as e: | ||||
|                 print("Błąd podczas sprawdzania paczki uploadu:", e) | ||||
|         self.after(100, self.join_check_uploaded_files_thread) | ||||
|  | ||||
|     def update_button_states(self): | ||||
|         """ | ||||
| @@ -774,7 +801,11 @@ class ImageBrowser(tk.Tk): | ||||
|             # Uaktualnij widget PNG Tags | ||||
|             self.update_png_tags_widget(parsed_parameters.split()) | ||||
|             # Uruchom Taggera w osobnym wątku | ||||
|             threading.Thread(target=self.run_tagger, daemon=True).start() | ||||
|             thread_name = f"tagger{self.tagger_thread_idx}" | ||||
|             thread = threading.Thread(target=self.run_tagger, args=(thread_name,)) | ||||
|             self.run_tagger_threads[thread_name] = thread | ||||
|             self.tagger_thread_idx += 1 | ||||
|             thread.start() | ||||
|         except Exception as e: | ||||
|             messagebox.showerror("Błąd", f"Nie można załadować obrazka:\n{e}") | ||||
|  | ||||
| @@ -1196,6 +1227,8 @@ class ImageBrowser(tk.Tk): | ||||
|     def process_tagger_queue(self): | ||||
|         """Przetwarza wszystkie obrazki w tle (pomijając aktualnie wybrany).""" | ||||
|         for file_path in self.image_files: | ||||
|             if self.process_tagger_queue_stop_event.is_set(): | ||||
|                 break | ||||
|             # Jeśli obrazek jest aktualnie wybrany, pomijamy – on będzie przetwarzany w foreground | ||||
|             if ( | ||||
|                 self.current_index is not None | ||||
| @@ -1203,8 +1236,9 @@ class ImageBrowser(tk.Tk): | ||||
|             ): | ||||
|                 continue | ||||
|             self.process_tagger_for_image(file_path) | ||||
|         self.after(100, self.join_process_tagger_queue_thread) | ||||
|  | ||||
|     def run_tagger(self): | ||||
|     def run_tagger(self, thread_name): | ||||
|         """ | ||||
|         Przetwarza aktualnie wybrany obrazek. | ||||
|         Jeśli wynik jest już w cache, wykorzystuje go; w przeciwnym razie uruchamia Taggera | ||||
| @@ -1223,9 +1257,15 @@ class ImageBrowser(tk.Tk): | ||||
|         self.rating_var.set(new_rating) | ||||
|         self.after(0, lambda: self.update_tagger_tags_widget(result)) | ||||
|         self.update_listbox_item_color_by_rating(file_path, result.rating) | ||||
|         self.after(100, lambda: self.run_tagger_threads[thread_name].join()) | ||||
|  | ||||
|     def upload_file( | ||||
|         self, file_path, final_tags=None, final_rating=None, progress_queue=None | ||||
|         self, | ||||
|         file_path, | ||||
|         final_tags=None, | ||||
|         final_rating=None, | ||||
|         progress_queue=None, | ||||
|         cancel_event=None, | ||||
|     ): | ||||
|         base_file_name = os.path.basename(file_path) | ||||
|         if progress_queue: | ||||
| @@ -1258,7 +1298,9 @@ class ImageBrowser(tk.Tk): | ||||
|                     progress_queue.put(("progress", percentage)) | ||||
|  | ||||
|             with open(file_path, "rb") as f: | ||||
|                 wrapped_file = ProgressFile(f, progress_callback, total_size) | ||||
|                 wrapped_file = ProgressFile( | ||||
|                     f, progress_callback, total_size, cancel_event | ||||
|                 ) | ||||
|                 files = {"file": (base_file_name, wrapped_file, "image/png")} | ||||
|                 response = requests.post(url, data=fields, files=files) | ||||
|             if progress_queue: | ||||
| @@ -1293,25 +1335,21 @@ class ImageBrowser(tk.Tk): | ||||
|                 self.uploaded[file_path] = post_id | ||||
|                 self.uploaded_count += 1 | ||||
|                 self.after(0, self.update_status_bar) | ||||
|             self.after(0, self.update_button_states) | ||||
|         except Exception as e: | ||||
|             self.upload_button.after(0, self.update_button_states) | ||||
|             messagebox.showerror("Błąd uploadu", str(e)) | ||||
|         finally: | ||||
|             self.upload_button.after(0, self.update_button_states) | ||||
|  | ||||
|     def edit_file( | ||||
|         self, file_path, final_tags=None, final_rating=None, progress_queue=None | ||||
|         self, | ||||
|         file_path, | ||||
|         final_tags=None, | ||||
|         final_rating=None, | ||||
|         progress_queue=None, | ||||
|         cancel_event=None, | ||||
|     ): | ||||
|         """ | ||||
|         Update tags and rating for an existing post without uploading the file. | ||||
|  | ||||
|         Args: | ||||
|             file_path: Path to the image file associated with the post | ||||
|             final_tags: Optional override tags (space-separated string) | ||||
|             final_rating: Optional override rating | ||||
|             progress_queue: Progress communication queue | ||||
|  | ||||
|         Requires: | ||||
|             - The file must have an existing post ID in self.uploaded | ||||
|         """ | ||||
|         base_file_name = os.path.basename(file_path) | ||||
|         post_id = self.uploaded.get(file_path) | ||||
| @@ -1328,17 +1366,28 @@ class ImageBrowser(tk.Tk): | ||||
|             progress_queue.put(("label", f"Aktualizuję tagi dla {base_file_name}...")) | ||||
|  | ||||
|         try: | ||||
|             # Check for cancellation before starting the operation. | ||||
|             if cancel_event is not None and cancel_event.is_set(): | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put(("label", "Operacja anulowana")) | ||||
|                 return | ||||
|  | ||||
|             # Get authentication session and token | ||||
|             session = login(self.settings) | ||||
|             auth_token = get_auth_token(session, self.settings) | ||||
|  | ||||
|             # Check cancellation after login if needed. | ||||
|             if cancel_event is not None and cancel_event.is_set(): | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put(("label", "Operacja anulowana")) | ||||
|                 return | ||||
|  | ||||
|             # Prepare tags and rating | ||||
|             tags = ( | ||||
|                 self.final_tags_text.get("1.0", tk.END).strip() | ||||
|                 if final_tags is None | ||||
|                 else final_tags | ||||
|             ) | ||||
|  | ||||
|             rating_value = self.rating_map.get( | ||||
|                 self.rating_var.get() if final_rating is None else final_rating, "?" | ||||
|             ) | ||||
| @@ -1358,6 +1407,12 @@ class ImageBrowser(tk.Tk): | ||||
|             if progress_queue: | ||||
|                 progress_queue.put(("progress", 50)) | ||||
|  | ||||
|             # Check for cancellation before sending the update request. | ||||
|             if cancel_event is not None and cancel_event.is_set(): | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put(("label", "Operacja anulowana")) | ||||
|                 return | ||||
|  | ||||
|             # Send update request | ||||
|             response = session.post(url, data=payload, allow_redirects=False) | ||||
|  | ||||
| @@ -1365,11 +1420,9 @@ class ImageBrowser(tk.Tk): | ||||
|             if response.status_code == 302: | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put(("progress", 100)) | ||||
|  | ||||
|                 message = "Tagi zostały zaktualizowane!" | ||||
|                 if not final_tags:  # Only show success if not bulk operation | ||||
|                     messagebox.showinfo("Sukces edycji", message) | ||||
|  | ||||
|                 # Update UI state | ||||
|                 self.after(0, self.update_button_states) | ||||
|                 return | ||||
| @@ -1400,8 +1453,22 @@ class ImageBrowser(tk.Tk): | ||||
|         ): | ||||
|             return | ||||
|  | ||||
|         def worker(): | ||||
|         def worker(progress_queue: queue = None, cancel_event: threading.Event = None): | ||||
|             files_count = len(self.image_files) | ||||
|             if progress_queue: | ||||
|                 progress_queue.put(("mode", "determinate")) | ||||
|                 progress_queue.put(("max", 100)) | ||||
|             file_idx = 0 | ||||
|             for file_path in self.image_files: | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put(("progress", file_idx * 1.0 / files_count)) | ||||
|                     progress_queue.put( | ||||
|                         ("label", f"Wysyłam plik {file_idx+1}/{files_count}...") | ||||
|                     ) | ||||
|                 if cancel_event is not None and cancel_event.is_set(): | ||||
|                     if progress_queue: | ||||
|                         progress_queue.put(("label", f"Anulowano operację!")) | ||||
|                     return | ||||
|                 if not self.uploaded.get(file_path, False): | ||||
|                     final_tags, final_rating = ( | ||||
|                         self.compute_final_tags_and_rating_for_file(file_path) | ||||
| @@ -1412,5 +1479,8 @@ class ImageBrowser(tk.Tk): | ||||
|                     self.upload_file( | ||||
|                         file_path, final_tags=final_tags, final_rating=final_rating | ||||
|                     ) | ||||
|             if progress_queue: | ||||
|                 progress_queue.put(("label", f"Przesłano pliki!")) | ||||
|                 progress_queue.put(("progress", 100)) | ||||
|  | ||||
|         threading.Thread(target=worker, daemon=True).start() | ||||
|         self.processing_dialog = ProcessingDialog(self, worker) | ||||
|   | ||||
| @@ -1,12 +1,17 @@ | ||||
| # Klasa pomocnicza do monitorowania postępu uploadu | ||||
| class ProgressFile: | ||||
|     def __init__(self, f, callback, total_size): | ||||
|     def __init__(self, f, callback, total_size, cancel_event=None): | ||||
|         self.f = f | ||||
|         self.callback = callback | ||||
|         self.cancel_event = cancel_event | ||||
|         self.total_size = total_size | ||||
|         self.read_bytes = 0 | ||||
|  | ||||
|     def read(self, size=-1): | ||||
|         # Check for cancellation before reading more data | ||||
|         if self.cancel_event is not None and self.cancel_event.is_set(): | ||||
|             raise Exception("Upload cancelled by user.") | ||||
|  | ||||
|         data = self.f.read(size) | ||||
|         self.read_bytes += len(data) | ||||
|         self.callback(self.read_bytes, self.total_size) | ||||
|   | ||||
| @@ -134,40 +134,67 @@ class TagsRepo: | ||||
|         except Exception as e: | ||||
|             print("Błąd przy inicjalizacji bazy tagów:", e) | ||||
|  | ||||
|     def regenerate_db(self): | ||||
|         # Połączenie z bazą SQLite i pobranie tagów | ||||
|     def regenerate_db(self, progress_queue=None, cancel_event=None): | ||||
|         """ | ||||
|         Regenerate the database of tags, aliases, and tag implications. | ||||
|  | ||||
|         Optionally updates a progress_queue with status messages and checks | ||||
|         cancel_event (a threading.Event) to cancel the operation. | ||||
|         """ | ||||
|         # Connect to SQLite and clear old data | ||||
|         conn = sqlite3.connect(self.db_path) | ||||
|         cursor = conn.cursor() | ||||
|  | ||||
|         if progress_queue: | ||||
|             progress_queue.put(("label", "Czyszczenie bazy danych...")) | ||||
|         cursor.execute("DELETE FROM tags") | ||||
|         cursor.execute("DELETE FROM tag_aliases") | ||||
|         conn.commit() | ||||
|  | ||||
|         rate_limit = 10  # requests per second | ||||
|         min_interval = 1.0 / rate_limit  # minimum seconds between requests (0.1 sec) | ||||
|         min_interval = 1.0 / rate_limit  # minimum seconds between requests | ||||
|  | ||||
|         # ----------------------- | ||||
|         # Fetch Tags | ||||
|         # ----------------------- | ||||
|         data_list = [] | ||||
|  | ||||
|         page = 0 | ||||
|         last_id = 0 | ||||
|         while True: | ||||
|             print(f"Tagi - Pobieranie od id {page}...") | ||||
|             if cancel_event and cancel_event.is_set(): | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put(("label", "Anulowano pobieranie tagów.")) | ||||
|                 conn.close() | ||||
|                 return | ||||
|  | ||||
|             if progress_queue: | ||||
|                 progress_queue.put(("label", f"Pobieranie tagów (od ID {last_id})...")) | ||||
|             start_time = time.monotonic() | ||||
|             url = f"https://danbooru.donmai.us/tags.json?limit=1000&page=a{page}" | ||||
|             url = f"https://danbooru.donmai.us/tags.json?limit=1000&page=a{last_id}" | ||||
|             response = requests.get(url) | ||||
|             if response.status_code != 200: | ||||
|                 print( | ||||
|                     f"Błąd przy pobieraniu strony {page}: HTTP {response.status_code}" | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put( | ||||
|                         ( | ||||
|                             "label", | ||||
|                             f"Błąd przy pobieraniu tagów od ID {last_id}: HTTP {response.status_code}", | ||||
|                         ) | ||||
|                     ) | ||||
|                 break | ||||
|  | ||||
|             data = response.json() | ||||
|             if not data: | ||||
|                 break | ||||
|             page = None | ||||
|  | ||||
|             last_id = None | ||||
|             for item in data: | ||||
|                 id = item.get("id") | ||||
|                 if not page: | ||||
|                     page = id | ||||
|                 if cancel_event and cancel_event.is_set(): | ||||
|                     if progress_queue: | ||||
|                         progress_queue.put(("label", "Anulowano przetwarzanie tagów.")) | ||||
|                     conn.close() | ||||
|                     return | ||||
|                 tag_id = item.get("id") | ||||
|                 if not last_id: | ||||
|                     last_id = tag_id | ||||
|                 name = item.get("name") | ||||
|                 post_count = item.get("post_count") | ||||
|                 category = item.get("category") | ||||
| @@ -177,7 +204,7 @@ class TagsRepo: | ||||
|                 words = json.dumps(item.get("words")) | ||||
|                 data_list.append( | ||||
|                     ( | ||||
|                         id, | ||||
|                         tag_id, | ||||
|                         name, | ||||
|                         post_count, | ||||
|                         category, | ||||
| @@ -191,15 +218,15 @@ class TagsRepo: | ||||
|             if len(data) < 1000: | ||||
|                 break | ||||
|  | ||||
|             # Calculate elapsed time and sleep if necessary to enforce the rate limit | ||||
|             elapsed = time.monotonic() - start_time | ||||
|             if elapsed < min_interval: | ||||
|                 time.sleep(min_interval - elapsed) | ||||
|  | ||||
|         print(f"Tagi - Pobrano {len(data_list)} tagów...") | ||||
|         if progress_queue: | ||||
|             progress_queue.put(("label", f"Pobrano {len(data_list)} tagów...")) | ||||
|  | ||||
|         data_list = sorted(data_list, key=lambda x: x[0]) | ||||
|         data_list = [(idx,) + row for idx, row in enumerate(data_list)] | ||||
|  | ||||
|         cursor.executemany( | ||||
|             """ | ||||
|             INSERT INTO tags ("index", id, name, post_count, category, created_at, updated_at, is_deprecated, words) | ||||
| @@ -210,26 +237,53 @@ class TagsRepo: | ||||
|         conn.commit() | ||||
|         data_list = [] | ||||
|  | ||||
|         page = 0 | ||||
|         # ----------------------- | ||||
|         # Fetch Tag Aliases | ||||
|         # ----------------------- | ||||
|         last_id = 0 | ||||
|         while True: | ||||
|             print(f"Aliasy tagów - Pobieranie od id {page}...") | ||||
|             if cancel_event and cancel_event.is_set(): | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put(("label", "Anulowano pobieranie aliasów tagów.")) | ||||
|                 conn.close() | ||||
|                 return | ||||
|  | ||||
|             if progress_queue: | ||||
|                 progress_queue.put( | ||||
|                     ("label", f"Pobieranie aliasów tagów (od ID {last_id})...") | ||||
|                 ) | ||||
|             start_time = time.monotonic() | ||||
|             url = f"https://danbooru.donmai.us/tag_aliases.json?limit=1000&only=id,antecedent_name,consequent_name&search[status]=active&page=a{page}" | ||||
|             url = ( | ||||
|                 f"https://danbooru.donmai.us/tag_aliases.json?limit=1000" | ||||
|                 f"&only=id,antecedent_name,consequent_name&search[status]=active&page=a{last_id}" | ||||
|             ) | ||||
|             response = requests.get(url) | ||||
|             if response.status_code != 200: | ||||
|                 print( | ||||
|                     f"Błąd przy pobieraniu strony {page}: HTTP {response.status_code}" | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put( | ||||
|                         ( | ||||
|                             "label", | ||||
|                             f"Błąd przy pobieraniu aliasów tagów od ID {last_id}: HTTP {response.status_code}", | ||||
|                         ) | ||||
|                     ) | ||||
|                 break | ||||
|  | ||||
|             data = response.json() | ||||
|             if not data: | ||||
|                 break | ||||
|             page = None | ||||
|  | ||||
|             last_id = None | ||||
|             for item in data: | ||||
|                 id = item.get("id") | ||||
|                 if not page: | ||||
|                     page = id | ||||
|                 if cancel_event and cancel_event.is_set(): | ||||
|                     if progress_queue: | ||||
|                         progress_queue.put( | ||||
|                             ("label", "Anulowano przetwarzanie aliasów tagów.") | ||||
|                         ) | ||||
|                     conn.close() | ||||
|                     return | ||||
|                 tag_id = item.get("id") | ||||
|                 if not last_id: | ||||
|                     last_id = tag_id | ||||
|                 antecedent = item.get("antecedent_name") | ||||
|                 consequent = item.get("consequent_name") | ||||
|                 data_list.append((antecedent, consequent)) | ||||
| @@ -237,15 +291,15 @@ class TagsRepo: | ||||
|             if len(data) < 1000: | ||||
|                 break | ||||
|  | ||||
|             # Calculate elapsed time and sleep if necessary to enforce the rate limit | ||||
|             elapsed = time.monotonic() - start_time | ||||
|             if elapsed < min_interval: | ||||
|                 time.sleep(min_interval - elapsed) | ||||
|  | ||||
|         print(f"Aliasy tagów - Pobrano {len(data_list)} aliasów tagów...") | ||||
|         if progress_queue: | ||||
|             progress_queue.put(("label", f"Pobrano {len(data_list)} aliasów tagów...")) | ||||
|  | ||||
|         data_list = sorted(data_list, key=lambda x: x[0]) | ||||
|         data_list = [(idx,) + row for idx, row in enumerate(data_list)] | ||||
|  | ||||
|         cursor.executemany( | ||||
|             """ | ||||
|             INSERT INTO tag_aliases ("index", alias, tag) | ||||
| @@ -256,19 +310,15 @@ class TagsRepo: | ||||
|         conn.commit() | ||||
|         data_list = [] | ||||
|  | ||||
|         # Pobranie tagów kategorii "character" (category = 4) | ||||
|         # ----------------------- | ||||
|         # Fetch Tag Categories for Implications | ||||
|         # ----------------------- | ||||
|         cursor.execute("SELECT name FROM tags WHERE category = 4") | ||||
|         character_tags = {row[0] for row in cursor.fetchall()} | ||||
|  | ||||
|         # Pobranie tagów kategorii "copyright" (category = 3) | ||||
|         cursor.execute("SELECT name FROM tags WHERE category = 3") | ||||
|         copyright_tags = {row[0] for row in cursor.fetchall()} | ||||
|  | ||||
|         # Pobranie tagów kategorii "meta" (category = 5) | ||||
|         cursor.execute("SELECT name FROM tags WHERE category = 5") | ||||
|         meta_tags = {row[0] for row in cursor.fetchall()} | ||||
|  | ||||
|         # Pobranie tagów kategorii "artist" (category = 1) | ||||
|         cursor.execute("SELECT name FROM tags WHERE category = 1") | ||||
|         artist_tags = {row[0] for row in cursor.fetchall()} | ||||
|  | ||||
| @@ -276,36 +326,60 @@ class TagsRepo: | ||||
|         cursor.execute("DELETE FROM tag_closure")  # Optional: reset table | ||||
|         conn.commit() | ||||
|  | ||||
|         # Budujemy strukturę implikacji: słownik, gdzie | ||||
|         # kluczem jest antecedent_name, a wartością zbiór consequent_name. | ||||
|         # ----------------------- | ||||
|         # Fetch Tag Implications | ||||
|         # ----------------------- | ||||
|         tag_dict = {} | ||||
|  | ||||
|         page = 0 | ||||
|         last_id = 0 | ||||
|         while True: | ||||
|             print(f"Implikacje tagów - Pobieranie od id {page}...") | ||||
|             url = f"https://danbooru.donmai.us/tag_implications.json?limit=1000&page=a{page}" | ||||
|             if cancel_event and cancel_event.is_set(): | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put( | ||||
|                         ("label", "Anulowano pobieranie implikacji tagów.") | ||||
|                     ) | ||||
|                 conn.close() | ||||
|                 return | ||||
|  | ||||
|             if progress_queue: | ||||
|                 progress_queue.put( | ||||
|                     ("label", f"Pobieranie implikacji tagów (od ID {last_id})...") | ||||
|                 ) | ||||
|             start_time = time.monotonic() | ||||
|             url = f"https://danbooru.donmai.us/tag_implications.json?limit=1000&page=a{last_id}" | ||||
|             response = requests.get(url) | ||||
|             if response.status_code != 200: | ||||
|                 print( | ||||
|                     f"Błąd przy pobieraniu strony {page}: HTTP {response.status_code}" | ||||
|                 if progress_queue: | ||||
|                     progress_queue.put( | ||||
|                         ( | ||||
|                             "label", | ||||
|                             f"Błąd przy pobieraniu implikacji tagów od ID {last_id}: HTTP {response.status_code}", | ||||
|                         ) | ||||
|                     ) | ||||
|                 break | ||||
|  | ||||
|             data = response.json() | ||||
|             if not data: | ||||
|                 break | ||||
|             page = None | ||||
|  | ||||
|             last_id = None | ||||
|             for item in data: | ||||
|                 id = item.get("id") | ||||
|                 if not page: | ||||
|                     page = id | ||||
|                 if cancel_event and cancel_event.is_set(): | ||||
|                     if progress_queue: | ||||
|                         progress_queue.put( | ||||
|                             ("label", "Anulowano przetwarzanie implikacji tagów.") | ||||
|                         ) | ||||
|                     conn.close() | ||||
|                     return | ||||
|                 tag_id = item.get("id") | ||||
|                 if not last_id: | ||||
|                     last_id = tag_id | ||||
|                 if item.get("status") != "active": | ||||
|                     continue | ||||
|  | ||||
|                 antecedent = item.get("antecedent_name") | ||||
|                 consequent = item.get("consequent_name") | ||||
|  | ||||
|                 # Dodanie prefiksu, jeżeli tag należy do jednej z kategorii | ||||
|                 # Prefix tags based on their category | ||||
|                 if antecedent in character_tags: | ||||
|                     antecedent = f"character:{antecedent}" | ||||
|                 elif antecedent in copyright_tags: | ||||
| @@ -330,13 +404,19 @@ class TagsRepo: | ||||
|  | ||||
|             if len(data) < 1000: | ||||
|                 break | ||||
|             # Calculate elapsed time and sleep if necessary to enforce the rate limit | ||||
|  | ||||
|             elapsed = time.monotonic() - start_time | ||||
|             if elapsed < min_interval: | ||||
|                 time.sleep(min_interval - elapsed) | ||||
|  | ||||
|         print(f"Implikacje tagów - Pobrano {len(tag_dict)} implikacji tagów...") | ||||
|         # Batch insert all unique pairs | ||||
|         if progress_queue: | ||||
|             progress_queue.put( | ||||
|                 ("label", f"Pobrano implikacje dla {len(tag_dict)} tagów...") | ||||
|             ) | ||||
|  | ||||
|         # ----------------------- | ||||
|         # Insert Tag Implications and Build Transitive Closure | ||||
|         # ----------------------- | ||||
|         for antecedent, consequents in tag_dict.items(): | ||||
|             for consequent in consequents: | ||||
|                 if antecedent != consequent: | ||||
| @@ -345,13 +425,18 @@ class TagsRepo: | ||||
|                         (antecedent, consequent), | ||||
|                     ) | ||||
|         conn.commit() | ||||
|  | ||||
|         closure_data = self.build_transitive_closure(tag_dict) | ||||
|         cursor.executemany( | ||||
|             "INSERT INTO tag_closure VALUES (?, ?, ?)", | ||||
|             self.build_transitive_closure(tag_dict), | ||||
|             closure_data, | ||||
|         ) | ||||
|         conn.commit() | ||||
|         conn.close() | ||||
|  | ||||
|         if progress_queue: | ||||
|             progress_queue.put(("label", "Regeneracja bazy zakończona.")) | ||||
|  | ||||
|     def build_transitive_closure(self, tag_dict): | ||||
|         closure = set() | ||||
|         for antecedent in tag_dict: | ||||
|   | ||||
| @@ -180,9 +180,16 @@ class TagManager(tk.Frame): | ||||
|     """ | ||||
|  | ||||
|     def __init__( | ||||
|         self, master, settings: Settings, tags_repo: TagsRepo, *args, **kwargs | ||||
|         self, | ||||
|         master, | ||||
|         settings: Settings, | ||||
|         tags_repo: TagsRepo, | ||||
|         tag_change_callback=None, | ||||
|         *args, | ||||
|         **kwargs, | ||||
|     ): | ||||
|         super().__init__(master, *args, **kwargs) | ||||
|         self.tag_change_callback = tag_change_callback | ||||
|         self.tags_repo = tags_repo | ||||
|         self.settings = settings | ||||
|         self.manual_tags = []  # List to hold manually entered tags | ||||
| @@ -234,6 +241,8 @@ class TagManager(tk.Frame): | ||||
|             self.tags_display.tag_bind(tag_name, "<Button-3>", self.open_tag_wiki_url) | ||||
|             self.tags_display.insert(tk.INSERT, " ") | ||||
|         self.tags_display.config(state=tk.DISABLED) | ||||
|         if self.tag_change_callback: | ||||
|             self.tag_change_callback() | ||||
|  | ||||
|     def remove_tag(self, event): | ||||
|         """Remove the clicked tag from the list and update the display.""" | ||||
|   | ||||
| @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" | ||||
|  | ||||
| [project] | ||||
| name = "kapitanbooru-uploader" | ||||
| version = "0.2.0" | ||||
| version = "0.3.0" | ||||
| description = "A GUI application for uploading images to KapitanBooru" | ||||
| authors = [ | ||||
|     {name = "Michał Leśniak", email = "kapitan@mlesniak.pl"} | ||||
|   | ||||
		Reference in New Issue
	
	Block a user