From 0aa853cea3d94e7d6e11245c9a43eebc000a3bd0 Mon Sep 17 00:00:00 2001 From: Kapitan Date: Mon, 26 Feb 2024 17:39:11 +0100 Subject: [PATCH] Initial commit --- .gitignore | 53 ++ .vscode/launch.json | 25 + LICENSE.txt | 21 + README.md | 3 + anime_easy_sync/__init__.py | 32 + anime_easy_sync/aes.py | 611 +++++++++++++++++++ anime_easy_sync/model.py | 366 +++++++++++ anime_easy_sync/oauth.py | 80 +++ anime_easy_sync/templates/_formhelpers.jinja | 12 + anime_easy_sync/templates/aes/index.jinja | 212 +++++++ anime_easy_sync/templates/aes/kitsu.jinja | 12 + anime_easy_sync/templates/base.jinja | 13 + instance/config.py | 5 + uwsgi.ini | 21 + 14 files changed, 1466 insertions(+) create mode 100644 .gitignore create mode 100644 .vscode/launch.json create mode 100644 LICENSE.txt create mode 100644 README.md create mode 100644 anime_easy_sync/__init__.py create mode 100644 anime_easy_sync/aes.py create mode 100644 anime_easy_sync/model.py create mode 100644 anime_easy_sync/oauth.py create mode 100644 anime_easy_sync/templates/_formhelpers.jinja create mode 100644 anime_easy_sync/templates/aes/index.jinja create mode 100644 anime_easy_sync/templates/aes/kitsu.jinja create mode 100644 anime_easy_sync/templates/base.jinja create mode 100644 instance/config.py create mode 100644 uwsgi.ini diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..49a20aa --- /dev/null +++ b/.gitignore @@ -0,0 +1,53 @@ +.DS_Store +.huskyrc.json +out +log.log +**/node_modules +*.pyc +*.vsix +envVars.txt +**/.vscode/.ropeproject/** +**/testFiles/**/.cache/** +*.noseids +.nyc_output +.vscode-test +__pycache__ +npm-debug.log +**/.mypy_cache/** +!yarn.lock +coverage/ +cucumber-report.json +**/.vscode-test/** +**/.vscode test/** +**/.vscode-smoke/** +**/.venv*/ +venv/ +port.txt +precommit.hook +pythonFiles/lib/** +pythonFiles/get-pip.py +debug_coverage*/** +languageServer/** +languageServer.*/** +bin/** +obj/** +.pytest_cache +tmp/** +.python-version +.vs/ +test-results*.xml +xunit-test-results.xml +build/ci/performance/performance-results.json +!build/ +debug*.log +debugpy*.log +pydevd*.log +nodeLanguageServer/** +nodeLanguageServer.*/** +dist/** +# translation files +*.xlf +package.nls.*.json +l10n/ +# config file +**/config.py diff --git a/.vscode/launch.json b/.vscode/launch.json new file mode 100644 index 0000000..8ba6aed --- /dev/null +++ b/.vscode/launch.json @@ -0,0 +1,25 @@ +{ + // Use IntelliSense to learn about possible attributes. + // Hover to view descriptions of existing attributes. + // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387 + "version": "0.2.0", + "configurations": [ + { + "name": "Python: Flask", + "type": "python", + "request": "launch", + "module": "flask", + "env": { + "FLASK_APP": "anime_easy_sync", + "FLASK_ENV": "development", + "FLASK_DEBUG": "0" + }, + "args": [ + "run", + "--no-debugger", + "--no-reload" + ], + "jinja": true + } + ] +} \ No newline at end of file diff --git a/LICENSE.txt b/LICENSE.txt new file mode 100644 index 0000000..7d521bf --- /dev/null +++ b/LICENSE.txt @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2024 Michał Leśniak + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/README.md b/README.md new file mode 100644 index 0000000..9036e08 --- /dev/null +++ b/README.md @@ -0,0 +1,3 @@ +# AnimeEasySync + +AnimeEasySync is a Flask application that synchronizes data between Anilist, MyAnimeList and Kitsu. diff --git a/anime_easy_sync/__init__.py b/anime_easy_sync/__init__.py new file mode 100644 index 0000000..6b24459 --- /dev/null +++ b/anime_easy_sync/__init__.py @@ -0,0 +1,32 @@ +"""Anime Easy Sync Flask server""" +import os + +from flask import Flask + + +def create_app(test_config=None): + """Creates and configures the app""" + # create and configure the app + app = Flask(__name__, instance_relative_config=True) + app.config.from_mapping( + SECRET_KEY=b'\x10\xc3\xf11\x9e\x17\x1f\xf0l\x84~A\x82{TP5Z\xe3\x8b\xbaa\x14\x03' + ) + + if test_config is None: + # load the instance config, if it exists, when not testing + app.config.from_pyfile('config.py', silent=True) + else: + # load the test config if passed in + app.config.from_mapping(test_config) + + # ensure the instance folder exists + try: + os.makedirs(app.instance_path) + except OSError: + pass + + from . import aes # pylint: disable=import-outside-toplevel + app.register_blueprint(aes.bp) + app.add_url_rule('/aes', endpoint='index') + + return app diff --git a/anime_easy_sync/aes.py b/anime_easy_sync/aes.py new file mode 100644 index 0000000..98ef623 --- /dev/null +++ b/anime_easy_sync/aes.py @@ -0,0 +1,611 @@ +"""AnimeEasySync module""" +from hashlib import sha256 +from random import choice +from string import ascii_lowercase, digits +from json import dumps, JSONEncoder, JSONDecoder +from base64 import b64decode, b64encode +from requests import Request +from requests.auth import HTTPBasicAuth +from flask import ( + Blueprint, render_template, request, session, redirect, url_for, current_app +) +from flask_wtf import FlaskForm +from wtforms import ( + StringField, PasswordField, SubmitField, HiddenField, SelectField +) +from wtforms.validators import DataRequired +from .model import ( + read_mal, read_anilist, read_kitsu, get_kitsu_user_id, UPDATE_FUNCS +) +from .oauth import get_oauth_token, MAL_TOKEN_URL, ANILIST_TOKEN_URL, KITSU_TOKEN_URL + +bp = Blueprint('aes', __name__, url_prefix='/aes') + + +MAL_AUTHORIZE_URL = 'https://myanimelist.net/v1/oauth2/authorize' +ANILIST_AUTHORIZE_URL = 'https://anilist.co/api/v2/oauth/authorize' + +BROKEN_MAL = ['35459', '27821', '31561', '38360', '36480', '40807'] + +BROKEN_KITSU = ['42279', '42090', '41963'] + +BROKEN_ANILIST = ['16932', '106169'] + +KITSU_MAL_FIX = { + '13298': '37897', + '42196': '39551', + '42867': '40852', + '13066': '34662', + '12770': '34451', + '8791': '21495' +} + +KITSU_ANILIST_FIX = { + '41963': '104243', + '12770': '97886', + '8791': '102471', + '43142': '117756' +} + + +def generate_pkce(): + """Generate random string""" + letters = ascii_lowercase + digits + return ''.join(choice(letters) for i in range(100)) + + +def hash_dict(dic): + """Hash dict""" + json = dumps(dic) + return sha256(json.encode()).hexdigest() + + +def at_least(count, iterable): + """Return True if bool(x) is True for at least `count` values x in the `iterable`.""" + i = 0 + for elem in iterable: + if elem: + i += 1 + if i == count: + return True + return False + + +def get_tokens(state): + """Return dict of tokens""" + ret = {} + for key in state: + if '_token' in key: + ret[key.rstrip('_token')] = state[key] + return ret + + +@bp.route('/', methods=('GET', 'POST')) +def index(): + """AnimeEasySync index page""" + if 'state' in session: + state = session['state'] + else: + state = dict() + session['pkce'] = generate_pkce() + can_sync = at_least(2, (bool(state.get(x)) + for x in ['mal_token', 'anilist_token', 'kitsu_token'])) + form = SyncForm() + if form.validate_on_submit() and can_sync and form.disc.data: + try: + disc_data = b64decode( + form.disc.data.encode('ascii')).decode('utf-8') + disc_lst = JSONDecoder( + object_hook=from_json).decode(disc_data) + tokens = { + 'mal': state.get('mal_token'), + 'anilist': state.get('anilist_token'), + 'kitsu': state.get('kitsu_token') + } + source = '' + if form.mal.data: + source = 'mal' + elif form.anilist.data: + source = 'anilist' + elif form.kitsu.data: + source = 'kitsu' + if source and tokens[source]: + target_tokens_dict = {k: tokens[k] + for k in tokens if k != source} + sync_anime(disc_lst, source, target_tokens_dict) + except Exception as err: + print(err) + + discrepancies = DiscrepancyCollection() + disc = b'' + + if can_sync: + if 'kitsu_user_id' in state: + kitsu_user_id = state['kitsu_user_id'] + elif 'kitsu_token' in state: + kitsu_user_id = get_kitsu_user_id( + state['kitsu_token']) + state['kitsu_user_id'] = kitsu_user_id + mal_entries = read_mal( + state['mal_token']) if 'mal_token' in state else [] + anilist_entries = read_anilist( + state['anilist_token']) if 'anilist_token' in state else [] + kitsu_entries = read_kitsu( + state['kitsu_token'], kitsu_user_id) if 'kitsu_token' in state else [] + discrepancies = get_discrepancies( + mal_entries, anilist_entries, kitsu_entries) + disc = b64encode( + dumps(discrepancies.get(False), cls=AESEncoder).encode('utf-8')) + + for token_name in ['mal_token', 'anilist_token', 'kitsu_token']: + if token_name in state and not state.get(token_name): + del state[token_name] + + session['state'] = state + + mal_params = { + 'response_type': 'code', + 'client_id': current_app.config['MAL_CLIENT_ID'], + 'state': hash_dict(state), + 'redirect_uri': url_for('aes.mal_callback', _external=True), + 'code_challenge': session['pkce'], + 'code_challenge_method': 'plain' + } + mal_url = Request('GET', MAL_AUTHORIZE_URL, + params=mal_params).prepare().url + + anilist_params = { + 'response_type': 'code', + 'client_id': current_app.config['ANILIST_CLIENT_ID'], + 'redirect_uri': url_for('aes.anilist_callback', _external=True) + } + anilist_url = Request('GET', ANILIST_AUTHORIZE_URL, + params=anilist_params).prepare().url + + modals_lst = [] + if 'mal_token' in state: + modals_lst.append('MAL') + if 'anilist_token' in state: + modals_lst.append('Anilist') + if 'kitsu_token' in state: + modals_lst.append('Kitsu') + modals = read_modals(discrepancies, modals_lst) + form.disc.data = disc.decode('ascii') + + return render_template('aes/index.jinja', state=state, mal_url=mal_url, + anilist_url=anilist_url, discrepancies=discrepancies.get(), + modals=modals, form=form) + + +@bp.route('/callback/anilist') +def anilist_callback(): + """AnimeEasySync Anilist callback""" + code = request.args.get('code', default=None, type=str) + state = dict() + if 'state' in session: + state = session['state'] + if code: + refresh_token = None + if 'anilist_token' in state: + refresh_token = state['anilist_token'].get('refresh_token') + token = get_oauth_token(ANILIST_TOKEN_URL, 'authorization_code', refresh_token, + client_id=current_app.config['ANILIST_CLIENT_ID'], + client_secret=current_app.config['ANILIST_CLIENT_SECRET'], + redirect_uri=url_for( + 'aes.anilist_callback', _external=True), + code=code) + if token: + state['anilist_token'] = token + session['state'] = state + return redirect(url_for('index')) + + +@bp.route('/callback/mal') +def mal_callback(): + """AnimeEasySync MAL callback""" + code = request.args.get('code', default=None, type=str) + state_hash = request.args.get('state', default=None, type=str) + state = dict() + if 'state' in session: + state = session['state'] + if code and state_hash and state_hash == hash_dict(state): + pkce = '' + if 'pkce' in session: + pkce = session['pkce'] + auth = HTTPBasicAuth(current_app.config['MAL_CLIENT_ID'], + current_app.config['MAL_CLIENT_SECRET']) + refresh_token = None + if 'mal_token' in state: + refresh_token = state['mal_token'].get('refresh_token') + token = get_oauth_token(MAL_TOKEN_URL, 'authorization_code', refresh_token, auth=auth, + client_id=current_app.config['MAL_CLIENT_ID'], code=code, + redirect_uri=url_for( + 'aes.mal_callback', _external=True), + code_verifier=pkce) + if token: + state['mal_token'] = token + session['state'] = state + return redirect(url_for('index')) + + +@bp.route('/kitsu', methods=['GET', 'POST']) +def kitsu(): + """Login page for Kitsu""" + state = {} + if 'state' in session: + state = session['state'] + if 'kitsu_token' in state: + return redirect(url_for('index')) + form = KitsuForm() + if form.validate_on_submit(): + token = get_oauth_token(KITSU_TOKEN_URL, 'password', None, + username=form.username.data, password=form.password.data) + if token: + state['kitsu_token'] = token + session['state'] = state + if 'kitsu_token' in state: + return redirect(url_for('index')) + return render_template('aes/kitsu.jinja', form=form) + + +def sync_valid(src, trgt): + """Check if entries are valid to sync""" + return src != trgt and is_valid_id(src.anime_id) and is_valid_id(trgt.anime_id) + + +def get_update_list(disc_lst, source, target): + """Gets list of update parameters""" + ret = [] + for dis in [ent for ent in disc_lst if sync_valid(ent.entries[source], ent.entries[target])]: + kwargs = {} + if dis.entries[source].score != dis.entries[target].score: + kwargs['score'] = dis.entries[source].score + if dis.entries[source].status != dis.entries[target].status: + kwargs['status'] = dis.entries[source].status + if 'completed' not in [dis.entries[source].status, dis.entries[target].status] and \ + dis.entries[source].progress != dis.entries[target].progress: + kwargs['progress'] = dis.entries[source].progress + anime_id = dis.entries[target].library_entry_id if hasattr( + dis.entries[target], 'library_entry_id') else dis.entries[target].anime_id + ret.append((anime_id, kwargs)) + return ret + + +def get_update_func(target): + """Gets an update func""" + return UPDATE_FUNCS.get(target) + + +def sync_anime(discrepancies, source, target_tokens_dict): + """Sync anime to source""" + for target in target_tokens_dict: + update = get_update_func(target) + if update: + for anime_id, kwargs in get_update_list(discrepancies, source, target): + update(target_tokens_dict[target], anime_id, **kwargs) + + +class KitsuForm(FlaskForm): + """Kitsu login form""" + username = StringField('Email/Username', validators=[DataRequired()]) + password = PasswordField('Password', validators=[DataRequired()]) + submit = SubmitField('Login') + + +class SyncForm(FlaskForm): + """Sync form""" + disc = HiddenField('disc', validators=[DataRequired()]) + mal = SubmitField('Sync to MAL values') + anilist = SubmitField('Sync to Anilist values') + kitsu = SubmitField('Sync to Kitsu values') + + +def from_json(json_object): + """AnimeEasySync JSON decoder""" + if isinstance(json_object, dict) and '__type__' in json_object: + obj_type = json_object['__type__'] + if obj_type in globals(): + obj_class = globals()[obj_type] + if obj_type == 'Anime': + ret = obj_class(from_json(json_object['anime_id'])) + elif obj_type == 'KitsuAnime': + ret = obj_class(from_json(json_object['library_entry_id']), from_json(json_object['anime_id'])) + else: + ret = obj_class() + for key in ret.__dict__: + setattr(ret, key, from_json(json_object[key])) + return ret + return json_object + + +class AESEncoder(JSONEncoder): + """AnimeEasySync JSON encoder""" + + def default(self, o): + if isinstance(o, (Anime, Discrepancy, DiscrepancyCollection, KitsuAnime)): + ret = {'__type__': o.__class__.__name__} + for key in o.__dict__: + if isinstance(o.__dict__[key], list): + lst = [] + for elem in o.__dict__[key]: + lst.append(self.default(elem)) + ret[key] = lst + else: + ret[key] = self.default(o.__dict__[key]) + return ret + if isinstance(o, (int, str, float, dict)): + return o + if isinstance(o, list): + ret = [] + for elem in o: + ret.append(self.default(elem)) + return ret + print(o) + return super().default(o) + + +class Anime: + """Class representing Anime entry""" + + def __init__(self, anime_id): + self.anime_id = anime_id + self.title = '' + self.broken = False + self.progress = 'none' + self.score = 0 + self.status = 'none' + self.link = '' + + def fill(self, entry, broken): + """Fill fields with data""" + self.progress = entry['progress'] + self.score = entry['score'] + self.status = entry['status'] + self.title = entry['title'] + self.link = entry['link'] + self.broken = broken + + def __eq__(self, other): + check_progress = self.progress == other.progress + check_status = self.status == other.status + check_score = self.score == other.score + all_completed = self.status == 'completed' and check_status + if not (check_progress or all_completed): + return False + if not check_score: + return False + if not check_status: + return False + return True + + +class KitsuAnime(Anime): + """Class representing Kitsu Anime entry""" + + def __init__(self, library_entry_id, *args): + super().__init__(*args) + self.library_entry_id = library_entry_id + + +def is_valid_id(anime_id): + """Check if anime id is valid""" + return anime_id and int(anime_id) > 0 + + +class Discrepancy: + """Class representing a discrepancy""" + + def __init__(self): + self.entries = {} + + def fill_mal(self, mal_entry): + """Fill fields from MAL entry""" + entry = Anime(mal_entry['mal_id']) + entry.fill(mal_entry, mal_entry['mal_id'] in BROKEN_MAL) + self.entries['mal'] = entry + + def fill_anilist(self, anilist_entry): + """Fill fields from Anilist entry""" + mal_entry = self.entries.get('mal') + if not mal_entry and anilist_entry['mal_id']: + mal_entry = Anime(anilist_entry['mal_id']) + self.entries['mal'] = mal_entry + entry = Anime(anilist_entry['anilist_id']) + entry.fill(anilist_entry, + anilist_entry['anilist_id'] in BROKEN_ANILIST) + self.entries['anilist'] = entry + + def fill_kitsu(self, kitsu_entry): + """Fill fields from Kitsu entry""" + mal_entry = self.entries.get('mal') + if not mal_entry and kitsu_entry['mal_id']: + mal_entry = Anime(kitsu_entry['mal_id']) + self.entries['mal'] = mal_entry + anilist_entry = self.entries.get('anilist') + if not anilist_entry and kitsu_entry['anilist_id']: + anilist_entry = Anime(kitsu_entry['anilist_id']) + self.entries['anilist'] = anilist_entry + entry = KitsuAnime(kitsu_entry['library_entry_id'], + kitsu_entry['kitsu_id']) + entry.fill(kitsu_entry, kitsu_entry['kitsu_id'] in BROKEN_KITSU) + self.entries['kitsu'] = entry + + def is_discrepancy(self): + """Check if there is a discrepancy""" + if any(not is_valid_id(x.anime_id) for x in self.entries.values()): + return True + if not self.entries.values(): + return False + return any(x != list(self.entries.values())[0] for x in self.entries.values()) + + def is_broken(self): + """Gets broken status""" + return any(x.broken for x in self.entries.values()) + + def get_title(self): + """Gets anime title""" + return next((x.title for x in self.entries.values() if x.title), '') + + +class DiscrepancyCollection: + """Class representing a collection of Discrepancies""" + + def __init__(self): + self.collection = [] + self.kitsu = {} + self.anilist = {} + self.mal = {} + + def update_mal(self, mal_entry): + """Add or update a MAL entry to collection""" + if mal_entry['mal_id'] in KITSU_MAL_FIX.values(): + mal_entry['kitsu_id'] = list(KITSU_MAL_FIX.keys())[list( + KITSU_MAL_FIX.values()).index(mal_entry['mal_id'])] + if mal_entry['mal_id'] in self.mal: + entry = self.collection[self.mal[mal_entry['mal_id']]] + else: + entry = Discrepancy() + self.collection.append(entry) + self.mal[mal_entry['mal_id']] = len(self.collection) - 1 + entry.fill_mal(mal_entry) + + def update_anilist(self, anilist_entry): + """Add or update a Anilist entry to collection""" + if anilist_entry['anilist_id'] in self.anilist: + entry = self.collection[self.anilist[anilist_entry['anilist_id']]] + elif anilist_entry['mal_id'] and anilist_entry['mal_id'] in self.mal: + entry = self.collection[self.mal[anilist_entry['mal_id']]] + self.anilist[anilist_entry['anilist_id'] + ] = self.mal[anilist_entry['mal_id']] + else: + entry = Discrepancy() + self.collection.append(entry) + if anilist_entry['mal_id']: + self.mal[anilist_entry['mal_id']] = len(self.collection) - 1 + self.anilist[anilist_entry['anilist_id']] = len( + self.collection) - 1 + entry.fill_anilist(anilist_entry) + + def update_kitsu(self, kitsu_entry): + """Add or update a Kitsu entry to collection""" + # WORKAROUND: fix broken ids + if kitsu_entry['kitsu_id'] in KITSU_ANILIST_FIX: + kitsu_entry['anilist_id'] = KITSU_ANILIST_FIX[kitsu_entry['kitsu_id']] + if kitsu_entry['kitsu_id'] in KITSU_MAL_FIX: + kitsu_entry['mal_id'] = KITSU_MAL_FIX[kitsu_entry['kitsu_id']] + + if kitsu_entry['kitsu_id'] in self.kitsu: + entry = self.collection[self.anilist[kitsu_entry['kitsu_id']]] + if kitsu_entry['anilist_id'] in self.anilist: + entry = self.collection[self.anilist[kitsu_entry['anilist_id']]] + self.kitsu[kitsu_entry['kitsu_id'] + ] = self.anilist[kitsu_entry['anilist_id']] + if kitsu_entry['mal_id'] and kitsu_entry['mal_id'] not in self.mal: + self.mal[kitsu_entry['mal_id'] + ] = self.anilist[kitsu_entry['anilist_id']] + elif kitsu_entry['mal_id'] and kitsu_entry['mal_id'] in self.mal: + entry = self.collection[self.mal[kitsu_entry['mal_id']]] + self.kitsu[kitsu_entry['kitsu_id'] + ] = self.mal[kitsu_entry['mal_id']] + if kitsu_entry['anilist_id']: + self.anilist[kitsu_entry['anilist_id'] + ] = self.mal[kitsu_entry['mal_id']] + else: + entry = Discrepancy() + self.collection.append(entry) + if kitsu_entry['mal_id']: + self.mal[kitsu_entry['mal_id']] = len(self.collection) - 1 + if kitsu_entry['anilist_id']: + self.anilist[kitsu_entry['anilist_id']] = len( + self.collection) - 1 + self.kitsu[kitsu_entry['kitsu_id']] = len(self.collection) - 1 + entry.fill_kitsu(kitsu_entry) + + def get(self, with_broken=True): + """Get entries that are broken or with a discrepancy""" + if with_broken: + return [x for x in self.collection if x.is_discrepancy() or x.is_broken()] + return [x for x in self.collection if x.is_discrepancy() and not x.is_broken()] + + +def get_discrepancies(mal_entries, anilist_entries, kitsu_entries): + """Get discrepancies between entries""" + discrepancies = DiscrepancyCollection() + for entry in mal_entries: + discrepancies.update_mal(entry) + for entry in anilist_entries: + discrepancies.update_anilist(entry) + for entry in kitsu_entries: + discrepancies.update_kitsu(entry) + return discrepancies + + +def read_modals(discrepancies, lst): + """Get modal info""" + ret = [] + disc = discrepancies.get(False) + for target in lst: + src_lst = list(lst) + src_lst.remove(target) + ret.append({ + 'type': target, + 'msg': changes_syncing(disc, target, *src_lst) + }) + return ret + + +def changes_syncing_to_mal(lst): + """Get changes to lists while syncing to MAL""" + return changes_syncing(lst, 'MAL', 'Anilist', 'Kitsu') + + +def changes_syncing(lst, target, *args): + """Get changes to lists while syncinc to target""" + ret = '' + trgt_ent = target.lower() + for src in args: + src_ent = src.lower() + broken = [ent for ent in lst if src_ent not in ent.entries] + if broken: + print('broken entries detected:') + for ent in broken: + print(dumps(ent, cls=AESEncoder)) + lst.remove(ent) + + entries = [ent for ent in lst if ent.entries[trgt_ent] + != ent.entries[src_ent]] + if entries: + ret += f'

Changes to {src}:

' + return ret + + +def changes_syncing_to_anilist(lst): + """Get changes to lists while syncing to Anilist""" + return changes_syncing(lst, 'Anilist', 'MAL', 'Kitsu') + + +def changes_syncing_to_kitsu(lst): + """Get changes to lists while syncing to Kitsu""" + return changes_syncing(lst, 'Kitsu', 'MAL', 'Anilist') + + +def get_changes(changes, src, trgt): + """Get changes between 2 entries""" + if not sync_valid(src, trgt): + return + if src.score != trgt.score: + changes.append(f'
  • Score: {src.score} -> {trgt.score}
    • ') + if src.status != trgt.status: + changes.append(f'
  • Status: {src.status} -> {trgt.status}
    • ') + if src.progress != trgt.progress and (src.status != 'completed' or trgt.status != 'completed'): + changes.append( + f'
  • Progress: {src.progress} -> {trgt.progress}
    • ') diff --git a/anime_easy_sync/model.py b/anime_easy_sync/model.py new file mode 100644 index 0000000..026e166 --- /dev/null +++ b/anime_easy_sync/model.py @@ -0,0 +1,366 @@ +"""AnimeEasySync model module""" +from base64 import b64decode +from json import loads +from time import sleep +import requests +from .oauth import refresh_mal_token, refresh_anilist_token, refresh_kitsu_token + + +ANILIST_TO_STATUS = { + 'CURRENT': 'watching', + 'COMPLETED': 'completed', + 'PAUSED': 'on_hold', + 'DROPPED': 'dropped', + 'PLANNING': 'plan_to_watch', + 'REPEATING': 'watching' +} + +KITSU_TO_STATUS = { + 'current': 'watching', + 'completed': 'completed', + 'on_hold': 'on_hold', + 'dropped': 'dropped', + 'planned': 'plan_to_watch' +} + +MAL_API_URL = 'https://api.myanimelist.net/v2' +KITSU_API_URL = 'https://kitsu.io/api/edge' +ANILIST_GRAPHQL_URL = 'https://graphql.anilist.co' + + +def get_userid_from_token(token): + """Get userId from Anilist token""" + payload = token['access_token'].split( + '.')[1].replace('-', '+').replace('_', '/') + payload = payload + '='*(len(payload) % 4) + return int(loads(b64decode(payload))['sub']) + + +class BearerAuth(requests.auth.AuthBase): + """Bearer Auth header""" + + def __init__(self, token): + self.token = token['access_token'] + + def __call__(self, r): + r.headers["authorization"] = "Bearer " + self.token + return r + + +def mal_to_entry(mal_entry): + """Map MAL entry to AES entry""" + return { + 'mal_id': str(mal_entry['node']['id']), + 'title': mal_entry['node']['title'], + 'score': mal_entry['list_status']['score'], + 'status': mal_entry['list_status']['status'], + 'progress': mal_entry['list_status']['num_episodes_watched'], + 'link': f'https://myanimelist.net/anime/{mal_entry["node"]["id"]}' + } + + +def read_mal(token): + """Read MAL entries""" + entries = [] + url = MAL_API_URL+'/users/@me/animelist?fields=list_status&limit=1000&nsfw=true' + auth = BearerAuth(token) + while True: + response = requests.get(url=url, auth=auth) + if response.status_code == requests.codes.unauthorized and token: # pylint: disable=no-member + refresh_mal_token(token) + if not token: + return entries + auth = BearerAuth(token) + continue + mal = response.json().get('data') + if mal is None: + print('read_mal exception') + print(response.json()) + return entries + for mal_entry in mal: + entries.append(mal_to_entry(mal_entry)) + paging = response.json().get('paging') + if paging: + url = paging.get('next') + if url: + continue + break + return entries + + +def anilist_to_entry(anilist_entry): + """Map Anilist entry to AES entry""" + return { + 'anilist_id': str(anilist_entry['media']['id']), + 'mal_id': str(anilist_entry['media']['idMal']) if anilist_entry['media']['idMal'] else '', + 'title': f"{anilist_entry['media']['title']['english']} " + + f"({anilist_entry['media']['title']['romaji']})", + 'score': anilist_entry['score'], + 'status': ANILIST_TO_STATUS[anilist_entry['status']], + 'progress': anilist_entry['progress'], + 'link': anilist_entry['media']['siteUrl'] + } + + +def read_anilist(token): + """Read Anilist entries""" + entries = [] + auth = BearerAuth(token) + anilist_query = ''' +query ($chunk: Int, $userId: Int) { + MediaListCollection(userId: $userId, type: ANIME, chunk: $chunk, perChunk: 500) { + hasNextChunk + lists { + name + isCustomList + isSplitCompletedList + status + entries { + status + score(format: POINT_10) + progress + media { + id + idMal + title { + english + romaji + } + siteUrl + } + } + } + } +} +''' + chunk = 1 + while True: + variables = { + 'chunk': chunk, + 'userId': get_userid_from_token(token) + } + response = requests.post( + ANILIST_GRAPHQL_URL, json={ + 'query': anilist_query, 'variables': variables}, + auth=auth) + if response.status_code == requests.codes.unauthorized and token: # pylint: disable=no-member + refresh_anilist_token(token) + if not token: + return entries + auth = BearerAuth(token) + continue + anilist = response.json().get('data') + if anilist is None: + print('read_anilist exception') + print(response.json()) + return entries + for lst in anilist['MediaListCollection']['lists']: + for anilist_entry in lst['entries']: + entries.append(anilist_to_entry(anilist_entry)) + if anilist['MediaListCollection']['hasNextChunk']: + chunk += 1 + else: + break + return entries + + +def get_kitsu_user_id(token): + """Get user's Kitsu user id""" + url = KITSU_API_URL+'/users?filter[self]=true&fields[users]=name' + auth = BearerAuth(token) + while True: + response = requests.get(url, auth=auth) + if response.ok: + user_id = response.json()['data'][0]['id'] + return user_id + if response.status_code == requests.codes.unauthorized and token: # pylint: disable=no-member + refresh_kitsu_token(token) + if not token: + return None + auth = BearerAuth(token) + continue + print('get_kitsu_user_id exception') + print(response.json()) + return None + + +def kitsu_to_entry(kitsu_entry, kitsu_anime, kitsu_mappings, kitsu_library_entry_id): + """Map Kitsu entry to AES entry""" + mal_id = next((str(x['attributes']['externalId']) + for x in kitsu_mappings + if x['attributes']['externalSite'] == 'myanimelist/anime'), '') + anilist_id = next((str(x['attributes']['externalId']) + for x in kitsu_mappings + if x['attributes']['externalSite'] == 'anilist/anime'), '') + if not anilist_id: + anilist_id = next((str(x['attributes']['externalId'][6:] + if x['attributes']['externalId'][:6] == 'anime/' + else x['attributes']['externalId']) + if x['attributes']['externalId'] else 0 + for x in kitsu_mappings + if x['attributes']['externalSite'] == 'anilist'), '') + return { + 'kitsu_id': str(kitsu_anime['id']), + 'mal_id': str(mal_id) if mal_id else '', + 'anilist_id': str(anilist_id) if anilist_id else '', + 'title': kitsu_anime['attributes']['canonicalTitle'], + 'score': (int(kitsu_entry['ratingTwenty']) if kitsu_entry['ratingTwenty'] else 0)//2, + 'status': KITSU_TO_STATUS[kitsu_entry['status']], + 'progress': kitsu_entry['progress'], + 'library_entry_id': kitsu_library_entry_id, + 'link': f'https://kitsu.io/anime/{kitsu_anime["id"]}' + } + + +def read_kitsu(token, user_id): + """Read Kitsu entries""" + entries = [] + url = KITSU_API_URL+f"/library-entries?filter[user_id]={user_id}" + \ + "&filter[kind]=anime&include=anime,anime.mappings" + \ + "&fields[anime]=canonicalTitle,episodeCount,slug,mappings" + \ + "&fields[mappings]=externalSite,externalId&page[limit]=500&page[offset]=0" + auth = BearerAuth(token) + retries = 2 + while True: + response = requests.get(url, auth=auth) + if response.status_code == requests.codes.unauthorized and token: # pylint: disable=no-member + refresh_kitsu_token(token) + if not token: + return entries + auth = BearerAuth(token) + continue + if response.status_code == requests.codes.server_error and retries > 0: # pylint: disable=no-member + retries -= 1 + sleep(1) + continue + kitsu = response.json().get('data') + if kitsu is None: + print('read_kitsu exception') + print(response.json()) + return entries + anime = [x for x in response.json()['included'] + if x['type'] == 'anime'] + mappings = [x for x in response.json()['included'] + if x['type'] == 'mappings'] + for data in kitsu: + kitsu_anime = next( + y for y in anime if y['id'] == data['relationships']['anime']['data']['id']) + kitsu_mappings = [] + for anime_entry in kitsu_anime['relationships']['mappings']['data']: + kitsu_mappings.append( + next(z for z in mappings if anime_entry['id'] == z['id'])) + entries.append(kitsu_to_entry( + data['attributes'], kitsu_anime, kitsu_mappings, data['id'])) + links = response.json().get('links') + if links: + url = links.get('next') + if url: + retries = 2 + continue + break + return entries + + +def update_mal(token, mal_id, status=None, score=None, progress=None): + """Update a MAL anime entry on user's list""" + auth = BearerAuth(token) + url = MAL_API_URL+f'/anime/{mal_id}/my_list_status' + data = {} + if status: + data['status'] = status + if score is not None: + data['score'] = score + if progress is not None: + data['num_watched_episodes'] = progress + + while True: + response = requests.put(url=url, data=data, auth=auth) + if response.status_code == requests.codes.unauthorized: # pylint: disable=no-member + refresh_mal_token(token) + if not token: + return False + auth = BearerAuth(token) + continue + if not response.ok: + print(response.json()) + return False + return True + + +def update_kitsu(token, kitsu_library_entry_id, status=None, score=None, progress=None): + """Update a Kitsu anime entry on user's list""" + auth = BearerAuth(token) + headers = {'Content-Type': 'application/vnd.api+json'} + url = KITSU_API_URL+f'/library-entries/{kitsu_library_entry_id}' + data = { + 'data': { + 'type': 'libraryEntries', + 'id': str(kitsu_library_entry_id), + 'attributes': {} + } + } + if status is not None: + data['data']['attributes']['status'] = list(KITSU_TO_STATUS.keys())[ + list(KITSU_TO_STATUS.values()).index(status)] + if score is not None: + data['data']['attributes']['ratingTwenty'] = int(score*2) + if progress is not None: + data['data']['attributes']['progress'] = progress + while True: + response = requests.patch( + url=url, headers=headers, auth=auth, json=data) + if response.status_code == requests.codes.unauthorized: # pylint: disable=no-member + refresh_kitsu_token(token) + if not token: + return False + auth = BearerAuth(token) + continue + if not response.ok: + print(response.json()) + return False + return True + + +def update_anilist(token, anilist_id, status=None, score=None, progress=None): + """Add or update an Anilist anime entry on user's list""" + auth = BearerAuth(token) + anilist_query = ''' +mutation ($id: Int, $status: MediaListStatus, $score: Float, $progress: Int) { + SaveMediaListEntry(id: $id, status: $status, score: $score, progress: $progress) { + id + status + score(format: POINT_10) + progress + } +} +''' + variables = { + 'id': int(anilist_id) + } + if status is not None: + variables['status'] = list(ANILIST_TO_STATUS.keys())[ + list(ANILIST_TO_STATUS.values()).index(status)] + if score is not None: + variables['score'] = float(score) + if progress is not None: + variables['progress'] = int(progress) + data = {'query': anilist_query, 'variables': variables} + while True: + response = requests.post(url=ANILIST_GRAPHQL_URL, auth=auth, json=data) + if response.status_code == requests.codes.unauthorized: # pylint: disable=no-member + refresh_anilist_token(token) + if not token: + return False + auth = BearerAuth(token) + continue + if not response.ok: + print(response.json()) + return False + return True + + +UPDATE_FUNCS = { + 'mal': update_mal, + 'anilist': update_anilist, + 'kitsu': update_kitsu +} diff --git a/anime_easy_sync/oauth.py b/anime_easy_sync/oauth.py new file mode 100644 index 0000000..6343b90 --- /dev/null +++ b/anime_easy_sync/oauth.py @@ -0,0 +1,80 @@ +"""AnimeEasySync OAuth module""" +from requests import post, codes +from requests.auth import HTTPBasicAuth +from flask import current_app + +MAL_TOKEN_URL = 'https://myanimelist.net/v1/oauth2/token' +ANILIST_TOKEN_URL = 'https://anilist.co/api/v2/oauth/token' +KITSU_TOKEN_URL = 'https://kitsu.io/api/oauth/token' + +OAUTH_PARAMS = { + 'authorization_code': { + 'required': ['client_id', 'code', 'redirect_uri'], + 'optional': ['code_verifier', 'client_secret'] + }, + 'password': { + 'required': ['username', 'password'], + 'optional': [] + }, + 'refresh_token': { + 'required': [], + 'optional': [] + } +} + + +def get_oauth_token(url, grant_type, refresh_token, **kwargs): + """Gets OAuth Token""" + print(f'url:{url}, grant_type:{grant_type}, refresh_token:{refresh_token}, kwargs:{kwargs}') + if not url or grant_type not in OAUTH_PARAMS or None in kwargs.values(): + return None + data = {'grant_type': grant_type} + if refresh_token and grant_type == 'refresh_token': + data['refresh_token'] = refresh_token + refresh_token = None + for req in OAUTH_PARAMS[grant_type]['required']: + if req not in kwargs: + return None + data[req] = kwargs[req] + for opt in OAUTH_PARAMS[grant_type]['optional']: + if opt in kwargs: + data[opt] = kwargs[opt] + request_kwargs = {} + if 'auth' in kwargs: + request_kwargs['auth'] = kwargs['auth'] + print(f'data:{data}, request_kwargs:{request_kwargs}') + response = post(url=url, data=data, **request_kwargs) + ret = None + if response.status_code == codes.ok: # pylint: disable=no-member + ret = response.json() + elif response.status_code == codes.unauthorized and refresh_token: # pylint: disable=no-member + ret = get_oauth_token(url, 'refresh_token', refresh_token, **kwargs) + else: + print(response.json()) + return ret + + +def refresh_oauth_token(token, url, **kwargs): + """Refresh token""" + refresh_token = token['refresh_token'] + new_token = get_oauth_token(url, 'refresh_token', refresh_token, **kwargs) + token.clear() + if new_token: + token.update(new_token) + + +def refresh_mal_token(token): + """Refresh MAL token""" + mal_auth = HTTPBasicAuth( + current_app.config['MAL_CLIENT_ID'], current_app.config['MAL_CLIENT_SECRET']) + refresh_oauth_token(token, MAL_TOKEN_URL, auth=mal_auth) + + +def refresh_anilist_token(token): + """Refresh Anilist token""" + refresh_oauth_token(token, ANILIST_TOKEN_URL) + + +def refresh_kitsu_token(token): + """Refresh Kitsu token""" + refresh_oauth_token(token, KITSU_TOKEN_URL) diff --git a/anime_easy_sync/templates/_formhelpers.jinja b/anime_easy_sync/templates/_formhelpers.jinja new file mode 100644 index 0000000..5790894 --- /dev/null +++ b/anime_easy_sync/templates/_formhelpers.jinja @@ -0,0 +1,12 @@ +{% macro render_field(field) %} +
    {{ field.label }} +
    {{ field(**kwargs)|safe }} + {% if field.errors %} + + {% endif %} +
    +{% endmacro %} \ No newline at end of file diff --git a/anime_easy_sync/templates/aes/index.jinja b/anime_easy_sync/templates/aes/index.jinja new file mode 100644 index 0000000..5dd7802 --- /dev/null +++ b/anime_easy_sync/templates/aes/index.jinja @@ -0,0 +1,212 @@ +{% extends "base.jinja" %} +{% block head %} + +{% endblock %} +{% block content %} +Log in to MAL +Log in to Anilist +Log in to Kitsu +
    +{% if state %} +{% if 'mal_token' in state %} + +{% endif %} +{% if 'anilist_token' in state %} + +{% endif %} +{% if 'kitsu_token' in state %} + +{% endif %} +{% endif %} + + + {% set sites = 0 %} + {% if state %} + + + {% if 'mal_token' in state %} + {% set sites = sites + 1 %} + + {% endif %} + {% if 'anilist_token' in state %} + {% set sites = sites + 1 %} + + {% endif %} + {% if 'kitsu_token' in state %} + {% set sites = sites + 1 %} + + {% endif %} + + {% endif %} + + + {% for i in range(sites) %} + + + + + {% endfor %} + + + + {% for x in discrepancies %} + + + {% for y in ['mal', 'anilist', 'kitsu'] %} + {% if y in x.entries %} + + + + + {% elif y + '_token' in state %} + + + + + {% endif %} + {% endfor %} + + {% endfor %} + +
    MyAnimeListAnilistKitsu
    TitleProgressScoreStatusLinks
    {{ '[BROKEN] ' if x.is_broken() else '' }}{{ x.get_title() }}{{ x.entries[y].progress|default('') }}{{ x.entries[y].score|default('') }}{{ x.entries[y].status|default('') }}{% if x.entries[y].link %}Link{% endif %}
    +
    +
    +{{ form.hidden_tag() }} +{% for x in modals %} + +{% endfor %} +
    + +{% endblock %} \ No newline at end of file diff --git a/anime_easy_sync/templates/aes/kitsu.jinja b/anime_easy_sync/templates/aes/kitsu.jinja new file mode 100644 index 0000000..d8015d0 --- /dev/null +++ b/anime_easy_sync/templates/aes/kitsu.jinja @@ -0,0 +1,12 @@ +{% extends "base.jinja" %} +{% from "_formhelpers.jinja" import render_field %} +{% block content %} +
    + {{ form.hidden_tag() }} +
    + {{ render_field(form.username) }} + {{ render_field(form.password) }} +
    + {{ form.submit() }} +
    +{% endblock content %} \ No newline at end of file diff --git a/anime_easy_sync/templates/base.jinja b/anime_easy_sync/templates/base.jinja new file mode 100644 index 0000000..1f125f8 --- /dev/null +++ b/anime_easy_sync/templates/base.jinja @@ -0,0 +1,13 @@ + + + + + + + Anime Easy Sync + {% block head %}{% endblock %} + + + {% block content %}{% endblock %} + + \ No newline at end of file diff --git a/instance/config.py b/instance/config.py new file mode 100644 index 0000000..95a2a29 --- /dev/null +++ b/instance/config.py @@ -0,0 +1,5 @@ +"""Anime Easy Sync config file""" +MAL_CLIENT_ID = 'REPLACE_ME' +MAL_CLIENT_SECRET = 'REPLACE_ME' +ANILIST_CLIENT_ID = 'REPLACE_ME' +ANILIST_CLIENT_SECRET = 'REPLACE_ME' diff --git a/uwsgi.ini b/uwsgi.ini new file mode 100644 index 0000000..1cd8052 --- /dev/null +++ b/uwsgi.ini @@ -0,0 +1,21 @@ +[uwsgi] +chdir = /home/pi/AnimeEasySync +module = anime_easy_sync:create_app() + +master = true +processes = 1 +threads = 2 + +uid = www-data +gid = www-data + +socket = /tmp/AnimeEasySync.sock +chmod-socket = 664 +vacuum = true + +die-on-term = true + +buffer-size = 32768 + +virtualenv = /home/pi/AnimeEasySync/venv +