#!/usr/bin/env python # -*- coding: utf-8 -*- """ Stashr - Utility Functions """ """ MIT License Copyright (c) 2020 Andrew Vanderbye Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. """ """------------------------------------------------------------------------------------------- -- IMPORTS -------------------------------------------------------------------------------------------""" """ --- HUEY IMPORT --- """ """ --- PYTHON IMPORTS --- """ import datetime, time, pathlib, os, shutil, requests, json from slugify import slugify """ --- STASHR DEPENDENCY IMPORTS --- """ from werkzeug.utils import secure_filename """ --- STASHR CORE IMPORTS --- """ from stashr import log, database, parsefilename, paths, folders, naming from stashr.comicvine import cv from stashr.config import stashrconfig from flask import flash from flask_login import current_user from flask_bcrypt import generate_password_hash from sqlalchemy.orm import contains_eager """ --- CREATE LOGGER --- """ logger = log.stashr_logger(__name__) """------------------------------------------------------------------------------------------- -- UTILITY FUNCTIONS -------------------------------------------------------------------------------------------""" """ --- SUBSCRIPTION UTILS --- """ # DELETE VOLUME def delete_volume(volume_id): logger.debug(f'Delete Subscription for ID {volume_id}') # DELETE FROM VOLUMES database.session \ .query(database.Volumes) \ .filter(database.Volumes.volume_id == volume_id) \ .delete() # DELETE ISSUES WITH VOLUME_ID database.session \ .query(database.Issues) \ .filter(database.Issues.issue_volume_id == volume_id) \ .delete() # DELETE DIRECTORY LISTING WITH VOLUME ID database.session \ .query(database.Directories) \ .filter(database.Directories.directory_volume_id == volume_id) \ .delete() # DELETE READ LINKS database.session \ .query(database.ReadIssues) \ .filter(database.ReadIssues.read_volume_id == volume_id) \ .delete() # DELETE OWNED LINKS database.session \ .query(database.OwnedIssues) \ .filter(database.OwnedIssues.owned_volume_id == volume_id) \ .delete() database.session.commit() """ --- SERIES TASKS --- """ """ --- RELEASE LIST TASKS --- """ # UPDATE RELEASE LIST def update_release_list(): logger.debug('Update New Release') database.session \ .query(database.NewReleases) \ .delete() database.session.commit() weekday = 2 today_date = datetime.date.today() delta_days = today_date.weekday() - weekday if delta_days == 0: start_date = today_date if delta_days > 0: start_date = today_date - datetime.timedelta(delta_days) if delta_days < 0: start_date = today_date - datetime.timedelta(delta_days) - datetime.timedelta(7) end_date = start_date + datetime.timedelta(7) print(f'{start_date}|{end_date}') filter = { 'store_date': f'{start_date}|{end_date}' } new_releases = cv.get_issues(filters=filter, sort="name") releases = [] total_results = new_releases.number_of_total_results while(len(releases) < total_results): for release in new_releases.results: new_release = database.NewReleases( new_release_issue_id=release['id'], new_release_volume_id=release['volume']['id'], new_release_comic_name=release['volume']['name'], new_release_issue_number=release['issue_number'], new_release_publish_date=release['store_date'], new_release_item_url=release['site_detail_url'], new_release_image_url=release['image']['medium_url'], ) releases.append(new_release) new_releases = cv.get_issues(filters=filter, sort="name", offset=len(releases)) database.session.bulk_save_objects(releases) database.session.commit() """ --- COMIC STATUS TASKS --- """ """ --- READING LIST TASKS --- """ """ --- COLLECTION TASKS --- """ # CREATE NEW COLLECTION def create_new_collection(user_id, collection_name): logger.debug('Create New Collection') collection_slug = slugify(collection_name) check_collection_slug = database.session \ .query(database.Collections) \ .filter(database.Collections.collection_slug == collection_slug) \ .first() if check_collection_slug is not None: collection_slug = slugify(f'{collection_slug}-{user_id}') articles = [ 'a', 'an', 'the' ] s = collection_slug.split('-', 1) while s[0] in articles: s.pop(0) sort_title = ' '.join(s) new_collection = database.Collections( collection_user_id=user_id, collection_name=collection_name, collection_slug=collection_slug, collection_sort_title=sort_title, ) database.session.add(new_collection) database.session.commit() return new_collection.collection_id # ADD ISSUE TO COLLECTION def add_issue_to_collection(collection_id, issue_id): current_list = database.session \ .query(database.CollectionLinks) \ .filter(database.CollectionLinks.collection_link_collection_id == collection_id) \ .all() issue_position = len(current_list) + 1 new_collection_item = database.CollectionLinks ( collection_link_collection_id = collection_id, collection_link_issue_id = issue_id, collection_link_issue_position = issue_position ) database.session.add(new_collection_item) database.session.commit() """ --- SERVER TASKS --- """ """ --- PLUGIN TASKS --- """ """" --- FILE TASKS --- """ """ --- DATABASE TASKS --- """ """ --- TO SORT --- """ def register_new_user(form): new_user = database.Users( username=form.username.data, password=generate_password_hash(form.reg_password.data).decode('utf-8'), email=form.email.data, is_active=True ) database.session.add(new_user) try: database.session.commit() except Exception as e: logger.error(e) database.session.rollback() def create_new_user(new_user_form): new_user = database.Users( username = new_user_form.username.data, password = generate_password_hash(new_user_form.password.data).decode('utf-8'), role = new_user_form.role.data, rating_allowed = new_user_form.age_rating.data, email = new_user_form.email.data, is_active = True, confirmed_at = datetime.date.today() ) database.session.add(new_user) try: database.session.commit() except Exception as e: logger.error(e) database.session.rollback() flash(e, 'error') finally: flash('User Successfully Created', 'success') def change_user_password(change_password_form): setattr(change_password_form.user, 'password', generate_password_hash(change_password_form.new_password.data).decode('utf-8')) database.session.merge(change_password_form.user) database.session.commit() def reset_user_password(reset_password_form): setattr(reset_password_form.user, 'password', generate_password_hash(reset_password_form.password.data).decode('utf-8')) database.session.merge(reset_password_form.user) database.session.commit() def edit_user_account(edit_user_form): setattr(edit_user_form.user, 'email', edit_user_form.email.data) setattr(edit_user_form.user, 'role', edit_user_form.role.data) setattr(edit_user_form.user, 'rating_allowed', edit_user_form.age_rating.data) database.session.merge(edit_user_form.user) database.session.commit() def delete_user_account(delete_user_form): database.session.delete(delete_user_form.user) database.session.commit() def update_app_settings(settings_form): stashrconfig['APP']['server_port'] = settings_form.server_port.data stashrconfig['APP']['open_registration'] = settings_form.open_registration.data stashrconfig['APP']['comicvine_api_key'] = settings_form.comicvine_api_key.data stashrconfig['APP']['log_level'] = settings_form.log_level.data stashrconfig.write() cv.update_api_key(settings_form.comicvine_api_key.data) def update_directory_settings(directories_form): stashrconfig['DIRECTORY']['temp'] = directories_form.temp_directory.data stashrconfig['DIRECTORY']['comics'] = directories_form.comics_directory.data stashrconfig['DIRECTORY']['log'] = directories_form.log_directory.data stashrconfig['DIRECTORY']['backup'] = directories_form.backup_directory.data stashrconfig['DIRECTORY']['plugins'] = directories_form.plugins_directory.data stashrconfig['DIRECTORY']['images'] = directories_form.images_directory.data stashrconfig.write() def update_mail_settings(mail_form): stashrconfig['MAIL']['mail_use'] = mail_form.mail_use.data stashrconfig['MAIL']['mail_username'] = mail_form.mail_username.data stashrconfig['MAIL']['mail_password'] = mail_form.mail_password.data stashrconfig['MAIL']['mail_default_sender'] = mail_form.mail_default_sender.data stashrconfig['MAIL']['mail_server'] = mail_form.mail_server.data stashrconfig['MAIL']['mail_port'] = mail_form.mail_port.data stashrconfig['MAIL']['mail_use_ssl'] = mail_form.mail_use_ssl.data stashrconfig.write() def populate_statuses(user_id, volume_id, issues): all_issues = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_volume_id == volume_id) \ .all() for item in issues: if item in all_issues: all_issues.remove(item) for item in all_issues: new_read_status = database.ReadIssues( read_user_id = user_id, read_issue_id = item.issue_id, read_volume_id = item.issue_volume_id, read_status = False ) database.session.merge(new_read_status) new_owned_status = database.OwnedIssues( owned_user_id = user_id, owned_issue_id = item.issue_id, owned_volume_id = item.issue_volume_id, owned_status = False ) database.session.merge(new_owned_status) database.session.commit() def complete_first_run(first_run_form): if stashrconfig['APP']['first_run'] == False: return admin_user = database.session \ .query(database.Users) \ .filter(database.Users.id == 1) \ .first() admin_user.username = first_run_form.username.data admin_user.email = first_run_form.email.data admin_user.password = generate_password_hash(first_run_form.confirm_password.data).decode('utf-8') stashrconfig['APP']['log_level'] = first_run_form.logging_level.data stashrconfig['APP']['open_registration'] = first_run_form.open_registration.data stashrconfig['APP']['comicvine_api_key'] = first_run_form.comicvine_api_key.data stashrconfig['APP']['first_run'] = False stashrconfig.write() cv.update_api_key(first_run_form.comicvine_api_key.data) """ --------------------- REWRITES 3/8/21 --------------------- """ """ Refactor Notes removing auto_scrape() in favor of scan_directories() -> match_directories() -> want to rename sub_from_scraped() => add_sub_bt_directory() rename add_issues_by_volume_id() => scrape_volume_issues() rename scrape_issue_files() => scan_volume_files() rename add_subscription => add_volume_to_library() SCRAPE: - scan_directories() - match_directories() - add_sub_by_directory() add_subscription(volume_id) """ def scan_directories(): logger.debug('Scanning Directories') stored_directories = database.session \ .query(database.Directories) \ .all() stored_directories = database.DirectoriesSchema(many=True).dump(stored_directories) volumes_folder = os.path.join( folders.StashrFolders().comic_folder(), ) new_directories = [] existing_directories = [] for item in os.listdir(volumes_folder): if os.path.isdir(os.path.join(volumes_folder, item)): existing_directories.append(item) for item in existing_directories: if not any(path['directory_path'] == item for path in stored_directories): new_directory_entry = database.Directories( directory_path = item ) new_directories.append(new_directory_entry) database.session.bulk_save_objects(new_directories) database.session.commit() create_scrape_entries() def match_directories(): missing_links = database.session \ .query(database.Directories) \ .filter(database.Directories.directory_volume_id == None) \ .all() for directory in missing_links: try: time.sleep(1) matches = cv.search(directory.directory_path, limit=10, resources=['volume']) total_results = len(matches.results) except Exception as e: logger.error(e) break if matches.status_code != 1: continue for match in matches.results: if str(match['id']) in directory.directory_path: directory.directory_volume_id = match['id'] matches.results.pop(matches.results.index(match)) break if len(matches.results) != total_results: continue for match in matches.results: if str(match['start_year']) in directory.directory_path: directory.directory_volume_id = match['id'] break database.session.commit() def add_sub_by_directory(): # Look to possibly rename? directories = database.session \ .query(database.Directories) \ .filter(database.Directories.directory_volume_id != None) \ .filter(database.Directories.directory_in_library == False) \ .all() for directory in directories: add_volume_to_library(directory.directory_volume_id) scrape_volume_issues(directory.directory_volume_id) scan_volume_files(directory.directory_volume_id) def add_volume_to_library(volume_id): check_library = database.session \ .query(database.Volumes) \ .filter(database.Volumes.volume_id == volume_id) \ .first() if check_library is not None: return volume = cv.get_volume(volume_id) add_publisher_to_library(volume.results['publisher']['id']) volume_slug = slugify(volume.results['name']) check_slug = database.session \ .query(database.Volumes) \ .filter(database.Volumes.volume_slug == volume_slug) \ .first() if check_slug is not None: volume_slug = slugify(f'{volume_slug} {volume_id}') """ --- v --- REWORK --- v --- """ # THIS SECTION CAUSES DUPLICATES IN DIRECTORIES IN DATABASE check_directory = database.session \ .query(database.Directories) \ .filter(database.Directories.directory_volume_id == volume_id) \ .first() volume_path = naming.volume_folder_name_from_cv_item(volume) if check_directory is None: new_directory = database.Directories( directory_volume_id = volume_id, directory_path = volume_path, directory_in_library = True, ) database.session.merge(new_directory) else: if check_directory.directory_path is None: setattr(check_directory, 'directory_path', volume_path) if not check_directory.directory_in_library: setattr(check_directory, 'directory_in_library', True) """ --- ^ --- REWORK --- ^ --- """ articles = [ 'a', 'an', 'the' ] s = volume_slug.split('-', 1) while s[0] in articles: s.pop(0) sort_title = ' '.join(s) new_library_entry = database.Volumes( volume_id=volume.results['id'], volume_name=volume.results['name'], volume_description=volume.results['description'], volume_year=volume.results['start_year'], volume_publisher_id=volume.results['publisher']['id'], volume_url=volume.results['site_detail_url'], volume_image=volume.results['image']['small_url'], volume_latest=volume.results['last_issue']['issue_number'], volume_have=0, volume_total=volume.results['count_of_issues'], volume_status=True, volume_last_update=datetime.date.today(), volume_path=volume_path, volume_slug=volume_slug, volume_sort_title=sort_title, ) database.session.merge(new_library_entry) try: database.session.commit() except Exception as e: database.session.rollback() logger.error(e) download_image(volume.results['id'], 'volume', volume.results['image']['small_url']) def add_publisher_to_library(publisher_id): check_publisher = database.session \ .query(database.Publishers) \ .filter(database.Publishers.publisher_id == publisher_id) \ .first() if check_publisher is not None: return publisher = cv.get_publisher(publisher_id) new_publisher = database.Publishers( publisher_id=publisher_id, publisher_image=publisher.results['image']['medium_url'], publisher_name=publisher.results['name'], publisher_description=publisher.results['description'], ) database.session.merge(new_publisher) try: database.session.commit() except Exception as e: database.session.rollback() logger.error(e) download_image(publisher_id, 'publisher', publisher.results['image']['small_url']) def scrape_volume_issues(volume_id): filter = { 'volume': volume_id } existing_issues = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_volume_id == volume_id) \ .all() issues = cv.get_issues(filters=filter) total_results = issues.number_of_total_results all_issues = [] while(len(all_issues) < total_results): for issue in issues.results: new_issue = database.Issues( issue_id=issue['id'], issue_volume_id=volume_id, issue_name=issue['name'], issue_number=issue['issue_number'], issue_release_date=issue['store_date'], issue_cover_url=issue['image']['medium_url'], issue_description=issue['description'], ) all_issues.append(new_issue) issues = cv.get_issues(filters=filter, offset=len(all_issues)) data = database.IssuesSchema(many=True).dump(existing_issues) for item in all_issues: if any(issue['issue_id'] == item.issue_id for issue in data): all_issues[all_issues.index(item)] = 'deleteMe' all_issues[:] = [x for x in all_issues if x != 'deleteMe'] for issue in all_issues: download_image(issue.issue_id, 'issue', issue.issue_cover_url) database.session.bulk_save_objects(all_issues) database.session.commit() def scan_volume_files(volume_id): print('scanning') directory = database.session \ .query(database.Directories) \ .filter(database.Directories.directory_volume_id == volume_id) \ .first() volume_folder = os.path.join( folders.StashrFolders().comic_folder(), directory.directory_path ) if not os.path.isdir(volume_folder): print('folder not found') return issues = [] allowed_extensions = [ '.cbr', '.cbz', '.cbt', '.cb7' ] for item in os.listdir(volume_folder): if os.path.isfile(os.path.join(volume_folder, item)): if pathlib.Path(os.path.join(volume_folder, item)).suffix in allowed_extensions: issues.append(item) for filename in issues: issue_number = parsefilename.getIssueNumber(filename) issue = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_volume_id == volume_id) \ .filter(database.Issues.issue_number == issue_number) \ .first() if issue is None: continue issue.issue_file_path = filename issue.issue_file_status = 1 issue.issue_file_date == datetime.date.today() database.session.commit() have_count = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_volume_id == volume_id) \ .filter(database.Issues.issue_file_status == 1) \ .count() database.session \ .query(database.Volumes) \ .filter(database.Volumes.volume_id == volume_id) \ .update({'volume_have': have_count}) database.session.commit() # REFRESH COMIC INFO FROM WEB def refresh_single_series(volume_id): print('NOTHING CALLS THIS?') return check_volume = database.session \ .query(database.Volumes) \ .filter(database.Volumes.volume_id == volume_id) \ .first() if check_volume is None: return """ issue_count = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_volume_id == volume_id) \ .filter(database.Issues.issue_file_status == True) \ .count() """ volume = cv.get_volume(check_volume.volume_id) check_volume.volume_description = volume.results['description'] check_volume.volume_total = volume.results['count_of_issues'] add_volume_to_library(check_volume.volume_id) def refresh_single_volume(volume_id): check_volume = database.session \ .query(database.Volumes) \ .filter(database.Volumes.volume_id == volume_id) \ .first() if check_volume is None: return scrape_volume_issues(volume_id) scan_volume_files(volume_id) have_issue_count = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_volume_id == volume_id) \ .filter(database.Issues.issue_file_status == True) \ .count() volume = cv.get_volume(check_volume.volume_id) check_volume.volume_description = volume.results['description'] check_volume.volume_total = volume.results['count_of_issues'] check_volume.volume_have = have_issue_count database.session.merge(check_volume) database.session.commit() # DELETE COMIC FILE def delete_comic_file(issue_id): issue = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_id == issue_id) \ .first() issue_file = os.path.join( folders.StashrFolders().comic_folder(), issue.volume.directory.directory_path, issue.issue_file_path ) if not os.path.isfile(issue_file): return os.remove(issue_file) issue.issue_file_status = 0 issue.issue_file_date = None database.session.commit() # ADD ISSUE TO DATABASE def add_issue(issue_id): logger.debug('NOTHING SHOULD BE CALLING THIS') print('NOTHING SHOULD BE CALLING THIS') pass # UPDATE ISSUE BY ID def update_issue(issue_id): logger.debug('ONLY CALL THIS FOR COMIC METADATA') print('ONLY CALL THIS FOR COMIC METADATA') pass def rename_issue_file(issue_id, file_path): issue = database.session \ .query(database.Issues) \ .filter(database.Issues.issue_id == issue_id) \ .first() if issue is None: return 404 extension = pathlib.Path(file_path).suffix new_path = os.path.join( folders.StashrFolders().comic_folder(), issue.volume.directory.directory_path, f'{naming.file_name_by_db(issue)}{extension}' ) shutil.move(file_path, new_path) issue.issue_file_path = pathlib.Path(new_path).name issue.issue_file_status = True issue.issue_file_date = datetime.date.today() database.session.commit() return 200 def download_image(id, image_type, url): allowed_types = [ 'volume', 'issue', 'publisher' ] foldername = None if image_type.lower() not in allowed_types: return if image_type.lower() == 'volume': foldername = 'volumes' elif image_type.lower() == 'issue': foldername = 'issues' elif image_type.lower() == 'publisher': foldername = 'publishers' if foldername is None: return folderpath = os.path.join( folders.StashrFolders().images_folder(), foldername ) if not os.path.isdir(folderpath): os.makedirs(folderpath) filepath = os.path.join( folderpath, f'{id}.jpg' ) if os.path.isfile(filepath): return r = requests.get(url) with open(filepath, 'wb') as f: f.write(r.content) def update_volumes_from_release_list(): updates_needed = database.session \ .query(database.NewReleases, database.Volumes) \ .filter(database.NewReleases.new_release_volume_id == database.Volumes.volume_id) \ .all() for update in updates_needed: refresh_single_volume(update.NewReleases.new_release_volume_id) """ --------------------- SCRAPE FOLDERS (RE)WRITE --------------------- """ def create_scrape_entries(): missing_links = database.session \ .query(database.Directories) \ .filter(database.Directories.directory_in_library == False) \ .all() # for item in missing_links: # create_empty_scrape_entry(item) # get_scrape_candidates(item) for item in missing_links: candidates = cv.search(item.directory_path, limit=10, resources=['volume']) match_found = False scrape_candidate = candidates.results[0]['id'] for candidate in candidates.results: if str(candidate['id']) in item.directory_path: scrape_candidate = candidate['id'] match_found = True break if not match_found: if str(candidate['start_year']) in item.directory_path: scrape_candidate = candidate['id'] match_found = True new_scrape_item = database.ScrapeItems( scrape_directory = item.directory_path, scrape_directory_id = item.directory_id, scrape_json = json.dumps(candidates.results), scrape_candidate = scrape_candidate, scrape_match = match_found ) try: database.session.merge(new_scrape_item) database.session.commit() except: database.session.rollback() logger.warning('Scrape Item in database') def get_scrape_candidates(directory): scrape_item = database.session \ .query(database.ScrapeItems) \ .filter(database.ScrapeItems.scrape_directory == directory) \ .first() candidates = cv.search(directory, limit=10, resources=['volume']) setattr(scrape_item, 'scrape_json', candidates) database.session.merge(scrape_item) database.session.commit() def add_scrape_match(match_item): match_item.directory.directory_volume_id = match_item.scrape_candidate match_item.directory.directory_in_library = 1 database.session.merge(match_item) database.session.delete(match_item) database.session.commit() add_volume_to_library(match_item.scrape_candidate) scrape_volume_issues(match_item.scrape_candidate) scan_volume_files(match_item.scrape_candidate) def create_empty_scrape_entry(item): new_scrape_item = database.ScrapeItems( scrape_directory = item.directory_path, scrape_directory_id = item.directory_id ) try: database.session.merge(new_scrape_item) database.session.commit() except: database.session.rollback() logger.warning('Scrape Item in database') """ --------------------- SCRAPE FOLDERS (RE)(RE)WRITE --------------------- """ def new_scan_directories(): logger.debug('Scanning Directories') stored_directories = database.session \ .query(database.Directories) \ .all() stored_directories = database.DirectoriesSchema(many=True).dump(stored_directories) volumes_folder = os.path.join( folders.StashrFolders().comic_folder(), ) new_directories = [] existing_directories = [] for item in os.listdir(volumes_folder): if os.path.isdir(os.path.join(volumes_folder, item)): existing_directories.append(item) for item in existing_directories: if not any(path['directory_path'] == item for path in stored_directories): new_directory_entry = database.Directories( directory_path = item ) new_directories.append(new_directory_entry) database.session.bulk_save_objects(new_directories) database.session.commit() # create_scrape_entries() def new_create_scrape_entries(): missing_links = database.session \ .query(database.Directories) \ .filter(database.Directories.directory_in_library == False) \ .all() for item in missing_links: new_create_empty_scrape_entry(item) new_get_scrape_candidates(item) # SLEEP HERE time.sleep(0.5) def new_create_empty_scrape_entry(item): new_scrape_item = database.ScrapeItems( scrape_directory = item.directory_path, scrape_directory_id = item.directory_id ) try: database.session.merge(new_scrape_item) database.session.commit() except: database.session.rollback() logger.warning('Scrape Item in database') def new_get_scrape_candidates(item): scrape_item = database.session \ .query(database.ScrapeItems) \ .filter(database.ScrapeItems.scrape_directory == item.directory_path) \ .first() candidates = cv.search(item.directory_path, limit=10, resources=['volume']) scrape_candidate = candidates.results[0]['id'] match_found = False for candidate in candidates.results: if str(candidate['id']) in item.directory_path: scrape_candidate = candidate['id'] match_found = True break if not match_found: if str(candidate['start_year']) in item.directory_path: scrape_candidate = candidate['id'] match_found = False scrape_item.scrape_json = json.dumps(candidates.results) scrape_item.scrape_candidate = scrape_candidate scrape_item.scrape_match = match_found scrape_item.scrape_add = match_found database.session.merge(scrape_item) database.session.commit() def new_add_scraped_matches(): matched_directories = database.session \ .query(database.ScrapeItems) \ .filter(database.ScrapeItems.scrape_add == 1) \ .all() for item in matched_directories: new_add_scrape_match(item) def new_add_scrape_match(item): item.directory.directory_volume_id = item.scrape_candidate item.directory.directory_in_library = 1 database.session.merge(item) database.session.delete(item) try: database.session.commit() except: logger.warning(f'Volume {item.scrape_candidate} in database') database.session.rollback() return add_volume_to_library(item.scrape_candidate) scrape_volume_issues(item.scrape_candidate) scan_volume_files(item.scrape_candidate)