Updates for better scraping

stashr/api.py

@@ -2122,6 +2122,45 @@ def api_get_directories():
     return create_json_return('200', results=data, total_results=total_results)
 
 
+@api.route('/scrape/directories/candidates/<scrape_id>', methods=['GET'])
+def api_get_directories_candidates(scrape_id):
+    """To Update Later
+    This is using docstrings for specifications.
+    ---
+    tags:
+        - scrape
+    """
+
+    user = current_user
+
+    if not user.is_authenticated:
+        api_key = request.args.get('api_key')
+        if api_key == "":
+            return jsonify(create_json_return('100'))
+        user = database.session \
+            .query(database.Users) \
+            .filter(database.Users.api_key == api_key) \
+            .first()
+
+    if user is None:
+        return jsonify(create_json_return('100'))
+
+    if user.role != 'admin':
+        return jsonify(create_json_return('401'))
+    
+    scrape_item = database.session \
+        .query(database.ScrapeItems) \
+        .filter(database.ScrapeItems.scrape_id == scrape_id) \
+        .first()
+
+    if scrape_item is None:
+        return jsonify(create_json_return('404'))
+
+    data = scrape_item.scrape_json
+
+    return create_json_return('200', results=data)
+
+
 @api.route('/scrape/directories/scan', methods=['POST'])
 def api_post_directories_scan():
     """To Update Later
@@ -2238,7 +2277,8 @@ def api_put_directories_edit(scrape_id):
     
     allowed_keys = [
         'scrape_add',
-        'scrape_candidate'
+        'scrape_candidate',
+        'scrape_ignore_directory'
     ]
 
     check_directory = database.session \
@@ -2250,14 +2290,18 @@ def api_put_directories_edit(scrape_id):
          return jsonify(create_json_return('404'))
 
     for key, value in request.json['data'].items():
+        print(f'{key} - {value}')
         if key not in allowed_keys:
-            database.session.rollbak()
+            database.session.rollback()
             return jsonify(create_json_return('400'))
         setattr(check_directory, key, value)
-
-    database.session.merge(check_directory)
-    database.session.commit()
-    # print(check_directory.scrape_id)
+    
+    try:
+        database.session.merge(check_directory)
+        database.session.commit()
+    except Exception as e:
+        print(e)
+        database.session.rollback()
 
     return jsonify(create_json_return('200'))
 
@@ -2296,15 +2340,19 @@ def api_post_directories_extend(scrape_id):
         'search_terms'
     ]
 
-    for key, value in request.json['data'].items():
-        if key not in allowed_keys:
-            return jsonify(create_json_return('400'))
+    print(f'JSON: {request.data}')
 
+    if not request.json:
+        return jsonify(create_json_return('400'))
     if "data" not in request.json:
         return jsonify(create_json_return('400'))
     if "search_terms"  not in request.json['data']:
         return jsonify(create_json_return('400'))
 
+    for key, value in request.json['data'].items():
+        if key not in allowed_keys:
+            return jsonify(create_json_return('400'))
+
     search_terms = request.json['data']['search_terms']
 
     # DO THINGS HERE

stashr/database.py

@@ -49,6 +49,7 @@ from flask_bcrypt import generate_password_hash
 
 """ --- SQLALCHEMY IMPORTS --- """
 from sqlalchemy import *
+from sqlalchemy import exc
 from sqlalchemy.ext.declarative import declarative_base
 from sqlalchemy.orm import *
 
@@ -441,6 +442,8 @@ class ScrapeItems(Base):
     scrape_candidate = Column(Integer)
     scrape_json = Column(String)
 
+    scrape_ignore_directory = Column(Boolean, server_default='0')
+
     directory = relationship('Directories', 
                              primaryjoin='ScrapeItems.scrape_directory_id == Directories.directory_id',
                              backref='scrape_items',
@@ -653,8 +656,13 @@ def migrate_database():
     # CHECK FOR NEW DATABASE TABLES
     if not engine.dialect.has_table(engine.connect(), "directory_link"):
         Directories.__table__.create(bind=engine)
-    # CHECK FOR NEW TABLE COLUMNS
 
+    # CHECK FOR NEW TABLE COLUMNS
+    try:
+        session.query(exists().where(ScrapeItems.scrape_ignore_directory)).scalar()
+    except exc.OperationalError:
+        with engine.connect() as conn:
+            conn.execute("ALTER TABLE scrape_items ADD column 'scrape_ignore_directory' BOOLEAN DEFAULT 0")
 
 # Create Database
 if not os.path.exists(folders.StashrPaths().db_path()):

stashr/templates/base.html

@@ -7,7 +7,7 @@
 <title>Stashr - {{ title }}</title>
 
 <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/bootstrap.css') }}">
-<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/stashr.css') }}">
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/stashr.css') }}?v=0.1.0">
 
 <link href="https://cdn.jsdelivr.net/npm/vue-toast-notification/dist/theme-sugar.css" rel="stylesheet">
 
@@ -158,11 +158,13 @@ Vue.use(VueToast);
     {% endif %}
 {% endwith %}
 
+/*
 window.addEventListener("load", () => {
     if ("serviceWorker" in navigator) {
     navigator.serviceWorker.register("service-worker.js");
     }
 });
+*/
 
 {% block script %}{% endblock %}
 

stashr/templates/scrape_page.html

@@ -11,7 +11,7 @@
 {% block content %}
 
 <div id="app">
-    <modal ref="modal" v-bind:individual="individual"></modal>
+    <modal ref="modal" v-bind:individual="individual" v-on:do-search="searchNewResults"></modal>
     <directories v-bind:directories='directories'></directories>
 </div>
 
@@ -67,7 +67,14 @@ Vue.component('modal', {
                             </div>
                             <div class="col-9">
                                 <div v-if='!this.individual.scrape_add'>
-                                    <input type="email" class="form-control" id="exampleFormControlInput1" placeholder="name@example.com">
+                                    <div class="my-3 p-0 input-group">
+                                        <input type="text" v-model="search" class="form-control" placeholder="Search Volumes..." />
+                                        <div class="input-group-append">
+                                            <button class="btn btn-success" type="button" @click="$emit('do-search', search)">
+                                                Search
+                                            </button>
+                                        </div>
+                                    </div>
                                     <select id="selectVolume" class="form-select" aria-label="Default select example" @change='doSomething($event)'>
                                         <candidate
                                             v-for="item in json"
@@ -85,6 +92,12 @@ Vue.component('modal', {
                         </div>
                     </div>
                     <div class="modal-footer">
+                        <span v-if="this.individual.scrape_ignore_directory">
+                            <button type="button" class="btn btn-success" data-bs-dismiss="modal" @click='toggleDirectoryIgnore'>Show Directory</button>
+                        </span>
+                        <span v-else>
+                            <button type="button" class="btn btn-danger" data-bs-dismiss="modal" @click='toggleDirectoryIgnore'>Hide Directory</button>
+                        </span>
                         <button type="button" class="btn btn-secondary" data-bs-dismiss="modal">Close</button>
                     </div>
                 </div>
@@ -117,6 +130,9 @@ Vue.component('modal', {
                 return item['id'] == this.individual.scrape_candidate
             })[0]
             document.getElementById('selectVolume').value = this.individual.scrape_candidate
+        },
+        toggleDirectoryIgnore() {
+            app.toggleDirectoryIgnore();
         }
     },
     delimiters: ["[[","]]"]
@@ -209,6 +225,12 @@ Vue.component('directories', {
                 <button type="button" class="btn-close text-reset" data-bs-dismiss="offcanvas" aria-label="Close"></button>
             </div>
             <div class="offcanvas-body">
+                <div class="form-check">
+                    <input class="form-check-input" type="checkbox" value="" id="showIgnoredDirectories" v-model="checked">
+                    <label class="form-check-label" for="flexCheckDefault">
+                        Show Hidden Directories
+                    </label>
+                </div>
                 <div class="text-center w-100 m-0 p-3">
                     <button type="button" class="btn btn-info m-1 w-100" data-bs-toggle="tooltip" data-bs-placement="top" title="Add Selected Directories" onclick="app.addDirectories()">
                         <i class="fas fa-plus"></i>
@@ -223,12 +245,17 @@ Vue.component('directories', {
         </div>
     </div>
     `,
-    data() { return { search: '', } },
+    data() { return { search: '', ignore: 'false', checked: false} },
     computed: {
         filteredList() {
-            return this.directories.filter(directory => {
-                return directory.scrape_directory.toLowerCase().includes(this.search.toLowerCase())
-            })
+            if (!this.checked) {
+                return this.directories
+                    .filter(directory => { return directory.scrape_directory.toLowerCase().includes(this.search.toLowerCase()) })
+                    .filter(directory => { return directory.scrape_ignore_directory.toString().includes(this.checked) })
+            } else {
+                return this.directories
+                    .filter(directory => { return directory.scrape_directory.toLowerCase().includes(this.search.toLowerCase()) })
+            }
         },
     },
     methods: {},
@@ -288,6 +315,44 @@ var app = new Vue({
                     }
                 })
         },
+        toggleDirectoryIgnore() {
+            console.log('toggle')
+            axios.put('{{ url_for('api.api_put_directories_edit', scrape_id='SCRAPEID') }}'.replace('SCRAPEID', this.individual.scrape_id), {
+                data: {
+                    scrape_ignore_directory: !this.individual.scrape_ignore_directory
+                }
+            })
+                .then(res=> {
+                    console.log(res)
+                    if(res.data.status_code == 200) {
+                        this.individual.scrape_ignore_directory = !this.individual.scrape_ignore_directory
+                        if(this.individual.scrape_ignore_directory) {
+                            stashrToast('Directory Hidden', 'success')
+                        } else {
+                            stashrToast('Directory Unhidden', 'success')
+                        }
+                    }
+                })
+                .catch(err => console.log(err))
+        },
+        searchNewResults(search) {
+            axios.post('{{ url_for('api.api_post_directories_extend', scrape_id='SCRAPEID') }}'.replace('SCRAPEID', this.individual.scrape_id), {
+                data: {
+                    search_terms: search
+                }
+            })
+                .then(res => {
+                    if(res.data.status_code == 200) {
+                        axios.get('{{ url_for('api.api_get_directories_candidates', scrape_id='SCRAPEID') }}'.replace('SCRAPEID', this.individual.scrape_id))
+                            .then(res => {
+                                if (res.data.status_code == 200) {
+                                    this.directories = [];
+                                    this.getDirectories();
+                                }
+                            })
+                    }
+                })
+        }
     },
     delimiters: ["[","]]"]
 })

stashr/templates/settings_page.html

@@ -8,7 +8,7 @@
 
 <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/bootstrap.css') }}">
 <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/new_base.css') }}">
-<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/stashr.css') }}">
+<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='css/stashr.css') }}?v=0.1.0">
 
 <link href="https://cdn.jsdelivr.net/npm/vue-toast-notification/dist/theme-sugar.css" rel="stylesheet">
 

stashr/utils.py

@@ -1094,7 +1094,9 @@ def new_scrape_extend_matches(scrape_id, search_terms=None):
 
     print(len(candidates.results))
     
-    scrape_item.scrape_json = json.dumps(candidates.results)
+    if len(candidates.results) > 0:
+        scrape_item.scrape_json = json.dumps(candidates.results)
+        scrape_item.scrape_candidate = candidates.results[0]['id']
 
     database.session.merge(scrape_item)
     database.session.commit()