diff --git a/pyShelf.py b/pyShelf.py old mode 100644 new mode 100755 index f6f61e2..7f5d7bd --- a/pyShelf.py +++ b/pyShelf.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 import asyncio -import websockets import sys from pathlib import Path -from loguru import logger -from src.backend.lib.config import Config -from src.backend.pyShelf_ScanLibrary import execute_scan -from src.backend.pyShelf_MakeCollections import MakeCollections +import websockets +from loguru import logger + +from src.backend.lib.config import Config +from src.backend.pyShelf_MakeCollections import MakeCollections +from src.backend.pyShelf_ScanLibrary import execute_scan root = Path.cwd() config = Config(root) @@ -18,34 +19,27 @@ tx = None async def runImport(): - execute_scan(PRG_PATH) - MakeCollections(PRG_PATH) + execute_scan(PRG_PATH, config=config) + MakeCollections(PRG_PATH, config=config) return "Import Complete" async def socketio(websocket, path): async for message in websocket: - if message == "import": - print("message from Con1 >> {}".format(message)) - tx = "ack->{}".format(message) - elif message == "Connection 2": - print("message from Con2 >> {}".format(message)) - tx = "ack->{}".format(message) - elif message == "ping": - print("<<[{}]".format(message)) + if message == "ping": + config.logger.info("<< Ping") tx = pong(message) elif message == "importBooks": - print("<<[{} cmd rcvd]\n Starting import".format(message)) + config.logger.info("Starting Import") tx = "Starting Import . . ." await websocket.send(tx) await runImport() tx = "complete" - await websocket.send(tx) def pong(message): - print('Ping Received') + config.logger.info(">> Pong") return "pong" diff --git a/requirements.txt b/requirements.txt index eaa8f3e..2cdef29 100755 --- a/requirements.txt +++ b/requirements.txt @@ -22,3 +22,4 @@ uwsgi jsonpickle django-widget-tweaks loguru +ptvsd diff --git a/src/backend/lib/config.py b/src/backend/lib/config.py index 69acc6a..026feb8 100755 --- a/src/backend/lib/config.py +++ b/src/backend/lib/config.py @@ -1,5 +1,6 @@ import json import pathlib + from loguru import logger @@ -8,49 +9,49 @@ class Config: Main System Configuration """ - _fp = "config.json" def __init__(self, root): """ Initialize main configuration options """ - _cp = pathlib.Path.joinpath(root, self._fp) - _data = self.open_file(_cp) + self._fp = "config.json" + self._cp = pathlib.Path.joinpath(root, self._fp) + self._data = self.open_file() self.root = root self.logger = self.get_logger() - self.book_path = _data["BOOKPATH"] - self.TITLE = _data["TITLE"] - self.VERSION = _data["VERSION"] + self.book_path = self._data["BOOKPATH"] + self.TITLE = self._data["TITLE"] + self.VERSION = self._data["VERSION"] self.TITLE = self.TITLE + " ver " + self.VERSION - self.book_shelf = _data["BOOKSHELF"] - self.catalogue_db = _data["DATABASE"] - self.user = _data["USER"] - self.password = _data["PASSWORD"] - self.db_host = _data["DB_HOST"] - self.db_port = _data["DB_PORT"] + self.book_shelf = self._data["BOOKSHELF"] + self.catalogue_db = self._data["DATABASE"] + self.user = self._data["USER"] + self.password = self._data["PASSWORD"] + self.db_host = self._data["DB_HOST"] + self.db_port = self._data["DB_PORT"] self.file_array = [ self.book_shelf, ] self.auto_scan = True - self.allowed_hosts = _data["ALLOWED_HOSTS"] - self.db_user = _data["USER"] - self.db_pass = _data["PASSWORD"] - self.SECRET = _data["SECRET"] + self.allowed_hosts = self._data["ALLOWED_HOSTS"] + self.db_user = self._data["USER"] + self.db_pass = self._data["PASSWORD"] + self.SECRET = self._data["SECRET"] def get_logger(self): _logger = logger - _logger.add(pathlib.PurePath(self.root, 'data','pyShelf_{time}.log'), rotation="10 MB", loop=None) + _logger.add(pathlib.PurePath(self.root, 'data','pyShelf_{time}.log'), + rotation="10 MB", enqueue=True, colorize=True) return _logger - @staticmethod - def open_file(_cp): + def open_file(self): """ Opens config.json and reads in configuration options """ - with open(str(_cp), "r") as read_file: + with open(str(self._cp), "r") as read_file: data = json.load(read_file) return data - def django_secret(self, _data): + def django_secret(self): pass diff --git a/src/backend/lib/library.py b/src/backend/lib/library.py index 353809f..145d00e 100755 --- a/src/backend/lib/library.py +++ b/src/backend/lib/library.py @@ -6,7 +6,6 @@ import re import zipfile from bs4 import BeautifulSoup - from mobi import Mobi from .api_hooks import DuckDuckGo @@ -14,7 +13,6 @@ from .config import Config from .storage import Storage - class Catalogue: """ Decodes book metadata for storage @@ -50,8 +48,6 @@ class Catalogue: self.file_list.append(self.scan_folder(_path)) else: self.file_list.append(_path) - self.config.logger.info(_path) - print(_path+"\n") def filter_books(self): """ @@ -65,7 +61,7 @@ class Catalogue: try: self.books = list(filter(regx.search, filter(None, self.file_list))) except TypeError as e: - print(e) + self.config.logger.error(e) """ for book in self.books: self._book_list_expanded[book] = self.process_by_filetype(book) @@ -73,8 +69,6 @@ class Catalogue: """ def process_by_filetype(self, book): - - print(str(book), end='\r', flush=True) if book.endswith(".epub"): epub = self.process_epub(book) return self.extract_metadata_epub(epub) @@ -210,6 +204,7 @@ class Catalogue: # ftags = ftags.replace(" ", ",") except KeyError: ftags = None + return [ title, author, @@ -286,7 +281,6 @@ class Catalogue: db.insert_book(book) inserted = db.commit() if inserted is not True: - print(inserted) - if input("Continue ? y/n") == "y": - pass + self.config.logger.error("Failed storing {} in database".format(str(book))) + pass db.close() diff --git a/src/backend/lib/pyShelf.py b/src/backend/lib/pyShelf.py index e29d542..f211e28 100755 --- a/src/backend/lib/pyShelf.py +++ b/src/backend/lib/pyShelf.py @@ -1,8 +1,10 @@ #!/usr/bin/env python3 +import asyncio import os import time -import asyncio + import websockets + from .config import Config from .storage import Storage @@ -11,16 +13,11 @@ class InitFiles: """First run file creation operations""" def __init__(self, file_array): - print("Checking for program files") for _pointer in file_array: time.sleep(1) if not os.path.isfile(_pointer): self.CreateFile(_pointer) - print("%s created" % _pointer) - else: - print("%s present" % _pointer) time.sleep(1) - print("File check complete.") def CreateFile(self, _pointer): """ diff --git a/src/backend/lib/storage.py b/src/backend/lib/storage.py index f82c1ec..6052f32 100755 --- a/src/backend/lib/storage.py +++ b/src/backend/lib/storage.py @@ -1,6 +1,7 @@ #!/usr/bin/python -import re import datetime +import re + import psycopg2 @@ -34,7 +35,7 @@ class Storage: set_perms.cursor.execute(_q) set_perms.close() except Exception as e: - print(e) + self.config.logger.error(e) set_perms.close() def create_tables(self): @@ -71,11 +72,12 @@ class Storage: book[4], # descr book[5], # ident book[6], # publisher - datetime.datetime.now(), + datetime.datetime.now(), book[8], # rights book[9], # tags ), ) + self.config.logger.info(book[0]) return True except Exception as e: if e.pgcode == '22007': # psycopg2's error code for invalid date @@ -92,7 +94,7 @@ class Storage: try: x = self.cursor.fetchall() except psycopg2.Error as e: - print(e) + self.config.logger.error(e) x = [] return x @@ -146,8 +148,9 @@ class Storage: (collection, book_id_id) VALUES ('%s',%s)""" % (_s, book[0]) ) + self.config.logger.info("Collection {} Added".format(_s)) except Exception as e: - print(e) + self.config.logger.error(e) _collections.append(_p) self.db.commit() self.close() diff --git a/src/backend/pyShelf_MakeCollections.py b/src/backend/pyShelf_MakeCollections.py index 139e394..18e69ca 100755 --- a/src/backend/pyShelf_MakeCollections.py +++ b/src/backend/pyShelf_MakeCollections.py @@ -11,13 +11,13 @@ from .lib.storage import Storage sys.path.append(os.path.abspath(".")) -def MakeCollections(root): +def MakeCollections(root, **kwargs): _t1 = time.time() - config = Config(root) # Get configuration settings + try: config = kwargs['config'] + except KeyError as e: config = Config(root) # InitFiles(config.file_array) # Initialize file system _storage = Storage(config) _storage.make_collections() _t2 = time.time() scan_time = round(_t2 - _t1) - print("Collections Made.") - print("Time %s seconds" % scan_time) + config.logger.info("Collections made in {}".format(scan_time)) diff --git a/src/backend/pyShelf_ScanLibrary.py b/src/backend/pyShelf_ScanLibrary.py index b8c5c33..53c2cc8 100755 --- a/src/backend/pyShelf_ScanLibrary.py +++ b/src/backend/pyShelf_ScanLibrary.py @@ -11,18 +11,18 @@ from .lib.storage import Storage sys.path.append(os.path.abspath(".")) -def execute_scan(root): +def execute_scan(root, **kwargs): """ Main scan execution :param root: Project root. Required to properly execute program. Sends to configuration. """ _t1 = time.time() - config = Config(root) # Get configuration settings + try: config = kwargs["config"]; + except KeyError as e: config = Config(root) # Get configuration settings InitFiles(config.file_array) # Initialize file system Storage(config).check_ownership() catalogue = Catalogue(config) # Open the Catalogue catalogue.import_books() _t2 = time.time() scan_time = round(_t2 - _t1) - print("Scan Completed.") - print("Scan Time %s seconds" % scan_time) + config.logger.info("Scan Completed in {} seconds".format(scan_time)) diff --git a/src/frontend/settings.py b/src/frontend/settings.py index 0cb56e8..6dfdac1 100755 --- a/src/frontend/settings.py +++ b/src/frontend/settings.py @@ -32,7 +32,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) # SECURITY WARNING: keep the secret key used in production secret! SECRET_KEY = CONFIG.SECRET # SECURITY WARNING: don't run with debug turned on in production! -DEBUG = TEMPLATE_DEBUG = False +DEBUG = TEMPLATE_DEBUG = True if DEBUG is True: from pudb.remote import set_trace ALLOWED_HOSTS = CONFIG.allowed_hosts