th3r00t/pyShelf
1
0
Fork 0
mirror of https://github.com/th3r00t/pyShelf.git synced 2026-04-28 01:59:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

Pre Refactor of main loop

Browse Source
This commit is contained in:
th3r00t
2022-11-25 13:41:06 -05:00
parent 555af145c5
commit 7066eb67e6
7 changed files with 112 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Pipfile vendored
View File

@@ -18,6 +18,9 @@ lxml = "*"
sqlalchemy = "==2.0.0b3" sqlalchemy = "==2.0.0b3"
pre-commit = "*" pre-commit = "*"
fastapi = {extras = ["all"], version = "*"} fastapi = {extras = ["all"], version = "*"}
debugpy = "*"
pudb = "*"
jinja2 = "*"
[dev-packages] [dev-packages]

29
configure vendored
View File

@@ -4,7 +4,9 @@ import json
from src.backend.lib.storage import Storage from src.backend.lib.storage import Storage
from src.backend.lib.config import Config from src.backend.lib.config import Config
def load_config(): def load_config():
"""Load program configuration."""
with open('config.json', "r") as file: with open('config.json', "r") as file:
config = json.load(file) config = json.load(file)
file.close() file.close()
@@ -12,20 +14,14 @@ def load_config():
def write_config(config): def write_config(config):
"""Write program configuration."""
with open('config.json', "w") as file: with open('config.json', "w") as file:
json.dump(config, file) json.dump(config, file)
file.close() file.close()
def set_secret(config=load_config()):
if config["SECRET"] == "":
config["SECRET"] = get_random_secret_key()
print(config["SECRET"])
else:
print("Secret already set, skipping.")
def set_book_directory(config=load_config(), *args): def set_book_directory(config=load_config(), *args):
"""Set book directory."""
if config["BOOKPATH"] == "": if config["BOOKPATH"] == "":
try: try:
config["BOOKPATH"] = args[0] config["BOOKPATH"] = args[0]
@@ -33,26 +29,9 @@ def set_book_directory(config=load_config(), *args):
config["BOOKPATH"] = input("Input Book Directory ") config["BOOKPATH"] = input("Input Book Directory ")
def init_django_database():
cmds = [
'python3 manage.py makemigrations',
'python3 manage.py makemigrations interface',
'python3 manage.py migrate',
'python3 manage.py migrate interface',
]
os.chdir("src")
for cmd in cmds:
os.system(cmd)
os.chdir("../")
config_file = load_config() config_file = load_config()
config = Config(os.path.split(os.path.realpath(__file__))[0]) config = Config(os.path.split(os.path.realpath(__file__))[0])
set_secret(config_file)
set_book_directory(config_file) set_book_directory(config_file)
write_config(config_file) write_config(config_file)
# TODO:: Refactor here to enable backend to handle database operations.
storage = Storage(config) storage = Storage(config)
storage.create_tables() storage.create_tables()
# init_django_database()
# Admin(Path.cwd()).createsuperuser()

45
pyShelf.py vendored
View File

@@ -6,14 +6,15 @@ from pathlib import Path
from threading import Thread from threading import Thread
import uvicorn import uvicorn
from fastapi import FastAPI from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse from fastapi.responses import HTMLResponse
from fastapi.routing import APIRoute from fastapi.routing import APIRoute
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from src.backend.lib.config import Config from src.backend.lib.config import Config
from src.backend.pyShelf_MakeCollections import MakeCollections from src.backend.pyShelf_MakeCollections import MakeCollections
from src.backend.pyShelf_ScanLibrary import execute_scan from src.backend.pyShelf_ScanLibrary import execute_scan
# import websockets # import websockets
@@ -22,7 +23,8 @@ config = Config(root)
PRG_PATH = Path.cwd().__str__() PRG_PATH = Path.cwd().__str__()
sys.path.insert(0, PRG_PATH) sys.path.insert(0, PRG_PATH)
app = FastAPI() app = FastAPI()
app.mount("/static", StaticFiles(directory="src/frontend/static"), name="static")
templates = Jinja2Templates(directory="src/frontend/templates")
def RunImport(): def RunImport():
"""Begin live import of books.""" """Begin live import of books."""
@@ -34,51 +36,54 @@ def RunImport():
def use_route_names_as_operation_ids(app: FastAPI) -> None: def use_route_names_as_operation_ids(app: FastAPI) -> None:
"""Use route name as operation id."""
for route in app.routes: for route in app.routes:
if isinstance(route, APIRoute): if isinstance(route, APIRoute):
route.operation_id = route.name route.operation_id = route.name
@app.get("/", response_class=HTMLResponse) @app.get("/", response_class=HTMLResponse)
async def index(): async def index(request: Request):
return """ """Home page responder."""
<html> return templates.TemplateResponse(
<head> "index.html",
<title>pyShelf eBook Server</title> {"request": request})
</head>
<body>
<h3>pyShelf Open Source Content Server</h3>
</body>
</html>
"""
@app.get("/users/me") @app.get("/users/me")
async def about_me(): async def about_me():
"""About me page responder."""
return {"user_id": "CurrentUser"} return {"user_id": "CurrentUser"}
@app.get("/users/{user_id}") @app.get("/users/{user_id}")
async def about_user(user_id: int): async def about_user(user_id: int):
"""About user page responder."""
return {"user_id": user_id} return {"user_id": user_id}
@app.get("/dev/test/echo/{_test_item_}") @app.get("/dev/test/echo/{_test_item_}")
async def echo_test(_test_item_): async def echo_test(_test_item_):
"""Test echo responder function."""
return {"Test Object": _test_item_} return {"Test Object": _test_item_}
async def fe_server(): async def fe_server():
"""Front end server entrypoint."""
config.logger.info("Starting FastAPI server.") config.logger.info("Starting FastAPI server.")
fe_config = uvicorn.Config("__main__:app", port=8080, log_level="info", reload=True) fe_config = uvicorn.Config("__main__:app", port=8080,
log_level="info", reload=True)
fe_server = uvicorn.Server(fe_config) fe_server = uvicorn.Server(fe_config)
await fe_server.serve() await fe_server.serve()
async def main(): async def main():
"""Program entrypoint."""
_import_thread = Thread(target=RunImport) _import_thread = Thread(target=RunImport)
_import_thread.start() _import_thread.start()
asyncio.create_task(fe_server()) _task = await asyncio.create_task(fe_server())
breakpoint()
return [_task, _import_thread]
if __name__ == "__main__": if __name__ == "__main__":
@@ -87,8 +92,8 @@ if __name__ == "__main__":
loop = asyncio.get_running_loop() loop = asyncio.get_running_loop()
except RuntimeError: except RuntimeError:
loop = asyncio.new_event_loop() loop = asyncio.new_event_loop()
loop.create_task(main()) _main_task = loop.create_task(main())
loop.run_forever() loop.run_until_complete()
loop.close() loop.close()
exit loop.shutdown_default_executor()
# asyncio.get_event_loop(asyncio.run(main())).run_forever() exit(0)

29
src/backend/lib/config.py vendored
View File

@@ -1,18 +1,15 @@
"""Pyshelf's Configuration Object."""
import json import json
import pathlib import pathlib
import re
import os import os
from loguru import logger from loguru import logger
class Config: class Config:
""" """Main System Configuration."""
Main System Configuration
"""
def __init__(self, root): def __init__(self, root):
""" """Initialize main configuration options."""
Initialize main configuration options
"""
self.root = root self.root = root
env = os.environ.copy() env = os.environ.copy()
self._fp = "config.json" self._fp = "config.json"
@@ -37,13 +34,15 @@ class Config:
self.db_port = env.get("DB_PORT", self._data["DB_PORT"]) self.db_port = env.get("DB_PORT", self._data["DB_PORT"])
self.file_array = [self.book_shelf] self.file_array = [self.book_shelf]
self.auto_scan = True self.auto_scan = True
self.allowed_hosts = env.get("ALLOWED_HOSTS", self._data["ALLOWED_HOSTS"]) self.allowed_hosts = env.get("ALLOWED_HOSTS",
self._data["ALLOWED_HOSTS"])
self.db_engine = env.get("DB_ENGINE", self._data["DB_ENGINE"])
self.db_user = env.get("USER", self._data["USER"]) self.db_user = env.get("USER", self._data["USER"])
self.db_pass = env.get("PASSWORD", self._data["PASSWORD"]) self.db_pass = env.get("PASSWORD", self._data["PASSWORD"])
self.SECRET = env.get("SECRET", self._data["SECRET"])
self.build_mode = env.get("BUILD_MODE", self._data["BUILD_MODE"]) self.build_mode = env.get("BUILD_MODE", self._data["BUILD_MODE"])
def get_logger(self): def get_logger(self):
"""Instantiate logging system."""
_logger = logger _logger = logger
_logger.add(pathlib.PurePath(self.root, 'data', 'pyshelf.log'), _logger.add(pathlib.PurePath(self.root, 'data', 'pyshelf.log'),
rotation="2 MB", rotation="2 MB",
@@ -52,17 +51,7 @@ class Config:
return _logger return _logger
def open_file(self): def open_file(self):
""" """Open config.json and reads in configuration options."""
Opens config.json and reads in configuration options
"""
with open(str(self._cp), "r") as read_file: with open(str(self._cp), "r") as read_file:
data = json.load(read_file) data = json.load(read_file)
return data return data
def path(self):
rstr = "pyShelf/src"
r = re.template(rstr)
_pathre = re.match("pyShelf/src")
def django_secret(self):
pass

36
src/backend/lib/library.py vendored
View File

@@ -40,35 +40,41 @@ class Catalogue:
folder = str(self.root_dir) + "/" + self.book_folder folder = str(self.root_dir) + "/" + self.book_folder
else: else:
folder = self.book_folder folder = self.book_folder
for f in os.listdir(folder): try:
_path = os.path.abspath(folder + "/" + f) for f in os.listdir(folder):
if os.path.isdir(_path.strip() + "/"): _path = os.path.abspath(folder + "/" + f)
self.file_list.append(self.scan_folder(_path)) if os.path.isdir(_path.strip() + "/"):
else: self.file_list.append(self.scan_folder(_path))
self.file_list.append(_path) else:
self.file_list.append(_path)
except FileNotFoundError as fnfe:
self.config.logger.error(fnfe)
def filter_books(self): def filter_books(self):
""" """Calls scan_folder and filters out book files.
Calls scan_folder and filters out book files
Proceeds to call process_book
:returns self._book_list_expanded: json string containing all book metadata :returns self._book_list_expanded: json string containing
all book metadata
""" """
self.scan_folder() # Populate file list self.scan_folder() # Populate file list
regx = re.compile(r"\.epub|\.mobi|\.pdf") regx = re.compile(r"\.epub|\.mobi|\.pdf")
try: try:
self.books = list(filter(regx.search, filter(None, self.file_list))) self.books = list(filter(
except TypeError as e: regx.search, filter(None, self.file_list)))
self.config.logger.error(e) except TypeError as error:
self.config.logger.error(error)
def process_by_filetype(self, book): def process_by_filetype(self, book):
"""Determine books filetype and process."""
if book.endswith(".epub"): if book.endswith(".epub"):
epub = self.process_epub(book) epub = self.process_epub(book)
return self.extract_metadata_epub(epub) return self.extract_metadata_epub(epub)
elif book.endswith(".mobi"): if book.endswith(".mobi"):
return self.extract_metadata_mobi(book) return self.extract_metadata_mobi(book)
elif book.endswith(".pdf"): if book.endswith(".pdf"):
return self.extract_metadata_pdf(book) return self.extract_metadata_pdf(book)
self.config.logger.error(f"Unknown Filetype {book}")
@staticmethod @staticmethod
def process_epub(book): def process_epub(book):

60
src/backend/lib/storage.py vendored
View File

@@ -1,6 +1,6 @@
#!/usr/bin/python """Pyshelf's Main Storage Class."""
import re import re
import os
from sqlalchemy import create_engine, select from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session from sqlalchemy.orm import Session
@@ -8,27 +8,47 @@ from .models import Book, Collection
class Storage: class Storage:
"""Contains all methods for system storage""" """Contains all methods for system storage."""
def __init__(self, config): def __init__(self, config):
self.sql = config.catalogue_db """Initialize storage object."""
self.user = config.user
self.password = config.password
self.db_host = config.db_host
self.db_port = config.db_port
self.engine = create_engine(
f"postgresql://{self.user}:{self.password}@{self.db_host}:{self.db_port}/{self.sql}"
)
self.config = config self.config = config
self.sql = self.config.catalogue_db
self.user = self.config.user
self.password = self.config.password
self.db_host = self.config.db_host
self.db_port = self.config.db_port
self.engine = create_engine(self.get_connection_string(),
pool_pre_ping=True)
def get_connection_string(self):
"""Get connection string.
Engine type references config.json:DB_ENGINE.
"""
if self.config.db_engine == "sqlite":
if os.path.exists(f"{self.config.root}/pyshelf.db"):
return f"sqlite:////{self.config.root}/pyshelf.db"
else:
sqlite_file = open(f'{self.config.root}/pyshelf.db', 'w')
sqlite_file.close()
return f"sqlite://{self.config.root}/pyshelf.db"
elif self.config.db_engine == "psql":
return f"postgresql://{self.user}:{self.password}\
@{self.db_host}:{self.db_port}/{self.sql}"
elif self.config.db_engine == "mysql":
return f"mysql://{self.user}:{self.password}\
@{self.db_host}:{self.db_port}/{self.sql}"
def create_tables(self): def create_tables(self):
"""Create table structure."""
tables = [Book, Collection] tables = [Book, Collection]
for table in tables: for table in tables:
table.metadata.create_all(self.engine) table.metadata.create_all(self.engine)
def insert_book(self, book): def insert_book(self, book):
""" """Insert a new book into the database.
Insert book in database
:returns: True if succeeds False if not :returns: True if succeeds False if not
""" """
with Session(self.engine) as session: with Session(self.engine) as session:
@@ -61,15 +81,14 @@ class Storage:
self.config.logger.error(f"{book[0][0:80]} :: {e}") self.config.logger.error(f"{book[0][0:80]} :: {e}")
def book_paths_list(self): def book_paths_list(self):
""" """Get file paths from database for comparison to system files."""
Get file paths from database for comparison to system files
"""
session = Session(self.engine) session = Session(self.engine)
_result = session.scalars(select(Book.file_name)).fetchall() _result = session.scalars(select(Book.file_name)).fetchall()
session.close() session.close()
return _result return _result
def make_collections(self): def make_collections(self):
"""Make collections."""
# TODO: Check this still works with the switch to sqlalchemy # TODO: Check this still works with the switch to sqlalchemy
self.config.logger.info("Making collections.") self.config.logger.info("Making collections.")
_title_regx = re.compile(r"^[0-9][0-9]*|-|\ \B") _title_regx = re.compile(r"^[0-9][0-9]*|-|\ \B")
@@ -98,14 +117,17 @@ class Storage:
) )
_sess.close() _sess.close()
if _q.fetchone() is None: if _q.fetchone() is None:
_collection = Collection(collection=_s, book_id=book.book_id) _collection = Collection(
collection=_s, book_id=book.book_id)
with Session(self.engine) as _sess: with Session(self.engine) as _sess:
try: try:
_sess.add(_collection) _sess.add(_collection)
_sess.commit() _sess.commit()
_sess.close() _sess.close()
self.config.logger.info(f"Collection {_s} added.") self.config.logger.info(
f"Collection {_s} added.")
except Exception as e: except Exception as e:
self.config.logger.error(f"Collection {_s} failed: {e}") self.config.logger.error(
f"Collection {_s} failed: {e}")
_collections.append(_p) _collections.append(_p)
self.config.logger.info("Finished making collections.") self.config.logger.info("Finished making collections.")

9
src/frontend/templates/index.html Normal file
View File

@@ -0,0 +1,9 @@
<html>
<head>
<title>pyShelf Content Server</title>
<link href="{{ url_for('static', path='/styles.css') }}" rel="stylesheet">
</head>
<body>
<h1>ID: {{ id }}</h1>
</body>
</html>