th3r00t/pyShelf
1
0
Fork 0
mirror of https://github.com/th3r00t/pyShelf.git synced 2026-04-28 01:59:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

Pre Refactor of main loop

Browse Source
This commit is contained in:
th3r00t
2022-11-25 13:41:06 -05:00
parent 555af145c5
commit 7066eb67e6
7 changed files with 112 additions and 99 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
Pipfile vendored
View File

@@ -18,6 +18,9 @@ lxml = "*"
sqlalchemy = "==2.0.0b3"
pre-commit = "*"
fastapi = {extras = ["all"], version = "*"}
debugpy = "*"
pudb = "*"
jinja2 = "*"
[dev-packages]

29
configure vendored
View File

@@ -4,7 +4,9 @@ import json
from src.backend.lib.storage import Storage
from src.backend.lib.config import Config
def load_config():
"""Load program configuration."""
with open('config.json', "r") as file:
config = json.load(file)
file.close()
@@ -12,20 +14,14 @@ def load_config():
def write_config(config):
"""Write program configuration."""
with open('config.json', "w") as file:
json.dump(config, file)
file.close()
def set_secret(config=load_config()):
if config["SECRET"] == "":
config["SECRET"] = get_random_secret_key()
print(config["SECRET"])
else:
print("Secret already set, skipping.")
def set_book_directory(config=load_config(), *args):
"""Set book directory."""
if config["BOOKPATH"] == "":
try:
config["BOOKPATH"] = args[0]
@@ -33,26 +29,9 @@ def set_book_directory(config=load_config(), *args):
config["BOOKPATH"] = input("Input Book Directory ")
def init_django_database():
cmds = [
'python3 manage.py makemigrations',
'python3 manage.py makemigrations interface',
'python3 manage.py migrate',
'python3 manage.py migrate interface',
]
os.chdir("src")
for cmd in cmds:
os.system(cmd)
os.chdir("../")
config_file = load_config()
config = Config(os.path.split(os.path.realpath(__file__))[0])
set_secret(config_file)
set_book_directory(config_file)
write_config(config_file)
# TODO:: Refactor here to enable backend to handle database operations.
storage = Storage(config)
storage.create_tables()
# init_django_database()
# Admin(Path.cwd()).createsuperuser()

45
pyShelf.py vendored
View File

@@ -6,14 +6,15 @@ from pathlib import Path
from threading import Thread
import uvicorn
from fastapi import FastAPI
from fastapi import FastAPI, Request
from fastapi.responses import HTMLResponse
from fastapi.routing import APIRoute
from fastapi.staticfiles import StaticFiles
from fastapi.templating import Jinja2Templates
from src.backend.lib.config import Config
from src.backend.pyShelf_MakeCollections import MakeCollections
from src.backend.pyShelf_ScanLibrary import execute_scan
# import websockets
@@ -22,7 +23,8 @@ config = Config(root)
PRG_PATH = Path.cwd().__str__()
sys.path.insert(0, PRG_PATH)
app = FastAPI()
app.mount("/static", StaticFiles(directory="src/frontend/static"), name="static")
templates = Jinja2Templates(directory="src/frontend/templates")
def RunImport():
"""Begin live import of books."""
@@ -34,51 +36,54 @@ def RunImport():
def use_route_names_as_operation_ids(app: FastAPI) -> None:
"""Use route name as operation id."""
for route in app.routes:
if isinstance(route, APIRoute):
route.operation_id = route.name
@app.get("/", response_class=HTMLResponse)
async def index():
return """
<html>
<head>
<title>pyShelf eBook Server</title>
</head>
<body>
<h3>pyShelf Open Source Content Server</h3>
</body>
</html>
"""
async def index(request: Request):
"""Home page responder."""
return templates.TemplateResponse(
"index.html",
{"request": request})
@app.get("/users/me")
async def about_me():
"""About me page responder."""
return {"user_id": "CurrentUser"}
@app.get("/users/{user_id}")
async def about_user(user_id: int):
"""About user page responder."""
return {"user_id": user_id}
@app.get("/dev/test/echo/{_test_item_}")
async def echo_test(_test_item_):
"""Test echo responder function."""
return {"Test Object": _test_item_}
async def fe_server():
"""Front end server entrypoint."""
config.logger.info("Starting FastAPI server.")
fe_config = uvicorn.Config("__main__:app", port=8080, log_level="info", reload=True)
fe_config = uvicorn.Config("__main__:app", port=8080,
log_level="info", reload=True)
fe_server = uvicorn.Server(fe_config)
await fe_server.serve()
async def main():
"""Program entrypoint."""
_import_thread = Thread(target=RunImport)
_import_thread.start()
asyncio.create_task(fe_server())
_task = await asyncio.create_task(fe_server())
breakpoint()
return [_task, _import_thread]
if __name__ == "__main__":
@@ -87,8 +92,8 @@ if __name__ == "__main__":
loop = asyncio.get_running_loop()
except RuntimeError:
loop = asyncio.new_event_loop()
loop.create_task(main())
loop.run_forever()
_main_task = loop.create_task(main())
loop.run_until_complete()
loop.close()
exit
# asyncio.get_event_loop(asyncio.run(main())).run_forever()
loop.shutdown_default_executor()
exit(0)

29
src/backend/lib/config.py vendored
View File

@@ -1,18 +1,15 @@
"""Pyshelf's Configuration Object."""
import json
import pathlib
import re
import os
from loguru import logger
class Config:
"""
Main System Configuration
"""
"""Main System Configuration."""
def __init__(self, root):
"""
Initialize main configuration options
"""
"""Initialize main configuration options."""
self.root = root
env = os.environ.copy()
self._fp = "config.json"
@@ -37,13 +34,15 @@ class Config:
self.db_port = env.get("DB_PORT", self._data["DB_PORT"])
self.file_array = [self.book_shelf]
self.auto_scan = True
self.allowed_hosts = env.get("ALLOWED_HOSTS", self._data["ALLOWED_HOSTS"])
self.allowed_hosts = env.get("ALLOWED_HOSTS",
self._data["ALLOWED_HOSTS"])
self.db_engine = env.get("DB_ENGINE", self._data["DB_ENGINE"])
self.db_user = env.get("USER", self._data["USER"])
self.db_pass = env.get("PASSWORD", self._data["PASSWORD"])
self.SECRET = env.get("SECRET", self._data["SECRET"])
self.build_mode = env.get("BUILD_MODE", self._data["BUILD_MODE"])
def get_logger(self):
"""Instantiate logging system."""
_logger = logger
_logger.add(pathlib.PurePath(self.root, 'data', 'pyshelf.log'),
rotation="2 MB",
@@ -52,17 +51,7 @@ class Config:
return _logger
def open_file(self):
"""
Opens config.json and reads in configuration options
"""
"""Open config.json and reads in configuration options."""
with open(str(self._cp), "r") as read_file:
data = json.load(read_file)
return data
def path(self):
rstr = "pyShelf/src"
r = re.template(rstr)
_pathre = re.match("pyShelf/src")
def django_secret(self):
pass

36
src/backend/lib/library.py vendored
View File

@@ -40,35 +40,41 @@ class Catalogue:
folder = str(self.root_dir) + "/" + self.book_folder
else:
folder = self.book_folder
for f in os.listdir(folder):
_path = os.path.abspath(folder + "/" + f)
if os.path.isdir(_path.strip() + "/"):
self.file_list.append(self.scan_folder(_path))
else:
self.file_list.append(_path)
try:
for f in os.listdir(folder):
_path = os.path.abspath(folder + "/" + f)
if os.path.isdir(_path.strip() + "/"):
self.file_list.append(self.scan_folder(_path))
else:
self.file_list.append(_path)
except FileNotFoundError as fnfe:
self.config.logger.error(fnfe)
def filter_books(self):
"""
Calls scan_folder and filters out book files
Proceeds to call process_book
"""Calls scan_folder and filters out book files.
:returns self._book_list_expanded: json string containing all book metadata
:returns self._book_list_expanded: json string containing
all book metadata
"""
self.scan_folder() # Populate file list
regx = re.compile(r"\.epub|\.mobi|\.pdf")
try:
self.books = list(filter(regx.search, filter(None, self.file_list)))
except TypeError as e:
self.config.logger.error(e)
self.books = list(filter(
regx.search, filter(None, self.file_list)))
except TypeError as error:
self.config.logger.error(error)
def process_by_filetype(self, book):
"""Determine books filetype and process."""
if book.endswith(".epub"):
epub = self.process_epub(book)
return self.extract_metadata_epub(epub)
elif book.endswith(".mobi"):
if book.endswith(".mobi"):
return self.extract_metadata_mobi(book)
elif book.endswith(".pdf"):
if book.endswith(".pdf"):
return self.extract_metadata_pdf(book)
self.config.logger.error(f"Unknown Filetype {book}")
@staticmethod
def process_epub(book):

60
src/backend/lib/storage.py vendored
View File

@@ -1,6 +1,6 @@
#!/usr/bin/python
"""Pyshelf's Main Storage Class."""
import re
import os
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session
@@ -8,27 +8,47 @@ from .models import Book, Collection
class Storage:
"""Contains all methods for system storage"""
"""Contains all methods for system storage."""
def __init__(self, config):
self.sql = config.catalogue_db
self.user = config.user
self.password = config.password
self.db_host = config.db_host
self.db_port = config.db_port
self.engine = create_engine(
f"postgresql://{self.user}:{self.password}@{self.db_host}:{self.db_port}/{self.sql}"
)
"""Initialize storage object."""
self.config = config
self.sql = self.config.catalogue_db
self.user = self.config.user
self.password = self.config.password
self.db_host = self.config.db_host
self.db_port = self.config.db_port
self.engine = create_engine(self.get_connection_string(),
pool_pre_ping=True)
def get_connection_string(self):
"""Get connection string.
Engine type references config.json:DB_ENGINE.
"""
if self.config.db_engine == "sqlite":
if os.path.exists(f"{self.config.root}/pyshelf.db"):
return f"sqlite:////{self.config.root}/pyshelf.db"
else:
sqlite_file = open(f'{self.config.root}/pyshelf.db', 'w')
sqlite_file.close()
return f"sqlite://{self.config.root}/pyshelf.db"
elif self.config.db_engine == "psql":
return f"postgresql://{self.user}:{self.password}\
@{self.db_host}:{self.db_port}/{self.sql}"
elif self.config.db_engine == "mysql":
return f"mysql://{self.user}:{self.password}\
@{self.db_host}:{self.db_port}/{self.sql}"
def create_tables(self):
"""Create table structure."""
tables = [Book, Collection]
for table in tables:
table.metadata.create_all(self.engine)
def insert_book(self, book):
"""
Insert book in database
"""Insert a new book into the database.
:returns: True if succeeds False if not
"""
with Session(self.engine) as session:
@@ -61,15 +81,14 @@ class Storage:
self.config.logger.error(f"{book[0][0:80]} :: {e}")
def book_paths_list(self):
"""
Get file paths from database for comparison to system files
"""
"""Get file paths from database for comparison to system files."""
session = Session(self.engine)
_result = session.scalars(select(Book.file_name)).fetchall()
session.close()
return _result
def make_collections(self):
"""Make collections."""
# TODO: Check this still works with the switch to sqlalchemy
self.config.logger.info("Making collections.")
_title_regx = re.compile(r"^[0-9][0-9]*|-|\ \B")
@@ -98,14 +117,17 @@ class Storage:
)
_sess.close()
if _q.fetchone() is None:
_collection = Collection(collection=_s, book_id=book.book_id)
_collection = Collection(
collection=_s, book_id=book.book_id)
with Session(self.engine) as _sess:
try:
_sess.add(_collection)
_sess.commit()
_sess.close()
self.config.logger.info(f"Collection {_s} added.")
self.config.logger.info(
f"Collection {_s} added.")
except Exception as e:
self.config.logger.error(f"Collection {_s} failed: {e}")
self.config.logger.error(
f"Collection {_s} failed: {e}")
_collections.append(_p)
self.config.logger.info("Finished making collections.")

9
src/frontend/templates/index.html Normal file
View File

@@ -0,0 +1,9 @@
<html>
<head>
<title>pyShelf Content Server</title>
<link href="{{ url_for('static', path='/styles.css') }}" rel="stylesheet">
</head>
<body>
<h1>ID: {{ id }}</h1>
</body>
</html>