Merge pull request #83 from th3r00t/0.8.0--dev-table_refactor

Fixed collection system.

pyShelf.py

@@ -22,7 +22,9 @@ def run_import():
     config.logger.info("Begining book import.")
     execute_scan(PRG_PATH, config=config)
     config.logger.info("Finished book import.")
+    storage = Storage(config=config)
     # MakeCollections(PRG_PATH, config=config)
+    storage.make_collections()
     return "Import Complete"
 
 

src/backend/lib/models.py

@@ -5,6 +5,7 @@ from sqlalchemy import func, ForeignKey
 from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship
 import datetime
 
+# Timestamp annotation
 timestamp = Annotated[
     datetime.datetime,
     mapped_column(nullable=False, server_default=func.CURRENT_TIMESTAMP()),
@@ -14,7 +15,6 @@ timestamp = Annotated[
 class Base(DeclarativeBase):
     """Base class for all models."""
 
-from sqlalchemy.orm import relationship
 
 class Book(Base):
     """Book model."""
@@ -36,8 +36,10 @@ class Book(Base):
     identifier: Mapped[Optional[str]]
     publisher: Mapped[Optional[str]]
 
-    # One book → many collection entries
+    # Relationship to join table
-    collections = relationship("Collection", back_populates="book", cascade="all, delete-orphan")
+    book_collections = relationship(
+        "BookCollection", back_populates="book", cascade="all, delete-orphan"
+    )
 
 
 class Collection(Base):
@@ -46,8 +48,23 @@ class Collection(Base):
     __tablename__ = "Collection"
 
     id: Mapped[int] = mapped_column(primary_key=True)
-    collection: Mapped[str]
+    name: Mapped[str] = mapped_column(unique=True)
-    book_id: Mapped[int] = mapped_column(ForeignKey("Book.id"))
 
-    # Each collection entry points to one book
+    # Relationship to join table
-    book = relationship("Book", back_populates="collections")
+    book_collections = relationship(
+        "BookCollection", back_populates="collection", cascade="all, delete-orphan"
+    )
+
+
+class BookCollection(Base):
+    """Association table linking Books and Collections."""
+
+    __tablename__ = "BookCollection"
+
+    id: Mapped[int] = mapped_column(primary_key=True)
+    book_id: Mapped[int] = mapped_column(ForeignKey("Book.id"))
+    collection_id: Mapped[int] = mapped_column(ForeignKey("Collection.id"))
+
+    # Relationships
+    book = relationship("Book", back_populates="book_collections")
+    collection = relationship("Collection", back_populates="book_collections")

src/backend/lib/storage.py

@@ -4,7 +4,7 @@ from sqlalchemy import create_engine, select
 from sqlalchemy.orm import Session
 from pathlib import Path
 
-from .models import Book, Collection
+from .models import Book, Collection, BookCollection
 
 
 class Storage:
@@ -135,89 +135,132 @@ class Storage:
         collections : list()
             List of collections.
         """
+        # collections = []
+        # title_regx = re.compile(r"^[0-9][0-9]*|-|\ \B")
+        # book_path: Path = Path(book[3])
+        # store_path: Path = Path(self.config.book_path)
+        # relative_book_path: Path = book_path.relative_to(store_path)
+        # for path in relative_book_path.parts:
+        #     collections.append(re.sub(title_regx, "", path).strip())
+        # collections.pop(-1)
+        # return collections
         collections = []
         title_regx = re.compile(r"^[0-9][0-9]*|-|\ \B")
         book_path: Path = Path(book[3])
         store_path: Path = Path(self.config.book_path)
         relative_book_path: Path = book_path.relative_to(store_path)
-        for path in relative_book_path.parts:
+        # Keep all folder names except the actual file
-            collections.append(re.sub(title_regx, "", path).strip())
+        for folder in relative_book_path.parts[:-1]:
-        collections.pop(-1)
+            clean_name = re.sub(title_regx, "", folder).strip()
+            if clean_name:
+                collections.append(clean_name)
         return collections
 
     def make_collections(self):
-        """Parse book path's to determine common folder structure.
+        """Ensure collections exist and link them to books (many-to-many)."""
-
-        Stores collections based on shared paths.
-        """
-        # TODO: Check this still works with the switch to sqlalchemy
         self.config.logger.info("Making collections.")
-        _title_regx = re.compile(r"^[0-9][0-9]*|-|\ \B")
         session = Session(self.engine)
-        _set = session.execute(select(Book.id, Book.file_name)).all()
+
-        if _set.__len__() > 0:
+        # get all books and paths
-            for book in _set:
+        books = session.execute(select(Book.id, Book.file_name)).all()
-                path = self.config.book_path + "/"
+        
-                _collections = []
+        for book_id, file_name in books:
-                try:
+            try:
-                    _pathing = book[1].split(path)[1].split("/")
+                relative_parts = Path(file_name).relative_to(self.config.book_path).parts
-                    _pathing.pop(0)
+            except ValueError:
-                    _pathing.pop(-1)
+                continue  # skip books outside the configured path
-                except IndexError:
+
-                    continue  # Skip if path is invalid eg. a book with no con-
+            # exclude the actual file name
-                              # taining folder
+            folder = relative_parts[1]
-                for _p in _pathing:
+            # for folder in folders:
-                    _s = _p.replace("'", "")
+            #     clean_name = re.sub(r"^[0-9][0-9]*|-|\ \B", "", folder).strip()
-                    _x = re.sub(_title_regx, "", _s)
+            #     if not clean_name:
-                    _s = _x.strip()
+            #         continue
-                    _sess = Session(self.engine)
+
-                    _q = _sess.execute(
+                # check if collection exists
-                            select(Collection.id).where(
+            collection = session.execute(
-                                Collection.collection == _s,
+                select(Collection).where(Collection.name == folder)
-                                # BUG: book.id is not the correct identifier.
+            ).scalar_one_or_none()
-                                Collection.book_id == book.id,
+            if not collection:
-                                )
+                collection = Collection(name=folder)
-                            )
+                session.add(collection)
-                    _sess.close()
+                session.flush()  # ensures collection.id is available
-                    if _q.fetchone() is None:
+
-                        _collection = Collection(collection=_s, book_id=book.id)
+            # check link
-                        with Session(self.engine) as _sess:
+            link_exists = session.execute(
-                            try:
+                select(BookCollection).where(
-                                _sess.add(_collection)
+                    BookCollection.book_id == book_id,
-                                _sess.commit()
+                    BookCollection.collection_id == collection.id
-                                _sess.close()
+                )
-                                # self.config.logger.info(f"Collection {_s} added.")
+            ).first()
-                            except Exception as e:
+
-                                self.config.logger.error(f"Collection {_s} failed: {e}")
+            if not link_exists:
-                    _collections.append(_p)
+                session.add(BookCollection(book_id=book_id, collection_id=collection.id))
+
+        session.commit()
+        session.close()
         self.config.logger.info("Finished making collections.")
 
+
+
+    # def get_books(self, collection=None, skip=None, limit=None):
+    #     """Get books from database.
+    #
+    #     Parameters
+    #     ----------
+    #     collection : str
+    #         Collection to filter by.
+    #
+    #     Returns
+    #     -------
+    #     _result : ScalarResult Object
+    #     """
+    #     session = Session(self.engine)
+    #     if collection:
+    #         _result = session.execute(
+    #                 select(Book)
+    #                 .join(Collection)
+    #                 # .where(Collection.id == collection)
+    #                 .where(Collection.name == collection)
+    #                 .offset(skip)
+    #                 .limit(limit)
+    #                 ).all()
+    #     else:
+    #         _result = session.execute(select(Book).offset(skip).limit(limit)).all()
+    #     session.close()
+    #     return _result
+
+
     def get_books(self, collection=None, skip=None, limit=None):
         """Get books from database.
 
         Parameters
         ----------
-        collection : str
+        collection : int or None
-            Collection to filter by.
+            Collection ID to filter by.
-
+        skip : int or None
-        Returns
+            Number of records to skip (offset).
-        -------
+        limit : int or None
-        _result : ScalarResult Object
+            Maximum number of records to return.
         """
-        session = Session(self.engine)
+        with Session(self.engine) as session:
-        if collection:
+            if collection is not None:
-            _result = session.execute(
+                # Join through BookCollection to filter books in a collection
+                result = session.execute(
                     select(Book)
-                    .join(Collection)
+                    .join(BookCollection)
-                    # .where(Collection.id == collection)
+                    .where(BookCollection.collection_id == collection)
-                    .where(Collection.collection == collection)
+                    .offset(skip or 0)
-                    .offset(skip)
+                    .limit(limit or 100)
-                    .limit(limit)
+                ).scalars().all()
-                    ).all()
+            else:
-        else:
+                result = session.execute(
-            _result = session.execute(select(Book).offset(skip).limit(limit)).all()
+                    select(Book)
-        session.close()
+                    .offset(skip or 0)
-        return _result
+                    .limit(limit or 100)
+                ).scalars().all()
+        return result
+
 
     def get_book(self, id):
         """Get book from database.
@@ -243,10 +286,15 @@ class Storage:
         -------
         _result : ScalarResult Object
         """
-        session = Session(self.engine)
+        with Session(self.engine) as session:
-        _result = session.execute(select(Collection).join(Book)).all()
+            result = session.execute(
-        session.close()
+                select(Collection).join(BookCollection).distinct()
-        return _result
+            ).scalars().all()
+        return result
+        # session = Session(self.engine)
+        # _result = session.execute(select(Collection).join(Book)).all()
+        # session.close()
+        # return _result
     
     def get_collection(self, name):
         """Get collection from database.
@@ -256,7 +304,6 @@ class Storage:
         _result : ScalarResult Object
         """
         session = Session(self.engine)
-        _result = session.execute(select(Collection).where(Collection.collection == name).join(Book)).all()
+        _result = session.execute(select(Collection).where(Collection.name == name).join(Book)).all()
-        breakpoint()
         session.close()
         return _result

src/frontend/lib/FastAPIServer.py

@@ -64,20 +64,20 @@ def books_tojson(obj) -> dumps:
     for book in obj:
         convert_none = lambda x: x if x is not None else "None"
         _books.append({
-            "book_id": book[0].id,
+            "book_id": book.id,
-            "title": book[0].title,
+            "title": book.title,
-            "author": book[0].author,
+            "author": book.author,
-            "categories": convert_none(book[0].categories),
+            "categories": convert_none(book.categories),
-            "cover": base64decode(book[0].cover),
+            "cover": base64decode(book.cover),
-            "pages": convert_none(book[0].pages),
+            "pages": convert_none(book.pages),
-            "progress": convert_none(book[0].progress),
+            "progress": convert_none(book.progress),
-            "file_name": book[0].file_name,
+            "file_name": book.file_name,
-            "description": convert_none(book[0].description),
+            "description": convert_none(book.description),
-            "date": convertDateTime(book[0].date),
+            "date": convertDateTime(book.date),
-            "rights": convert_none(book[0].rights),
+            "rights": convert_none(book.rights),
-            "tags": convert_none(book[0].tags),
+            "tags": convert_none(book.tags),
-            "identifier": convert_none(book[0].identifier),
+            "identifier": convert_none(book.identifier),
-            "publisher": convert_none(book[0].publisher),
+            "publisher": convert_none(book.publisher),
         })
     return dumps(_books)
 
@@ -85,20 +85,20 @@ def books_tojson(obj) -> dumps:
 def book_tojson(book) -> dumps:
     """Convert a book object to a json."""
     return dumps({
-            "book_id": book[0].id,
+            "book_id": book.id,
-            "title": book[0].title,
+            "title": book.title,
-            "author": book[0].author,
+            "author": book.author,
-            "categories": book[0].categories,
+            "categories": book.categories,
-            "cover": base64decode(book[0].cover),
+            "cover": base64decode(book.cover),
-            "pages": book[0].pages,
+            "pages": book.pages,
-            "progress": book[0].progress,
+            "progress": book.progress,
-            "file_name": book[0].file_name,
+            "file_name": book.file_name,
-            "description": book[0].description,
+            "description": book.description,
-            "date": convertDateTime(book[0].date),
+            "date": convertDateTime(book.date),
-            "rights": book[0].rights,
+            "rights": book.rights,
-            "tags": book[0].tags,
+            "tags": book.tags,
-            "identifier": book[0].identifier,
+            "identifier": book.identifier,
-            "publisher": book[0].publisher,
+            "publisher": book.publisher,
         })
 
 def tojson(obj) -> dumps:
@@ -109,13 +109,13 @@ def collections_tojson(collection) -> dumps:
     _collections = []
     _collection_id_set = set()
     for _collection in collection:
-        if _collection[0].id in _collection_id_set:
+        if _collection.id in _collection_id_set:
             pass
         else:
-            _collection_id_set.add(_collection[0].id)
+            _collection_id_set.add(_collection.id)
             _collections.append({
-                "collection_id": _collection[0].id,
+                "collection_id": _collection.id,
-                "collection": _collection[0].collection,
+                "collection": _collection.name,
             })
     return dumps(_collections)
 
@@ -167,8 +167,13 @@ class FastAPIServer():
 
     @app.get("/", response_class=HTMLResponse)
     async def index(request: Request, skip: int = 0, limit: int = 30):
+        if skip <= 0:
+            skip_num = 0
+            skip = 0
+        else:
+            skip_num = skip * limit
         storage = Storage(Config(os.path.abspath(os.getcwd())))
-        books = storage.get_books(collection=None, skip=skip*limit, limit=limit)
+        books = storage.get_books(collection=None, skip=skip_num, limit=limit)
         collections = storage.get_collections()
         """Home page responder."""
         context = {"request": request, "books": books, "collections": collections, "page": skip, "limit": limit}
@@ -208,16 +213,25 @@ class FastAPIServer():
         return JSONResponse(content=collections_tojson(collections))
 
     @app.get("/api/collection/{collection}", response_class=JSONResponse)
-    async def collection(request: Request, collection: str, skip=0, limit=30):
+    async def collection(request: Request, collection: str, skip:int=0, limit:int=30):
-        storage = Storage(Config(os.path.abspath(os.getcwd())))
-        # collection = storage.get_collection(collection_name)
-        collection = storage.get_books(collection)
         """Collection file responder."""
+        storage = Storage(Config(os.path.abspath(os.getcwd())))
+        if skip <= 0:
+            skip_num = 0
+            skip = 0
+        else:
+            skip_num = skip * limit
+        books = storage.get_books(collection, skip=skip_num, limit=limit)
         collections = storage.get_collections()
-        # books = JSONResponse(content=books_tojson(collection))
+        context = {
-        context = {"request": request, "books": collection, "collections": collections, "page": skip, "limit": limit}
+            "request": request,
-        return templates.TemplateResponse("index.html", context)
+            "books": books,
-        # return JSONResponse(content=collections_tojson(collection))
+            "collections": collections,
+            "collection": collection,
+            "page": skip,
+            "limit": limit
+        }
+        return templates.TemplateResponse("collection.html", context)
 
     async def run(self):
         """Front end server entrypoint."""

src/frontend/templates/collection.html

@@ -0,0 +1,52 @@
+<!DOCTYPE html>
+{% block javascript %}
+<script type="text/javascript" src={{ url_for('static', path='script/pako.min.js') }}>
+	<script type="text/javascript">
+		const books = {{ books|books_tojson }};
+		let inflatedJSON = {};
+		const pako = require('pako');
+		inflatedJSON = JSON.parse(pako.inflate(books, { to: 'string'}));
+	</script>
+	{% endblock %}
+	{% include 'header.html' %}
+	{% include 'navigation.html' %}
+	<section id="master-container">
+		<!-- <div id="book-shelf" class="container is-dark"> -->
+		<div id="book-shelf">
+			{% for book in books %}
+			{% set cover = book.cover|b64decode %}
+			{% if cover != 'None' %}
+			<div class="is-dark book" id="{{book.id}}" onclick="window.location.href='/api/get_book/{{ book.id }}'">
+				<div class="image book-thumbnail">
+					<figure class="image is-4by3">
+						<img src="data:;base64,{{ book.cover|b64decode }}" alt="{{ book.title }}">
+					</figure>
+				</div>
+			</div>
+			{% else %}
+			<div class="is-dark book" id="{{book.id}}" onclick="window.location.href='/api/get_book/{{ book.id }}'">
+				<div class="image book-thumbnail"
+						 style="
+										background-image: url("{{ url_for('static', path='images/no-cover.jpg') }}");
+										background-size: cover;
+										background-position: center;
+										background-repeat: no-repeat;
+										">
+					<figure class="image is-4by3">
+						<div class="no-image-title">{{ book.title }}</div>
+						<!-- alt="{{ book.title }}" -->
+					</figure>
+				</div>
+				<!-- <p class="content">{{ book.description|summarize }}</p> -->
+			</div>
+			{% endif %}
+			{% endfor %}
+		</div>
+		<div id="pagination" class="container is-dark">
+			<nav class="pagination is-centered" role="navigation" aria-label="pagination">
+				<a class="pagination-previous" href="/api/collection/{{collection}}?skip={{ page|int - 1 }}" id="prev-page">Previous</a>
+				<a class="pagination-next" href="/api/collection/{{collection}}?skip={{ page|int + 1 }}" id="next-page">Next</a>
+			</nav>
+		</div>
+	</section>
+	{% include 'footer.html' %}

src/frontend/templates/index.html

@@ -1,6 +1,6 @@
 <!DOCTYPE html>
 {% block javascript %}
-<script type="text/javascript" src=static/script/pako.min.js>
+<script type="text/javascript" src={{ url_for('static', path='script/pako.min.js') }}>
 	<script type="text/javascript">
 		const books = {{ books|books_tojson }};
 		let inflatedJSON = {};
@@ -11,34 +11,32 @@
 	{% include 'header.html' %}
 	{% include 'navigation.html' %}
 	<section id="master-container">
-		<p>Total books: {{ books|length }}</p>
+		<div id="book-shelf">
-		<!-- <div id="book-shelf" class="container is-dark"> -->
-		<div id="book-shelf" class="is-dark">
 			{% for book in books %}
-			{% set cover = book[0].cover|b64decode %}
+			{% set cover = book.cover|b64decode %}
 			{% if cover != 'None' %}
-			<div class="is-dark book" id="{{book[0].id}}" onclick="window.location.href='/api/get_book/{{ book[0].id }}'">
+			<div class="is-dark book" id="{{book.id}}" onclick="window.location.href='/api/get_book/{{ book.id }}'">
 				<div class="image book-thumbnail">
 					<figure class="image is-4by3">
-						<img src="data:;base64,{{ book[0].cover|b64decode }}" alt="{{ book[0].title }}">
+						<img src="data:;base64,{{ book.cover|b64decode }}" alt="{{ book.title }}">
 					</figure>
 				</div>
 			</div>
 			{% else %}
-			<div class="is-dark book" id="{{book[0].id}}" onclick="window.location.href='/api/get_book/{{ book[0].id }}'">
+			<div class="is-dark book" id="{{book.id}}" onclick="window.location.href='/api/get_book/{{ book.id }}'">
 				<div class="image book-thumbnail"
 						 style="
-										background-image: url('static/images/no-cover.jpg');
+										background-image: url("{{ url_for('static', path='images/no-cover.jpg') }}");
 										background-size: cover;
 										background-position: center;
 										background-repeat: no-repeat;
 										">
 					<figure class="image is-4by3">
-						<div class="no-image-title">{{ book[0].title }}</div>
+						<div class="no-image-title">{{ book.title }}</div>
-						<!-- alt="{{ book[0].title }}" -->
+						<!-- alt="{{ book.title }}" -->
 					</figure>
 				</div>
-				<!-- <p class="content">{{ book[0].description|summarize }}</p> -->
+				<!-- <p class="content">{{ book.description|summarize }}</p> -->
 			</div>
 			{% endif %}
 			{% endfor %}

src/frontend/templates/navigation.html

@@ -50,7 +50,7 @@
 				<!-- <select id="collection_select" onchange="window.location.href='/api/collection/' + this.value"> -->
 				<select id="collection_select">
           {% for collection in collections %}
-						<option	value="{{collection[0].collection}}" class="collection_selection">{{collection[0].collection}}</option>
+						<option	value="{{collection.id}}" class="collection_selection">{{collection.name}}</option>
           {% endfor %}
         </select>
 				<script>