th3r00t/DosVault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Updated theming

Browse Source
This commit is contained in:
th3r00t
2025-09-06 18:51:10 -04:00
parent dc7b538be6
commit dae849bb90
11 changed files with 704 additions and 111 deletions
Download Patch File Download Diff File Expand all files Collapse all files

30
src/__main__.py
View File

@@ -9,7 +9,7 @@ from typing import Optional, List
from sqlalchemy import create_engine
from sqlalchemy.orm import Session
from libs.config import Config
from libs.database import (Base, ingest_roms, get_existing_rom_paths)
from libs.database import (ingest_roms, get_existing_rom_paths)
from libs.objects import Metadata, Game, Roms
from libs.functions import extract_year_from_title, clean_title, download_image, get_image_filename
from libs.apis import Credentials, IGDB
@@ -84,7 +84,7 @@ async def make_romlist(dir: Optional[Path] = None, roms: Optional[Roms] = None)
for pointer in rompath.rglob("*"):
if pointer.is_file():
title = pointer.stem
title = pointer.stem.strip('\'"') # Remove quotes from filename
romList.list.append(Game(title=title, path=pointer, metadata=Metadata()))
return romList
@@ -102,12 +102,12 @@ async def inject_metadata(roms: Roms) -> Roms:
except ValueError:
scrape_errors.append(game.title)
md = Metadata(title=game.title, year=extract_year_from_title(game.title))
# print each item as its done to the top of the screen
# log each item as its done
results[i] = md
print("\033[F\033[K", end='')
logging.info(f"Scraped: {game.title} # {i+1}/{len(roms.list)}")
# Log recent errors
for err in scrape_errors[-5:]:
print(f"Error: {err}")
print(f"Scraped: {game.title} # {i+1}/{len(roms.list)}")
logging.warning(f"Scraping error: {err}")
tasks = [asyncio.create_task(_job(i, game)) for i, game in enumerate(roms.list)]
await asyncio.gather(*tasks)
@@ -125,9 +125,9 @@ async def filter_new_roms(romlist: Roms, session: Session) -> Roms:
if game.path.resolve() not in existing_paths:
new_roms.list.append(game)
print(f"Found {len(romlist.list)} total ROMs")
print(f"Found {len(existing_paths)} existing ROMs in database")
print(f"Will scrape {len(new_roms.list)} new ROMs")
logging.info(f"Found {len(romlist.list)} total ROMs")
logging.info(f"Found {len(existing_paths)} existing ROMs in database")
logging.info(f"Will scrape {len(new_roms.list)} new ROMs")
return new_roms
@@ -145,11 +145,15 @@ async def main():
new_romlist = await inject_metadata(new_romlist)
ingest_roms(new_romlist, s)
else:
print("No new ROMs to scrape!")
logging.info("No new ROMs to scrape!")
print("Done\nError list:")
for err in scrape_errors:
print(f" - {err}")
logging.info("ROM scanning completed")
if scrape_errors:
logging.warning(f"Total scraping errors: {len(scrape_errors)}")
for err in scrape_errors:
logging.warning(f"Failed to scrape: {err}")
else:
logging.info("ROM scanning completed with no errors")
if __name__ == "__main__":
# Initialize logging

8
src/backfill_images.py
View File

@@ -217,7 +217,7 @@ class ImageBackfillManager:
# Commit batch
db_session.commit()
print(f" Batch committed to database")
print(" Batch committed to database")
async def run(self, limit: Optional[int] = None, dry_run: bool = False):
"""Run the image backfill process."""
@@ -226,7 +226,7 @@ class ImageBackfillManager:
# Show current statistics
stats = self.get_stats()
print(f"Database Statistics:")
print("Database Statistics:")
print(f" Total games: {stats['total_games']}")
print(f" Games with cover URLs: {stats['games_with_cover_urls']}")
print(f" Games with local covers: {stats['games_with_local_covers']}")
@@ -287,12 +287,12 @@ class ImageBackfillManager:
await self.process_batch(games)
# Show final results
print(f"\n✅ Backfill Complete!")
print("\n✅ Backfill Complete!")
print(f" Successfully downloaded: {self.successful_downloads} images")
print(f" Failed downloads: {len(self.failed_downloads)}")
if self.failed_downloads:
print(f"\nFailed Downloads:")
print("\nFailed Downloads:")
for failure in self.failed_downloads[:10]: # Show first 10
print(f" - {failure}")
if len(self.failed_downloads) > 10:

2
src/create_admin.py
View File

@@ -2,7 +2,7 @@
from sqlalchemy import create_engine
from sqlalchemy.orm import sessionmaker
from libs.config import Config
from libs.database import Base, User_table, UserRole
from libs.database import User_table, UserRole
from libs.auth import AuthManager
import sys

6
src/libs/functions.py
View File

@@ -1,7 +1,7 @@
from typing import Optional
import re
import asyncio
import aiohttp
import logging
from pathlib import Path
import hashlib
@@ -42,10 +42,10 @@ async def download_image(url: str, save_path: Path, session: aiohttp.ClientSessi
f.write(content)
return True
else:
print(f"Failed to download {url}: HTTP {response.status}")
logging.warning(f"Failed to download {url}: HTTP {response.status}")
return False
except Exception as e:
print(f"Error downloading {url}: {e}")
logging.error(f"Error downloading {url}: {e}")
return False
def get_image_filename(url: str, game_title: str, image_type: str) -> str:

2
src/migrate.py
View File

@@ -7,8 +7,6 @@ import argparse
from pathlib import Path
from alembic.config import Config
from alembic import command
from alembic.script import ScriptDirectory
from alembic.runtime.environment import EnvironmentContext
from sqlalchemy import create_engine, inspect
# Add current directory to path for imports

8
src/refresh_covers.py
View File

@@ -9,7 +9,7 @@ import asyncio
import aiohttp
from pathlib import Path
from typing import List, Optional
from sqlalchemy import create_engine, select, func
from sqlalchemy import create_engine, select
from sqlalchemy.orm import Session, selectinload
try:
@@ -191,7 +191,7 @@ class CoverRefreshManager:
# Commit batch
db_session.commit()
print(f" Batch committed to database")
print(" Batch committed to database")
async def run(self, limit: Optional[int] = None, dry_run: bool = False):
"""Run the cover refresh process."""
@@ -246,13 +246,13 @@ class CoverRefreshManager:
await self.process_batch(games)
# Show final results
print(f"\n✅ Refresh Complete!")
print("\n✅ Refresh Complete!")
print(f" Games with updated metadata: {self.refreshed_count}")
print(f" Images successfully downloaded: {self.download_success_count}")
print(f" Failed refreshes: {len(self.failed_refreshes)}")
if self.failed_refreshes:
print(f"\nFailed Refreshes (first 10):")
print("\nFailed Refreshes (first 10):")
for failure in self.failed_refreshes[:10]:
print(f" - {failure}")
if len(self.failed_refreshes) > 10:

157
src/webapp.py
View File

@@ -5,7 +5,6 @@ from typing import Optional, Annotated
from datetime import timedelta, datetime, timezone
import re
import asyncio
import subprocess
from pathlib import Path
from fastapi import FastAPI, Depends, HTTPException, status, Request, Form, Query, BackgroundTasks
@@ -26,7 +25,7 @@ try:
except ImportError:
# Fall back to absolute imports (when run directly)
from libs.config import Config
from libs.database import Base, Game_table, Metadata_table, User_table, UserRole, user_favorites, Tags_table, Genre_table
from libs.database import Game_table, Metadata_table, User_table, UserRole, user_favorites, Tags_table, Genre_table
from libs.auth import AuthManager, ACCESS_TOKEN_EXPIRE_MINUTES
from libs.logging import get_log_manager
@@ -259,6 +258,53 @@ async def get_game(
})
@app.get("/api/games/{game_id}")
async def get_game_json(
game_id: int,
db: Session = Depends(get_db),
current_user: Optional[User_table] = Depends(get_current_user)
):
"""Get game details as JSON for the overlay"""
game = db.get(Game_table, game_id)
if not game:
raise HTTPException(status_code=404, detail="Game not found")
is_favorite = False
if current_user and current_user.role != UserRole.DEMO.value:
is_favorite = game in current_user.favorites
# Build game data structure
game_data = {
"id": game.id,
"title": game.metadata_obj.title if game.metadata_obj and game.metadata_obj.title else game.title,
"filename": game.path.name,
"filepath": str(game.path),
"is_favorite": is_favorite,
"can_download": current_user and current_user.role != UserRole.DEMO.value,
"is_demo": current_user is None or current_user.role == UserRole.DEMO.value,
"is_super": current_user and current_user.role == UserRole.SUPER.value,
"metadata": {}
}
if game.metadata_obj:
metadata = game.metadata_obj
game_data["metadata"] = {
"description": metadata.description,
"year": metadata.year,
"developer": metadata.developer,
"publisher": metadata.publisher,
"players": metadata.players,
"cover_image": metadata.cover_image_path.name if metadata.cover_image_path else metadata.cover_image,
"screenshot": metadata.screenshot_path.name if metadata.screenshot_path else metadata.screenshot,
"cover_image_local": bool(metadata.cover_image_path),
"screenshot_local": bool(metadata.screenshot_path),
"genres": [{"name": genre.name} for genre in metadata.genre] if metadata.genre else [],
"tags": [{"name": tag.name} for tag in metadata.tags] if metadata.tags else []
}
return game_data
@app.post("/games/{game_id}/favorite")
async def toggle_favorite(
game_id: int,
@@ -301,6 +347,8 @@ async def download_game(
# Create a clean filename using the game title
game_title = game.metadata_obj.title if game.metadata_obj and game.metadata_obj.title else game.title
# Strip quotes from the title first
game_title = game_title.strip('\'"') # Remove leading/trailing quotes
# Clean the title for use as filename
clean_title = re.sub(r'[^\w\s-]', '', game_title).strip()
clean_title = re.sub(r'[-\s]+', '-', clean_title)
@@ -532,7 +580,7 @@ async def get_cover_url(game_id: int, db: Session = Depends(get_db)):
db.commit()
return {"cover_url": cover_url}
except Exception as e:
print(f"Error converting image ID to URL: {e}")
logging.error(f"Error converting image ID to URL: {e}")
# Try to fetch cover from IGDB
try:
@@ -555,7 +603,7 @@ async def get_cover_url(game_id: int, db: Session = Depends(get_db)):
return {"cover_url": cover_url}
except Exception as e:
print(f"Error fetching cover for game {game_id}: {e}")
logging.error(f"Error fetching cover for game {game_id}: {e}")
return {"cover_url": None}
@@ -564,13 +612,13 @@ async def get_cover_url(game_id: int, db: Session = Depends(get_db)):
async def browse_by_tag(
tag_name: str,
request: Request,
page: int = Query(1, ge=1),
per_page: int = Query(20, ge=1, le=100),
view: str = Query("grid", pattern="^(grid|list)$"),
db: Session = Depends(get_db),
current_user: Optional[User_table] = Depends(get_current_user)
):
"""Browse games by tag"""
page = int(request.query_params.get("page", 1))
per_page = int(request.query_params.get("per_page", 20))
view = request.query_params.get("view", "grid")
per_page = max(10, min(per_page, 100))
offset = (page - 1) * per_page
@@ -624,6 +672,7 @@ async def browse_by_genre(
request: Request,
page: int = Query(1, ge=1),
per_page: int = Query(20, ge=1, le=100),
view: str = Query("grid", pattern="^(grid|list)$"),
db: Session = Depends(get_db),
current_user: Optional[User_table] = Depends(get_current_user)
):
@@ -657,18 +706,15 @@ async def browse_by_genre(
"request": request,
"current_user": current_user,
"games": games,
"page": page,
"current_page": page,
"per_page": per_page,
"total": total,
"total_games": total,
"total_pages": total_pages,
"search": f"genre:{genre_name}",
"show_pagination": True,
"current_url": f"/browse/genres/{genre_name}",
"view": view,
"browse_type": "genre",
"browse_value": genre_name,
"is_demo": current_user is None or current_user.role == UserRole.DEMO.value,
"view": "grid", # Default to grid view for genre browsing
"current_page": page,
"user_favorites": user_favorites
})
@@ -807,7 +853,7 @@ async def admin_system_stats(
"recent_users": recent_users,
"disk_usage": disk_usage,
"running_tasks": {
task_name: not task.done() if task_name in running_tasks else False
task_name: not running_tasks[task_name].done() if task_name in running_tasks else False
for task_name in ["rom_scan", "metadata_refresh", "image_sync"]
}
}
@@ -815,43 +861,110 @@ async def admin_system_stats(
async def run_rom_scan():
"""Run the ROM scanner subprocess"""
try:
logging.info("Starting ROM scan subprocess")
process = await asyncio.create_subprocess_exec(
"python", "-m", "src",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await process.communicate()
return {"success": process.returncode == 0, "output": stdout.decode(), "error": stderr.decode()}
# Log the output for visibility
if stdout:
for line in stdout.decode().strip().split('\n'):
if line.strip():
logging.info(f"ROM Scanner: {line.strip()}")
if stderr:
for line in stderr.decode().strip().split('\n'):
if line.strip():
logging.error(f"ROM Scanner Error: {line.strip()}")
success = process.returncode == 0
logging.info(f"ROM scan subprocess completed with exit code: {process.returncode}")
return {
"success": success,
"output": stdout.decode(),
"error": stderr.decode(),
"returncode": process.returncode
}
except Exception as e:
return {"success": False, "error": str(e)}
error_msg = f"Failed to start ROM scan subprocess: {str(e)}"
logging.error(error_msg)
return {"success": False, "error": error_msg}
async def run_metadata_refresh():
"""Refresh metadata for games without complete metadata"""
try:
# Run ROM scanner with metadata refresh flag
logging.info("Starting metadata refresh subprocess")
process = await asyncio.create_subprocess_exec(
"python", "-m", "src", "--refresh-metadata",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await process.communicate()
return {"success": process.returncode == 0, "output": stdout.decode(), "error": stderr.decode()}
# Log the output for visibility
if stdout:
for line in stdout.decode().strip().split('\n'):
if line.strip():
logging.info(f"Metadata Refresh: {line.strip()}")
if stderr:
for line in stderr.decode().strip().split('\n'):
if line.strip():
logging.error(f"Metadata Refresh Error: {line.strip()}")
success = process.returncode == 0
logging.info(f"Metadata refresh subprocess completed with exit code: {process.returncode}")
return {
"success": success,
"output": stdout.decode(),
"error": stderr.decode(),
"returncode": process.returncode
}
except Exception as e:
return {"success": False, "error": str(e)}
error_msg = f"Failed to start metadata refresh subprocess: {str(e)}"
logging.error(error_msg)
return {"success": False, "error": error_msg}
async def run_image_sync():
"""Download missing cover images and screenshots"""
try:
# Run ROM scanner with image sync flag
logging.info("Starting image sync subprocess")
process = await asyncio.create_subprocess_exec(
"python", "-m", "src", "--sync-images",
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE
)
stdout, stderr = await process.communicate()
return {"success": process.returncode == 0, "output": stdout.decode(), "error": stderr.decode()}
# Log the output for visibility
if stdout:
for line in stdout.decode().strip().split('\n'):
if line.strip():
logging.info(f"Image Sync: {line.strip()}")
if stderr:
for line in stderr.decode().strip().split('\n'):
if line.strip():
logging.error(f"Image Sync Error: {line.strip()}")
success = process.returncode == 0
logging.info(f"Image sync subprocess completed with exit code: {process.returncode}")
return {
"success": success,
"output": stdout.decode(),
"error": stderr.decode(),
"returncode": process.returncode
}
except Exception as e:
return {"success": False, "error": str(e)}
error_msg = f"Failed to start image sync subprocess: {str(e)}"
logging.error(error_msg)
return {"success": False, "error": error_msg}
@app.get("/api/admin/system-logs")
async def admin_system_logs(