th3r00t/DosVault
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Added new rom import system utilizing WAL to avoid locking the database and freezing the frontend

Browse Source 
Also added new logging setup to hopefully stream the scrape process
This commit is contained in:
th3r00t
2025-09-07 12:50:05 -04:00
parent c94c0554df
commit 7e4c194c1f
19 changed files with 205 additions and 1057 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
src/libs/database.py
View File

@@ -199,43 +199,62 @@ def get_existing_rom_paths(session: Session) -> set[Path]:
return {game.path.resolve() for game in session.scalars(select(Game_table)).all()}
def ingest_roms(roms: Roms, session: Session, *, batch: int = 200) -> int:
import logging
n = 0
for g in roms.list:
game = session.scalar(select(Game_table).where(Game_table.path == g.path))
if game is None:
game = Game_table(title=g.title, path=g.path)
session.add(game)
else:
game.title = g.title
mdto = g.metadata
md = game.metadata_obj
if md is None:
md = Metadata_table(game=game, title=mdto.title or g.title)
session.add(md)
try:
game = session.scalar(select(Game_table).where(Game_table.path == g.path))
if game is None:
game = Game_table(title=g.title, path=g.path)
session.add(game)
logging.info(f"Adding new game: {g.title}")
else:
game.title = g.title
logging.info(f"Updating existing game: {g.title}")
mdto = g.metadata
md = game.metadata_obj
if md is None:
md = Metadata_table(game=game, title=mdto.title or g.title)
session.add(md)
md.title = mdto.title or g.title
md.description = mdto.description
md.year = mdto.year if mdto.year is not None else extract_year_from_title(md.title)
md.developer = mdto.developer
md.publisher = mdto.publisher
md.players = mdto.players
md.cover_image = mdto.cover_image
md.screenshot = mdto.screenshot
md.cover_image_path = mdto.cover_image_path
md.screenshot_path = mdto.screenshot_path
md.title = mdto.title or g.title
md.description = mdto.description
md.year = mdto.year if mdto.year is not None else extract_year_from_title(md.title)
md.developer = mdto.developer
md.publisher = mdto.publisher
md.players = mdto.players
md.cover_image = mdto.cover_image
md.screenshot = mdto.screenshot
md.cover_image_path = mdto.cover_image_path
md.screenshot_path = mdto.screenshot_path
try: genres = sorted({s.strip() for s in (mdto.genre or []) if s and s.strip()})
except: genres = []
try: tags = sorted({s.strip() for s in (mdto.tags or []) if s and s.strip()})
except: tags = []
try: genres = sorted({s.strip() for s in (mdto.genre or []) if s and s.strip()})
except: genres = []
try: tags = sorted({s.strip() for s in (mdto.tags or []) if s and s.strip()})
except: tags = []
md.genre = [_get_or_create_by_name(session, Genre_table, name) for name in genres]
md.tags = [_get_or_create_by_name(session, Tags_table, name) for name in tags]
md.genre = [_get_or_create_by_name(session, Genre_table, name) for name in genres]
md.tags = [_get_or_create_by_name(session, Tags_table, name) for name in tags]
n += 1
if n % batch == 0:
session.flush()
n += 1
# Use more frequent flushes and commits to reduce lock time
if n % batch == 0:
session.commit() # Commit more frequently to reduce lock duration
logging.info(f"Committed batch of {batch} games to database ({n} total)")
except Exception as e:
logging.error(f"Failed to ingest game {g.title}: {e}")
session.rollback() # Rollback on error to prevent corruption
continue
session.commit()
# Final commit for remaining items
try:
session.commit()
logging.info(f"Successfully ingested {n} games to database")
except Exception as e:
logging.error(f"Failed final commit during ROM ingestion: {e}")
session.rollback()
return n