Added new rom import system utilizing WAL to avoid locking the database and freezing the frontend

Also added new logging setup to hopefully stream the scrape process

devenv.nix

@@ -1,103 +0,0 @@
-{ pkgs, lib, config, inputs, ... }:
-
-{
-  # https://devenv.sh/basics/
-
-  # https://devenv.sh/packages/
-	packages = with pkgs; [
-		git
-		curl
-		pkg-config
-		sqlite
-		pyright
-		pre-commit
-	];
-	languages.python = {
-		enable = true;
-		package = pkgs.python313;
-		libraries = with pkgs.python313Packages; [ ];
-		venv = {
-			enable = true;
-			requirements = ''
-				pudb
-				ptpython
-				ipython
-				pytest
-				pytest-cov
-				flake8
-				ptpython
-				ipython
-				isort
-				pynvim
-				ruff
-				black
-				sqlalchemy
-				requests
-				fastapi
-				uvicorn
-				jinja2
-				python-multipart
-				bcrypt
-				python-jose
-				passlib
-				alembic
-				aiohttp
-			'';
-		};
-		# uv = {
-		# 	enable = false;
-		# 	sync.enable = true;
-		# };
-	};
-	env = {
-		PYTHONBREAKPOINT = "pudb.set_trace";
-	};
-
-	# https://devenv.sh/variables/
-	# variables = {
-	# 	GREET = "world";
-	# };
-
-  # https://devenv.sh/scripts/
-	scripts = {
-		"tests".exec = "cd $REPO_ROOT && python -m pytest --rootdir=$REPO_ROOT -c $REPO_ROOT/pytest.ini";
-		"lint".exec = "cd $REPO_ROOT && ${pkgs.ruff}/bin/ruff check . && black --check .";
-		"fix".exec = "cd $REPO_ROOT && ${pkgs.ruff}/bin/ruff check . --fix && black .";
-		"typecheck".exec = "cd $REPO_ROOT && pyright";
-		"run".exec = ''cd $REPO_ROOT && ./src/__main__.py "$@"'';
-		"serve".exec = "cd $REPO_ROOT && python src/webapp.py";
-		"create-admin".exec = "cd $REPO_ROOT && python src/create_admin.py";
-		"migrate".exec = "cd $REPO_ROOT && python src/migrate.py";
-		"db-init".exec = "cd $REPO_ROOT && python src/migrate.py init";
-		"db-upgrade".exec = "cd $REPO_ROOT && python src/migrate.py upgrade";
-		"db-create".exec = "cd $REPO_ROOT && python src/migrate.py create";
-		"build".exec = "cd $REPO_ROOT && ./build.sh";
-		"backfill-images".exec = "cd $REPO_ROOT && python src/backfill_images.py";
-		"export-requirements".exec = "pip freeze > requirements.txt";
-		"export-reqs".exec = ''
-			cd "$REPO_ROOT"
-			printf "%s\n" '${config.languages.python.venv.requirements}' > requirements.txt
-			echo "Wrote requirements.txt"
-		'';
-	};
-  enterShell = ''
-		export REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null || pwd)"
-  '';
-
-  # https://devenv.sh/tasks/
-  # tasks = {
-  #   "myproj:setup".exec = "mytool build";
-  #   "devenv:enterShell".after = [ "myproj:setup" ];
-  # };
-
-  # https://devenv.sh/tests/
-  enterTest = ''
-    echo "Running tests"
-   	pytest -q 
-  '';
-
-  # https://devenv.sh/git-hooks/
-  # git-hooks.hooks.shellcheck.enable = true;
-
-  # See full reference at https://devenv.sh/reference/options/
-}