th3r00t/pyShelf
1
0
Fork 0
mirror of https://github.com/th3r00t/pyShelf.git synced 2026-04-28 01:59:35 -04:00
Code Issues Packages Projects Releases Wiki Activity

First working draft docker-compose

Browse Source
This commit is contained in:
MartenBE
2020-07-30 16:57:39 +02:00
parent be2c525d9a
commit d555d94be9
16 changed files with 77 additions and 541 deletions
Download Patch File Download Diff File Expand all files Collapse all files

47
Dockerfile
View File

@@ -1,20 +1,27 @@
FROM archlinux:latest FROM ubuntu
RUN pacman -Syy
RUN pacman -Syu --noconfirm EXPOSE 8000
RUN pacman -S --noconfirm python python-pip git postgresql sudo gcc
RUN sudo -u postgres initdb --locale=en_US.UTF-8 -E UTF8 -D /var/lib/postgres/data RUN apt-get update -y
RUN useradd pyshelf && chpasswd pyshelf:pyshelf RUN DEBIAN_FRONTEND=noninteractive apt-get install -y build-essential python3 python3-dev python3-pip python3-venv nginx-full
#RUN mkdir -p /srv/Books && mkdir -p /srv/http && mkdir -p /srv/logs/ && mkdir -p /run/postgresql && \
# touch /srv/logs/pgsql.log && chown postgres.postgres /run/postgresql && \ COPY . /pyshelf
# chown http.pyshelf /srv/Books && chown http.pyshelf /srv/http && chown postgres.postgres /srv/logs/pgsql.log
VOLUME /srv/Books ./Books WORKDIR /pyshelf/
VOLUME /srv/http . RUN python3 -m pip install -r requirements.txt
VOLUME /srv/logs ./logs
VOLUME /var/lib/postgres/data ./pgdata COPY ./uwsgi_params /etc/nginx/uwsgi_params
RUN sudo -u postgres pg_ctl -D /var/lib/postgres/data -l /srv/logs/pgsql.log start COPY ./pyshelf_nginx.conf /etc/nginx/sites-available/pyshelf_nginx.conf
RUN sudo -u postgres psql -f create_db.sql RUN ln -s /etc/nginx/sites-available/pyshelf_nginx.conf /etc/nginx/sites-enabled/
ENV PYTHONUNBUFFERED=1
WORKDIR /srv/http WORKDIR /pyshelf/
RUN pip install -r requirements.txt ENTRYPOINT cd src/ \
EXPOSE 80 8000 && python3 manage.py makemigrations \
CMD ["sh", "-c","/srv/http/entry.sh"] && python3 manage.py makemigrations interface \
&& python3 manage.py migrate \
&& python3 manage.py migrate interface \
&& cd .. \
&& python3 importBooks \
&& python3 makeCollections \
&& nginx -g "daemon on;" \
&& uwsgi --ini uwsgi.ini

16
config.json
View File

@@ -1 +1,15 @@
{"TITLE": "pyShelf E-Book Server", "VERSION": "0.6.0", "BOOKPATH": "/home/raelon/Books", "DB_HOST": "localhost", "DB_PORT": "5432", "DATABASE": "pyshelf", "USER": "pyshelf", "PASSWORD": "pyshelf", "BOOKSHELF": "data/shelf.json", "ALLOWED_HOSTS": "*", "hostname": "localhost", "webport": "8000", "wsgiport": "8001"} {
"TITLE": "pyShelf E-Book Server",
"VERSION": "0.6.0",
"BOOKPATH": "/books",
"DB_HOST": "db",
"DB_PORT": "5432",
"DATABASE": "pyshelf",
"USER": "pyshelf",
"PASSWORD": "pyshelf",
"BOOKSHELF": "data/shelf.json",
"ALLOWED_HOSTS": "*",
"hostname": "localhost",
"webport": "8000",
"wsgiport": "8001"
}

3
create_db.sql
View File

@@ -1,3 +0,0 @@
CREATE USER pyshelf WITH PASSWORD 'pyshelf';
CREATE DATABASE pyshelf;
GRANT ALL PRIVILEGES ON DATABASE pyshelf TO pyshelf;

13
database.sh
View File

@@ -1,13 +0,0 @@
#!/bin/bash
sudo -u postgres pg_ctl -D /var/lib/postgres/data -l /srv/logs/pgsql.log start &&
sudo -u postgres psql -f create_db.sql &&
rm /srv/http/database.sh &&
echo "sudo -u postgres pg_ctl -D /var/lib/postgres/data -l /srv/logs/lgsql.log start" > /srv/http/database.sh &&
chmod +x /srv/http/database.sh &&
cd src && \
python manage.py makemigrations && \
python manage.py makemigrations interface && \
python manage.py migrate && \
python manage.py migrate interface && \
cd ..
echo "pyShelf Env Started"

25
docker-compose.yml Normal file
View File

@@ -0,0 +1,25 @@
version: "3.7"
services:
db:
image: "postgres"
environment:
- "POSTGRES_PASSWORD=pyshelf"
- "POSTGRES_USER=pyshelf"
- "POSTGRES_DB=pyshelf"
volumes:
- "pgdata:/var/lib/postgresql/data/"
pyshelf:
build:
context: .
dockerfile: Dockerfile
ports:
- "8080:8000"
volumes:
- "./books:/books"
depends_on:
- db
volumes:
pgdata:

6
docker/.env vendored
View File

@@ -1,6 +0,0 @@
LOCAL_BOOK_DIR=/home/user/pyShelf/Books
PORT=8088
POSTGRES_VER=12.2
POSTGRES_USER=pyshelf
POSTGRES_PASSWORD=pyshelf
POSTGRES_DB=pyshelf

20
docker/Dockerfile vendored
View File

@@ -1,20 +0,0 @@
# docker build -t ubuntu1604py36
FROM ubuntu:18.04
RUN apt-get update && apt-get install -y software-properties-common
RUN add-apt-repository ppa:deadsnakes/ppa
RUN apt-get update -y
RUN apt-get install -y build-essential python3.8 python3.8-dev python3-pip python3.8-venv && apt-get install -y git
# update pip
RUN python3 -m pip install wheel
RUN git clone https://github.com/hat/pyShelf.git /usr/src/app/
RUN python3 -m pip install -r /usr/src/app/requirements.txt
EXPOSE 8000
WORKDIR /usr/src/app/src/
#RUN ../docker/scripts/wait-for-it.sh db:5432
RUN python3 manage.py migrate
#RUN python3 manage.py migrate interface
CMD ["python3", "manage.py", "runserver", "0.0.0.0:8000"]

30
docker/docker-compose.yml vendored
View File

@@ -1,30 +0,0 @@
version: "3.7"
services:
db:
image: "postgres:${POSTGRES_VER}"
environment:
- "POSTGRES_PASSWORD=${POSTGRES_PASSWORD}"
- "POSTGRES_USER=${POSTGRES_USER}"
- "POSTGRES_DB=${POSTGRES_DB}"
volumes:
- ./postgres_data/:/var/lib/postgresql/data/
pyshelf:
build:
context: ./
dockerfile: Dockerfile
volumes:
- "${LOCAL_BOOK_DIR}:/usr/src/app/Books"
# - "./cron/root:/etc/crontabs/root"
# - "./scripts/wait-for-it.sh:/usr/src/app/src/"
ports:
- "${PORT}:8000"
depends_on:
- db
networks:
default:
driver: bridge
ipam:
config:
- subnet: 10.20.20.0/24

4
entry.sh
View File

@@ -1,4 +0,0 @@
#!/bin/bash
#sh /srv/http/database.sh
sudo -u postgres pg_ctl -D /var/lib/postgres/data -l /srv/logs/pgsql.log start &&
#gunicorn pyShelf.wsgi 8000:8000

0
importBooks.pstat
View File

4
install.sh
View File

@@ -1,4 +0,0 @@
#!/usr/bin/env bash
eval python3 preinstall
eval "pip install -r requirements.txt"
eval python3 installer

395
installer
View File

@@ -1,395 +0,0 @@
#!/usr/bin/ env python
import json
import os
import pathlib
import platform
import pprint
import subprocess as sp
import sys
from shutil import copyfile
import psutil
from src.backend.lib.display import TerminalDisplay
log_file = "installer.log"
messages = []
class Configuration:
def __init__(self):
self._cp = pathlib.Path("config.json")
self._data = self.open_file()
self.system = platform.system()
def open_file(self):
"""
Try to open and then backup the configuration file.
Fail and return false if initial configuration is not found.
# TODO: More specific error handling
"""
try:
with open(str(self._cp), "r") as read_file:
data = json.load(read_file)
with open("config.backup.json", "w") as backup_file:
json.dump(data, backup_file)
return data
except Exception as e:
print(e)
return False
def write_file(self, data):
"""
Write the provided data to the new configuration file
"""
with open(str(self._cp), "w") as write_file:
json.dump(data, write_file)
return True
class RequiredServices:
def check_ps(self, service_list):
"""
Check service_list against running processes
by calling self.process_list, remove found
services from the list and return
"""
# Get the matched processes
_matches = self.process_list().intersection(set(service_list))
for r in _matches:
service_list.remove(r)
return service_list
@staticmethod
def process_list():
"""
Iterate running processes returning the name of each
make it a set and return
"""
_processes = []
for p in psutil.process_iter():
_processes.append(p.name())
return set(_processes)
@staticmethod
def web_server_found(service_list):
# Determine whether or not both possible webservers are missing
_c = 0
for r in service_list:
if r == "nginx" or r == "httpd":
_c = _c + 1
if _c > 1:
return False # Return false if neither are found
else:
return True # Return true if one is found
@staticmethod
def db_server_found(service_list):
_c = 0
for r in service_list:
if r == "postgres":
_c = _c + 1
if _c > 0:
return False
else:
return True
class SystemInstaller:
def __init__(self):
self.bin = self.get()
self.site_dirs = ["/etc/nginx/sites-available", "/etc/nginx/sites-enabled"]
self.nginx_conf = "pyshelf_nginx.conf"
def get(self):
platfrm = platform.platform().split("-")
if platfrm[0].lower() == "linux":
installers = [
{"bin": "apt", "options": [], "search": "search", "install": "install"},
{"bin": "pacman", "options": [], "search": "-Ss", "install": "-S"},
{"bin": "yum", "options": [], "search": "search", "install": "install"},
{"bin": "docker", "options": []},
]
_paths = os.environ["PATH"].split(":")
for p in _paths:
for _installer in installers:
_fp = p + "/" + str(_installer["bin"])
if os.path.isfile(_fp):
global messages
messages = messages + [
"Found system installer binary " + str(_installer["bin"])
]
return _installer
def copy_config(self, _file=None, _dirs=None):
if _file is None:
_file = self.nginx_conf
if _dirs is None:
_dirs = self.site_dirs
outfile = "/%s" % _file.__str__()
if os.path.isdir(_dirs[0]):
os.system("sudo cp %s %s" % (_file, _dirs[0] + outfile))
else:
os.system("sudo mkdir %s" % _dirs[0])
os.system("sudo cp %s %s" % (_file, _dirs[0] + outfile))
try:
if os.path.isdir(_dirs[1]):
ln_string = str(_dirs[0] + outfile + " " + _dirs[1] + outfile)
os.system("sudo ln -s %s" % ln_string)
""" TODO check for sites-enabled, create it if it doesnt exist,
then symlink the config """
except Exception as e:
pass
return True
def make_nginx_config(self, answers):
root = os.path.abspath(".")
_fp = "pyshelf_nginx.conf"
for r in answers:
if r["name"] == "hostname":
hostname = r["answer"]
elif r["name"] == "webport":
port = r["answer"]
elif r["name"] == "wsgiport":
wsgiport = r["answer"]
nginx_conf_str = """
# pyshelf_nginx.conf
upstream django {server unix:///tmp/pyshelf_wsgi.sock;}
server {
listen %s;
server_name %s;
access_log /var/log/nginx/pyshelf.access.log;
error_log /var/log/nginx/pyshelf.error.log;
charset utf-8;
client_max_body_size 75M;
location /media {root %s/src/interface;}
location /static {root %s/src/interface;}
location /books {internal; alias %s;}
location / {uwsgi_pass django; include %s/uwsgi_params;}
}
""" % (
port,
hostname,
root,
root,
root,
root,
)
with open(_fp, "w") as write_file:
write_file.write(nginx_conf_str)
global messages
messages = messages + ["Generated new pyshelf_nginx.conf", nginx_conf_str]
def make_wsgi_config(self, answers):
root = os.path.abspath(".")
_fp = "uwsgi.ini"
for r in answers:
if r["name"] == "hostname":
hostname = r["answer"]
elif r["name"] == "wsgiport":
wsgiport = r["answer"]
wsgi_conf_str = """
[uwsgi]
chdir=%s/src
module=frontend.wsgi
master=True
pidfile=/tmp/pyShelf.pid
vacuum=True
socket=/tmp/pyshelf_wsgi.sock
chmod-socket=666
""" % (
root,
)
with open(_fp, "w") as write_file:
write_file.write(wsgi_conf_str)
global messages
messages = messages + ["Generated uwsgi.ini", wsgi_conf_str]
def log(self):
global log_file
global messages
with open(log_file, "w") as write_file:
write_file.write(TerminalDisplay().banner_render())
for message in messages:
write_file.write(message + "\n")
messages = messages + ["Log file written to " + log_file.__str__()]
config = Configuration().open_file()
sysinstall = SystemInstaller()
installer = sysinstall.bin
# Get user configuration options
install_answers = TerminalDisplay().installer()
for key in install_answers:
config[key["name"]] = key["answer"]
# config["USER"] = os.environ["USER"]
config["USER"] = "pyshelf"
# Write configuration
Configuration().write_file(config)
# Start checking for our list of required services
service_list = ["postgres", "nginx", "httpd"]
req = RequiredServices().check_ps(service_list)
# Does user have either nginx || apache?
if RequiredServices().web_server_found(req) is False:
web_prompt = [
{
"message": " You must have either apache or nginx\n would you like \
us to try and install nginx now? Enter for default 'no' > ",
"options": "nginx",
"name": "NGINX",
"answer": None,
"default": "no",
}
]
install_prompt = TerminalDisplay().prompt(web_prompt)
if install_prompt[0]["answer"] == "yes":
if installer is None:
installer = SystemInstaller().bin
if installer["bin"] == "pacman":
package = "nginx-mainline"
else:
package = "nginx"
options = ""
for o in installer["options"]:
options = options + " " + o
cmd = (
"sudo "
+ installer["bin"]
+ " "
+ installer["install"]
+ " "
+ options
+ package
)
install_status = os.system(cmd)
os.system("sudo systemctl start nginx")
messages = messages + [
"Nginx installed and started",
"To enable autostart you must run",
" sudo systemctl enable nginx",
"\n",
]
# Does user have postgreSQL?
if RequiredServices().db_server_found(req) is False:
db_prompt = [
{
"message": " You must have PostgreSQL\n would you like us to try \
and install it now? Enter for default 'no' > ",
"options": "postgres",
"name": "postgresql",
"answer": None,
"default": "no",
}
]
install_prompt = TerminalDisplay().prompt(db_prompt)
if install_prompt[0]["answer"] == "yes":
if installer is None:
installer = SystemInstaller().bin
options = ""
for o in installer["options"]:
options = options + " " + o
package = "postgresql"
cmd = (
"sudo "
+ installer["bin"]
+ " "
+ installer["install"]
+ " "
+ options
+ package
)
install_status = os.system(cmd)
for r in install_answers:
if r["name"] == "PASSWORD":
sql_pass = r["answer"]
sql_user = config["USER"]
# sql_user = "pyshelf"
db_name = "pyshelf"
psql_cmd = (
"CREATE DATABASE %s; CREATE USER %s WITH PASSWORD '%s'; \
GRANT ALL PRIVILEGES ON DATABASE %s TO %s;"
% (db_name, sql_user, sql_pass, db_name, sql_user)
)
_sql_file = "/tmp/create_db.sql"
with open(_sql_file, "w") as sql_file_open:
sql_file_open.write(psql_cmd)
sql_file_open.close()
os.system(
"sudo -u postgres initdb --locale=en_US.UTF-8 -E UTF8 \
-D /var/lib/postgres/data"
)
os.system("sudo systemctl start postgresql")
os.system("sudo -u postgres psql -f %s" % _sql_file)
# os.system("sudo -u postgres psql -c \'%s\'"%psql_cmd)
messages = messages + [
"PostgreSQL installed and started",
"To enable autostart you must run",
" sudo systemctl enable nginx",
"\n",
"Database cluster initialized at /var/lib/postgres",
"pyShelf database and user created",
psql_cmd,
]
else:
for r in install_answers:
if r["name"] == "PASSWORD":
sql_pass = r["answer"]
sql_user = config["USER"]
db_name = "pyshelf"
psql_cmd = (
"CREATE DATABASE %s; CREATE USER %s WITH PASSWORD '%s'; \
GRANT ALL PRIVILEGES ON DATABASE %s TO %s;"
% (db_name, sql_user, sql_pass, db_name, sql_user)
)
_sql_file = "/tmp/create_db.sql"
with open(_sql_file, "w") as sql_file_open:
sql_file_open.write(psql_cmd)
sql_file_open.close()
os.system("sudo systemctl start postgresql")
os.system("sudo -u postgres psql -f %s" % _sql_file)
# os.system("sudo -u postgres psql -c \'%s\'"%psql_cmd)
messages = messages + [
"pyShelf database and user created",
psql_cmd,
]
# Post install configurations
sysinstall.make_nginx_config(install_answers)
try:
os.chdir("src/")
os.system("python manage.py makemigrations")
os.system("python manage.py makemigrations interface")
os.system("python manage.py migrate")
os.system("python manage.py migrate interface")
os.chdir("../")
except Exception as e:
print("-" * 80)
print(" E:" + e)
try:
copy_config = sysinstall.copy_config()
if copy_config:
messages = messages + [
"pyShelf site config copied to sites-available, and symlinked to sites-enabled"
]
except Exception as e:
messages = messages + [
"nginx site config not copied",
'A nginx config for your install has been created "pyshelf_nginx.conf"',
]
sysinstall.make_wsgi_config(install_answers)
messages = messages + [
"You should now import your books by running importBooks",
"You can then start the interface with uwsgi --ini uwsgi.ini",
]
# Display end screen
sysinstall.log()
TerminalDisplay().clear()
TerminalDisplay().banner()
for message in messages:
print(" " + message)
print()
TerminalDisplay().h_rule()

33
preinstall
View File

@@ -1,33 +0,0 @@
#!python
import os
from subprocess import run
class RequiredPackages:
def __init__(self, bins):
self.required = bins
self.to_be_installed = []
self.locate()
def locate(self):
for bin in self.required:
if os.path.isfile(bin[1]):
continue
else:
self.to_be_installed.append(bin[0])
return self.to_be_installed
# Package List
package_list = [["gcc", "/usr/bin/gcc"]]
packages, required_packages = "", RequiredPackages(package_list)
for package in required_packages.to_be_installed:
packages = packages + package + " "
try:
ret = run(["sudo", "pacman", "-S", packages[0:-1]])
print(ret)
except Exception as e:
ret = run(["pacman", "-S", packages[0:-1]])
print(ret)

2
requirements.txt
View File

@@ -18,6 +18,6 @@ prompt_toolkit
psutil psutil
pyfiglet pyfiglet
mobi-python mobi-python
pudb uwsgi
jsonpickle jsonpickle
django-widget-tweaks django-widget-tweaks

2
src/frontend/settings.py
View File

@@ -33,7 +33,7 @@ BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
SECRET_KEY = "@(9b9jslgg41u1u=mr)-2*-n2x0vef0zsy39*z@sz18&tvow18" SECRET_KEY = "@(9b9jslgg41u1u=mr)-2*-n2x0vef0zsy39*z@sz18&tvow18"
# SECURITY WARNING: don't run with debug turned on in production! # SECURITY WARNING: don't run with debug turned on in production!
DEBUG = TEMPLATE_DEBUG = True DEBUG = TEMPLATE_DEBUG = False
if DEBUG is True: if DEBUG is True:
from pudb.remote import set_trace from pudb.remote import set_trace
ALLOWED_HOSTS = CONFIG.allowed_hosts ALLOWED_HOSTS = CONFIG.allowed_hosts

18
uwsgi.ini
View File

@@ -1,10 +1,8 @@
[uwsgi]
[uwsgi] chdir=/pyshelf/src
chdir=/home/raelon/Projects/pyShelf/src module=frontend.wsgi
module=frontend.wsgi master=True
master=True pidfile=/tmp/pyShelf.pid
pidfile=/tmp/pyShelf.pid vacuum=True
vacuum=True socket=/tmp/pyshelf_wsgi.sock
socket=/tmp/pyshelf_wsgi.sock chmod-socket=666
chmod-socket=666