Initial commit.

host-files/docker-compose.yml

+version: "3"
+
+services:
+
+  dbadmin:
+    image: postgres:12
+    ports:
+      - "5432:5432"
+    environment:
+      - POSTGRES_USER=${DBADMIN_USER}
+      - POSTGRES_PASSWORD=${DBADMIN_PASSWORD}
+      - POSTGRES_DB=dbadmin
+
+  dbproject1:
+    image: postgres:12
+    expose:
+      - "5433"
+    ports:
+      - "5433:5433"
+    command: -p 5433
+    environment:
+      - POSTGRES_USER=${DBPROJECT_USER}
+      - POSTGRES_PASSWORD=${DBPROJECT_PASSWORD}
+
+  esap_api:
+    image: esap_api:latest
+    container_name: esap-api
+    environment:
+      - OIDC_RP_CLIENT_ID=669d7bef-32c0-4980-ae35-d8ede56bd5ef
+      - OIDC_RP_CLIENT_SECRET
+      - OIDC_OP_JWKS_ENDPOINT=https://iam-escape.cloud.cnaf.infn.it/jwk
+      - OIDC_OP_AUTHORIZATION_ENDPOINT=https://iam-escape.cloud.cnaf.infn.it/authorize
+      - OIDC_OP_TOKEN_ENDPOINT=https://iam-escape.cloud.cnaf.infn.it/token
+      - OIDC_OP_USER_ENDPOINT=https://iam-escape.cloud.cnaf.infn.it/userinfo
+      - LOGIN_REDIRECT_URL=http://localhost:3000/esap-gui/login
+      - LOGOUT_REDIRECT_URL=http://localhost:3000/esap-gui/logout
+      - LOGIN_REDIRECT_URL_FAILURE=http://localhost:3000/esap-gui/error
+    build:
+      context: ${ESAP_ROOT}/esap-api-gateway/esap
+    ports:
+      - 8000:8000
+    restart: always
+    command: "python manage.py runserver 0.0.0.0:8000 --settings=esap.settings.dev"
+
+  esap_db:
+    image: esap_db:latest
+    container_name: esap-db
+    stdin_open: true  # docker run -i
+    tty: true         # docker run -t
+    environment:
+      - SERVER_NAME=${DOMAIN?Variable not set}
+      - SERVER_HOST=https://${DOMAIN?Variable not set}
+      - SENTRY_DSN=https://nowhere.com
+      - PROJECT_NAME=ESAP-DB
+      - DBADMIN_SERVER=dbadmin
+      - DBADMIN_USER
+      - DBADMIN_PASSWORD
+      - DBADMIN_DB=dbadmin
+      - DBPROJECT_USER
+      - DBPROJECT_PASSWORD
+      - DBPROJECT_SERVERS=["dbproject1:5433"]
+      - FIRST_SUPERUSER=esapadmin@nowhere.com
+      - FIRST_SUPERUSER_PASSWORD=esapadmin
+    build:
+      context: ${ESAP_ROOT}/esap-db
+      args:
+        INSTALL_DEV: ${INSTALL_DEV-false}
+    command: bash -c "scripts/prepare-app.sh && uvicorn app.main:app --port 8001 --host 0.0.0.0 --reload"
+    volumes:
+      - ${ESAP_ROOT}/esap-db:/code
+    ports:
+      - "8001:8001"
+    depends_on:
+      - dbadmin
+
+  pgadmin:
+    container_name: pgadmin
+    image: dpage/pgadmin4:latest
+    environment:
+      - PGADMIN_DEFAULT_EMAIL=pgadmin4@pgadmin.org
+      - PGADMIN_DEFAULT_PASSWORD=admin
+    ports:
+      - "5050:80"
+    depends_on:
+      - dbadmin

host-files/run-tests.sh

+#! /usr/bin/env bash
+
+# Exit in case of error
+set -e
+
+export ESAP_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. &> /dev/null && pwd )"
+export ESAP_DB_PATH=${ESAP_ROOT}/esap-db/host-files
+
+ESAP_RUN_OPTIONS="-d" ${ESAP_DB_PATH}/run.sh
+docker-compose -f docker-stack.yml exec -T esap_db bash /code/scripts/run-tests.sh "$@"
+docker-compose -f docker-stack.yml down -v --remove-orphans

host-files/run.sh

+#! /usr/bin/env bash
+
+# Exit in case of error
+set -e
+
+export ESAP_ROOT="$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../.. &> /dev/null && pwd )"
+export ESAP_DB_PATH=${ESAP_ROOT}/esap-db/host-files
+
+DOMAIN=backend \
+SMTP_HOST="" \
+TRAEFIK_PUBLIC_NETWORK_IS_EXTERNAL=false \
+INSTALL_DEV=true \
+docker-compose -f ${ESAP_DB_PATH}/docker-compose.yml --env-file ${ESAP_DB_PATH}/.env config > docker-stack.yml
+
+docker-compose -f docker-stack.yml build
+docker-compose -f docker-stack.yml down -v --remove-orphans # Remove possibly previous broken stacks left hanging after an error
+docker-compose -f docker-stack.yml up ${ESAP_RUN_OPTIONS}

migrations/README

+Generic single-database configuration.

migrations/env.py

+# ---------------- added code here -------------------------#
+import os
+import sys
+from logging.config import fileConfig
+from pathlib import Path
+
+from alembic import context
+from dotenv import load_dotenv
+from sqlalchemy import engine_from_config, pool
+
+# add your model's MetaData object here
+# for 'autogenerate' support
+# from myapp import mymodel
+# target_metadata = mymodel.Base.metadata
+from app.config import settings
+from app.db import models
+
+BASE_DIR = Path(__file__).parents[1].absolute()
+load_dotenv(BASE_DIR / '.env')
+sys.path.append(str(BASE_DIR))
+# ------------------------------------------------------------#
+
+# this is the Alembic Config object, which provides
+# access to the values within the .ini file in use.
+config = context.config
+
+# ---------------- added code here -------------------------#
+# this will overwrite the ini-file sqlalchemy.url path
+# with the path given in the config of the main code
+config.set_main_option('sqlalchemy.url', settings.SQLALCHEMY_DBADMIN_URI)
+# ------------------------------------------------------------#
+
+# Interpret the config file for Python logging.
+# This line sets up loggers basically.
+fileConfig(config.config_file_name)
+
+target_metadata = models.Base.metadata
+
+# other values from the config, defined by the needs of env.py,
+# can be acquired:
+# my_important_option = config.get_main_option("my_important_option")
+# ... etc.
+
+
+def run_migrations_offline():
+    """Run migrations in 'offline' mode.
+
+    This configures the context with just a URL
+    and not an Engine, though an Engine is acceptable
+    here as well.  By skipping the Engine creation
+    we don't even need a DBAPI to be available.
+
+    Calls to context.execute() here emit the given string to the
+    script output.
+
+    """
+    url = config.get_main_option('sqlalchemy.url')
+    context.configure(
+        url=url,
+        target_metadata=target_metadata,
+        literal_binds=True,
+        dialect_opts={'paramstyle': 'named'},
+    )
+
+    with context.begin_transaction():
+        context.run_migrations()
+
+
+def run_migrations_online():
+    """Run migrations in 'online' mode.
+
+    In this scenario we need to create an Engine
+    and associate a connection with the context.
+
+    """
+    connectable = engine_from_config(
+        config.get_section(config.config_ini_section),
+        prefix='sqlalchemy.',
+        poolclass=pool.NullPool,
+    )
+
+    with connectable.connect() as connection:
+        context.configure(connection=connection, target_metadata=target_metadata)
+
+        with context.begin_transaction():
+            context.run_migrations()
+
+
+if context.is_offline_mode():
+    run_migrations_offline()
+else:
+    run_migrations_online()

migrations/script.py.mako

+"""${message}
+
+Revision ID: ${up_revision}
+Revises: ${down_revision | comma,n}
+Create Date: ${create_date}
+
+"""
+from alembic import op
+import sqlalchemy as sa
+${imports if imports else ""}
+
+# revision identifiers, used by Alembic.
+revision = ${repr(up_revision)}
+down_revision = ${repr(down_revision)}
+branch_labels = ${repr(branch_labels)}
+depends_on = ${repr(depends_on)}
+
+
+def upgrade():
+    ${upgrades if upgrades else "pass"}
+
+
+def downgrade():
+    ${downgrades if downgrades else "pass"}

migrations/versions/5dd23cffa253_table_project_servers.py

+"""Table project_servers.
+
+Revision ID: 5dd23cffa253
+Revises: 6e7e210ad4b8
+Create Date: 2021-07-06 16:33:36.478623
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '5dd23cffa253'
+down_revision = '6e7e210ad4b8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'project_servers',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('uri', sa.String(), nullable=False),
+        sa.Column('max_size', sa.BigInteger(), nullable=False),
+        sa.Column('available_size', sa.BigInteger(), nullable=False),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('uri'),
+    )
+    op.create_index(
+        op.f('ix_project_servers_id'), 'project_servers', ['id'], unique=False
+    )
+    op.create_table(
+        'projects',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('project_server_id', sa.Integer(), nullable=True),
+        sa.Column('name', sa.String(length=256), nullable=False),
+        sa.Column('description', sa.String(), nullable=False),
+        sa.Column('uri', sa.String(), nullable=False),
+        sa.Column('max_size', sa.BigInteger(), nullable=False),
+        sa.ForeignKeyConstraint(
+            ['project_server_id'],
+            ['project_servers.id'],
+        ),
+        sa.PrimaryKeyConstraint('id'),
+    )
+    op.create_index(op.f('ix_projects_id'), 'projects', ['id'], unique=False)
+    op.create_index(op.f('ix_projects_name'), 'projects', ['name'], unique=False)
+    op.add_column('users', sa.Column('is_superuser', sa.Boolean(), nullable=False))
+    op.alter_column('users', 'first_name', existing_type=sa.VARCHAR(), nullable=False)
+    op.alter_column('users', 'last_name', existing_type=sa.VARCHAR(), nullable=False)
+    op.alter_column('users', 'email', existing_type=sa.VARCHAR(), nullable=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.alter_column('users', 'email', existing_type=sa.VARCHAR(), nullable=True)
+    op.alter_column('users', 'last_name', existing_type=sa.VARCHAR(), nullable=True)
+    op.alter_column('users', 'first_name', existing_type=sa.VARCHAR(), nullable=True)
+    op.drop_column('users', 'is_superuser')
+    op.drop_index(op.f('ix_projects_name'), table_name='projects')
+    op.drop_index(op.f('ix_projects_id'), table_name='projects')
+    op.drop_table('projects')
+    op.drop_index(op.f('ix_project_servers_id'), table_name='project_servers')
+    op.drop_table('project_servers')
+    # ### end Alembic commands ###

migrations/versions/6e7e210ad4b8_first_migration.py

+"""First migration
+
+Revision ID: 6e7e210ad4b8
+Revises:
+Create Date: 2021-06-14 00:12:40.340546
+
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '6e7e210ad4b8'
+down_revision = None
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.create_table(
+        'users',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('first_name', sa.String(), nullable=True),
+        sa.Column('last_name', sa.String(), nullable=True),
+        sa.Column('email', sa.String(), nullable=True),
+        sa.PrimaryKeyConstraint('id'),
+    )
+    op.create_index(op.f('ix_users_id'), 'users', ['id'], unique=False)
+    # ### end Alembic commands ###
+
+
+def downgrade():
+    # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_users_id'), table_name='users')
+    op.drop_table('users')
+    # ### end Alembic commands ###

pyproject.toml

+[tool.poetry]
+name = "app"
+version = "0.1.0"
+description = "ESAP-DB Backend."
+authors = ["Pierre Chanial <pierre.chanial@ego-gw.it>"]
+
+[tool.poetry.dependencies]
+python = "^3.9"
+requests = "^2.25.1"
+psycopg2 = "^2.9.1"
+fastapi = "^0.66.0"
+alembic = "^1.6.5"
+python-dotenv = "^0.18.0"
+pandas = "^1.3.0"
+FastAPI-SQLAlchemy = "^0.2.1"
+tenacity = "^7.0.0"
+gunicorn = "^20.1.0"
+uvicorn = "^0.14.0"
+pydantic = {extras = ["email"], version = "^1.8.2"}
+SQLAlchemy = {extras = ["mypy"], version = "^1.4.20"}
+
+[tool.poetry.dev-dependencies]
+ipython = "^7.25.0"
+pre-commit = "^2.13.0"
+sqlalchemy-stubs = "^0.4"
+types-requests = "^2.25.0"
+pytest = "^6.2.4"
+pytest-cov = "^2.12.1"
+
+[build-system]
+requires = ["poetry-core>=1.0.0a5"]
+build-backend = "poetry.core.masonry.api"
+
+[tool.black]
+skip-string-normalization = true
+
+[tool.isort]
+profile = "black"
+known_third_party = ["alembic", "dotenv", "fastapi", "pandas", "pydantic", "pytest", "requests", "sqlalchemy", "starlette", "tenacity"]
+
+[tool.mypy]
+python_version = "3.9"
+plugins = ["pydantic.mypy", "sqlalchemy.ext.mypy.plugin"]
+ignore_missing_imports = true
+disallow_untyped_defs = true

scripts/initialize_dbadmin.py

+"""This script initializes the admin database."""
+import logging
+
+from sqlalchemy.exc import IntegrityError
+
+from app.config import settings
+from app.db import AdminSession, DBProjectServer
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+
+def initialize_dbadmin() -> None:
+    """Initializes the admin database."""
+    session = AdminSession()
+    for server_name in settings.DBPROJECT_SERVERS:
+        logger.info(f'Adding project server: {server_name}')
+        uri = f'postgresql://{settings.DBPROJECT_USER}:{settings.DBPROJECT_PASSWORD}@{server_name}'  # noqa
+        max_size = 2 ** 50  # 1 PiB
+        server = DBProjectServer(uri=uri, max_size=max_size, available_size=max_size)
+        try:
+            with session.begin():
+                session.add(server)
+        except IntegrityError:
+            # The project server is already in the database
+            pass
+
+
+def main() -> None:
+    """Logs and initializes the admin database."""
+    logger.info('Creating initial data')
+    initialize_dbadmin()
+    logger.info('Initial data created')
+
+
+if __name__ == '__main__':
+    main()

scripts/prepare-app.sh

+#! /usr/bin/env bash
+
+# Let the DB start
+python /code/scripts/wait_for_dbadmin.py
+
+# Run migrations
+alembic upgrade head
+
+# Create initial data in DB
+python /code/scripts/initialize_dbadmin.py

scripts/run-tests.sh

+#! /usr/bin/env bash
+set -ex
+
+# Let the DB start
+python /code/scripts/wait_for_initialized_dbadmin.py
+
+pytest --cov=app --cov-report=term-missing tests "${@}"

scripts/wait_for_dbadmin.py

+"""This script waits until a connection is established to the admin database."""
+import logging
+
+from sqlalchemy import select
+from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed
+
+from app.db import AdminSession
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+max_tries = 60 * 5  # 5 minutes
+wait_seconds = 1
+
+
+@retry(
+    stop=stop_after_attempt(max_tries),
+    wait=wait_fixed(wait_seconds),
+    before=before_log(logger, logging.INFO),
+    after=after_log(logger, logging.WARN),
+)
+def execute_command() -> None:
+    """Attempts to execute a command in the admin database."""
+    with AdminSession.begin() as session:
+        try:
+            # Try to create session to check if DB is awake
+            session.execute(select(1))
+        except Exception as e:
+            logger.error(e)
+            raise e
+
+
+def main() -> None:
+    """Logs and connects to the admin database."""
+    logger.info('Initializing service')
+    execute_command()
+    logger.info('Service finished initializing')
+
+
+if __name__ == '__main__':
+    main()

scripts/wait_for_initialized_dbadmin.py

+"""This script waits until a connection is established to the admin database."""
+import logging
+
+from sqlalchemy import select
+from tenacity import after_log, before_log, retry, stop_after_attempt, wait_fixed
+
+from app.db import AdminSession, DBProjectServer
+
+logging.basicConfig(level=logging.INFO)
+logger = logging.getLogger(__name__)
+
+max_tries = 60 * 5  # 5 minutes
+wait_seconds = 1
+
+
+@retry(
+    stop=stop_after_attempt(max_tries),
+    wait=wait_fixed(wait_seconds),
+    before=before_log(logger, logging.INFO),
+    after=after_log(logger, logging.WARN),
+)
+def execute_command() -> None:
+    """Ensures the project database has at least one entry."""
+    with AdminSession.begin() as session:
+        try:
+            stmt = select(DBProjectServer)
+            session.execute(stmt).one()
+        except Exception as e:
+            logger.error(e)
+            raise e
+
+
+def main() -> None:
+    """Logs and connects to the admin database."""
+    logger.info('Initializing service')
+    execute_command()
+    logger.info('Service finished initializing')
+
+
+if __name__ == '__main__':
+    main()

tests/conftest.py

+from typing import Generator
+
+import pytest
+from sqlalchemy.orm import Session
+
+from app.db.access import AdminSession
+
+
+@pytest.fixture(scope='session')
+def db() -> Generator[Session, None, None]:
+    yield AdminSession()

tests/functional/conftest.py

+from typing import Generator
+
+import pytest
+from fastapi.testclient import TestClient
+
+from app.main import app
+
+
+@pytest.fixture(scope='module')
+def client() -> Generator[TestClient, None, None]:
+    with TestClient(app) as c:
+        yield c