From 1e6864d548e5d0be7bd7a7a143e113eda4834374 Mon Sep 17 00:00:00 2001
From: Alissa Cheng <cheng@astron.nl>
Date: Wed, 17 Jan 2024 17:24:17 +0100
Subject: [PATCH] update readme, add set up instructions
---
.gitignore | 1 +
README.md | 97 ++++++++++++++++++++++++++++++++-------------
atdb/run.sh.example | 4 ++
3 files changed, 74 insertions(+), 28 deletions(-)
create mode 100644 atdb/run.sh.example
diff --git a/.gitignore b/.gitignore
index c78f1afc..7a0e2826 100644
--- a/.gitignore
+++ b/.gitignore
@@ -3,3 +3,4 @@
**/.idea/
**/__pycache__/
*.sql
+/atdb/run.sh
diff --git a/README.md b/README.md
index 6f3b237b..fa624a77 100644
--- a/README.md
+++ b/README.md
@@ -8,7 +8,7 @@ Communication is done through the REST API.
## Micro Services (in separate repo)
* https://git.astron.nl/astron-sdc/ldv-services
-## Project Documentation (Confluence)
+## Project Documentation
### Confluence Page:
* https://support.astron.nl/confluence/display/LDV/LOFAR+Data+Valorization+Home
@@ -17,9 +17,6 @@ These diagrams roughly serves as the specifications for adapting ATDB for LDV.
* workflow: https://support.astron.nl/confluence/display/LDV/WORKFLOW
* datamodel: https://dbdiagram.io/d/5ffc5fb180d742080a35d560
-## Overview Diagrams (current implementation)
-These diagrams show the current implementation and are kept up-to-date.
-
### Datamodel:
* https://drive.google.com/file/d/1v5hMBQS0jT8DQJwySVISfRa1zF4o0fCQ/view?usp=sharing
@@ -37,21 +34,21 @@ These diagrams show the current implementation and are kept up-to-date.
## Deployed Instances
-### main GUI:
+### Main GUI:
* test: https://sdc-dev.astron.nl:5554/atdb/
* prod: https://sdc.astron.nl:5554/atdb/
-### admin interface:
+### Admin interface:
* test: https://sdc-dev.astron.nl:5554/atdb/admin/
* prod: https://sdc.astron.nl:5554/atdb/admin/
### REST API (prod)
-serializers:
+#### serializers:
* workflows: http://sdc.astron.nl:5554/atdb/workflows/
* tasks: http://sdc.astron.nl:5554/atdb/tasks/
-get_size:
-Return the sum the sizes of all tasks with a given list of statusses
+#### get_size:
+Return the sum the sizes of all tasks with a given list of statuses
* https://sdc.astron.nl:5554/atdb/tasks/get_size
* https://sdc.astron.nl:5554/atdb/tasks/get_size?status__in=staged,processing,processed,validating,validated,ingesting,removing,removed
@@ -90,13 +87,6 @@ Master can be deployed to sdc-dev (test) or sdc (production)
When the `models.py` is changed, then the database must be migrated.
This is the procedure for that.
- on local dev:
-
- > python manage.py makemigrations --settings=atdb.settings.dev
- > python manage.py migrate --settings=atdb.settings.dev
- - add new migration file to git
- - commit & push
-
on CI/CD page: https://git.astron.nl/astron-sdc/atdb-ldv/-/pipelines
- when automatic build is finished, push >> to deploy
@@ -107,21 +97,72 @@ This is the procedure for that.
(this should say 'No changes detected', but do this step anyway as a check)
> docker exec -it atdb-ldv python manage.py migrate --settings atdb.settings.docker_sdc
-
-#### Integrating Astronauth
-
-See the integration instructions in the [astronauth repo](https://git.astron.nl/astron-sdc/astronauth)
+#### CI tests
+Tests are executed in the CI pipeline through the test stage in the `.gitlab-ci.yml` file.
+For running the tests in the CI pipeline, the settings file `atdb/settings/test_ci.dev` is used.
+
+
+## Local development
+### Set up
+1. Clone the repo
+<br/><br/>
+2. Install Postgres and make a database:
+ ```shell
+ sudo apt install postgresql postgresql-contrib libpq-dev python3-dev
+ sudo -u postgres psql
+ ```
+ The following database credentials should be identical to those in `atdb/atdb/settings/dev.py`:
+ ```postgresql
+ CREATE DATABASE atdb_ldv_12jan2024;
+ CREATE USER atdb_admin WITH PASSWORD 'atdb123';
+ ALTER ROLE atdb_admin SET client_encoding TO 'utf8';
+ ALTER ROLE atdb_admin SET default_transaction_isolation TO 'read committed';
+ ALTER ROLE atdb_admin SET timezone TO 'UTC';
+ GRANT ALL PRIVILEGES ON DATABASE atdb_ldv_12jan2024 TO atdb_admin;
+ CREATE ROLE ldvstats;
+ \q
+ ```
+ Import the database dump `atdbdb.sql` from its parent directory:
+ ```shell
+ sudo pg_restore -h localhost -U atdb_admin -d atdb_ldv_12jan2024 < atdbdb.sql
+ ```
+ Note: The required password here is `atdb123`, not your sudo password.
+<br/><br/>
+3. Make sure you have Python installed. Then go to the `atdb` directory (not the second `atdb/atdb`) - it's where the `manage.py` lives. Create a virtual environment and install dependencies:
+ ```shell
+ python3 -m venv venv
+ source venv/bin/activate
+ python -m pip install -r requirements/dev.txt
+ python manage.py migrate --settings=atdb.settings.dev
+ ```
+
+### Start the server
+To start the server, run:
+```shell
+python manage.py runserver --settings=atdb.settings.dev
+```
+However, if you need to login to the local server, then you will need some environment variables for Keycloak.
+In `atdb` directory where the file `run.sh.example` lives, duplicate it and rename it `run.sh`.
+Fill in the Keycloak client secret, you can find that in the Keycloak dashboard. Do not commit this file.
+Then you can start the server by running the file: `source ./run.sh`.
+
+### Migrations
+When the models.py is changed, then the database must be migrated.
+In the `atdb` directory, run:
+```shell
+python manage.py makemigrations --settings=atdb.settings.dev
+python manage.py migrate --settings=atdb.settings.dev
+```
+Then add and commit the resulting migration file.
### Running tests
-To run tests, you can spin up a dedicated test database locally using the the provided `docker-compose-test-local.yml` file. This test database will not interfere with your local development database. For example, you can run this command from the `atdb/docker` folder:
-
-` docker compose -f .\docker-compose-test-local.yml up -d`
-
-After spinning up the database, you can execute the tests with the following command:
-
-`python manage.py test --settings atdb.settings.test_local`
+To run tests, you can spin up a dedicated test database locally with docker.
+This test database will not interfere with your local development database.
+```shell
+docker compose -f docker/docker-compose-test-local.yml up -d
+python manage.py test --settings atdb.settings.test_local
+```
Dedicated settings for running the tests are provided in the `atdb/settings/test_local.dev` file. For convenience, `test.bat` is provided to run the above command (Windows only).
-Finally, these tests are also executed in the CI pipeline through the test stage in the `.gitlab-ci.yml` file. For running the tests in the CI pipeline, the settings file `atdb/settings/test_ci.dev` is used.
\ No newline at end of file
diff --git a/atdb/run.sh.example b/atdb/run.sh.example
new file mode 100644
index 00000000..0679736b
--- /dev/null
+++ b/atdb/run.sh.example
@@ -0,0 +1,4 @@
+export KEYCLOAK_URL=https://keycloak-sdc.astron.nl
+export KEYCLOAK_CLIENT_ID=ATDB-LDV-DEV
+export KEYCLOAK_CLIENT_SECRET=
+python manage.py runserver --settings=atdb.settings.dev
--
GitLab