Skip to content
Snippets Groups Projects
Commit 315471d4 authored by Nico Vermaas's avatar Nico Vermaas
Browse files

Merge branch 'dev-nico' into 'master'

add optional private key and private key password

See merge request !9
parents 411ab9ea 1287392d
Branches
No related tags found
1 merge request!9add optional private key and private key password
...@@ -96,6 +96,35 @@ pip install -e "git+https://git.astron.nl/ldv/ldv_utils.git#egg=ldvspec-migratio ...@@ -96,6 +96,35 @@ pip install -e "git+https://git.astron.nl/ldv/ldv_utils.git#egg=ldvspec-migratio
``` ```
### Configuration
The database and tunnel configuration are in a local file on the host that can be given as a `--configuration` parameter.
The parameter file can contain a link to a private key file, and password.
When those keys are not given, the script will try to read the local SSH_CONFIG file `~/.ssh/config`. (Note that this does not work on Windows)
See for more documentation about the sshtunnel mechanism:
https://pypi.org/project/sshtunnel/
The following example shows a local configuration using private key.
```
[postgresql-local]
host=localhost
port=5433
database=ldv-spec-db
user=postgres
password=xxxxx
[postgresql-ldv]
tunnelhost=dop821.astron.nl
tunnelusername=sdco
host=sdc-db.astron.nl
port=5432
database=ldvadmin
user=ldvrbow
password=xxxxx
ssh_pkey = "C:\\Program Files Nico\\putty\\astron_private_key.ppk"
ssh_private_key_password = "xxxxx"
```
### Running ### Running
To test if it works To test if it works
```bash ```bash
...@@ -132,4 +161,7 @@ Some examples: ...@@ -132,4 +161,7 @@ Some examples:
ldv_migrate --limit 50000 --max_nbr_dps_to_insert_per_request 10000 ldv_migrate --limit 50000 --max_nbr_dps_to_insert_per_request 10000
- Import only 1000 records at production: - Import only 1000 records at production:
ldv_migrate --limit 1000 --host prod ldv_migrate --limit 1000 --host prod
- Import 10000 records into the ldv-spec-db:12000 test database on sdc-dev
ldv_migrate --limit 10000 --verbose --configuration ~/shared/ldv_migrate_sdc-dev.cfg --token 2cd0e124abf17e803e2ce0a664225b5e3dbaeaa6 --host test
``` ```
...@@ -48,6 +48,22 @@ def open_tunnel(configuration_params): ...@@ -48,6 +48,22 @@ def open_tunnel(configuration_params):
host = configuration_params.get('host', "no host given") host = configuration_params.get('host', "no host given")
port = int(configuration_params.get('port', "no port given")) port = int(configuration_params.get('port', "no port given"))
# check if a private key and password was given
ssh_pkey = configuration_params.get('ssh_pkey',None)
ssh_private_key_password = configuration_params.get('ssh_private_key_password',None)
logging.info("Creating ssh tunnel for %s and port %s with tunnel host %s and username %s", repr(host), port,
repr(tunnel_host), repr(tunnel_username))
if ssh_pkey:
ssh_tunnel = SSHTunnelForwarder(
ssh_address_or_host=tunnel_host,
ssh_username=tunnel_username,
remote_bind_address=(host, port),
ssh_pkey = ssh_pkey,
ssh_private_key_password = ssh_private_key_password
)
else:
try: try:
ssh_config_file = os.path.expanduser("~/.ssh/config") ssh_config_file = os.path.expanduser("~/.ssh/config")
except FileNotFoundError as exc: except FileNotFoundError as exc:
...@@ -55,14 +71,13 @@ def open_tunnel(configuration_params): ...@@ -55,14 +71,13 @@ def open_tunnel(configuration_params):
"Ssh config file not found on standard path '~/.ssh/config'. This is mandatory for opening the ssh tunnel" "Ssh config file not found on standard path '~/.ssh/config'. This is mandatory for opening the ssh tunnel"
) from exc ) from exc
logging.info("Creating ssh tunnel for %s and port %s with tunnel host %s and username %s", repr(host), port,
repr(tunnel_host), repr(tunnel_username))
ssh_tunnel = SSHTunnelForwarder( ssh_tunnel = SSHTunnelForwarder(
ssh_address_or_host=tunnel_host, ssh_address_or_host=tunnel_host,
ssh_username=tunnel_username, ssh_username=tunnel_username,
ssh_config_file=ssh_config_file, ssh_config_file=ssh_config_file,
remote_bind_address=(host, port) remote_bind_address=(host, port),
) )
ssh_tunnel.start() ssh_tunnel.start()
return ssh_tunnel return ssh_tunnel
......
...@@ -19,13 +19,13 @@ import datetime ...@@ -19,13 +19,13 @@ import datetime
from urllib.parse import urlparse, urlunparse from urllib.parse import urlparse, urlunparse
# ============================================================== # ==============================================================
# The request header # The request header
ATDB_HEADER = { REQUEST_HEADER = {
'content-type': "application/json", 'content-type': "application/json",
'cache-control': "no-cache" 'cache-control': "no-cache"
} }
LDV_HOST_DEV = "http://localhost:8000/ldvspec/api/v1" # your local development environment with Django webserver LDV_HOST_DEV = "http://localhost:8000/ldvspec/api/v1" # your local development environment with Django webserver
LDV_HOST_TEST = "https://sdc-dev.astron.nl:5554/ldvspec/api/v1" # the ldv sdc test environment. LDV_HOST_TEST = "https://sdc-dev.astron.nl/ldvspec/api/v1" # the ldv sdc test environment.
LDV_HOST_PROD = "https://sdc.astron.nl:5554/ldvspec/api/v1" # the ldv sdc production environment. LDV_HOST_PROD = "https://sdc.astron.nl/ldvspec/api/v1" # the ldv sdc production environment.
class APIException(Exception): class APIException(Exception):
...@@ -79,7 +79,7 @@ class LDVSpecInterface(): ...@@ -79,7 +79,7 @@ class LDVSpecInterface():
self.host = LDV_HOST_PROD self.host = LDV_HOST_PROD
if not self.host.endswith('/'): if not self.host.endswith('/'):
self.host += '/' self.host += '/'
self.header = ATDB_HEADER self.header = REQUEST_HEADER
self.header['Authorization'] = f'Token {token}' self.header['Authorization'] = f'Token {token}'
self._session = None self._session = None
......
...@@ -70,8 +70,7 @@ def main(): ...@@ -70,8 +70,7 @@ def main():
parser.add_argument("--version", default=False, help="Show current version of this program", action="store_true") parser.add_argument("--version", default=False, help="Show current version of this program", action="store_true")
parser.add_argument("-v", "--verbose", default=False, help="More information at run time.", action="store_true") parser.add_argument("-v", "--verbose", default=False, help="More information at run time.", action="store_true")
parser.add_argument("-l", "--limit", default=0, type=int, help="Limit on the number of queries (0 is no limit)", action="store") parser.add_argument("-l", "--limit", default=0, type=int, help="Limit on the number of queries (0 is no limit)", action="store")
parser.add_argument("-t", "--token", default="ad9b37a24380948601257f9c1f889b07a00ac81e",
help="Token to access the REST API of ldvspec", action="store")
parser.add_argument("--host", nargs="?", default='dev', parser.add_argument("--host", nargs="?", default='dev',
help="The ldv-spec-db host. Presets are 'dev' (default), 'test', 'prod', otherwise give a full url like https://sdc.astron.nl:5554/ldvspec/api/v1") help="The ldv-spec-db host. Presets are 'dev' (default), 'test', 'prod', otherwise give a full url like https://sdc.astron.nl:5554/ldvspec/api/v1")
parser.add_argument("--configuration", default='~/shared/ldv_migrate.cfg', parser.add_argument("--configuration", default='~/shared/ldv_migrate.cfg',
...@@ -82,6 +81,7 @@ def main(): ...@@ -82,6 +81,7 @@ def main():
# tested with 10.000 results in 90 seconds so # 11 mil. will be at least 28 hours # tested with 10.000 results in 90 seconds so # 11 mil. will be at least 28 hours
parser.add_argument("-r", "--max_nbr_dps_to_insert_per_request", default=1000, type=int, parser.add_argument("-r", "--max_nbr_dps_to_insert_per_request", default=1000, type=int,
help="The number of dataproducts to insert per REST request (0 is no limit)", action="store") help="The number of dataproducts to insert per REST request (0 is no limit)", action="store")
parser.add_argument("--token", default="ca1a247b2d9ccb556f450e541874e714e6d04eba", help="Token for ldvspec")
args = parser.parse_args() args = parser.parse_args()
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Please register or to comment