diff --git a/README.md b/README.md
index 0728c44e64497859f7ad3f72457f07fafd45a4e5..814cd6c0225998c75921c14f93e10d93399cc3c4 100644
--- a/README.md
+++ b/README.md
@@ -96,6 +96,35 @@ pip install -e "git+https://git.astron.nl/ldv/ldv_utils.git#egg=ldvspec-migratio
```
+### Configuration
+The database and tunnel configuration are in a local file on the host that can be given as a `--configuration` parameter.
+The parameter file can contain a link to a private key file, and password.
+When those keys are not given, the script will try to read the local SSH_CONFIG file `~/.ssh/config`. (Note that this does not work on Windows)
+
+See for more documentation about the sshtunnel mechanism:
+https://pypi.org/project/sshtunnel/
+
+The following example shows a local configuration using private key.
+```
+[postgresql-local]
+host=localhost
+port=5433
+database=ldv-spec-db
+user=postgres
+password=xxxxx
+
+[postgresql-ldv]
+tunnelhost=dop821.astron.nl
+tunnelusername=sdco
+host=sdc-db.astron.nl
+port=5432
+database=ldvadmin
+user=ldvrbow
+password=xxxxx
+ssh_pkey = "C:\\Program Files Nico\\putty\\astron_private_key.ppk"
+ssh_private_key_password = "xxxxx"
+```
+
### Running
To test if it works
```bash
@@ -132,4 +161,7 @@ Some examples:
ldv_migrate --limit 50000 --max_nbr_dps_to_insert_per_request 10000
- Import only 1000 records at production:
ldv_migrate --limit 1000 --host prod
+
+- Import 10000 records into the ldv-spec-db:12000 test database on sdc-dev
+ ldv_migrate --limit 10000 --verbose --configuration ~/shared/ldv_migrate_sdc-dev.cfg --token 2cd0e124abf17e803e2ce0a664225b5e3dbaeaa6 --host test
```
diff --git a/ldv_migrate/ldv_migrate/connection/retrieve_db_connection.py b/ldv_migrate/ldv_migrate/connection/retrieve_db_connection.py
index 52f1184a10c23fb8df52522a198605e8c8d17251..20d13f7b012437e8eef4261a530ca48220e0f26f 100644
--- a/ldv_migrate/ldv_migrate/connection/retrieve_db_connection.py
+++ b/ldv_migrate/ldv_migrate/connection/retrieve_db_connection.py
@@ -48,21 +48,36 @@ def open_tunnel(configuration_params):
host = configuration_params.get('host', "no host given")
port = int(configuration_params.get('port', "no port given"))
- try:
- ssh_config_file = os.path.expanduser("~/.ssh/config")
- except FileNotFoundError as exc:
- raise FileNotFoundError(
- "Ssh config file not found on standard path '~/.ssh/config'. This is mandatory for opening the ssh tunnel"
- ) from exc
+ # check if a private key and password was given
+ ssh_pkey = configuration_params.get('ssh_pkey',None)
+ ssh_private_key_password = configuration_params.get('ssh_private_key_password',None)
logging.info("Creating ssh tunnel for %s and port %s with tunnel host %s and username %s", repr(host), port,
repr(tunnel_host), repr(tunnel_username))
- ssh_tunnel = SSHTunnelForwarder(
- ssh_address_or_host=tunnel_host,
- ssh_username=tunnel_username,
- ssh_config_file=ssh_config_file,
- remote_bind_address=(host, port)
- )
+
+ if ssh_pkey:
+ ssh_tunnel = SSHTunnelForwarder(
+ ssh_address_or_host=tunnel_host,
+ ssh_username=tunnel_username,
+ remote_bind_address=(host, port),
+ ssh_pkey = ssh_pkey,
+ ssh_private_key_password = ssh_private_key_password
+ )
+ else:
+ try:
+ ssh_config_file = os.path.expanduser("~/.ssh/config")
+ except FileNotFoundError as exc:
+ raise FileNotFoundError(
+ "Ssh config file not found on standard path '~/.ssh/config'. This is mandatory for opening the ssh tunnel"
+ ) from exc
+
+ ssh_tunnel = SSHTunnelForwarder(
+ ssh_address_or_host=tunnel_host,
+ ssh_username=tunnel_username,
+ ssh_config_file=ssh_config_file,
+ remote_bind_address=(host, port),
+ )
+
ssh_tunnel.start()
return ssh_tunnel
diff --git a/ldv_migrate/ldv_migrate/ldv_specification_interface.py b/ldv_migrate/ldv_migrate/ldv_specification_interface.py
index d0469ad552c8646e191c6bdebc55387e75e72451..1ef670ff3284e5da8deff620211cb81d3c0ecb4e 100644
--- a/ldv_migrate/ldv_migrate/ldv_specification_interface.py
+++ b/ldv_migrate/ldv_migrate/ldv_specification_interface.py
@@ -19,13 +19,13 @@ import datetime
from urllib.parse import urlparse, urlunparse
# ==============================================================
# The request header
-ATDB_HEADER = {
+REQUEST_HEADER = {
'content-type': "application/json",
'cache-control': "no-cache"
}
LDV_HOST_DEV = "http://localhost:8000/ldvspec/api/v1" # your local development environment with Django webserver
-LDV_HOST_TEST = "https://sdc-dev.astron.nl:5554/ldvspec/api/v1" # the ldv sdc test environment.
-LDV_HOST_PROD = "https://sdc.astron.nl:5554/ldvspec/api/v1" # the ldv sdc production environment.
+LDV_HOST_TEST = "https://sdc-dev.astron.nl/ldvspec/api/v1" # the ldv sdc test environment.
+LDV_HOST_PROD = "https://sdc.astron.nl/ldvspec/api/v1" # the ldv sdc production environment.
class APIException(Exception):
@@ -79,7 +79,7 @@ class LDVSpecInterface():
self.host = LDV_HOST_PROD
if not self.host.endswith('/'):
self.host += '/'
- self.header = ATDB_HEADER
+ self.header = REQUEST_HEADER
self.header['Authorization'] = f'Token {token}'
self._session = None
diff --git a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py
index 0086bfd7055f0a1604d09d606a7d054cbe517452..9cfb88dcd0219c5f90ddb53a19d8faa131355a0c 100644
--- a/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py
+++ b/ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py
@@ -70,8 +70,7 @@ def main():
parser.add_argument("--version", default=False, help="Show current version of this program", action="store_true")
parser.add_argument("-v", "--verbose", default=False, help="More information at run time.", action="store_true")
parser.add_argument("-l", "--limit", default=0, type=int, help="Limit on the number of queries (0 is no limit)", action="store")
- parser.add_argument("-t", "--token", default="ad9b37a24380948601257f9c1f889b07a00ac81e",
- help="Token to access the REST API of ldvspec", action="store")
+
parser.add_argument("--host", nargs="?", default='dev',
help="The ldv-spec-db host. Presets are 'dev' (default), 'test', 'prod', otherwise give a full url like https://sdc.astron.nl:5554/ldvspec/api/v1")
parser.add_argument("--configuration", default='~/shared/ldv_migrate.cfg',
@@ -82,6 +81,7 @@ def main():
# tested with 10.000 results in 90 seconds so # 11 mil. will be at least 28 hours
parser.add_argument("-r", "--max_nbr_dps_to_insert_per_request", default=1000, type=int,
help="The number of dataproducts to insert per REST request (0 is no limit)", action="store")
+ parser.add_argument("--token", default="ca1a247b2d9ccb556f450e541874e714e6d04eba", help="Token for ldvspec")
args = parser.parse_args()