Merge branch 'dev-nico' into 'master'

add optional private key and private key password

See merge request !9

README.md

@@ -96,6 +96,35 @@ pip install -e "git+https://git.astron.nl/ldv/ldv_utils.git#egg=ldvspec-migratio
 
 ```
 
+### Configuration
+The database and tunnel configuration are in a local file on the host that can be given as a `--configuration` parameter.
+The parameter file can contain a link to a private key file, and password. 
+When those keys are not given, the script will try to read the local SSH_CONFIG file `~/.ssh/config`. (Note that this does not work on Windows)
+
+See for more documentation about the sshtunnel mechanism:
+https://pypi.org/project/sshtunnel/ 
+
+The following example shows a local configuration using private key.
+```
+[postgresql-local]
+host=localhost
+port=5433
+database=ldv-spec-db
+user=postgres
+password=xxxxx
+
+[postgresql-ldv]
+tunnelhost=dop821.astron.nl
+tunnelusername=sdco
+host=sdc-db.astron.nl
+port=5432
+database=ldvadmin
+user=ldvrbow
+password=xxxxx
+ssh_pkey = "C:\\Program Files Nico\\putty\\astron_private_key.ppk"
+ssh_private_key_password = "xxxxx"
+```
+
 ### Running
 To test if it works
 ```bash
@@ -132,4 +161,7 @@ Some examples:
      ldv_migrate --limit 50000 --max_nbr_dps_to_insert_per_request 10000
 - Import only 1000 records at production:
      ldv_migrate --limit 1000 --host prod
+
+- Import 10000 records into the ldv-spec-db:12000 test database on sdc-dev
+  ldv_migrate --limit 10000 --verbose --configuration ~/shared/ldv_migrate_sdc-dev.cfg --token 2cd0e124abf17e803e2ce0a664225b5e3dbaeaa6 --host test
 ```

ldv_migrate/ldv_migrate/connection/retrieve_db_connection.py

@@ -48,6 +48,22 @@ def open_tunnel(configuration_params):
     host = configuration_params.get('host', "no host given")
     port = int(configuration_params.get('port', "no port given"))
 
+    # check if a private key and password was given
+    ssh_pkey = configuration_params.get('ssh_pkey',None)
+    ssh_private_key_password = configuration_params.get('ssh_private_key_password',None)
+
+    logging.info("Creating ssh tunnel for %s and port %s with tunnel host %s and username %s", repr(host), port,
+                 repr(tunnel_host), repr(tunnel_username))
+
+    if ssh_pkey:
+        ssh_tunnel = SSHTunnelForwarder(
+            ssh_address_or_host=tunnel_host,
+            ssh_username=tunnel_username,
+            remote_bind_address=(host, port),
+            ssh_pkey = ssh_pkey,
+            ssh_private_key_password = ssh_private_key_password
+        )
+    else:
         try:
             ssh_config_file = os.path.expanduser("~/.ssh/config")
         except FileNotFoundError as exc:
@@ -55,14 +71,13 @@ def open_tunnel(configuration_params):
                 "Ssh config file not found on standard path '~/.ssh/config'. This is mandatory for opening the ssh tunnel"
             ) from exc
 
-    logging.info("Creating ssh tunnel for %s and port %s with tunnel host %s and username %s", repr(host), port,
-                 repr(tunnel_host), repr(tunnel_username))
         ssh_tunnel = SSHTunnelForwarder(
             ssh_address_or_host=tunnel_host,
             ssh_username=tunnel_username,
             ssh_config_file=ssh_config_file,
-        remote_bind_address=(host, port)
+            remote_bind_address=(host, port),
         )
+
     ssh_tunnel.start()
     return ssh_tunnel
 

ldv_migrate/ldv_migrate/ldv_specification_interface.py

@@ -19,13 +19,13 @@ import datetime
 from urllib.parse import urlparse, urlunparse
 # ==============================================================
 # The request header
-ATDB_HEADER = {
+REQUEST_HEADER = {
     'content-type': "application/json",
     'cache-control': "no-cache"
 }
 LDV_HOST_DEV = "http://localhost:8000/ldvspec/api/v1"  # your local development environment with Django webserver
-LDV_HOST_TEST = "https://sdc-dev.astron.nl:5554/ldvspec/api/v1"  # the ldv sdc test environment.
-LDV_HOST_PROD = "https://sdc.astron.nl:5554/ldvspec/api/v1"  # the ldv sdc production environment.
+LDV_HOST_TEST = "https://sdc-dev.astron.nl/ldvspec/api/v1"  # the ldv sdc test environment.
+LDV_HOST_PROD = "https://sdc.astron.nl/ldvspec/api/v1"  # the ldv sdc production environment.
 
 
 class APIException(Exception):
@@ -79,7 +79,7 @@ class LDVSpecInterface():
             self.host = LDV_HOST_PROD
         if not self.host.endswith('/'):
             self.host += '/'
-        self.header = ATDB_HEADER
+        self.header = REQUEST_HEADER
         self.header['Authorization'] = f'Token {token}'
         self._session = None
 

ldv_migrate/ldv_migrate/migrate_ldvadmin_to_ldvspec.py

@@ -70,8 +70,7 @@ def main():
     parser.add_argument("--version", default=False, help="Show current version of this program", action="store_true")
     parser.add_argument("-v", "--verbose", default=False, help="More information at run time.", action="store_true")
     parser.add_argument("-l", "--limit", default=0, type=int, help="Limit on the number of queries (0 is no limit)", action="store")
-    parser.add_argument("-t", "--token", default="ad9b37a24380948601257f9c1f889b07a00ac81e",
-                        help="Token to access the REST API of ldvspec", action="store")
+
     parser.add_argument("--host", nargs="?", default='dev',
                         help="The ldv-spec-db host. Presets are 'dev' (default), 'test', 'prod', otherwise give a full url like https://sdc.astron.nl:5554/ldvspec/api/v1")
     parser.add_argument("--configuration", default='~/shared/ldv_migrate.cfg',
@@ -82,6 +81,7 @@ def main():
     # tested with 10.000 results in 90 seconds so # 11 mil. will be at least 28 hours
     parser.add_argument("-r", "--max_nbr_dps_to_insert_per_request", default=1000, type=int,
                         help="The number of dataproducts to insert per REST request (0 is no limit)", action="store")
+    parser.add_argument("--token", default="ca1a247b2d9ccb556f450e541874e714e6d04eba", help="Token for ldvspec")
 
     args = parser.parse_args()