Merge branch 'update_design_doc_picas_api' into 'master'

Update design doc picas api

See merge request !5

GRID_LRT/__init__.py

@@ -6,7 +6,6 @@ from subprocess import call, Popen, PIPE, STDOUT
 if sys.version_info[0:2] != (2, 6):
     from subprocess import check_output
 
-
 __all__ = ["storage", 'auth', "application", "Staging", 'token']
 __version__ = "1.0.7"
 __author__ = "Alexandar P. Mechev"

GRID_LRT/application/submit.py

@@ -18,6 +18,7 @@ import tempfile
 from GRID_LRT.auth.get_picas_credentials import picas_cred as pc
 import GRID_LRT
 from GRID_LRT.auth import grid_credentials
+from subprocess import Popen
 
 class SafePopen(Popen):
     def __init__(self, *args, **kwargs):
@@ -25,6 +26,7 @@ class SafePopen(Popen):
             kwargs['encoding'] = 'utf8'
         return super(SafePopen, self).__init__(*args, **kwargs)
 
+
 #class job_launcher(object):
 #    """Generic Job Launcher
 #    """
@@ -376,20 +378,19 @@ class SpiderLauncher(JdlLauncher):
         if not os.path.exists(self.launch_file):
             raise IOError("Launch file doesn't exist! "+self.launch_file)
         slurmfile = '#!/usr/bin/env bash\n'
+        slurmfile += '#SBATCH --nodes=1 --ntasks 1 --cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=job-%A_%a.out --error=job-%A_%a.err'
         if self.wholenodes:
-            slurmfile += '#SBATCH --exclusive --nodes=1 --ntasks 1 --cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=spiderjob-%A_%a.out --error=spiderjob-%A_%a.err\n'
-        else:
-            slurmfile += '#SBATCH --nodes=1 --ntasks 1 --cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=spiderjob-%A_%a.out --error=spiderjob-%A_%a.err\n'
+            slurmfile += " ".join(slurmfile, '--exclusive')
+        slurmfile += '\n'
         slurmfile += """
 echo Job landed on $(hostname)
-JOBDIR=$(mktemp -d -p $TMPDIR)
-cd $TMPDIR
+JOBDIR=$TMPDIR
 export JOBDIR
 echo Created job directory $JOBDIR
 cd $JOBDIR
 {launcher:s} {db:s} {usr:s} {pw:s} {tt:s}
 """
-        slurmfile = slurmfile.format(ncpu=int(self.ncpu),
+	slurmfile = slurmfile.format(ncpu=int(self.ncpu),
         queue=str(self.queue),
         njobs=self.numjobs,
         concurrent=self.parameter_step,

GRID_LRT/auth/grid_credentials.py

@@ -4,15 +4,14 @@ activated
 """
 import subprocess
 from functools import wraps
-
+from GRID_LRT.application.submit import SafePopen
 
 def check_uberftp():
     """Checks if the gfal-ls executable
     exists on the system. Returns True if it exists
 
     :returns: bool"""
-    process = subprocess.Popen(['which', 'gfal-ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-            encoding='utf8')
+    process = SafePopen(['which', 'gfal-ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE) 
     output = process.communicate()
     if output[0] == '' and output[1] == '':
         return False
@@ -26,11 +25,10 @@ def grid_credentials_enabled():
     This requires gfal-ls!"""
     if not check_uberftp():
         return False
-    process = subprocess.Popen([
+    process = SafePopen([
         'gfal-ls', '-l',
         'gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lofar/user/sksp/diskonly'
-        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
-        encoding='utf8')
+        ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     res = process.communicate()
     if type(res[1])==bytes:
         error = res[1].decode('utf8')

GRID_LRT/data/launchers/run_remote_sandbox.sh

@@ -20,9 +20,9 @@ export PICAS_API_VERSION=$(curl -X GET https://"$2":"$3"@picas-lofar.grid.surfsa
 set -x
 if type git &> /dev/null
 then
- git clone https://github.com/apmechev/GRID_PiCaS_Launcher.git p_tools_git
+ git clone https://git.astron.nl/astron-sdc/GRID_PiCaS_Launcher.git p_tools_git
  cd p_tools_git || exit -100 
- git checkout v"$PICAS_API_VERSION"
+ git checkout "$PICAS_API_VERSION"
  git describe
  cd ../ || exit -100
 else  #move this to testpy3

GRID_LRT/storage/gsifile.py

@@ -9,14 +9,9 @@ import re
 import GRID_LRT.auth.grid_credentials as grid_creds
 from GRID_LRT.auth.get_picas_credentials import picas_cred
 from GRID_LRT.Staging.srmlist import srmlist
+from GRID_LRT.application.submit import SafePopen
 import humanfriendly
 
-class SafePopen(Popen):
-    def __init__(self, *args, **kwargs):
-        if sys.version_info.major == 3 :
-            kwargs['encoding'] = 'utf8'
-        return super(SafePopen, self).__init__(*args, **kwargs)
-
 import pdb
 class GSIFile(object):
     def __init__(self, location, parent_dir=None):

GRID_LRT/token.py

@@ -129,14 +129,14 @@ class Token(dict):
         using the python __get__ method, i.e. token['field']. Each token has a token_id that is unique
         in the database, which it is part of, however the database interface is only defined in the 
         child classes of Token"""
-        self.__setitem__("type", token_type)
+        self['type'] = token_type
         if not token_id:
-            self.__setitem__("_id", token_type)
+            self['_id'] = token_type
         else:
-            self.__setitem__("_id", token_id)
-        self.__setitem__("PICAS_API_VERSION", GRID_LRT.__version__)
-        self.__setitem__("lock", 0)
-        self.__setitem__("done", 0)
+            self['_id'] = token_id
+        self['PICAS_API_VERSION'] = GRID_LRT.__version__
+        self['lock'] = 0
+        self['done'] = 0
 
     @property
     def database(self):
@@ -173,15 +173,15 @@ class Token(dict):
         """General interface to 'reset' a Token as defined in PiCaS Standards. 
         The lock and done fields are timestamps and if are set to 0, the token will be
         considered in the 'todo' state"""
-        self.__setitem__("lock", 0)
-        self.__setitem__("done", 0)
+        self['lock'] = 0
+        self['done'] = 0
         scrub_count = self.get(
             "scrub_count", 0
         )  ##TODO: Get function override to get from db
-        self.__setitem__("scrub_count", scrub_count + 1)
-        self.__setitem__("hostname", "")
-        self.__setitem__("output", "")
-        self.__setitem__("status", "reset")
+        self['scrub_count'] = scrub_count + 1
+        self['hostname'] = ''
+        self['output'] = ''
+        self['status'] = 'reset'
 
     def archive(self, delete=False):
         """Archives all tokens, given a database. Deletes them if the delete flag is set to True"""
@@ -330,7 +330,7 @@ class TokenList(list):
         if design_doc_name not in self.database:
             self._design_doc.save()
         self._design_doc.fetch()
-        self._design_doc.__setitem__("PICAS_API_VERSION", GRID_LRT.__version__)
+        self._design_doc['PICAS_API_VERSION'] = GRID_LRT.__version__
         self._design_doc.save()
 
     def add_attachment(self, filename, attachment_name):
@@ -345,6 +345,10 @@ class TokenList(list):
 
     def append(self, item):
         if isinstance(item, Token):
+            if 'PICAS_API_VERSION' in item:
+                # Custom version specified in the config file, update design doc PICAS_API_VERSION from token.
+                self._design_doc['PICAS_API_VERSION'] = item['PICAS_API_VERSION']
+                self._design_doc.save()
             if not hasattr(self, 'database'):
                 self.database=item.database  
             if not hasattr(self, 'token_type'):