Commit 9f12ae35 authored by meyer's avatar meyer

modified such that it runs on spider

parent 11195982
......@@ -5,13 +5,6 @@ import socket
from subprocess import call, Popen, PIPE, STDOUT
if sys.version_info[0:2] != (2, 6):
from subprocess import check_output
from subprocess import Popen
class SafePopen(Popen):
def __init__(self, *args, **kwargs):
if sys.version_info.major == 3 :
kwargs['encoding'] = 'utf8'
return super(SafePopen, self).__init__(*args, **kwargs)
__all__ = ["storage", 'auth', "application", "Staging", 'token']
......
......@@ -18,7 +18,12 @@ import tempfile
from GRID_LRT.auth.get_picas_credentials import picas_cred as pc
import GRID_LRT
from GRID_LRT.auth import grid_credentials
from GRID_LRT import SafePopen
class SafePopen(Popen):
def __init__(self, *args, **kwargs):
if sys.version_info.major == 3 :
kwargs['encoding'] = 'utf8'
return super(SafePopen, self).__init__(*args, **kwargs)
#class job_launcher(object):
# """Generic Job Launcher
......@@ -371,20 +376,21 @@ class SpiderLauncher(JdlLauncher):
if not os.path.exists(self.launch_file):
raise IOError("Launch file doesn't exist! "+self.launch_file)
slurmfile = '#!/usr/bin/env bash\n'
slurmfile += '#SBATCH --nodes=1 --ntasks 1 --cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=job-%A_%a.out --error=job-%A_%a.err'
if self.wholenodes:
slurmfile += " ".join(slurmfile, '--exclusive')
slurmfile += '\n'
slurmfile += '#SBATCH --exclusive --nodes=1 --ntasks 1--cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=spiderjob-%A_%a.out --error=spiderjob-%A_%a.err\n'
else:
slurmfile += '#SBATCH --nodes=1 --ntasks 1 --cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=spiderjob-%A_%a.out --error=spiderjob-%A_%a.err\n'
slurmfile += """
echo Job landed on $(hostname)
JOBDIR=$TMPDIR
JOBDIR=$(mktemp -d -p $TMPDIR)
cd $TMPDIR
export JOBDIR
echo Created job directory $JOBDIR
cd $JOBDIR
{launcher:s} {db:s} {usr:s} {pw:s} {tt:s}
"""
slurmfile = slurmfile.format(ncpu=int(self.ncpu),
queue=str(self.queue),
slurmfile = slurmfile.format(ncpu=int(self.ncpu),
queueu=str(self.queue),
njobs=self.numjobs,
concurrent=self.parameter_step,
launcher=str(self.launch_file),
......
......@@ -4,14 +4,15 @@ activated
"""
import subprocess
from functools import wraps
from GRID_LRT import SafePopen
def check_uberftp():
"""Checks if the gfal-ls executable
exists on the system. Returns True if it exists
:returns: bool"""
process = SafePopen(['which', 'gfal-ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
process = subprocess.Popen(['which', 'gfal-ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
encoding='utf8')
output = process.communicate()
if output[0] == '' and output[1] == '':
return False
......@@ -25,10 +26,11 @@ def grid_credentials_enabled():
This requires gfal-ls!"""
if not check_uberftp():
return False
process = SafePopen([
process = subprocess.Popen([
'gfal-ls', '-l',
'gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lofar/user/sksp/diskonly'
], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
encoding='utf8')
res = process.communicate()
if type(res[1])==bytes:
error = res[1].decode('utf8')
......
......@@ -20,9 +20,9 @@ export PICAS_API_VERSION=$(curl -X GET https://"$2":"$3"@picas-lofar.grid.surfsa
set -x
if type git &> /dev/null
then
git clone https://git.astron.nl/astron-sdc/GRID_PiCaS_Launcher.git p_tools_git
git clone https://github.com/apmechev/GRID_PiCaS_Launcher.git p_tools_git
cd p_tools_git || exit -100
git checkout "$PICAS_API_VERSION"
git checkout v"$PICAS_API_VERSION"
git describe
cd ../ || exit -100
else #move this to testpy3
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment