Commit 9159f365 authored by Frits's avatar Frits

Revert "modified such that it runs on spider"

This reverts commit 9f12ae35.
parent 9f12ae35
......@@ -5,6 +5,13 @@ import socket
from subprocess import call, Popen, PIPE, STDOUT
if sys.version_info[0:2] != (2, 6):
from subprocess import check_output
from subprocess import Popen
class SafePopen(Popen):
def __init__(self, *args, **kwargs):
if sys.version_info.major == 3 :
kwargs['encoding'] = 'utf8'
return super(SafePopen, self).__init__(*args, **kwargs)
__all__ = ["storage", 'auth', "application", "Staging", 'token']
......
......@@ -18,12 +18,7 @@ import tempfile
from GRID_LRT.auth.get_picas_credentials import picas_cred as pc
import GRID_LRT
from GRID_LRT.auth import grid_credentials
class SafePopen(Popen):
def __init__(self, *args, **kwargs):
if sys.version_info.major == 3 :
kwargs['encoding'] = 'utf8'
return super(SafePopen, self).__init__(*args, **kwargs)
from GRID_LRT import SafePopen
#class job_launcher(object):
# """Generic Job Launcher
......@@ -376,21 +371,20 @@ class SpiderLauncher(JdlLauncher):
if not os.path.exists(self.launch_file):
raise IOError("Launch file doesn't exist! "+self.launch_file)
slurmfile = '#!/usr/bin/env bash\n'
slurmfile += '#SBATCH --nodes=1 --ntasks 1 --cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=job-%A_%a.out --error=job-%A_%a.err'
if self.wholenodes:
slurmfile += '#SBATCH --exclusive --nodes=1 --ntasks 1--cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=spiderjob-%A_%a.out --error=spiderjob-%A_%a.err\n'
else:
slurmfile += '#SBATCH --nodes=1 --ntasks 1 --cpus-per-task={ncpu:d} -p {queue:s} --array 1-{njobs:d}%{concurrent:d} --output=spiderjob-%A_%a.out --error=spiderjob-%A_%a.err\n'
slurmfile += " ".join(slurmfile, '--exclusive')
slurmfile += '\n'
slurmfile += """
echo Job landed on $(hostname)
JOBDIR=$(mktemp -d -p $TMPDIR)
cd $TMPDIR
JOBDIR=$TMPDIR
export JOBDIR
echo Created job directory $JOBDIR
cd $JOBDIR
{launcher:s} {db:s} {usr:s} {pw:s} {tt:s}
"""
slurmfile = slurmfile.format(ncpu=int(self.ncpu),
queueu=str(self.queue),
slurmfile = slurmfile.format(ncpu=int(self.ncpu),
queue=str(self.queue),
njobs=self.numjobs,
concurrent=self.parameter_step,
launcher=str(self.launch_file),
......
......@@ -4,15 +4,14 @@ activated
"""
import subprocess
from functools import wraps
from GRID_LRT import SafePopen
def check_uberftp():
"""Checks if the gfal-ls executable
exists on the system. Returns True if it exists
:returns: bool"""
process = subprocess.Popen(['which', 'gfal-ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
encoding='utf8')
process = SafePopen(['which', 'gfal-ls'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
output = process.communicate()
if output[0] == '' and output[1] == '':
return False
......@@ -26,11 +25,10 @@ def grid_credentials_enabled():
This requires gfal-ls!"""
if not check_uberftp():
return False
process = subprocess.Popen([
process = SafePopen([
'gfal-ls', '-l',
'gsiftp://gridftp.grid.sara.nl:2811/pnfs/grid.sara.nl/data/lofar/user/sksp/diskonly'
], stdout=subprocess.PIPE, stderr=subprocess.PIPE,
encoding='utf8')
], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
res = process.communicate()
if type(res[1])==bytes:
error = res[1].decode('utf8')
......
......@@ -20,9 +20,9 @@ export PICAS_API_VERSION=$(curl -X GET https://"$2":"$3"@picas-lofar.grid.surfsa
set -x
if type git &> /dev/null
then
git clone https://github.com/apmechev/GRID_PiCaS_Launcher.git p_tools_git
git clone https://git.astron.nl/astron-sdc/GRID_PiCaS_Launcher.git p_tools_git
cd p_tools_git || exit -100
git checkout v"$PICAS_API_VERSION"
git checkout "$PICAS_API_VERSION"
git describe
cd ../ || exit -100
else #move this to testpy3
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment