diff --git a/CEP/Pipeline/framework/lofarpipe/support/lofarnode.py b/CEP/Pipeline/framework/lofarpipe/support/lofarnode.py index 031e589a754f3e9fa542bd52991fd918da4bbc82..bd7ee143d349e8c8337bc93d00e579ba208448e0 100644 --- a/CEP/Pipeline/framework/lofarpipe/support/lofarnode.py +++ b/CEP/Pipeline/framework/lofarpipe/support/lofarnode.py @@ -119,22 +119,38 @@ class LOFARnodeTCP(LOFARnode): else: break - def __fetch_arguments(self): + def __fetch_arguments(self, tries=5, min_timeout=1.0, max_timeout=5.0): """ Connect to a remote job dispatch server (an instance of jobserver.JobSocketReceive) and obtain all the details necessary to run this job. """ - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - self.__try_connect(s) - message = "GET %d" % self.job_id - s.sendall(struct.pack(">L", len(message)) + message) - chunk = s.recv(4) - slen = struct.unpack(">L", chunk)[0] - chunk = s.recv(slen) - while len(chunk) < slen: - chunk += s.recv(slen - len(chunk)) - self.arguments = pickle.loads(chunk) + while True: + tries -= 1 + try: + s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.__try_connect(s) + message = "GET %d" % self.job_id + s.sendall(struct.pack(">L", len(message)) + message) + chunk = s.recv(4) + slen = struct.unpack(">L", chunk)[0] + chunk = s.recv(slen) + while len(chunk) < slen: + chunk += s.recv(slen - len(chunk)) + self.arguments = pickle.loads(chunk) + except socket.error, e: + print "Failed to get recipe arguments from server" + if tries > 0: + timeout = random.uniform(min_timeout, max_timeout) + print("Retrying in %f seconds (%d more %s)." % + (timeout, tries, "try" if tries == 1 else "tries")) + time.sleep(timeout) + else: + # we tried 5 times, abort with original exception + raise + else: + # no error, thus break the loop + break # def __send_results(self): """ diff --git a/CEP/Pipeline/visual_studio/Pipeline.v12.suo b/CEP/Pipeline/visual_studio/Pipeline.v12.suo index 384cf1194369d06dc525ac091e858e9645f1aaf7..1d2c795786e59d4613d66e8a6b0d804edd2f3d22 100644 Binary files a/CEP/Pipeline/visual_studio/Pipeline.v12.suo and b/CEP/Pipeline/visual_studio/Pipeline.v12.suo differ