Task #7458: Merge into release branch. Reviewed by Jan David

CEP/Pipeline/framework/lofarpipe/support/lofarnode.py

@@ -119,12 +119,15 @@ class LOFARnodeTCP(LOFARnode):
             else:
                 break
 
-    def __fetch_arguments(self):
+    def __fetch_arguments(self, tries=5, min_timeout=1.0, max_timeout=5.0):
         """
         Connect to a remote job dispatch server (an instance of
         jobserver.JobSocketReceive) and obtain all the details necessary to
         run this job.
         """
+        while True:
+            tries -= 1
+            try:
                 s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
                 self.__try_connect(s)
                 message = "GET %d" % self.job_id
@@ -135,6 +138,19 @@ class LOFARnodeTCP(LOFARnode):
                 while len(chunk) < slen:
                     chunk += s.recv(slen - len(chunk))
                 self.arguments = pickle.loads(chunk)
+            except socket.error, e:
+                print "Failed to get recipe arguments from server"
+                if tries > 0:
+                    timeout = random.uniform(min_timeout, max_timeout)
+                    print("Retrying in %f seconds (%d more %s)." %
+                          (timeout, tries, "try" if tries == 1 else "tries"))
+                    time.sleep(timeout)
+                else:
+                    # we tried 5 times, abort with original exception
+                    raise 
+            else:
+                # no error, thus break the loop
+                break  #
 
     def __send_results(self):
         """