Compare revisions

Mario Raciti · Mario Raciti · Mario Raciti · Mario Raciti · Mario Raciti · Mario Raciti
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -202,10 +202,10 @@ dockerize_TMSS:
    - cd SAS/TMSS/backend/test/oidc/docker-test-mozilla-django-oidc
    - docker build -t tmss_testprovider:$CI_COMMIT_SHORT_SHA -f dockerfiles/oidc_testprovider .
    - docker login -u $CI_NEXUS_REGISTRY_USERNAME -p $CI_NEXUS_REGISTRY_PASSWORD $CI_NEXUS_REGISTRY
-    - docker tag tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
-    - docker push nexus.cep4.control.lofar:18080/tmss_django:$CI_COMMIT_SHORT_SHA
-    - docker tag tmss_testprovider:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
-    - docker push nexus.cep4.control.lofar:18080/tmss_testprovider:$CI_COMMIT_SHORT_SHA
+    - docker tag tmss_django:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/tmss_django:$CI_COMMIT_SHORT_SHA
+    - docker push $CI_NEXUS_REGISTRY_LOCATION/tmss_django:$CI_COMMIT_SHORT_SHA
+    - docker tag tmss_testprovider:$CI_COMMIT_SHORT_SHA $CI_NEXUS_REGISTRY_LOCATION/tmss_testprovider:$CI_COMMIT_SHORT_SHA
+    - docker push $CI_NEXUS_REGISTRY_LOCATION/tmss_testprovider:$CI_COMMIT_SHORT_SHA
    - docker logout $CI_NEXUS_REGISTRY
  interruptible: true
  needs:
@@ -286,10 +286,11 @@ deploy-tmss-test:
    - chmod 644 ~/.ssh/known_hosts
  script:
    - ssh lofarsys@scu199.control.lofar "supervisorctl -u user -p 123 stop TMSS:*"
-    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
-    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA"
-    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_testprovider:latest"
-    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY}/tmss_django:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY}/tmss_django:latest"
+    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA"
+    - ssh lofarsys@scu199.control.lofar "docker pull ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA"
+    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:$CI_COMMIT_SHORT_SHA ${CI_NEXUS_REGISTRY_LOCATION}/tmss_testprovider:latest"
+    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA nexus.cep4.control.lofar:18080/tmss_django:latest"
+    - ssh lofarsys@scu199.control.lofar "docker tag ${CI_NEXUS_REGISTRY_LOCATION}/tmss_django:$CI_COMMIT_SHORT_SHA tmss_django:latest"
    - ssh lofarsys@scu199.control.lofar "supervisorctl -u user -p 123 start TMSS:*"
  needs:
    - dockerize_TMSS

--- a/MAC/Deployment/data/OTDB/OnlineControl.comp
+++ b/MAC/Deployment/data/OTDB/OnlineControl.comp
@@ -12,7 +12,7 @@ par  _hostname          I    text   -      10    0    'CCU001'
 par  applications       I    vtext  -      10    0    ["CorrAppl"]    -     "The applications the controller should manage."
 par  applOrder          I    vtext  -      10    0    ["CorrAppl"]    -     "The application depencies if any!"
 par  inspectionProgram 	I    text   -     100	0	'launch-msplots.sh'		-	"Script to start the inspection"
-par  inspectionHost	    I    text   -     100	0	'master.cep4.control.lofar'	-	"Machine the inspection-script should be started"
+par  inspectionHost	    I    text   -     100	0	'head.cep4.control.lofar'	-	"Machine the inspection-script should be started"


 uses  CorrAppl      4.0.0  development 1                  "CN Application"

--- a/MAC/Deployment/data/OTDB/PythonControl.comp
+++ b/MAC/Deployment/data/OTDB/PythonControl.comp
@@ -10,7 +10,7 @@ node  PythonControl  4.0.0  development 'node constraint'  "Controller for the o
 #--------------------------------------------------------------------------------------------------------
 par  _hostname               I    text   -      100	0	'CCU001'                   			-       "Machine the PythonController should run on"
 par  pythonProgram           I    text   -      100	0	'startPipeline.py'				-	"Python script to start"
-par  pythonHost              I    text   -      100	0	'master.cep4.control.lofar'			-	"Machine the Pythonscript should be started"
+par  pythonHost              I    text   -      100	0	'head.cep4.control.lofar'			-	"Machine the Pythonscript should be started"
 par  canCommunicate          I    bool   -      10	0	'true'						-	"Temp flag to tell MAC if the current PythonController can respond to CONTROL_xxx messages"
 par  softwareVersion         I    text   -      100     0       ''                                              -       "the LOFAR software version to use for the pipeline (literally the sub-directory name in lofar_versions)"


--- a/MAC/Services/src/PipelineControl.py
+++ b/MAC/Services/src/PipelineControl.py
@@ -102,15 +102,17 @@ def runCommand(cmdline, input=None):
        cmdline,
        stdin=subprocess.PIPE if input else None,
        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT,
+        stderr=subprocess.PIPE,
        shell=True,
        universal_newlines=True
    )

    # Feed input and wait for termination
    logger.debug("runCommand input: %s", input)
-    stdout, _ = communicate_returning_strings(proc, input)
+    stdout, stderr = communicate_returning_strings(proc, input)
    logger.debug("runCommand output: %s", stdout)
+    if stderr:
+        logger.warn("runCommand stderr output: %s", stderr)

    # Check exit status, bail on error
    if proc.returncode != 0:

--- a/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
+++ b/SAS/ResourceAssignment/RAtoOTDBTaskSpecificationPropagator/lib/translator.py
@@ -495,7 +495,7 @@ class RAtoOTDBTranslator():
                # (Inspection plots from station subband stats are independent from this and always avail.)
                if any(key.endswith('.locations') and 'CEP4:' in val for key, val in list(parset.items())):
                    logging.info("CreateParset: Adding inspection plot commands to parset")
-                    parset[PREFIX+'ObservationControl.OnlineControl.inspectionHost'] = 'head01.cep4.control.lofar'
+                    parset[PREFIX+'ObservationControl.OnlineControl.inspectionHost'] = 'head.cep4.control.lofar'
                    parset[PREFIX+'ObservationControl.OnlineControl.inspectionProgram'] = 'inspection-plots-observation.sh'

                #special case for dynspec projects for Richard Fallows

--- a/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/webservice.py
+++ b/SAS/ResourceAssignment/ResourceAssignmentEditor/lib/webservice.py
@@ -948,7 +948,7 @@ def getTaskLogHtml(task_id):

    cmd = []
    if task['type'] == 'pipeline':
-        cmd = ['ssh', 'lofarsys@head01.cep4.control.lofar', 'cat /data/log/pipeline-%s-*.log' % task['otdb_id']]
+        cmd = ['ssh', 'lofarsys@head.cep4.control.lofar', 'cat /data/log/pipeline-%s-*.log' % task['otdb_id']]
    else:
        cmd = ['ssh', 'mcu001.control.lofar', 'cat /opt/lofar/var/log/mcu001\\:ObservationControl\\[0\\]\\{%s\\}.log*' % task['otdb_id']]


--- a/SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/constraints/template_constraints_v1.py
@@ -251,6 +251,9 @@ def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: mod
    Checks whether it is possible to place the scheduling unit arbitrarily in the given time window, i.e. the sky constraints must be met over the full time window.
    :return: True if all sky constraints are met over the entire time window, else False.
    """
+    # TODO: remove this shortcut after demo
+    return True
+
    constraints = scheduling_unit.draft.scheduling_constraints_doc
    if not "sky" in constraints:
        return True

--- a/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/lib/dynamic_scheduling.py
@@ -275,7 +275,7 @@ class TMSSDynamicSchedulingMessageHandler(TMSSEventMessageHandler):
                    if models.Setting.objects.get(name=models.SystemSettingFlag.Choices.DYNAMIC_SCHEDULING_ENABLED.value).value:
                        do_dynamic_schedule()
                    else:
-                        logger.warning("Skipping update of dynamic schedule because the setting %s=%s", models.SystemSettingFlag.Choices.DYNAMIC_SCHEDULING_ENABLED.value, models.Setting.objects.get(name=models.Flag.Choices.DYNAMIC_SCHEDULING_ENABLED.value).value)
+                        logger.warning("Skipping update of dynamic schedule because the setting %s=%s", models.SystemSettingFlag.Choices.DYNAMIC_SCHEDULING_ENABLED.value, models.Setting.objects.get(name=models.SystemSettingFlag.Choices.DYNAMIC_SCHEDULING_ENABLED.value).value)
                except Exception as e:
                    logger.exception(str(e))
                    # just continue processing events. better luck next time...

--- a/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
+++ b/SAS/TMSS/backend/services/scheduling/test/t_dynamic_scheduling.py
@@ -26,6 +26,9 @@ from astropy.coordinates import Angle
 import logging
 logger = logging.getLogger(__name__)

+#TODO: remove after demo
+exit(3)
+
 from lofar.common.test_utils import skip_integration_tests
 if skip_integration_tests():
    exit(3)

--- a/SAS/TMSS/backend/src/tmss/tmssapp/adapters/parset.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/adapters/parset.py
@@ -395,7 +395,7 @@ def _convert_to_parset_dict_for_observationcontrol_schema(subtask: models.Subtas
        parset[prefix+"ObservationControl.OnlineControl._hostname"] = 'CCU001'
        parset[prefix+"ObservationControl.OnlineControl.applOrder"] = '["CorrAppl"]'
        parset[prefix+"ObservationControl.OnlineControl.applications"] = '["CorrAppl"]'
-        parset[prefix+"ObservationControl.OnlineControl.inspectionHost"] = 'head01.cep4.control.lofar'
+        parset[prefix+"ObservationControl.OnlineControl.inspectionHost"] = 'head.cep4.control.lofar'
        parset[prefix+"ObservationControl.OnlineControl.inspectionProgram"] = 'inspection-plots-observation.sh'
        parset[prefix+"ObservationControl.StationControl._hostname"] = parset["Observation.VirtualInstrument.stationList"]
        parset[prefix+"ObservationControl.StationControl.aartfaacPiggybackAllowed"] = False

--- a/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/backend/src/tmss/tmssapp/subtasks.py
@@ -758,11 +758,11 @@ def create_cleanup_subtask_from_task_blueprint(task_blueprint: TaskBlueprint) ->
    subtask_data = {"start_time": None,
                    "stop_time": None,
                    "state": SubtaskState.objects.get(value=SubtaskState.Choices.DEFINING.value),
-                    "task_blueprint": task_blueprint,
                    "specifications_template": subtask_template,
                    "specifications_doc": subtask_specs,
                    "cluster": Cluster.objects.get(name=cluster_name)}
    subtask = Subtask.objects.create(**subtask_data)
+    subtask.task_blueprints.set([task_blueprint])

    # step 2: create and link subtask input
    # for this cleanup subtask an 'input' seems a bit weird, but it actually makes sense!

--- a/SAS/TMSS/scripts/notebooks/project_report_poc.ipynb
+++ b/SAS/TMSS/scripts/notebooks/project_report_poc.ipynb
No results found