diff --git a/CMake/LofarPackageList.cmake b/CMake/LofarPackageList.cmake
index d4f6966e12814caac01dda87311fdbea2535433f..e9c1c4bc0f8d36043bf178fbdbe02f3884f5fc69 100644
--- a/CMake/LofarPackageList.cmake
+++ b/CMake/LofarPackageList.cmake
@@ -1,7 +1,7 @@
# - Create for each LOFAR package a variable containing the absolute path to
# its source directory.
#
-# Generated by gen_LofarPackageList_cmake.sh at do 28 mei 2020 11:22:44 CEST
+# Generated by gen_LofarPackageList_cmake.sh at vr 27 nov 2020 16:08:48 CET
#
# ---- DO NOT EDIT ----
#
@@ -210,6 +210,7 @@ if(NOT DEFINED LOFAR_PACKAGE_LIST_INCLUDED)
set(TMSSSchedulingService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/services/scheduling)
set(TMSSFeedbackHandlingService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/services/feedback_handling)
set(TMSSPostgresListenerService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/services/tmss_postgres_listener)
+ set(TMSSWorkflowService_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TMSS/services/workflow_service)
set(TriggerEmailServiceCommon_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TriggerEmailService/Common)
set(TriggerEmailServiceServer_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SAS/TriggerEmailService/Server)
set(CCU_MAC_SOURCE_DIR ${CMAKE_SOURCE_DIR}/SubSystems/CCU_MAC)
diff --git a/SAS/TMSS/client/lib/populate.py b/SAS/TMSS/client/lib/populate.py
index bb39f6967617e077aa4c8d00f425534cbfc4d95c..63a148b2eeaa2cb59b7f2a77ecc9b9405d67283e 100644
--- a/SAS/TMSS/client/lib/populate.py
+++ b/SAS/TMSS/client/lib/populate.py
@@ -72,7 +72,7 @@ def populate_schemas(schema_dir: str=None, templates_filename: str=None):
else:
template['schema'] = json_schema
- logger.info("Uploading template name='%s' version='%s'", name, version)
+ logger.info("Uploading template with name='%s' version='%s' template='%s' ", name, version, template)
client.post_template(template_path=template_name,
name=name,
diff --git a/SAS/TMSS/frontend/tmss_webapp/package.json b/SAS/TMSS/frontend/tmss_webapp/package.json
index 8230388bc6f2c1f16805cf9a25ba4f05f6b6b000..705a5183e8801882a780f93350b30333e5bd3582 100644
--- a/SAS/TMSS/frontend/tmss_webapp/package.json
+++ b/SAS/TMSS/frontend/tmss_webapp/package.json
@@ -37,6 +37,7 @@
"react-calendar-timeline": "^0.27.0",
"react-dom": "^16.13.1",
"react-frame-component": "^4.1.2",
+ "react-json-to-table": "^0.1.7",
"react-json-view": "^1.19.1",
"react-loader-spinner": "^3.1.14",
"react-router-dom": "^5.2.0",
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js b/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
index e9c0b245f5eaeeaf5bd579a4c42c6b67e46eae44..1a4f0d4290cde1e5be6ce84333dad9622678693a 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/JSONEditor/JEditor.js
@@ -173,15 +173,15 @@ function Jeditor(props) {
}
editor = new JSONEditor(element, editorOptions);
// editor.getEditor('root').disable();
- if (props.disabled) {
- editor.on('ready',() => {
- editor.disable();
- });
- }
- if (props.parentFunction) {
- props.parentFunction(editorFunction);
- }
editorRef.current = editor;
+ editor.on('ready',() => {
+ if (props.disabled) {
+ editor.disable();
+ }
+ if (props.parentFunction) {
+ props.parentFunction(editorFunction);
+ }
+ });
editor.on('change', () => {setEditorOutput()});
};
@@ -208,8 +208,13 @@ function Jeditor(props) {
/**
* Function called by the parent component to perform certain action ib JEditor
*/
- function editorFunction() {
- editorRef.current.destroy();
+ function editorFunction(name, params) {
+ if (name === "setValue") {
+ const newValue = updateInput(_.cloneDeep(params[0]));
+ editorRef.current.setValue(newValue);
+ } else {
+ editorRef.current.destroy();
+ }
}
/**
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
index d611b7857cf365ed32c8eee56b0936e8eb7e5b87..180744f4df30abb3edb16851ca9db17313ef356d 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/components/Timeline/CalendarTimeline.js
@@ -157,6 +157,12 @@ export class CalendarTimeline extends Component {
setInterval(function(){setCurrentUTC(true)}, 60000);
// Update UTC clock every second to keep the clock display live
setInterval(function(){setCurrentUTC()}, 1000);
+ if (this.state.viewType === UIConstants.timeline.types.WEEKVIEW) {
+ this.addWeekSunTimes(this.state.defaultStartTime, this.state.defaultEndTime, this.state.group, this.state.items)
+ .then(items => {
+ this.setState({items: items});
+ });
+ }
}
shouldComponentUpdate() {
@@ -251,22 +257,21 @@ export class CalendarTimeline extends Component {
if (startMonth !== endMonth) {
monthDuration = `(${startMonth}-${endMonth})`;
}
- return (<div {...getRootProps()}
- style={{color: '#ffffff', textAlign: "right", width: `${this.state.sidebarWidth}px`,
- paddingRight: '10px', backgroundColor: '#8ba7d9'}}>
- <div style={{height:'30px'}}>{this.state.viewType===UIConstants.timeline.types.NORMAL?
+ return (<div {...getRootProps()} className="sidebar-header"
+ style={{width: `${this.state.sidebarWidth}px`}}>
+ <div className="sidebar-header-row">{this.state.viewType===UIConstants.timeline.types.NORMAL?
(this.state.dayHeaderVisible?`Day${monthDuration}`:`Week${monthDuration}`)
:`Week (${this.state.timelineStartDate.week()}) / Day`}</div>
- <div style={{height:'30px'}}>{this.state.dayHeaderVisible?`UTC(Hr)`:`UTC(Day)`}</div>
- <div style={{height:'30px'}}>{this.state.dayHeaderVisible?`LST(Hr)`:`LST(Day)`}</div>
- {this.state.viewType === UIConstants.timeline.types.NORMAL &&
- <div className="p-grid"
- style={{height:this.props.showSunTimings?'30px':'0px', paddingTop:'10px', paddingLeft:'10px'}}>
- <div className="col-4" style={{marginTop:'2px', paddingLeft:'5px', backgroundColor:'yellow', color: '#212529'}}>Sunrise</div>
- <div className="col-4" style={{marginTop:'2px', paddingLeft:'5px', backgroundColor:'orange', color: '#212529'}}>Sunset</div>
- <div className="col-4" style={{marginTop:'2px', paddingLeft:'5px', backgroundColor:'blue'}}>Night</div>
+ <div className="sidebar-header-row">{this.state.dayHeaderVisible?`UTC(Hr)`:`UTC(Day)`}</div>
+ <div className="sidebar-header-row">{this.state.dayHeaderVisible?`LST(Hr)`:`LST(Day)`}</div>
+ {/* {this.state.viewType === UIConstants.timeline.types.NORMAL && */}
+ <div className="p-grid legend-row"
+ style={{height:this.props.showSunTimings?'30px':'0px'}}>
+ <div className="col-4 legend-suntime legend-sunrise">Sunrise</div>
+ <div className="col-4 legend-suntime legend-sunset">Sunset</div>
+ <div className="col-4 legend-suntime legend-night">Night</div>
</div>
- }
+ {/* } */}
</div>
);
}
@@ -449,8 +454,8 @@ export class CalendarTimeline extends Component {
}
const nightStyle = {
lineHeight: '30px',
- backgroundColor: 'blue',
- color: 'blue'
+ backgroundColor: 'grey',
+ color: 'grey'
}
const sunriseStyle = {
lineHeight: '30px',
@@ -463,27 +468,25 @@ export class CalendarTimeline extends Component {
color: 'orange'
}
// Get the intervals UTC date format and time
- const intervalDate = interval.startTime.clone().utc().format("YYYYMMDDT12:00:00");
+ const intervalDate = interval.startTime.clone().utc().format("YYYY-MM-DD");
const intervalTime = interval.startTime.clone().utc();
// Get the suntime for the UTC date
const intervalDateSunTime = sunTimeMap[intervalDate];
let intervalStyle = dayStyle;
// If suntime is available display suntime blocks
if (intervalDateSunTime) {
- // Set 15 minutes duration for sunrise and sunset and create blocks accordingly
- if (intervalTime.isBefore(intervalDateSunTime.sunrise) ||
- intervalTime.isAfter(intervalDateSunTime.sunset.clone().add(14, 'minutes'))) {
- intervalStyle = nightStyle;
- } else if (intervalTime.isSame(intervalDateSunTime.sunrise) ||
- intervalTime.isBefore(intervalDateSunTime.sunrise.clone().add(15, 'minutes'))) {
+ if (intervalTime.isBefore(intervalDateSunTime.sunrise.start) ||
+ intervalTime.isAfter(intervalDateSunTime.sunset.end)) {
+ intervalStyle = nightStyle;
+ } else if (intervalTime.isSameOrAfter(intervalDateSunTime.sunrise.start) &&
+ intervalTime.isSameOrBefore(intervalDateSunTime.sunrise.end)) {
intervalStyle = sunriseStyle;
- } else if (intervalTime.isSame(intervalDateSunTime.sunset) ||
- (intervalTime.isAfter(intervalDateSunTime.sunset) &&
- intervalTime.isBefore(intervalDateSunTime.sunset.clone().add(15, 'minutes')))) {
+ } else if (intervalTime.isSameOrAfter(intervalDateSunTime.sunset.start) &&
+ intervalTime.isSameOrBefore(intervalDateSunTime.sunset.end)) {
intervalStyle = sunsetStyle;
}
return (
- <div
+ <div clasName={`suntime-header, ${intervalStyle}`}
{...getIntervalProps({
interval,
style: intervalStyle
@@ -500,46 +503,133 @@ export class CalendarTimeline extends Component {
}
/**
- * Function to render sunrise timings on the timeline view in normal view.
+ * Function to render sunrise and before sunrise timings on the timeline view in normal view.
* @param {Array} sunRiseTimings
*/
renderSunriseMarkers(sunRiseTimings) {
+ let endPoint = 0;
return (
<>
{sunRiseTimings && sunRiseTimings.length>0 && sunRiseTimings.map((item, index) => (
- <CustomMarker key={"sunrise-"+index} date={item}>
- {({ styles, date }) => {
- const customStyles = {
- ...styles,
- backgroundColor: 'yellow',
- width: '3px'
- }
- return <div style={customStyles} />
- }}
- </CustomMarker>
+ <>
+ {/* Marker to get the position of the sunrise end time */}
+ <CustomMarker key={"sunrise-"+index} date={item.end}>
+ {({ styles, date }) => {
+ endPoint = styles.left;
+ return ""
+ }}
+ </CustomMarker>
+ {/* Marker to represent dark light before sunrise on the day */}
+ <CustomMarker key={"sunrise-"+index} date={item.start.clone().hours(0).minutes(0).seconds(0)}>
+ {({ styles, date }) => {
+ const customStyles = {
+ ...styles,
+ backgroundColor: 'grey',
+ opacity:0.7,
+ zIndex: 10,
+ // width: '3px'
+ width: (endPoint-styles.left)
+ }
+ return <div style={customStyles} />
+ }}
+ </CustomMarker>
+ {/* Marker to represent the duration of sunrise */}
+ <CustomMarker key={"sunrise-"+index} date={item.start}>
+ {({ styles, date }) => {
+ const customStyles = {
+ ...styles,
+ backgroundColor: 'yellow',
+ opacity:0.7,
+ zIndex: 10,
+ // width: '3px'
+ width: (endPoint-styles.left)
+ }
+ return <div style={customStyles} />
+ }}
+ </CustomMarker>
+ </>
))}
</>
);
}
/**
- * Function to render sunrise timings on the timeline view in normal view.
+ * Function to render sunset & after sunset timings on the timeline view in normal view.
* @param {Array} sunSetTimings
*/
renderSunsetMarkers(sunSetTimings) {
+ let endPoint = 0;
return (
<>
{sunSetTimings && sunSetTimings.length>0 && sunSetTimings.map((item, index) => (
- <CustomMarker key={"sunset-"+index} date={item}>
- {({ styles, date }) => {
- const customStyles = {
- ...styles,
- backgroundColor: 'orange',
- width: '3px'
- }
- return <div style={customStyles} />
- }}
- </CustomMarker>
+ <>
+ {/* Marker to get the position of the sunset end time */}
+ <CustomMarker key={"sunset-"+index} date={item.end}>
+ {({ styles, date }) => {
+ endPoint = styles.left;
+ return ""
+ }}
+ </CustomMarker>
+ {/* Marker to represent the dark light after sunset */}
+ <CustomMarker key={"sunset-"+index} date={item.start.clone().hours(23).minutes(59).seconds(59)}>
+ {({ styles, date }) => {
+ const customStyles = {
+ ...styles,
+ backgroundColor: 'grey',
+ opacity:0.7,
+ zIndex: 10,
+ left: endPoint,
+ width: styles.left-endPoint
+ }
+ return <div style={customStyles} />
+ }}
+ </CustomMarker>
+ {/* Marker to represent the actual sunset duration */}
+ <CustomMarker key={"sunset-"+index} date={item.start}>
+ {({ styles, date }) => {
+ const customStyles = {
+ ...styles,
+ backgroundColor: 'orange',
+ opacity:0.7,
+ zIndex: 10,
+ width: endPoint - styles.left
+ }
+ return <div style={customStyles} />
+ }}
+ </CustomMarker>
+ </>
+ ))}
+ </>
+ );
+ }
+
+ /**
+ * Function to render sunrise timings on the timeline view in normal view.
+ * @param {Array} sunSetTimings
+ */
+ renderNightMarkers(sunRiseTimings, sunSetTimings) {
+ let endPoint = 0;
+ return (
+ <>
+ {sunSetTimings && sunSetTimings.length>0 && sunSetTimings.map((item, index) => (
+ <>
+ <CustomMarker key={"sunset-"+index} date={item.end}>
+ {({ styles, date }) => {
+ endPoint = styles.left;
+ return ""
+ }}
+ </CustomMarker>
+ <CustomMarker key={"sunset-"+index} date={item.start}>
+ {({ styles, date }) => {
+ const customStyles = {
+ ...styles,
+ backgroundColor: 'orange',
+ width: endPoint - styles.left
+ }
+ return <div style={customStyles} />
+ }}
+ </CustomMarker>
+ </>
))}
</>
);
@@ -577,10 +667,16 @@ export class CalendarTimeline extends Component {
/** Custom function to pass to timeline component to render item */
renderItem({ item, timelineContext, itemContext, getItemProps, getResizeProps }) {
+ /* Reduce the item height so that the suntimings can be viewed above the item block.
+ Also suntimes are rendered as items with tiny height to represent as horizontal bar above the actual items */
+ if (item.type === "SUNTIME") {
+ itemContext.dimensions.height = 3;
+ } else {
+ itemContext.dimensions.height -= 3;
+ itemContext.dimensions.top += 3;
+ }
const { left: leftResizeProps, right: rightResizeProps } = getResizeProps();
const backgroundColor = itemContext.selected?item.bgColor:item.bgColor;
- // const backgroundColor = itemContext.selected ? (itemContext.dragging ? "red" : item.selectedBgColor) : item.bgColor;
- // const borderColor = itemContext.resizing ? "red" : item.color;
let itemContentStyle = {lineHeight: `${Math.floor(itemContext.dimensions.height)}px`,
fontSize: "14px",
overflow: "hidden", textOverflow: "ellipsis", whiteSpace: "nowrap",
@@ -603,7 +699,8 @@ export class CalendarTimeline extends Component {
borderRadius: 3,
borderLeftWidth: itemContext.selected ? 3 : 1,
borderRightWidth: itemContext.selected ? 3 : 1,
- opacity: item.type==="SUNTIME"?0.6:1
+ // opacity: item.type==="SUNTIME"?0.6:1
+ zIndex: item.type==="SUNTIME"?79:80
},
onMouseDown: () => {
if (item.type !== "SUNTIME") {
@@ -623,7 +720,7 @@ export class CalendarTimeline extends Component {
//whiteSpace: "nowrap"
}}
>
- { this.state.viewType===UIConstants.timeline.types.WEEKVIEW &&
+ { this.state.viewType===UIConstants.timeline.types.WEEKVIEW && item.type !== "SUNTIME" &&
<><div style={itemContentStyle}><i style={{fontSize:"12px"}} className="fa fa-user" title="Friend"></i><span>{item.project}</span></div>
<div style={itemContentStyle}><span>{item.duration}</span></div>
<div style={itemContentStyle}><span>{item.band}</span></div> </>}
@@ -719,14 +816,18 @@ export class CalendarTimeline extends Component {
const noOfDays = endTime.diff(startTime, 'days');
for (const number of _.range(noOfDays+1)) {
const date = startTime.clone().add(number, 'days').hours(12).minutes(0).seconds(0);
- const formattedDate = date.format("YYYYMMDDTHH:mm:ss");
- UtilService.getSunTimings(formattedDate+"Z").then(timings => {
- const sunriseTime = moment.utc(timings.sun_rise.split('.')[0]);
- const sunsetTime = moment.utc(timings.sun_set.split('.')[0]);
- if (moment.utc(timings.sun_rise).isAfter(startTime)) {
+ const formattedDate = date.format("YYYY-MM-DD");
+ UtilService.getSunTimings(formattedDate).then(timings => {
+ const sunriseStartTime = moment.utc(timings.sun_rise.start.split('.')[0]);
+ const sunriseEndTime = moment.utc(timings.sun_rise.end.split('.')[0]);
+ const sunsetStartTime = moment.utc(timings.sun_set.start.split('.')[0]);
+ const sunsetEndTime = moment.utc(timings.sun_set.end.split('.')[0]);
+ const sunriseTime = {start: sunriseStartTime, end: sunriseEndTime};
+ const sunsetTime = {start: sunsetStartTime, end: sunsetEndTime};
+ if (moment.utc(timings.sunriseEndTime).isAfter(startTime)) {
sunRiseTimings.push(sunriseTime);
}
- if (moment.utc(timings.sun_set).isBefore(endTime)) {
+ if (moment.utc(timings.sunsetStartTime).isBefore(endTime)) {
sunSetTimings.push(sunsetTime);
}
sunTimeMap[formattedDate] = {sunrise: sunriseTime, sunset: sunsetTime};
@@ -735,33 +836,62 @@ export class CalendarTimeline extends Component {
}
}
+ /**
+ *
+ * @param {moment} startTime
+ * @param {moment} endTime
+ * @param {Array} stationGroup - Array of station group objects
+ * @param {Array} items - Array of Item objects
+ */
async addStationSunTimes(startTime, endTime, stationGroup, items) {
const noOfDays = endTime.diff(startTime, 'days');
let sunItems = _.cloneDeep(items);
for (const number of _.range(noOfDays+1)) {
for (const station of stationGroup) {
const date = startTime.clone().add(number, 'days').hours(12).minutes(0).seconds(0);
- const timings = await UtilService.getSunTimings(date.format("YYYYMMDDTHH:mm:ss")+"Z", station.id);
- let sunriseItem = { id: `sunrise-${number}-${station.id}`,
- group: station.id,
- title: timings.sun_rise,
- project: "",
- name: "",
- duration: "",
- start_time: moment.utc(timings.sun_rise),
- end_time: moment.utc(timings.sun_rise).add(5, 'minutes'),
- bgColor: "yellow",
- selectedBgColor: "yellow",
- type: "SUNTIME"};
- sunItems.push(sunriseItem);
- let sunsetItem = _.cloneDeep(sunriseItem);
- sunsetItem.id = `sunset-${number}-${station.id}`;
- sunsetItem.start_time = moment.utc(timings.sun_set);
- sunsetItem.end_time = moment.utc(timings.sun_set).add(5, 'minutes');
- sunsetItem.bgColor = "orange";
- sunsetItem.selectedBgColor = "0range";
- sunItems.push(sunsetItem);
-
+ const timings = await UtilService.getSunTimings(date.format("YYYY-MM-DD"), station.id);
+ if (timings) {
+ let sunriseItem = { id: `sunrise-${number}-${station.id}`,
+ group: station.id,
+ // title: `${timings.sun_rise.start} to ${timings.sun_rise.end}`,
+ title: "",
+ project: "",
+ name: "",
+ duration: "",
+ start_time: moment.utc(timings.sun_rise.start),
+ end_time: moment.utc(timings.sun_rise.end),
+ bgColor: "yellow",
+ selectedBgColor: "yellow",
+ type: "SUNTIME"};
+ sunItems.push(sunriseItem);
+ let sunsetItem = _.cloneDeep(sunriseItem);
+ sunsetItem.id = `sunset-${number}-${station.id}`;
+ // sunsetItem.title = `${timings.sun_set.start} to ${timings.sun_set.end}`;
+ sunsetItem.title = "";
+ sunsetItem.start_time = moment.utc(timings.sun_set.start);
+ sunsetItem.end_time = moment.utc(timings.sun_set.end);
+ sunsetItem.bgColor = "orange";
+ sunsetItem.selectedBgColor = "orange";
+ sunItems.push(sunsetItem);
+ let befSunriseItem = _.cloneDeep(sunriseItem);
+ befSunriseItem.id = `bef-sunrise-${number}-${station.id}`;
+ // sunsetItem.title = `${timings.sun_set.start} to ${timings.sun_set.end}`;
+ befSunriseItem.title = "";
+ befSunriseItem.start_time = moment.utc(timings.sun_rise.start).hours(0).minutes(0).seconds(0);
+ befSunriseItem.end_time = moment.utc(timings.sun_rise.start);
+ befSunriseItem.bgColor = "grey";
+ befSunriseItem.selectedBgColor = "grey";
+ sunItems.push(befSunriseItem);
+ let afterSunsetItem = _.cloneDeep(sunriseItem);
+ afterSunsetItem.id = `aft-sunset-${number}-${station.id}`;
+ // sunsetItem.title = `${timings.sun_set.start} to ${timings.sun_set.end}`;
+ afterSunsetItem.title = "";
+ afterSunsetItem.start_time = moment.utc(timings.sun_set.end);
+ afterSunsetItem.end_time = moment.utc(timings.sun_set.end).hours(23).minutes(59).seconds(59);
+ afterSunsetItem.bgColor = "grey";
+ afterSunsetItem.selectedBgColor = "grey";
+ sunItems.push(afterSunsetItem);
+ }
}
}
if (!this.props.showSunTimings && this.state.viewType === UIConstants.timeline.types.NORMAL) {
@@ -770,6 +900,73 @@ export class CalendarTimeline extends Component {
return items;
}
+ /**
+ * To Render sunrise, sunset and night times as horizontal bar, new items are created and appended with actual items.
+ * @param {moment} startTime
+ * @param {moment} endTime
+ * @param {Array} weekGroup
+ * @param {Array} items
+ */
+ async addWeekSunTimes(startTime, endTime, weekGroup, items) {
+ const noOfDays = endTime.diff(startTime, 'days');
+ let sunItems = _.cloneDeep(items);
+ for (const weekDay of weekGroup) {
+ if (weekDay.value) {
+ const timings = await UtilService.getSunTimings(weekDay.value.format("YYYY-MM-DD"), 'CS001');
+ const sunriseStart = moment.utc(timings.sun_rise.start);
+ const sunriseEnd = moment.utc(timings.sun_rise.end);
+ const sunsetStart = moment.utc(timings.sun_set.start);
+ const sunsetEnd = moment.utc(timings.sun_set.end);
+ if (timings) {
+ let sunriseItem = { id: `sunrise-${weekDay.id}`,
+ group: weekDay.id,
+ // title: `${timings.sun_rise.start} to ${timings.sun_rise.end}`,
+ title: "",
+ project: "",
+ name: "",
+ duration: "",
+ start_time: startTime.clone().hours(sunriseStart.hours()).minutes(sunriseStart.minutes()).seconds(sunriseStart.seconds()),
+ end_time: startTime.clone().hours(sunriseEnd.hours()).minutes(sunriseEnd.minutes()).seconds(sunriseEnd.seconds()),
+ bgColor: "yellow",
+ selectedBgColor: "yellow",
+ type: "SUNTIME"};
+ sunItems.push(sunriseItem);
+ let sunsetItem = _.cloneDeep(sunriseItem);
+ sunsetItem.id = `sunset-${weekDay.id}`;
+ // sunsetItem.title = `${timings.sun_set.start} to ${timings.sun_set.end}`;
+ sunsetItem.title = "";
+ sunsetItem.start_time = startTime.clone().hours(sunsetStart.hours()).minutes(sunsetStart.minutes()).seconds(sunsetStart.seconds());
+ sunsetItem.end_time = startTime.clone().hours(sunsetEnd.hours()).minutes(sunsetEnd.minutes()).seconds(sunsetEnd.seconds());
+ sunsetItem.bgColor = "orange";
+ sunsetItem.selectedBgColor = "orange";
+ sunItems.push(sunsetItem);
+ let befSunriseItem = _.cloneDeep(sunriseItem);
+ befSunriseItem.id = `bef-sunrise-${weekDay.id}`;
+ // sunsetItem.title = `${timings.sun_set.start} to ${timings.sun_set.end}`;
+ befSunriseItem.title = "";
+ befSunriseItem.start_time = startTime.clone().hours(0).minutes(0).seconds(0);
+ befSunriseItem.end_time = startTime.clone().hours(sunriseStart.hours()).minutes(sunriseStart.minutes()).seconds(sunriseStart.seconds());;
+ befSunriseItem.bgColor = "grey";
+ befSunriseItem.selectedBgColor = "grey";
+ sunItems.push(befSunriseItem);
+ let afterSunsetItem = _.cloneDeep(sunriseItem);
+ afterSunsetItem.id = `aft-sunset-${weekDay.id}`;
+ // sunsetItem.title = `${timings.sun_set.start} to ${timings.sun_set.end}`;
+ afterSunsetItem.title = "";
+ afterSunsetItem.start_time = startTime.clone().hours(sunsetEnd.hours()).minutes(sunsetEnd.minutes()).seconds(sunsetEnd.seconds());
+ afterSunsetItem.end_time = startTime.clone().hours(23).minutes(59).seconds(59);
+ afterSunsetItem.bgColor = "grey";
+ afterSunsetItem.selectedBgColor = "grey";
+ sunItems.push(afterSunsetItem);
+ }
+ }
+ }
+ if (this.state.viewType === UIConstants.timeline.types.WEEKVIEW) {
+ items = sunItems;
+ }
+ return items;
+ }
+
/**
* Resets the timeline view to default zoom and move to the current timeline
*/
@@ -964,11 +1161,13 @@ export class CalendarTimeline extends Component {
dayHeaderVisible = rangeDays > 35?false: true;
weekHeaderVisible = rangeDays > 35?true: false;
lstDateHeaderUnit = rangeDays > 35?"day":"hour";
+ const items = await this.addWeekSunTimes(timelineStart, timelineEnd, group, result.items);
+ console.log(items);
this.setState({defaultStartTime: timelineStart, defaultEndTime: timelineEnd,
timelineStartDate: timelineStart, timelineEndDate: timelineEnd,
zoomLevel: this.ZOOM_LEVELS[this.ZOOM_LEVELS.length-1].name, isTimelineZoom: false,
dayHeaderVisible: dayHeaderVisible, weekHeaderVisible: weekHeaderVisible,
- lstDateHeaderUnit: lstDateHeaderUnit, group: group, items: result.items});
+ lstDateHeaderUnit: lstDateHeaderUnit, group: group, items: items});
this.loadLSTDateHeaderMap(startDate, endDate, lstDateHeaderUnit);
this.setState({isWeekLoading: false});
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss
index c2fb7b46ab00ed3721db70e61b834f5c9d17b412..76b02736fb1959096ef3067f75b67f2ad1a72336 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_suSummary.scss
@@ -1,5 +1,5 @@
.constraints-summary>div {
- overflow: scroll;
+ overflow-y: scroll;
max-height: 500px;
margin-bottom: 10px;
}
@@ -24,4 +24,25 @@
.task-summary #block_container {
margin-top: 0px;
-}
\ No newline at end of file
+}
+
+/*
+ * STYLE 3
+ */
+
+ .json-to-table::-webkit-scrollbar-track
+ {
+ -webkit-box-shadow: inset 0 0 6px rgba(248, 245, 245, 0.3);
+ background-color: #F5F5F5;
+ }
+
+ .json-to-table::-webkit-scrollbar
+ {
+ width: 6px;
+ background-color: #F5F5F5;
+ }
+
+ .json-to-table::-webkit-scrollbar-thumb
+ {
+ background-color: #0000007c;
+ }
\ No newline at end of file
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_timeline.scss b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_timeline.scss
index 25e0ca50ba4e4546260f615e0ccb3150dc01d50a..51b2e71b3a46bf57fa21eb4e54b3de8aaa0152a1 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_timeline.scss
+++ b/SAS/TMSS/frontend/tmss_webapp/src/layout/sass/_timeline.scss
@@ -1,7 +1,7 @@
.sticky {
position: sticky;
top:49px;
- z-index:1000;
+ z-index:999;
}
.rct-sidebar-row {
@@ -84,6 +84,66 @@
margin-left: 10px;
}
+.sidebar-header {
+ color: #ffffff;
+ text-align: right;
+ padding-right: 10px;
+ background-color: #8ba7d9;
+}
+
+.sidebar-header-row {
+ height: 30px;
+}
+
+.legend-row {
+ padding-top: 10px;
+ padding-left: 10px;
+ font-size: 14px;
+}
+
+.legend-suntime {
+ margin-top: 2px;
+ padding-left: 3px !important;
+}
+
+.legend-sunrise {
+ background-color: yellow;
+ color: #212529;
+}
+
+.legend-sunset {
+ background-color: orange;
+ color: #212529;
+}
+
+.legend-night {
+ background-color: grey;
+}
+
+.suntime-header {
+ line-height: 30px;
+}
+
+.suntime-header-day {
+ background-color: white;
+ color: white;
+}
+
+.suntime-header-night {
+ background-color: grey;
+ color: grey;
+}
+
+.suntime-header-sunrise {
+ background-color: yellow;
+ color: yellow;
+}
+
+.suntime-header-sunset {
+ background-color: orange;
+ color: orange;
+}
+
.resize-div,
.resize-div-min,
.resize-div-avg,
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
index 5de435d85b1f8e16a09f7e43c85ffaffd3da6227..1bcbcefcbd4a7cd6a5bbb2c2cf96eb7c4b7c2800 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/Scheduling.Constraints.js
@@ -1,5 +1,6 @@
import React, { useState, useEffect } from 'react';
import moment from 'moment';
+import _ from 'lodash';
import Jeditor from '../../components/JSONEditor/JEditor';
import UnitConversion from '../../utils/unit.converter';
/* eslint-disable react-hooks/exhaustive-deps */
@@ -138,7 +139,7 @@ export default (props) => {
};
const modifyInitiValue = () => {
- const initValue = { ...props.initValue }
+ const initValue = _.cloneDeep(props.initValue);
// For DateTime
for (let key in initValue.time) {
if (typeof initValue.time[key] === 'string') {
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
index f70b3eb5b38d6797bc9b578dd2e660c9a8379b54..4dba59e7cc582d9ff47663d3130e2dd9efdea3ab 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/create.js
@@ -396,7 +396,6 @@ export class SchedulingUnitCreate extends Component {
},
projectDisabled: (this.props.match.params.project? true:false),
observStrategy: {},
- selectedStations:{},
paramsOutput: null,
validEditor: false,
validFields: {},
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js
index 7263fc7b3441386d301f4bba6e5e1e82aedd368c..4b2ef70081a130dd75f9567e8c1ec2616b186c90 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Scheduling/summary.js
@@ -3,7 +3,7 @@ import { Link } from 'react-router-dom/cjs/react-router-dom.min';
import moment from 'moment';
import _ from 'lodash';
import ViewTable from '../../components/ViewTable';
-import { JSONToHTMLTable } from '@kevincobain2000/json-to-html-table'
+import { JsonToTable } from "react-json-to-table";
import SchedulingConstraints from './Scheduling.Constraints';
/**
@@ -42,7 +42,7 @@ export class SchedulingUnitSummary extends Component {
/* Format the object to remove empty values*/
const constraint = this.getFormattedConstraint(constraintsDoc[constraintKey]);
if (constraint) {
- orderedConstraints[constraintKey] = constraint;
+ orderedConstraints[constraintKey.replace('_',' ')] = constraint;
}
}
return orderedConstraints;
@@ -66,7 +66,7 @@ export class SchedulingUnitSummary extends Component {
break;
}
case "boolean": {
- constraint = constraint?constraint:null;
+ constraint = constraint?'Yes':null;
break;
}
case "object": {
@@ -88,7 +88,7 @@ export class SchedulingUnitSummary extends Component {
for (const objectKey of _.keys(constraint)) {
let object = this.getFormattedConstraint(constraint[objectKey]);
if (object) {
- newObject[objectKey] = object;
+ newObject[objectKey.replace(/_/g, ' ')] = object;
}
}
constraint = (!_.isEmpty(newObject))? newObject:null;
@@ -143,10 +143,12 @@ export class SchedulingUnitSummary extends Component {
</div>
{/* Scheduling Constraint Display in table format */}
{constraintsDoc &&
+ <>
<div className="col-12 constraints-summary">
<label>Constraints:</label>
- <JSONToHTMLTable data={constraintsDoc} tableClassName="table table-sm"/>
+ <JsonToTable json={constraintsDoc} />
</div>
+ </>
}
</>
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
index 5ab65e6a7b09886f107dc78943236999c1836156..2fd2e272fc587b7ed335de24b48ed3a6c31a4352 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/routes/Timeline/view.js
@@ -46,6 +46,7 @@ export class TimelineView extends Component {
isSummaryLoading: false
}
this.STATUS_BEFORE_SCHEDULED = ['defining', 'defined', 'schedulable']; // Statuses before scheduled to get station_group
+ this.allStationsGroup = [];
this.onItemClick = this.onItemClick.bind(this);
this.closeSUDets = this.closeSUDets.bind(this);
@@ -60,7 +61,8 @@ export class TimelineView extends Component {
ScheduleService.getSchedulingUnitBlueprint(),
ScheduleService.getSchedulingUnitDraft(),
ScheduleService.getSchedulingSets(),
- UtilService.getUTC()] ;
+ UtilService.getUTC(),
+ ScheduleService.getStations('All')] ;
Promise.all(promises).then(async(responses) => {
const projects = responses[0];
const suBlueprints = _.sortBy(responses[1].data.results, 'name');
@@ -105,7 +107,9 @@ export class TimelineView extends Component {
}
}
}
-
+ for (const station of responses[5]['stations']) {
+ this.allStationsGroup.push({id: station, title: station});
+ }
this.setState({suBlueprints: suBlueprints, suDrafts: suDrafts, group: group, suSets: suSets,
projects: projects, suBlueprintList: suList,
items: items, currentUTC: currentUTC, isLoading: false});
@@ -117,9 +121,6 @@ export class TimelineView extends Component {
* @param {Object} suBlueprint
*/
getTimelineItem(suBlueprint) {
- // Temporary for testing
- const diffOfCurrAndStart = moment().diff(moment(suBlueprint.stop_time), 'seconds');
- // suBlueprint.status = diffOfCurrAndStart>=0?"FINISHED":"DEFINED";
let item = { id: suBlueprint.id,
group: suBlueprint.suDraft.id,
title: `${suBlueprint.project} - ${suBlueprint.suDraft.name} - ${(suBlueprint.durationInSec/3600).toFixed(2)}Hrs`,
@@ -189,7 +190,7 @@ export class TimelineView extends Component {
if (this.state.stationView) {
const loadSubtasks = this.STATUS_BEFORE_SCHEDULED.indexOf(suBlueprint.status.toLowerCase()) < 0 ;
suBlueprint.tasks = await ScheduleService.getTaskBlueprintsBySchedulingUnit(suBlueprint, true, loadSubtasks);
- this.getStationItemGroups(suBlueprint, timelineItem, group, items);
+ this.getStationItemGroups(suBlueprint, timelineItem, this.allStationsGroup, items);
} else {
items.push(timelineItem);
if (!_.find(group, {'id': suBlueprint.suDraft.id})) {
@@ -207,7 +208,7 @@ export class TimelineView extends Component {
this.setState({suBlueprintList: _.filter(suBlueprintList, (suBlueprint) => {return suBlueprint.start_time!=null})});
// On range change close the Details pane
// this.closeSUDets();
- return {group: _.sortBy(group,'id'), items: items};
+ return {group: this.stationView?this.allStationsGroup:_.sortBy(group,'id'), items: items};
}
/**
@@ -245,9 +246,9 @@ export class TimelineView extends Component {
stationItem.id = `${stationItem.id}-${station}`;
stationItem.group = station;
items.push(stationItem);
- if (!_.find(group, {'id': station})) {
- group.push({'id': station, title: station});
- }
+ // if (!_.find(group, {'id': station})) {
+ // group.push({'id': station, title: station});
+ // }
}
}
@@ -281,7 +282,7 @@ export class TimelineView extends Component {
const suBlueprint = _.find(suBlueprints, {actionpath: data.actionpath});
let timelineItem = this.getTimelineItem(suBlueprint);
if (this.state.stationView) {
- this.getStationItemGroups(suBlueprint, timelineItem, group, items);
+ this.getStationItemGroups(suBlueprint, timelineItem, this.allStationsGroup, items);
} else {
items.push(timelineItem);
if (!_.find(group, {'id': suBlueprint.suDraft.id})) {
@@ -290,7 +291,7 @@ export class TimelineView extends Component {
}
}
if (this.timeline) {
- this.timeline.updateTimeline({group: _.sortBy(group,"id"), items: items});
+ this.timeline.updateTimeline({group: this.state.stationView?this.allStationsGroup:_.sortBy(group,"id"), items: items});
}
}
diff --git a/SAS/TMSS/frontend/tmss_webapp/src/services/util.service.js b/SAS/TMSS/frontend/tmss_webapp/src/services/util.service.js
index 5d629c58d0dbbc340bf084c0f57ff31f0868cee2..4b8d41ca2c1fd83d8476980d9ac4ef882a44af0c 100644
--- a/SAS/TMSS/frontend/tmss_webapp/src/services/util.service.js
+++ b/SAS/TMSS/frontend/tmss_webapp/src/services/util.service.js
@@ -39,8 +39,10 @@ const UtilService = {
console.error(error);
}
},
+ /** Function to fetch sun timings from the backend for single station. */
getSunTimings: async(timestamp, station) => {
try {
+ station = station?station:"CS001";
let stationTimestamp = (station?`${station}-`:"") + timestamp;
let localSunTimeMap = localStorage.getItem('SUN_TIME_MAP');
if (localSunTimeMap) {
@@ -51,14 +53,9 @@ const UtilService = {
} else {
localSunTimeMap = {};
}
- // const url = `/api/sun_rise_and_set/${timestamp}`;
- // const sunTimings = (await axios.get(url)).data;
- let sunTimings = {sun_rise: moment.utc(moment(timestamp, "YYYYMMDDTHH:mm:ss")).format('YYYY-MM-DDT06:30:00.sssss')+"Z",
- sun_set: moment.utc(moment(timestamp, "YYYYMMDDTHH:mm:ss")).format('YYYY-MM-DDT17:00:00.sssss')+"Z"};
- if (station==="CS001") {
- sunTimings = {sun_rise: moment.utc(moment(timestamp, "YYYYMMDDTHH:mm:ss")).format('YYYY-MM-DDT05:30:00.sssss')+"Z",
- sun_set: moment.utc(moment(timestamp, "YYYYMMDDTHH:mm:ss")).format('YYYY-MM-DDT16:00:00.sssss')+"Z"};
- }
+ const url = `/api/util/sun_rise_and_set?stations=${station?station:'CS001'}×tamps=${timestamp}`;
+ const stationSunTimings = (await axios.get(url)).data;
+ let sunTimings = {sun_rise: stationSunTimings[station]['sunrise'][0], sun_set: stationSunTimings[station]['sunset'][0]};
localSunTimeMap[stationTimestamp] = sunTimings;
localStorage.setItem('SUN_TIME_MAP', JSON.stringify(localSunTimeMap));
return sunTimings;
diff --git a/SAS/TMSS/services/CMakeLists.txt b/SAS/TMSS/services/CMakeLists.txt
index 7ca90e1a5220ba1c278a45e986029e408c2506d6..cc7f8cb954f815663766cb72e8950d78e621d84f 100644
--- a/SAS/TMSS/services/CMakeLists.txt
+++ b/SAS/TMSS/services/CMakeLists.txt
@@ -1,4 +1,5 @@
lofar_add_package(TMSSSchedulingService scheduling)
lofar_add_package(TMSSFeedbackHandlingService feedback_handling)
lofar_add_package(TMSSPostgresListenerService tmss_postgres_listener)
+lofar_add_package(TMSSWorkflowService workflow_service)
diff --git a/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py b/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py
index 5d5818896d27167f4ffe8526a955643689b349ce..3135cbc66549cdc84ebde7b143e953312823f7bc 100644
--- a/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py
+++ b/SAS/TMSS/services/scheduling/lib/constraints/template_constraints_v1.py
@@ -188,11 +188,7 @@ def can_run_anywhere_within_timewindow_with_sky_constraints(scheduling_unit: mod
Checks whether it is possible to place the scheduling unit arbitrarily in the given time window, i.e. the sky constraints must be met over the full time window.
:return: True if all sky constraints are met over the entire time window, else False.
"""
- '''evaluate the time contraint(s)'''
constraints = scheduling_unit.draft.scheduling_constraints_doc
- # TODO: TMSS-245 TMSS-250 (and more?), evaluate the constraints in constraints['sky']
- # maybe even split this method into sub methods for the very distinct sky constraints: min_calibrator_elevation, min_target_elevation, transit_offset & min_distance
-
beam = scheduling_unit.requirements_doc['tasks']['Observation']['specifications_doc']['tile_beam']
angle1 = beam['angle1']
angle2 = beam['angle2']
diff --git a/SAS/TMSS/services/workflow_service/CMakeLists.txt b/SAS/TMSS/services/workflow_service/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5c5d502c21bb7b28fb93be9f02f0498a44fda702
--- /dev/null
+++ b/SAS/TMSS/services/workflow_service/CMakeLists.txt
@@ -0,0 +1,7 @@
+lofar_package(TMSSWorkflowService 0.1 DEPENDS TMSSClient PyCommon pyparameterset PyMessaging)
+
+lofar_find_package(PythonInterp 3.4 REQUIRED)
+
+add_subdirectory(lib)
+add_subdirectory(bin)
+
diff --git a/SAS/TMSS/services/workflow_service/bin/CMakeLists.txt b/SAS/TMSS/services/workflow_service/bin/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..2e7ec964e60e2c8a2becb4db91c456e8b201a015
--- /dev/null
+++ b/SAS/TMSS/services/workflow_service/bin/CMakeLists.txt
@@ -0,0 +1,4 @@
+lofar_add_bin_scripts(tmss_workflow_service)
+
+# supervisord config files
+lofar_add_sysconf_files(tmss_workflow_service.ini DESTINATION supervisord.d)
diff --git a/SAS/TMSS/services/workflow_service/bin/tmss_workflow_service b/SAS/TMSS/services/workflow_service/bin/tmss_workflow_service
new file mode 100755
index 0000000000000000000000000000000000000000..51dd037a08aaa765c994f5aed0df7ca1f2d296e2
--- /dev/null
+++ b/SAS/TMSS/services/workflow_service/bin/tmss_workflow_service
@@ -0,0 +1,22 @@
+#!/usr/bin/python3
+
+# Copyright (C) 2012-2015 ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O. Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as published
+# by the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+
+if __name__ == '__main__':
+ from lofar.sas.tmss.services.workflow_service import main
+ main()
diff --git a/SAS/TMSS/services/workflow_service/bin/tmss_workflow_service.ini b/SAS/TMSS/services/workflow_service/bin/tmss_workflow_service.ini
new file mode 100644
index 0000000000000000000000000000000000000000..0f80770faf3c580ff8a0558e62399adb66e2fa76
--- /dev/null
+++ b/SAS/TMSS/services/workflow_service/bin/tmss_workflow_service.ini
@@ -0,0 +1,9 @@
+[program:tmss_workflow_service]
+command=/bin/bash -c 'source $LOFARROOT/lofarinit.sh;exec tmss_workflow_service'
+user=lofarsys
+stopsignal=INT ; KeyboardInterrupt
+stopasgroup=true ; bash does not propagate signals
+stdout_logfile=%(program_name)s.log
+redirect_stderr=true
+stderr_logfile=NONE
+stdout_logfile_maxbytes=0
diff --git a/SAS/TMSS/services/workflow_service/lib/CMakeLists.txt b/SAS/TMSS/services/workflow_service/lib/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..f18bbb8b7ccbc3554a0f0e1c5fb32a44145fab22
--- /dev/null
+++ b/SAS/TMSS/services/workflow_service/lib/CMakeLists.txt
@@ -0,0 +1,10 @@
+lofar_find_package(PythonInterp 3.4 REQUIRED)
+include(PythonInstall)
+
+set(_py_files
+ workflow_service.py
+ )
+
+python_install(${_py_files}
+ DESTINATION lofar/sas/tmss/services)
+
diff --git a/SAS/TMSS/services/workflow_service/lib/workflow_service.py b/SAS/TMSS/services/workflow_service/lib/workflow_service.py
new file mode 100644
index 0000000000000000000000000000000000000000..c38bde688e87903f9b66a4c9f2d6234814a4c808
--- /dev/null
+++ b/SAS/TMSS/services/workflow_service/lib/workflow_service.py
@@ -0,0 +1,88 @@
+#!/usr/bin/env python3
+
+# subtask_scheduling.py
+#
+# Copyright (C) 2015
+# ASTRON (Netherlands Institute for Radio Astronomy)
+# P.O.Box 2, 7990 AA Dwingeloo, The Netherlands
+#
+# This file is part of the LOFAR software suite.
+# The LOFAR software suite is free software: you can redistribute it
+# and/or modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation, either version 3 of the
+# License, or (at your option) any later version.
+#
+# The LOFAR software suite is distributed in the hope that it will be
+# useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along
+# with the LOFAR software suite. If not, see <http://www.gnu.org/licenses/>.
+#
+
+import os
+import logging
+logger = logging.getLogger(__name__)
+
+from lofar.sas.tmss.client.tmssbuslistener import *
+
+class SchedulingUnitEventMessageHandler(TMSSEventMessageHandler):
+
+ def onSchedulingUnitBlueprintStatusChanged(self, id: int, status: str):
+ try:
+ # import here and not at top of module because we need the django.setup() to be run first, either from this module's main, or from the TMSSTestEnvironment
+ from lofar.sas.tmss.tmss.workflowapp.signals import scheduling_unit_blueprint_signal
+ from lofar.sas.tmss.tmss.tmssapp.models import SchedulingUnitBlueprint
+
+ logger.info("SchedulingUnitBlueprint id=%s status changed to '%s', signalling workflow...", id, status)
+ scheduling_unit_blueprint = SchedulingUnitBlueprint.objects.get(pk=id)
+ scheduling_unit_blueprint_signal.send(sender=self.__class__, instance=scheduling_unit_blueprint, status=status)
+ except Exception as e:
+ logger.error(e)
+
+
+def create_workflow_service(exchange: str=DEFAULT_BUSNAME, broker: str=DEFAULT_BROKER):
+ return TMSSBusListener(handler_type=SchedulingUnitEventMessageHandler,
+ handler_kwargs={},
+ exchange=exchange, broker=broker)
+
+def main():
+ # make sure we run in UTC timezone
+ os.environ['TZ'] = 'UTC'
+
+ from optparse import OptionParser, OptionGroup
+ from lofar.common import dbcredentials
+
+ logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+ # Check the invocation arguments
+ parser = OptionParser('%prog [options]',
+ description='run the tmss_workflow_service which forwards TMSS events to the workflow engine.')
+
+ group = OptionGroup(parser, 'Messaging options')
+ group.add_option('-b', '--broker', dest='broker', type='string', default=DEFAULT_BROKER,
+ help='Address of the message broker, default: %default')
+ group.add_option('-e', "--exchange", dest="exchange", type="string", default=DEFAULT_BUSNAME,
+ help="Bus or queue where the TMSS messages are published. [default: %default]")
+ parser.add_option_group(group)
+
+ parser.add_option_group(dbcredentials.options_group(parser))
+ parser.set_defaults(dbcredentials=os.environ.get('TMSS_DBCREDENTIALS', 'TMSS'))
+ (options, args) = parser.parse_args()
+
+ dbcreds = dbcredentials.parse_options(options)
+ logger.info("Using TMSS dbcreds: %s" % dbcreds.stringWithHiddenPassword())
+
+ # setup django
+ os.environ["TMSS_DBCREDENTIALS"] = options.dbcredentials
+ os.environ["DJANGO_SETTINGS_MODULE"] = "lofar.sas.tmss.tmss.settings"
+ os.environ['TMSS_ENABLE_VIEWFLOW'] = 'True'
+ import django
+ django.setup()
+
+ with create_workflow_service(options.exchange, options.broker):
+ waitForInterrupt()
+
+if __name__ == '__main__':
+ main()
diff --git a/SAS/TMSS/src/tmss/exceptions.py b/SAS/TMSS/src/tmss/exceptions.py
index e45ba40745dbfac84a842d9334b3fd687ad2cc23..c918a64950632d8573d7e1cef3f2745f2383dcdc 100644
--- a/SAS/TMSS/src/tmss/exceptions.py
+++ b/SAS/TMSS/src/tmss/exceptions.py
@@ -14,6 +14,9 @@ class BlueprintCreationException(ConversionException):
class SubtaskCreationException(ConversionException):
pass
+class SubtaskException(TMSSException):
+ pass
+
class SchedulingException(TMSSException):
pass
diff --git a/SAS/TMSS/src/tmss/settings.py b/SAS/TMSS/src/tmss/settings.py
index 9ba919e02252205cd5b2d7c0e83565bd2cf088c4..e8b60ccede3319cf623eda0852283b367fbdc05a 100644
--- a/SAS/TMSS/src/tmss/settings.py
+++ b/SAS/TMSS/src/tmss/settings.py
@@ -92,8 +92,7 @@ INSTALLED_APPS = [
'drf_yasg',
'django_filters',
'material',
- 'material.frontend'
-]
+ 'material.frontend']
MIDDLEWARE = [
'django.middleware.gzip.GZipMiddleware',
@@ -114,7 +113,6 @@ if show_debug_toolbar():
INSTALLED_APPS.append('debug_toolbar')
MIDDLEWARE.insert(MIDDLEWARE.index('django.middleware.gzip.GZipMiddleware')+1, 'debug_toolbar.middleware.DebugToolbarMiddleware')
-
if bool(os.environ.get('TMSS_ENABLE_VIEWFLOW', False)):
INSTALLED_APPS.extend(['viewflow', 'viewflow.frontend', 'lofar.sas.tmss.tmss.workflowapp'])
diff --git a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
index f375f739ae5a435eff01474107753925b5b4208f..a657a6531ec6a608c64768ac2905a436e20bec7d 100644
--- a/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/tmssapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2020-11-17 13:44
+# Generated by Django 3.0.9 on 2020-11-30 08:59
from django.conf import settings
import django.contrib.postgres.fields
@@ -243,6 +243,19 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='DefaultReservationTemplate',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(max_length=128, unique=True)),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='DefaultSchedulingConstraintsTemplate',
fields=[
@@ -369,6 +382,7 @@ class Migration(migrations.Migration):
('name', models.CharField(help_text='Human-readable name of this object.', max_length=128, primary_key=True, serialize=False)),
('priority_rank', models.FloatField(help_text='Priority of this project w.r.t. other projects. Projects can interrupt observations of lower-priority projects.')),
('trigger_priority', models.IntegerField(default=1000, help_text='Priority of this project w.r.t. triggers.')),
+ ('auto_ingest', models.BooleanField(default=False, help_text='True if The data is ingested when the other scheduling unit tasks are finished. False if The data is ingested after approval in the QA validation workflow. At the end of this a flag is set that the data can be ingested.')),
('can_trigger', models.BooleanField(default=False, help_text='True if this project is allowed to supply observation requests on the fly, possibly interrupting currently running observations (responsive telescope).')),
('private_data', models.BooleanField(default=True, help_text='True if data of this project is sensitive. Sensitive data is not made public.')),
('expert', models.BooleanField(default=False, help_text='Expert projects put more responsibility on the PI.')),
@@ -404,6 +418,39 @@ class Migration(migrations.Migration):
'abstract': False,
},
),
+ migrations.CreateModel(
+ name='Reservation',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
+ ('description', models.CharField(help_text='Short description for this reservation, used in overviews', max_length=255)),
+ ('start_time', models.DateTimeField(help_text='Start of this reservation.')),
+ ('duration', models.IntegerField(help_text='Duration of this reservations.', null=True)),
+ ('specifications_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Properties of this reservation')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
+ migrations.CreateModel(
+ name='ReservationTemplate',
+ fields=[
+ ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
+ ('tags', django.contrib.postgres.fields.ArrayField(base_field=models.CharField(max_length=128), blank=True, default=list, help_text='User-defined search keywords for object.', size=8)),
+ ('created_at', models.DateTimeField(auto_now_add=True, help_text='Moment of object creation.')),
+ ('updated_at', models.DateTimeField(auto_now=True, help_text='Moment of last object update.')),
+ ('name', models.CharField(help_text='Human-readable name of this object.', max_length=128)),
+ ('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
+ ('version', models.IntegerField(editable=False, help_text='Version of this template (with respect to other templates of the same name)')),
+ ('schema', django.contrib.postgres.fields.jsonb.JSONField(help_text='Schema for the configurable parameters needed to use this template.')),
+ ],
+ options={
+ 'abstract': False,
+ },
+ ),
migrations.CreateModel(
name='ResourceType',
fields=[
@@ -506,6 +553,8 @@ class Migration(migrations.Migration):
('description', models.CharField(blank=True, default='', help_text='A longer description of this object.', max_length=255)),
('requirements_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling and/or quality requirements for this scheduling unit (IMMUTABLE).')),
('do_cancel', models.BooleanField()),
+ ('ingest_permission_required', models.BooleanField(default=False, help_text='Explicit permission is needed before the task.')),
+ ('ingest_permission_granted_since', models.DateTimeField(help_text='Moment of object creation.', null=True)),
],
options={
'abstract': False,
@@ -523,6 +572,7 @@ class Migration(migrations.Migration):
('requirements_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling and/or quality requirements for this run.')),
('generator_instance_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Parameter value that generated this run draft (NULLable).', null=True)),
('scheduling_constraints_doc', django.contrib.postgres.fields.jsonb.JSONField(help_text='Scheduling Constraints for this run.', null=True)),
+ ('ingest_permission_required', models.BooleanField(default=False, help_text='Explicit permission is needed before the task.')),
],
options={
'abstract': False,
@@ -1143,6 +1193,20 @@ class Migration(migrations.Migration):
name='quantity',
field=models.ForeignKey(help_text='The quantity of this resource type.', on_delete=django.db.models.deletion.PROTECT, to='tmssapp.Quantity'),
),
+ migrations.AddConstraint(
+ model_name='reservationtemplate',
+ constraint=models.UniqueConstraint(fields=('name', 'version'), name='reservationtemplate_unique_name_version'),
+ ),
+ migrations.AddField(
+ model_name='reservation',
+ name='project',
+ field=models.ForeignKey(help_text='Reservation will be accounted for this project.', on_delete=django.db.models.deletion.CASCADE, related_name='reservations', to='tmssapp.Project'),
+ ),
+ migrations.AddField(
+ model_name='reservation',
+ name='specifications_template',
+ field=models.ForeignKey(help_text='Schema used for specifications_doc.', on_delete=django.db.models.deletion.CASCADE, to='tmssapp.ReservationTemplate'),
+ ),
migrations.AddField(
model_name='projectquota',
name='project',
@@ -1207,6 +1271,11 @@ class Migration(migrations.Migration):
name='template',
field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.SchedulingConstraintsTemplate'),
),
+ migrations.AddField(
+ model_name='defaultreservationtemplate',
+ name='template',
+ field=models.ForeignKey(on_delete=django.db.models.deletion.PROTECT, to='tmssapp.ReservationTemplate'),
+ ),
migrations.AddField(
model_name='defaultgeneratortemplate',
name='template',
@@ -1376,6 +1445,10 @@ class Migration(migrations.Migration):
model_name='defaultschedulingconstraintstemplate',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_def_tags_480bbd_gin'),
),
+ migrations.AddIndex(
+ model_name='defaultreservationtemplate',
+ index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_def_tags_58d7a4_gin'),
+ ),
migrations.AddIndex(
model_name='defaultgeneratortemplate',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_def_tags_89c89d_gin'),
@@ -1400,4 +1473,4 @@ class Migration(migrations.Migration):
model_name='dataproduct',
index=django.contrib.postgres.indexes.GinIndex(fields=['tags'], name='tmssapp_dat_tags_5932a3_gin'),
),
- ]
+ ]
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/models/specification.py b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
index f2f04f15009885ff04473a3601256dd878108803..1f5a6e3b5be0631f4c5b99ee9e4b6a2176556623 100644
--- a/SAS/TMSS/src/tmss/tmssapp/models/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/models/specification.py
@@ -375,6 +375,44 @@ class DefaultTaskRelationSelectionTemplate(BasicCommon):
name = CharField(max_length=128, unique=True)
template = ForeignKey("TaskRelationSelectionTemplate", on_delete=PROTECT)
+
+class ReservationTemplate(Template):
+ pass
+
+
+class DefaultReservationTemplate(BasicCommon):
+ name = CharField(max_length=128, unique=True)
+ template = ForeignKey("ReservationTemplate", on_delete=PROTECT)
+
+
+#
+# DatabaseView objects
+#
+class TaskBlueprintSummary(Model):
+ taskblueprint_id = IntegerField()
+ subtask_id = IntegerField()
+ substate = CharField(max_length=128)
+ subtask_type = CharField(max_length=128)
+
+ class Meta:
+ managed = False
+ db_table = 'tmssapp_taskblueprintsummary'
+
+
+class SchedulingUnitBlueprintSummary(Model):
+ # Using in an id and ForeignKey is not common for a view BUT the id is a 'dummy' to be able to use in Django
+ # https://resources.rescale.com/using-database-views-in-django-orm/
+ # otherwise an exception will be thrown
+ id = IntegerField(primary_key=True)
+ sub_id = IntegerField()
+ taskblueprint_id = IntegerField()
+ task_type = CharField(max_length=128)
+ derived_task_status = CharField(max_length=128)
+
+ class Meta:
+ managed = False
+ db_table = 'tmssapp_schedulingunitblueprintsummary'
+
#
# Instance Objects
#
@@ -411,6 +449,7 @@ class Project(NamedCommonPK):
cycles = ManyToManyField('Cycle', related_name='projects', null=True, help_text='Cycles to which this project belongs (NULLable).')
priority_rank = FloatField(null=False, help_text='Priority of this project w.r.t. other projects. Projects can interrupt observations of lower-priority projects.') # todo: add if needed: validators=[MinValueValidator(0.0), MaxValueValidator(1.0)]
trigger_priority = IntegerField(default=1000, help_text='Priority of this project w.r.t. triggers.') # todo: verify meaning and add to help_text: "Triggers with higher priority than this threshold can interrupt observations of projects."
+ auto_ingest = BooleanField(default=False, help_text='True if The data is ingested when the other scheduling unit tasks are finished. False if The data is ingested after approval in the QA validation workflow. At the end of this a flag is set that the data can be ingested.')
can_trigger = BooleanField(default=False, help_text='True if this project is allowed to supply observation requests on the fly, possibly interrupting currently running observations (responsive telescope).')
private_data = BooleanField(default=True, help_text='True if data of this project is sensitive. Sensitive data is not made public.')
expert = BooleanField(default=False, help_text='Expert projects put more responsibility on the PI.')
@@ -516,6 +555,7 @@ class SchedulingUnitDraft(NamedCommon):
observation_strategy_template = ForeignKey('SchedulingUnitObservingStrategyTemplate', on_delete=PROTECT, null=True, help_text='Observation Strategy Template used to create the requirements_doc.')
scheduling_constraints_doc = JSONField(help_text='Scheduling Constraints for this run.', null=True)
scheduling_constraints_template = ForeignKey('SchedulingConstraintsTemplate', on_delete=CASCADE, null=True, help_text='Schema used for scheduling_constraints_doc.')
+ ingest_permission_required = BooleanField(default=False, help_text='Explicit permission is needed before the task.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
if self.requirements_doc is not None and self.requirements_template_id and self.requirements_template.schema is not None:
@@ -527,6 +567,11 @@ class SchedulingUnitDraft(NamedCommon):
if self.scheduling_constraints_doc is not None and self.scheduling_constraints_template_id and self.scheduling_constraints_template.schema is not None:
validate_json_against_schema(self.scheduling_constraints_doc, self.scheduling_constraints_template.schema)
+ # This code only happens if the objects is not in the database yet. self._state.adding is True creating
+ if self._state.adding and hasattr(self, 'scheduling_set') and self.scheduling_set.project.auto_ingest is False:
+ #When auto_ingest=False, the scheduling units will be created with ingest_permission_required = True
+ self.ingest_permission_required=True
+
annotate_validate_add_defaults_to_doc_using_template(self, 'requirements_doc', 'requirements_template')
annotate_validate_add_defaults_to_doc_using_template(self, 'scheduling_constraints_doc', 'scheduling_constraints_template')
super().save(force_insert, force_update, using, update_fields)
@@ -574,11 +619,19 @@ class SchedulingUnitBlueprint(NamedCommon):
requirements_doc = JSONField(help_text='Scheduling and/or quality requirements for this scheduling unit (IMMUTABLE).')
do_cancel = BooleanField()
+ ingest_permission_required = BooleanField(default=False, help_text='Explicit permission is needed before the task.')
+ ingest_permission_granted_since = DateTimeField(auto_now_add=False, null=True, help_text='Moment of object creation.')
requirements_template = ForeignKey('SchedulingUnitTemplate', on_delete=CASCADE, help_text='Schema used for requirements_doc (IMMUTABLE).')
draft = ForeignKey('SchedulingUnitDraft', related_name='scheduling_unit_blueprints', on_delete=CASCADE, help_text='Scheduling Unit Draft which this run instantiates.')
def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
annotate_validate_add_defaults_to_doc_using_template(self, 'requirements_doc', 'requirements_template')
+
+ # This code only happens if the objects is not in the database yet. self._state.adding is True creating
+ if self._state.adding and hasattr(self, 'draft') and self.draft.scheduling_set.project.auto_ingest is False:
+ #When auto_ingest=False, the scheduling units will be created with ingest_permission_required = True
+ self.ingest_permission_required=True
+
super().save(force_insert, force_update, using, update_fields)
@@ -1069,3 +1122,15 @@ class TaskSchedulingRelationDraft(BasicCommon):
raise ValidationError("Time_offset must be >= 0")
super().save(force_insert, force_update, using, update_fields)
+
+class Reservation(NamedCommon):
+ project = ForeignKey('Project', related_name='reservations', on_delete=CASCADE, help_text='Reservation will be accounted for this project.')
+ description = CharField(max_length=255, help_text='Short description for this reservation, used in overviews')
+ start_time = DateTimeField(help_text='Start of this reservation.')
+ duration = IntegerField(null=True, help_text='Duration of this reservations.')
+ specifications_doc = JSONField(help_text='Properties of this reservation')
+ specifications_template = ForeignKey('ReservationTemplate', on_delete=CASCADE, help_text='Schema used for specifications_doc.')
+
+ def save(self, force_insert=False, force_update=False, using=None, update_fields=None):
+ annotate_validate_add_defaults_to_doc_using_template(self, 'specifications_doc', 'specifications_template')
+ super().save(force_insert, force_update, using, update_fields)
diff --git a/SAS/TMSS/src/tmss/tmssapp/populate.py b/SAS/TMSS/src/tmss/tmssapp/populate.py
index 05ec07e83f2f102caa1f65d1bcadf8ffb3447935..73116db990983bddcc038740907cae326e5d75b3 100644
--- a/SAS/TMSS/src/tmss/tmssapp/populate.py
+++ b/SAS/TMSS/src/tmss/tmssapp/populate.py
@@ -68,6 +68,16 @@ def populate_test_data():
if 'Commissioning' not in tmss_project.tags:
continue
+ # for test purposes also add a reservation object
+ reservation_template = models.ReservationTemplate.objects.get(name="resource reservation")
+ reservation_template_spec = get_default_json_object_for_schema(reservation_template.schema)
+ Reservation.objects.create(name="DummyReservation",
+ description="Just A non-scheduled reservation as example",
+ project=tmss_project,
+ specifications_template=reservation_template,
+ specifications_doc=reservation_template_spec,
+ start_time=datetime.now())
+
for scheduling_set in tmss_project.scheduling_sets.all():
for unit_nr in range(2):
for strategy_template in [uc1_strategy_template, simple_strategy_template]:
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/reservation_template-reservation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/reservation_template-reservation-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..cdcbb306ad3f90ddde1701666ac2eb10845695a7
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/reservation_template-reservation-1.json
@@ -0,0 +1,132 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/tasktemplate/reservation/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "resource reservation",
+ "description": "This schema defines the parameters to reserve instrument resources, and to annotate the reservation.",
+ "version": 1,
+ "type": "object",
+ "properties": {
+ "activity": {
+ "title": "Activity",
+ "description": "Description of the activity during this reservation",
+ "type": "object",
+ "additonalProperties": false,
+ "default":{},
+ "properties": {
+ "type": {
+ "title": "Type",
+ "description": "Reason for this reservation",
+ "type": "string",
+ "enum": [ "maintenance", "test", "upgrade", "outage", "pr", "stand-alone mode", "test system", "other" ],
+ "default": "maintenance"
+ },
+ "description": {
+ "title": "Description",
+ "description": "Free-form explanation of the reason",
+ "type": "string",
+ "default": ""
+ },
+ "contact": {
+ "title": "Contact",
+ "description": "Who coordinates this maintenance",
+ "type": "string",
+ "default": ""
+ },
+ "subject": {
+ "title": "Subject",
+ "description": "What will be modified or affected (select 'system' if multiple)",
+ "type": "string",
+ "enum": [ "environment", "hardware", "firmware", "software", "system", "network", "nothing" ],
+ "default": "nothing"
+ },
+ "planned": {
+ "title": "Planned",
+ "description": "Was this planned?",
+ "type": "boolean",
+ "default": true
+ }
+ },
+ "required": [
+ "type"
+ ]
+ },
+ "resources": {
+ "title": "Resources",
+ "description": "Which resources are affected",
+ "type": "object",
+ "additonalProperties": false,
+ "default":{},
+ "properties": {
+ "stations": {
+ "title": "Stations",
+ "description": "List of stations",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1#/definitions/station_list"
+ }
+ },
+ "required": []
+ },
+ "schedulability": {
+ "title": "Schedulability",
+ "description": "Schedulability of the reserved resources",
+ "type": "object",
+ "additonalProperties": false,
+ "default":{},
+ "properties": {
+ "manual": {
+ "title": "Manual",
+ "description": "Manual scheduling is allowed",
+ "type": "boolean",
+ "default": true
+ },
+ "dynamic": {
+ "title": "Dynamic",
+ "description": "Dynamic scheduling is allowed",
+ "type": "boolean",
+ "default": false
+ },
+ "project_exclusive": {
+ "title": "Schedule only for this project",
+ "description": "Only tasks from this project can be scheduled",
+ "type": "boolean",
+ "default": true
+ }
+ }
+ },
+ "effects": {
+ "title": "Effect",
+ "description": "Effect the actions have during this reservation",
+ "type": "object",
+ "additonalProperties": false,
+ "default":{},
+ "properties": {
+ "lba_rfi": {
+ "title": "LBA RFI",
+ "description": "RFI increases in the LBA spectrum during this maintenance",
+ "type": "boolean",
+ "default": false
+ },
+ "hba_rfi": {
+ "title": "HBA RFI",
+ "description": "RFI increases in the HBA spectrum during this maintenance",
+ "type": "boolean",
+ "default": false
+ },
+ "expert": {
+ "title": "Expert mode",
+ "description": "Quality cannot be guaranteed",
+ "type": "boolean",
+ "default": true
+ }
+ },
+ "required": []
+ }
+ },
+ "required": [
+ "activity", "resources", "effects"
+ ]
+}
+
+
+
+
+
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-reservation-1.json b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-reservation-1.json
new file mode 100644
index 0000000000000000000000000000000000000000..dd1d1996959e6630bc0b78f5be64a6422ec6721f
--- /dev/null
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/subtask_template-reservation-1.json
@@ -0,0 +1,27 @@
+{
+ "$id": "http://tmss.lofar.org/api/schemas/subtasktemplate/reservation/1#",
+ "$schema": "http://json-schema.org/draft-06/schema#",
+ "title": "resource reservation",
+ "description": "This schema defines reserved resources",
+ "version": 1,
+ "type": "object",
+ "properties": {
+ "resources": {
+ "title": "Resources",
+ "description": "Which resources are reserved",
+ "type": "object",
+ "additonalProperties": false,
+ "properties": {
+ "stations": {
+ "title": "Stations",
+ "description": "List of stations",
+ "$ref": "http://tmss.lofar.org/api/schemas/commonschematemplate/stations/1#/definitions/station_list"
+ }
+ },
+ "required": []
+ }
+ },
+ "required": [
+ "resources"
+ ]
+}
diff --git a/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json b/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
index 6e1d2c710101efe1a396935340fcdee899fe3ded..55f5cd2d4401c5592f0faaa3b627557e29f6cace 100644
--- a/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
+++ b/SAS/TMSS/src/tmss/tmssapp/schemas/templates.json
@@ -126,14 +126,18 @@
"file_name": "sap_template-1.json",
"template": "sap_template"
},
- {
+ {
"file_name": "subtask_template-ingest-1.json",
"template": "subtask_template",
"type": "copy"
- },
- {
+ },
+ {
"file_name": "task_template-ingest-1.json",
"template": "task_template",
"type": "ingest"
+ },
+ {
+ "file_name": "reservation_template-reservation-1.json",
+ "template": "reservation_template"
}
]
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
index 0c215aa57d1915e0660bd31572775bd3992d00d9..19afb2076dfec6e7b2644021680e1750d90db36e 100644
--- a/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/serializers/specification.py
@@ -385,3 +385,20 @@ class TaskTypeSerializer(RelationalHyperlinkedModelSerializer):
model = models.TaskType
fields = '__all__'
+
+class ReservationTemplateSerializer(AbstractTemplateSerializer):
+ class Meta:
+ model = models.ReservationTemplate
+ fields = '__all__'
+
+
+class DefaultReservationTemplateSerializer(RelationalHyperlinkedModelSerializer):
+ class Meta:
+ model = models.DefaultReservationTemplate
+ fields = '__all__'
+
+
+class ReservationSerializer(RelationalHyperlinkedModelSerializer):
+ class Meta:
+ model = models.Reservation
+ fields = '__all__'
diff --git a/SAS/TMSS/src/tmss/tmssapp/subtasks.py b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
index 2c0a5a50dbce25b18ba327c0872c7fa31cc7ea56..468666fb8e83630762d693bade374662db92ba3e 100644
--- a/SAS/TMSS/src/tmss/tmssapp/subtasks.py
+++ b/SAS/TMSS/src/tmss/tmssapp/subtasks.py
@@ -9,7 +9,7 @@ from lofar.common import isProductionEnvironment
from lofar.common.json_utils import add_defaults_to_json_object_for_schema, get_default_json_object_for_schema
from lofar.common.lcu_utils import get_current_stations
-from lofar.sas.tmss.tmss.exceptions import SubtaskCreationException, SubtaskSchedulingException
+from lofar.sas.tmss.tmss.exceptions import SubtaskCreationException, SubtaskSchedulingException, SubtaskException
from datetime import datetime, timedelta
from lofar.common.datetimeutils import parseDatetime
@@ -19,6 +19,7 @@ from lofar.sas.resourceassignment.resourceassigner.rarpc import RARPC
from lofar.sas.resourceassignment.resourceassignmentservice.rpc import RADBRPC
from lofar.sas.tmss.tmss.tmssapp.adapters.parset import convert_to_parset_dict
from lofar.sas.resourceassignment.taskprescheduler.cobaltblocksize import CorrelatorSettings, BlockConstraints, BlockSize
+from lofar.sas.resourceassignment.resourceassigner.schedulers import ScheduleException
from lofar.sas.tmss.tmss.tmssapp.conversions import antennafields_for_antennaset_and_station
@@ -616,36 +617,161 @@ def check_prerequities_for_scheduling(subtask: Subtask) -> bool:
return True
-def _assign_or_unassign_resources(subtask: Subtask):
- if subtask.state.value not in [SubtaskState.Choices.SCHEDULING.value, SubtaskState.Choices.UNSCHEDULING.value]:
- raise SubtaskSchedulingException("Cannot assign resources for subtask id=%d because it is not in (UN)SCHEDULING state. "
+
+def _create_ra_specification(_subtask):
+ # Should we do something with station list, for 'detecting' conflicts it can be empty
+ parset_dict = convert_to_parset_dict(_subtask)
+ return { 'tmss_id': _subtask.id,
+ 'task_type': _subtask.specifications_template.type.value.lower(),
+ 'task_subtype': parset_dict.get("Observation.processSubtype","").lower(),
+ 'status': 'prescheduled' if _subtask.state.value == SubtaskState.Choices.SCHEDULING.value else 'approved',
+ 'starttime': _subtask.start_time,
+ 'endtime': _subtask.stop_time,
+ 'cluster': _subtask.cluster.name,
+ 'station_requirements': [],
+ 'specification': parset_dict }
+
+
+def assign_or_unassign_resources(subtask: Subtask):
+ """
+ :param subtask:
+ """
+ MAX_NBR_ASSIGNMENTS = 10
+
+ if subtask.state.value != SubtaskState.Choices.SCHEDULING.value:
+ raise SubtaskSchedulingException("Cannot assign resources for subtask id=%d because it is not in SCHEDULING state. "
"Current state=%s" % (subtask.pk, subtask.state.value))
- def create_ra_specification(_subtask):
- parset_dict = convert_to_parset_dict(_subtask)
- return { 'tmss_id': _subtask.id,
- 'task_type': _subtask.specifications_template.type.value.lower(),
- 'task_subtype': parset_dict.get("Observation.processSubtype","").lower(),
- 'status': 'prescheduled' if subtask.state.value == SubtaskState.Choices.SCHEDULING.value else 'approved',
- 'starttime': _subtask.start_time,
- 'endtime': _subtask.stop_time,
- 'cluster': _subtask.cluster.name,
- 'station_requirements': [],
- 'specification': parset_dict }
-
- ra_spec = create_ra_specification(subtask)
+ ra_spec = _create_ra_specification(subtask)
ra_spec['predecessors'] = []
for pred in subtask.predecessors.all():
try:
- ra_spec['predecessors'].append(create_ra_specification(pred))
+ ra_spec['predecessors'].append(_create_ra_specification(pred))
except:
pass
-
+ assigned = False
+ cnt_do_assignments = 1
with RARPC.create() as rarpc:
- assigned = rarpc.do_assignment(ra_spec)
-
+ while not assigned and cnt_do_assignments < MAX_NBR_ASSIGNMENTS:
+ try:
+ cnt_do_assignments += 1
+ assigned = rarpc.do_assignment(ra_spec)
+ except ScheduleException as e:
+ logger.info("Conflicts in assignment detected, lets check the stations in conflict and re-assign if possible")
+ # Try to re-assign if not assigned yet
+ if not assigned:
+ lst_stations_in_conflict = get_stations_in_conflict(subtask.id)
+ lst_stations = determine_stations_which_can_be_assigned(subtask, lst_stations_in_conflict)
+ ra_spec = update_specification(ra_spec, lst_stations)
+
+ # At the end still not possible to assign, give Exception.
if not assigned:
- raise SubtaskSchedulingException("Cannot schedule subtask id=%d because the required resources are not (fully) available." % (subtask.pk, ))
+ raise SubtaskSchedulingException("Cannot schedule subtask id=%d within %d number of attempts. "
+ "The required resources are not (fully) available." % (subtask.pk, cnt_do_assignments))
+
+
+def get_stations_in_conflict(subtask_id):
+ """
+ Retrieve a list of station names which RADB 'marked' as a resource in conflict after the last resource assignment
+ :param subtask_id: The subtask id
+ :return: lst_stations_in_conflict List of station names (string) which are in conflict
+ """
+ lst_stations_in_conflict = []
+ with RADBRPC.create() as radbrpc:
+ task_id = radbrpc.getTask(tmss_id=subtask_id)['id']
+ conflict_claims = radbrpc.getResourceClaims(task_ids=[task_id], status="conflict", extended=True)
+ # Conflicts_claims are resources which are in conflict. Determine the resource names in conflict which are
+ # for example ['CS001rcu', 'CS001chan0', 'CS001bw0', 'CS001chan1', 'CS001bw1']
+ resource_names_in_conflict = []
+ for resc in conflict_claims:
+ # cross check on status in conflict
+ if resc["status"] == "conflict":
+ resource_names_in_conflict.append(resc["resource_name"])
+ logger.info("Resource names with conflict %s" % resource_names_in_conflict)
+
+ # Now get for all the resources in conflict its parent_id. Check for all parent_id which is
+ # resource_group_type 'station', this will be the station name in conflict which we need
+ resource_group_memberships = radbrpc.getResourceGroupMemberships()
+ parent_ids = []
+ for resc in resource_group_memberships["resources"].values():
+ if resc["resource_name"] in resource_names_in_conflict:
+ parent_ids.extend(resc['parent_group_ids'])
+
+ logger.info("Parent group ids with conflict %s" % parent_ids)
+ for parent_id in list(set(parent_ids)):
+ resc_group_item = resource_group_memberships["groups"][parent_id]
+ if resc_group_item["resource_group_type"] == "station":
+ lst_stations_in_conflict.append(resc_group_item["resource_group_name"])
+ logger.info("Stations in conflict %s", lst_stations_in_conflict)
+ return lst_stations_in_conflict
+
+
+def determine_stations_which_can_be_assigned(subtask, lst_stations_in_conflict):
+ """
+ Determine which stations can be assigned when conflict of stations are occurred
+ Station in conflict should be removed.
+ Use the max_nr_missing from the task specifications and the conflicted station list to create a station list
+ which should be possible to assign. If the number of max missing in a station group is larger than the station
+ to be skipped, then new assignment is not possible so raise an SubtaskSchedulingException with context
+ :param subtask:
+ :param lst_stations_in_conflict:
+ :return: lst_stations: List of station which can be assigned
+ """
+ # Get the station list from specification and remove the conflict stations
+ lst_specified_stations = subtask.specifications_doc["stations"]["station_list"]
+ lst_stations = list(set(lst_specified_stations) - set(lst_stations_in_conflict))
+ logger.info("Determine stations which can be assigned %s" % lst_stations)
+
+ # Check whether the removing of the conflict station the requirements of max_nr_missing per station_group is
+ # still fulfilled. If that is OK then we are done otherwise we will raise an Exception
+ stations_groups = get_station_groups(subtask)
+ for sg in stations_groups:
+ nbr_missing = len(set(sg["stations"]) & set(lst_stations_in_conflict))
+ if nbr_missing > sg["max_nr_missing"]:
+ raise SubtaskSchedulingException("There are more stations in conflict than the specification is given "
+ "(%d is larger than %d). The stations that are in conflict are '%s'."
+ "Please check station of subtask %d " %
+ (nbr_missing, sg["max_nr_missing"], lst_stations_in_conflict, subtask.pk))
+ return lst_stations
+
+
+def get_station_groups(subtask):
+ """
+ Retrieve the stations_group specifications of the given subtask
+ Need to retrieve it from (related) Target Observation Task
+ Note list can be empty (some testcase) which result in no checking max_nr_missing
+ :param subtask:
+ :return: station_groups which is a list of dict. { station_list, max_nr_missing }
+ """
+ station_groups = []
+ if 'calibrator' in subtask.task_blueprint.specifications_template.name.lower():
+ # Calibrator requires related Target Task Observation for some specifications
+ target_task_blueprint = get_related_target_observation_task_blueprint(subtask.task_blueprint)
+ if target_task_blueprint is None:
+ raise SubtaskException("Cannot retrieve related target observation of task_blueprint %d (subtask %d)" %
+ (subtask.task_blueprint.id, subtask.id))
+ if "station_groups" in target_task_blueprint.specifications_doc.keys():
+ station_groups = target_task_blueprint.specifications_doc["station_groups"]
+ else:
+ if "station_groups" in subtask.task_blueprint.specifications_doc.keys():
+ station_groups = subtask.task_blueprint.specifications_doc["station_groups"]
+ return station_groups
+
+
+def update_specification(ra_spec, lst_stations):
+ """
+ Update the RA Specification dictionary with the correct list of stations
+ :param ra_spec: Dictionary of the RA specification
+ :param lst_stations: List of stations to 'assign'
+ :return: Dictionary with updated RA specification
+ """
+ if len(lst_stations) == 0:
+ raise SubtaskSchedulingException("Cannot re-assign resources after conflict for subtask id=%d "
+ "because there are no stations left to assign. " % ra_spec["tmss_id"])
+ updated_ra_spec = ra_spec
+ updated_ra_spec["specification"]["Observation.VirtualInstrument.stationList"] = "[%s]" % ','.join(s for s in lst_stations)
+ # ?? should the station_requirements also be updated or just leave that empty '[]' assume for now it can be empty
+ return updated_ra_spec
def schedule_qafile_subtask(qafile_subtask: Subtask):
@@ -854,7 +980,7 @@ def schedule_observation_subtask(observation_subtask: Subtask):
sap=sap) for sb_nr in pointing['subbands']])
# step 4: resource assigner (if possible)
- _assign_or_unassign_resources(observation_subtask)
+ assign_or_unassign_resources(observation_subtask)
# TODO: TMSS-382: evaluate the scheduled stations and see if the requiments given in the subtask.task_bluepring.specifications_doc are met for the station_groups and max_nr_missing.
@@ -948,7 +1074,7 @@ def schedule_pipeline_subtask(pipeline_subtask: Subtask):
DataproductTransform.objects.bulk_create(transforms)
# step 4: resource assigner (if possible)
- _assign_or_unassign_resources(pipeline_subtask)
+ assign_or_unassign_resources(pipeline_subtask)
# step 5: set state to SCHEDULED (resulting in the qaservice to pick this subtask up and run it)
pipeline_subtask.state = SubtaskState.objects.get(value=SubtaskState.Choices.SCHEDULED.value)
diff --git a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
index f4f1e95ddbe38152855429597c6360be6448e4dc..64c4e9e588e228a509c12be4f66a687b132ae096 100644
--- a/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
+++ b/SAS/TMSS/src/tmss/tmssapp/viewsets/specification.py
@@ -174,6 +174,21 @@ class DefaultTaskRelationSelectionTemplateViewSet(LOFARViewSet):
serializer_class = serializers.DefaultTaskRelationSelectionTemplateSerializer
+class DefaultReservationTemplateViewSet(LOFARViewSet):
+ queryset = models.DefaultReservationTemplate.objects.all()
+ serializer_class = serializers.DefaultReservationTemplateSerializer
+
+
+class ReservationTemplateViewSet(AbstractTemplateViewSet):
+ queryset = models.ReservationTemplate.objects.all()
+ serializer_class = serializers.ReservationTemplateSerializer
+
+
+class ReservationViewSet(LOFARViewSet):
+ queryset = models.Reservation.objects.all()
+ serializer_class = serializers.ReservationSerializer
+
+
class RoleViewSet(LOFARViewSet):
queryset = models.Role.objects.all()
serializer_class = serializers.RoleSerializer
@@ -914,4 +929,5 @@ class TaskRelationBlueprintNestedViewSet(LOFARNestedViewSet):
class TaskTypeViewSet(LOFARViewSet):
queryset = models.TaskType.objects.all()
- serializer_class = serializers.TaskTypeSerializer
\ No newline at end of file
+ serializer_class = serializers.TaskTypeSerializer
+
diff --git a/SAS/TMSS/src/tmss/urls.py b/SAS/TMSS/src/tmss/urls.py
index 623d43642732d4a11463f252adffb0938259d9c9..81258f3d806a1fd7937d85240769739c532fbe9e 100644
--- a/SAS/TMSS/src/tmss/urls.py
+++ b/SAS/TMSS/src/tmss/urls.py
@@ -127,12 +127,14 @@ router.register(r'scheduling_unit_template', viewsets.SchedulingUnitTemplateView
router.register(r'scheduling_constraints_template', viewsets.SchedulingConstraintsTemplateViewSet)
router.register(r'task_template', viewsets.TaskTemplateViewSet)
router.register(r'task_relation_selection_template', viewsets.TaskRelationSelectionTemplateViewSet)
+router.register(r'reservation_template', viewsets.ReservationTemplateViewSet)
router.register(r'task_connector_type', viewsets.TaskConnectorTypeViewSet)
router.register(r'default_generator_template', viewsets.DefaultGeneratorTemplateViewSet)
router.register(r'default_scheduling_unit_template', viewsets.DefaultSchedulingUnitTemplateViewSet)
router.register(r'default_scheduling_constraints_template', viewsets.DefaultSchedulingConstraintsTemplateViewSet)
router.register(r'default_task_template', viewsets.DefaultTaskTemplateViewSet)
router.register(r'default_task_relation_selection_template', viewsets.DefaultTaskRelationSelectionTemplateViewSet)
+router.register(r'default_reservation_template', viewsets.DefaultReservationTemplateViewSet)
# instances
router.register(r'cycle', viewsets.CycleViewSet)
@@ -141,6 +143,7 @@ router.register(r'project', viewsets.ProjectViewSet)
router.register(r'resource_type', viewsets.ResourceTypeViewSet)
router.register(r'project_quota', viewsets.ProjectQuotaViewSet)
router.register(r'setting', viewsets.SettingViewSet)
+router.register(r'reservation', viewsets.ReservationViewSet)
router.register(r'scheduling_set', viewsets.SchedulingSetViewSet)
router.register(r'scheduling_unit_draft', viewsets.SchedulingUnitDraftViewSet)
@@ -204,8 +207,6 @@ router.register(r'user', viewsets.UserViewSet)
router.register(r'sap', viewsets.SAPViewSet)
router.register(r'sip_identifier', viewsets.SIPidentifierViewSet)
-# ---
-
urlpatterns.extend(router.urls)
frontend_urlpatterns = [
@@ -222,21 +223,23 @@ urlpatterns = [url(r'^api$', RedirectView.as_view(url='/api/')),
]
-
-# ---
# QA Workflow steps
if bool(os.environ.get('TMSS_ENABLE_VIEWFLOW', False)):
from .workflowapp import viewsets as workflow_viewsets
+ viewflow_urlpatterns = []
viewflow_router = OptionalSlashRouter()
viewflow_router.APIRootView = TMSSAPIRootView
- viewflow_router.register('scheduling_unit_flow/su', workflow_viewsets.SchedulingUnitFlowViewSet, basename='su')
+ from .workflowapp import viewsets as workflow_viewsets
viewflow_router.register('scheduling_unit_flow/qa_reporting_to', workflow_viewsets.QAReportingTOViewSet, basename='qa_reporting_to')
viewflow_router.register('scheduling_unit_flow/qa_reporting_sos', workflow_viewsets.QAReportingSOSViewSet, basename='qa_reporting_sos')
viewflow_router.register('scheduling_unit_flow/qa_pi_verification', workflow_viewsets.PIVerificationViewSet, basename='qa_pi_verification')
viewflow_router.register('scheduling_unit_flow/qa_decide_acceptance', workflow_viewsets.DecideAcceptanceViewSet, basename='qa_decide_acceptance')
viewflow_router.register('scheduling_unit_flow/qa_scheduling_unit_process', workflow_viewsets.SchedulingUnitProcessViewSet, basename='qa_scheduling_unit_process')
- urlpatterns.extend([url(r'^workflow$', RedirectView.as_view(url='/workflow/', permanent=False)),
- url(r'^workflow_api/', include(viewflow_router.urls))])
+ viewflow_urlpatterns.extend(viewflow_router.urls)
+
+ urlpatterns.insert(0,url(r'^workflow$', RedirectView.as_view(url='/workflow/', permanent=False)))
+ #Doesn't work if it is at the end of urlpatterns
+ urlpatterns.insert(0,url(r'^workflow_api/', include(viewflow_urlpatterns)))
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/workflowapp/CMakeLists.txt b/SAS/TMSS/src/tmss/workflowapp/CMakeLists.txt
index e7c3171661a6fd3927e6b4214251c21f0240d0b1..495fd6fd253557a1af5b9ae7c8231db36c5d1083 100644
--- a/SAS/TMSS/src/tmss/workflowapp/CMakeLists.txt
+++ b/SAS/TMSS/src/tmss/workflowapp/CMakeLists.txt
@@ -5,7 +5,7 @@ set(_py_files
__init__.py
admin.py
apps.py
- tests.py
+ signals.py
)
python_install(${_py_files}
@@ -17,4 +17,5 @@ add_subdirectory(flows)
add_subdirectory(viewsets)
add_subdirectory(forms)
add_subdirectory(templates)
+add_subdirectory(tests)
add_subdirectory(serializers)
diff --git a/SAS/TMSS/src/tmss/workflowapp/apps.py b/SAS/TMSS/src/tmss/workflowapp/apps.py
index d70dc7921a32145aa2a76285c3362041e091a358..3ba8ab2cdb3c7e994ebf6bea7850ddc6128b9428 100644
--- a/SAS/TMSS/src/tmss/workflowapp/apps.py
+++ b/SAS/TMSS/src/tmss/workflowapp/apps.py
@@ -1,5 +1,8 @@
+import os
from django.apps import AppConfig
class WorkflowappConfig(AppConfig):
- name = 'workflowapp'
+
+ name = 'lofar.sas.tmss.tmss.workflowapp'
+
diff --git a/SAS/TMSS/src/tmss/workflowapp/flows/__init__.py b/SAS/TMSS/src/tmss/workflowapp/flows/__init__.py
index a0ae3713747c0b28c5595736d06f4bcb800da5b5..abd9afee878556c103eaad7ef61ce33695f58a50 100644
--- a/SAS/TMSS/src/tmss/workflowapp/flows/__init__.py
+++ b/SAS/TMSS/src/tmss/workflowapp/flows/__init__.py
@@ -1,2 +1,2 @@
-from .helloworldflow import *
+#from .helloworldflow import *
from .schedulingunitflow import *
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py
index 8d01c51a15bc840bdb775acce1297938234a1611..bcd8c2bdd182d7c82f438054dea08940bf124282 100644
--- a/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/flows/schedulingunitflow.py
@@ -11,8 +11,17 @@ from viewflow import mixins
from .. import models
from .. import viewsets
+from lofar.sas.tmss.tmss.tmssapp.models import Subtask
+
+from django.dispatch import receiver
+from lofar.sas.tmss.tmss.workflowapp.signals import scheduling_unit_blueprint_signal
+
from viewflow import frontend, ThisObject
from viewflow.activation import STATUS
+from viewflow.models import Process
+
+import logging
+logger = logging.getLogger(__name__)
class ConditionActivation(FuncActivation):
@classmethod
@@ -45,11 +54,11 @@ class Condition(Signal):
sent with Task instance.
"""
self.condition_check = condition_check
-
super(Condition, self).__init__(signal, self.signal_handler, sender, task_loader, **kwargs)
@method_decorator(flow.flow_signal)
def signal_handler(self, activation, sender, instance, **signal_kwargs):
+
if activation.get_status() == STATUS.DONE:
# race condition -- condition was true on activation but we also receive the signal now
return
@@ -60,28 +69,37 @@ class Condition(Signal):
def ready(self):
"""Resolve internal `this`-references. and subscribe to the signal."""
+
if isinstance(self.condition_check, ThisObject):
self.condition_check = getattr(self.flow_class.instance, self.condition_check.name)
super(Condition, self).ready()
+
@frontend.register
class SchedulingUnitFlow(Flow):
process_class = models.SchedulingUnitProcess
start = (
flow.StartSignal(
- post_save,
+ scheduling_unit_blueprint_signal,
this.on_save_can_start,
- sender=models.SchedulingUnit
- ).Next(this.wait_schedulable)
+ ).Next(this.wait_scheduled)
+ )
+
+ wait_scheduled = (
+ Condition(
+ this.check_condition_scheduled,
+ scheduling_unit_blueprint_signal,
+ task_loader=this.get_scheduling_unit_task
+ )
+ .Next(this.wait_processed)
)
- wait_schedulable = (
+ wait_processed = (
Condition(
- this.check_condition,
- post_save,
- sender=models.SchedulingUnit,
+ this.check_condition_processed,
+ scheduling_unit_blueprint_signal,
task_loader=this.get_scheduling_unit_task
)
.Next(this.qa_reporting_to)
@@ -154,21 +172,28 @@ class SchedulingUnitFlow(Flow):
this.do_mark_sub
).Next(this.end)
)
-
+
end = flow.End()
-
+
@method_decorator(flow.flow_start_signal)
- def on_save_can_start(self, activation, sender, instance, created, **signal_kwargs):
- if created:
- activation.prepare()
- activation.process.su = instance
-
- activation.done()
- print("workflow started")
- else:
- print("no workflow started")
- return activation
-
+ def on_save_can_start(self, activation, sender, instance, status, **signal_kwargs):
+
+ if status == "schedulable":
+ try:
+ process = models.SchedulingUnitProcess.objects.get(su=instance)
+
+ except Process.DoesNotExist:
+ activation.prepare()
+ activation.process.su = instance
+ activation.done()
+ logger.info("workflow started")
+
+ except Process.MultipleObjectsReturned:
+ logger.info("QA Workflow for process %s already exists",process)
+ else:
+ logger.info("no workflow started")
+ return activation
+
def do_mark_sub(self, activation):
@@ -177,24 +202,24 @@ class SchedulingUnitFlow(Flow):
and (activation.process.qa_reporting_sos is not None and activation.process.qa_reporting_sos.sos_accept_show_pi)
and (activation.process.decide_acceptance is not None and activation.process.decide_acceptance.sos_accept_after_pi))
- print("!!!!!!!!!!!END FLOW!!!!!!!!!!!")
- print ("can_delete:")
- print (activation.process.can_delete)
- print ("results_accepted:")
- print (activation.process.results_accepted)
-
+ logger.info("End of schedulingunit workflow: can_delete: %s, results_accepted: %s", activation.process.can_delete, activation.process.results_accepted)
return activation
- def check_condition(self, activation, instance):
+ def check_condition_scheduled(self, activation, instance):
if instance is None:
instance = activation.process.su
+
+ condition = instance.status == "scheduled"
+ return condition
- condition = instance.state == 5
- print("condition is ",condition)
+ def check_condition_processed(self, activation, instance):
+ if instance is None:
+ instance = activation.process.su
+
+ condition = instance.status == "processed"
return condition
def get_scheduling_unit_task(self, flow_task, sender, instance, **kwargs):
- print(kwargs)
process = models.SchedulingUnitProcess.objects.get(su=instance)
return Task.objects.get(process=process,flow_task=flow_task)
diff --git a/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py b/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py
index 1da372c3f5a8ea06e95f13d9861676f8bcdf8636..aa03e1a4d75db411590a57f4ec385e3e0d39bd10 100644
--- a/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py
+++ b/SAS/TMSS/src/tmss/workflowapp/migrations/0001_initial.py
@@ -1,4 +1,4 @@
-# Generated by Django 3.0.9 on 2020-11-02 14:31
+# Generated by Django 3.0.9 on 2020-11-26 09:54
from django.db import migrations, models
import django.db.models.deletion
@@ -45,14 +45,6 @@ class Migration(migrations.Migration):
('operator_accept', models.BooleanField(default=False)),
],
),
- migrations.CreateModel(
- name='SchedulingUnit',
- fields=[
- ('id', models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name='ID')),
- ('name', models.CharField(max_length=50)),
- ('state', models.IntegerField()),
- ],
- ),
migrations.CreateModel(
name='HelloWorldProcess',
fields=[
@@ -76,7 +68,7 @@ class Migration(migrations.Migration):
('pi_verification', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='workflowapp.PIVerification')),
('qa_reporting_sos', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='workflowapp.QAReportingSOS')),
('qa_reporting_to', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='workflowapp.QAReportingTO')),
- ('su', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='workflowapp.SchedulingUnit')),
+ ('su', models.ForeignKey(blank=True, null=True, on_delete=django.db.models.deletion.CASCADE, to='tmssapp.SchedulingUnitBlueprint')),
],
options={
'abstract': False,
diff --git a/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py
index 3e340fbf8c9713fbd37daec0dc977e3d453eb69f..c883b4fc9a3c857e2c2913df064c629d16239619 100644
--- a/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/models/schedulingunitflow.py
@@ -3,6 +3,9 @@
from django.db.models import CharField, IntegerField,BooleanField, ForeignKey, CASCADE, Model,NullBooleanField
from viewflow.models import Process
+from lofar.sas.tmss.tmss.tmssapp.models import SchedulingUnitBlueprint
+
+
class QAReportingTO(Model):
operator_report = CharField(max_length=150)
operator_accept = BooleanField(default=False)
@@ -23,16 +26,11 @@ class DecideAcceptance(Model):
sos_accept_after_pi = BooleanField(default=False)
-class SchedulingUnit(Model):
- name = CharField(max_length=50)
- state = IntegerField()
-
-
class SchedulingUnitProcess(Process):
- su = ForeignKey(SchedulingUnit, blank=True, null=True, on_delete=CASCADE)
+ su = ForeignKey(SchedulingUnitBlueprint, blank=True, null=True, on_delete=CASCADE)
qa_reporting_to=ForeignKey(QAReportingTO, blank=True, null=True, on_delete=CASCADE)
qa_reporting_sos=ForeignKey(QAReportingSOS, blank=True, null=True, on_delete=CASCADE)
pi_verification=ForeignKey(PIVerification, blank=True, null=True, on_delete=CASCADE)
decide_acceptance=ForeignKey(DecideAcceptance, blank=True, null=True, on_delete=CASCADE)
can_delete = BooleanField(default=False)
- results_accepted = BooleanField(default=False)
\ No newline at end of file
+ results_accepted = BooleanField(default=False)
diff --git a/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py
index e29cf3cb9796afcce95e94e63636fe300791f5b0..884061c224168ac706def57a833c5ee9cb7952bd 100644
--- a/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/serializers/schedulingunitflow.py
@@ -7,12 +7,6 @@ from django.forms.models import modelform_factory
from .. import forms
-#View to add a fake Scheduling Unit for the QA Workflow
-class SchedulingUnitSerializer(ModelSerializer):
- class Meta:
- model = models.SchedulingUnit
- fields = '__all__'
-
#Viewsets and serializers to access intermediate steps of the QA Workflow
#through DRF
class QAReportingTOSerializer(ModelSerializer):
diff --git a/SAS/TMSS/src/tmss/workflowapp/signals.py b/SAS/TMSS/src/tmss/workflowapp/signals.py
new file mode 100644
index 0000000000000000000000000000000000000000..6087fb1615c6b7a8a5c33f897a4e1cbcce36c6f2
--- /dev/null
+++ b/SAS/TMSS/src/tmss/workflowapp/signals.py
@@ -0,0 +1,3 @@
+import django.dispatch
+
+scheduling_unit_blueprint_signal = django.dispatch.Signal()
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/workflowapp/tests.py b/SAS/TMSS/src/tmss/workflowapp/tests.py
deleted file mode 100644
index 7ce503c2dd97ba78597f6ff6e4393132753573f6..0000000000000000000000000000000000000000
--- a/SAS/TMSS/src/tmss/workflowapp/tests.py
+++ /dev/null
@@ -1,3 +0,0 @@
-from django.test import TestCase
-
-# Create your tests here.
diff --git a/SAS/TMSS/src/tmss/workflowapp/tests/CMakeLists.txt b/SAS/TMSS/src/tmss/workflowapp/tests/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..5f18a8b929917fed948f46562c1af9077c484c17
--- /dev/null
+++ b/SAS/TMSS/src/tmss/workflowapp/tests/CMakeLists.txt
@@ -0,0 +1,7 @@
+# $Id: CMakeLists.txt 32679 2015-10-26 09:31:56Z schaap $
+
+if(BUILD_TESTING)
+ include(LofarCTest)
+
+ lofar_add_test(t_workflow_qaworkflow)
+endif()
diff --git a/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.py b/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.py
new file mode 100755
index 0000000000000000000000000000000000000000..342438051afe19d583061a88b29c5ae7a698066e
--- /dev/null
+++ b/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.py
@@ -0,0 +1,109 @@
+import os
+import unittest
+import requests
+
+import logging
+logger = logging.getLogger(__name__)
+logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s', level=logging.INFO)
+
+from lofar.common.test_utils import skip_integration_tests
+if skip_integration_tests():
+ exit(3)
+
+from lofar.messaging.messagebus import TemporaryExchange
+import uuid
+
+
+class SchedulingUnitFlowTest(unittest.TestCase):
+
+ @classmethod
+ def setUpClass(cls) -> None:
+ cls.TEST_UUID = uuid.uuid1()
+
+ cls.tmp_exchange = TemporaryExchange("%s_%s" % (cls.__name__, cls.TEST_UUID))
+ cls.tmp_exchange.open()
+
+ # override DEFAULT_BUSNAME
+ import lofar
+ lofar.messaging.config.DEFAULT_BUSNAME = cls.tmp_exchange.address
+
+ # import here, and not at top of module, because DEFAULT_BUSNAME needs to be set before importing
+ from lofar.sas.resourceassignment.resourceassigner.test.ra_test_environment import RATestEnvironment
+ from lofar.sas.tmss.test.test_utils import TMSSTestEnvironment
+ from lofar.sas.tmss.test.tmss_test_data_rest import TMSSRESTTestDataCreator
+
+ cls.ra_test_env = RATestEnvironment(exchange=cls.tmp_exchange.address)
+ cls.ra_test_env.start()
+
+ cls.tmss_test_env = TMSSTestEnvironment(exchange=cls.tmp_exchange.address, populate_schemas=True, populate_test_data=False,
+ start_subtask_scheduler=False, start_postgres_listener=True, start_ra_test_environment=True,
+ start_dynamic_scheduler=False, enable_viewflow=True, start_workflow_service=True)
+ cls.tmss_test_env.start()
+
+
+ @classmethod
+ def tearDownClass(cls) -> None:
+ cls.tmss_test_env.stop()
+ cls.ra_test_env.stop()
+ cls.tmp_exchange.close()
+
+
+ def test_qa_workflow(self):
+ from lofar.sas.tmss.tmss.workflowapp.flows.schedulingunitflow import SchedulingUnitFlow
+
+ from lofar.sas.tmss.tmss.tmssapp import models
+ from lofar.sas.tmss.tmss.tmssapp.tasks import create_task_blueprints_and_subtasks_from_scheduling_unit_draft
+ from lofar.sas.tmss.test.tmss_test_data_django_models import SchedulingSet_test_data
+
+ from lofar.sas.tmss.tmss.workflowapp.models.schedulingunitflow import SchedulingUnitProcess
+ from viewflow.models import Task
+
+ #check if one QA Workflow is created after scheduling unit blueprint creation
+ self.assertEqual(0, len(SchedulingUnitProcess.objects.all()))
+ strategy_template = models.SchedulingUnitObservingStrategyTemplate.objects.get(name="UC1 CTC+pipelines")
+
+ scheduling_unit_draft = models.SchedulingUnitDraft.objects.create(
+ name="Test Scheduling Unit UC1",
+ requirements_doc=strategy_template.template,
+ requirements_template=strategy_template.scheduling_unit_template,
+ observation_strategy_template=strategy_template,
+ copy_reason=models.CopyReason.objects.get(value='template'),
+ generator_instance_doc="para",
+ copies=None,
+ scheduling_set=models.SchedulingSet.objects.create(**SchedulingSet_test_data()))
+
+ create_task_blueprints_and_subtasks_from_scheduling_unit_draft(scheduling_unit_draft)
+
+ scheduling_unit_draft.refresh_from_db()
+ task_drafts = scheduling_unit_draft.task_drafts.all()
+ scheduling_unit_blueprints = scheduling_unit_draft.scheduling_unit_blueprints.all()
+ scheduling_unit_blueprint = scheduling_unit_blueprints[0]
+ task_blueprints = scheduling_unit_blueprint.task_blueprints.all()
+ qa_workflow = SchedulingUnitProcess.objects.all()
+ self.assertEqual(1, len(qa_workflow))
+
+ #test that QA workflow have two tasks
+ self.assertEqual(2, len(Task.objects.all()))
+ self.assertEqual(Task.objects.get(id=1).flow_task.name, 'start')
+ self.assertEqual(Task.objects.get(id=1).status, 'DONE')
+ self.assertEqual(Task.objects.get(id=2).flow_task.name, 'wait_scheduled')
+ self.assertEqual(Task.objects.get(id=2).status, 'NEW')
+
+ #Change subtask status to scheduled
+ for task_blueprint in task_blueprints:
+ for subtask in task_blueprint.subtasks.all():
+ subtask.state = models.SubtaskState.objects.get(value='scheduled')
+ subtask.save()
+
+ #Check the QA Workflow is now with 3 Task
+ self.assertEqual(3, len(Task.objects.all()))
+ self.assertEqual(Task.objects.get(id=2).flow_task.name, 'wait_scheduled')
+ self.assertEqual(Task.objects.get(id=2).status, 'DONE')
+ self.assertEqual(Task.objects.get(id=3).flow_task.name, 'wait_processed')
+ self.assertEqual(Task.objects.get(id=3).status, 'NEW')
+
+
+
+if __name__ == '__main__':
+ #run the unit tests
+ unittest.main()
diff --git a/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.run b/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.run
new file mode 100755
index 0000000000000000000000000000000000000000..f4f60358833b8b424de8c55201f3c1672720bef2
--- /dev/null
+++ b/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.run
@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# Run the unit test
+source python-coverage.sh
+python_coverage_test "*tmss*" t_workflow_qaworkflow.py
+
diff --git a/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.sh b/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.sh
new file mode 100755
index 0000000000000000000000000000000000000000..ec908c9e200cdce26adc79bcc75f33a3b44e9ae6
--- /dev/null
+++ b/SAS/TMSS/src/tmss/workflowapp/tests/t_workflow_qaworkflow.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+./runctest.sh t_workflow_qaworkflow
\ No newline at end of file
diff --git a/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py b/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py
index 1c70e87e110fd31d5f2533712165f973d0701733..530d4cb8f9c0207c4c12212c1a4bce7832aa8d29 100644
--- a/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py
+++ b/SAS/TMSS/src/tmss/workflowapp/viewsets/schedulingunitflow.py
@@ -14,15 +14,6 @@ from django.forms.models import modelform_factory
from .. import forms, models, serializers
-class SchedulingUnitFlowViewSet(viewsets.ModelViewSet):
- queryset = models.SchedulingUnit.objects.all()
- serializer_class = serializers.SchedulingUnitSerializer
-
- @action(methods=['get'], detail=True)
- def trigger(self, request, pk=None):
- SchedulingUnitFlow
- return Response("ok")
-
#Viewsets and serializers to access intermediate steps of the QA Workflow
#through DRF
class QAReportingTOViewSet(viewsets.ModelViewSet):
diff --git a/SAS/TMSS/test/t_scheduling.py b/SAS/TMSS/test/t_scheduling.py
index ef00fc0a9956c05a7ce6425db34220e3777165ff..c52ae56ed47dfa46c0a30a0b04f93e95a566dc3d 100755
--- a/SAS/TMSS/test/t_scheduling.py
+++ b/SAS/TMSS/test/t_scheduling.py
@@ -76,6 +76,27 @@ def create_subtask_object_for_testing(subtask_type_value, subtask_state_value):
return models.Subtask.objects.create(**subtask_data)
+def create_reserved_stations_for_testing(station_list):
+ """
+ Helper function to create stations in reservation, in other words assigned in Resource Assigner
+ :param station_list: List of station names to assign
+ """
+ with RARPC.create() as rarpc:
+ ra_spec = {'task_type': 'reservation',
+ 'task_subtype': 'maintenance',
+ 'status': 'prescheduled',
+ 'starttime': datetime.utcnow() - timedelta(hours=1),
+ 'endtime': datetime.utcnow() + timedelta(hours=2),
+ 'cluster': None,
+ 'specification': {}}
+ inner_spec = {'Observation.VirtualInstrument.stationList': station_list,
+ 'Observation.startTime': ra_spec['starttime'],
+ 'Observation.endTime': ra_spec['starttime']}
+ ra_spec['specification'] = inner_spec
+ assigned = rarpc.do_assignment(ra_spec)
+ return assigned
+
+
class SchedulingTest(unittest.TestCase):
def setUp(self):
# clean all specs/tasks/claims in RADB (cascading delete)
@@ -106,23 +127,47 @@ class SchedulingTest(unittest.TestCase):
self.assertEqual('scheduled', subtask['state_value'])
self.assertEqual('scheduled', tmss_test_env.ra_test_environment.radb.getTask(tmss_id=subtask_id)['status'])
- def test_schedule_observation_subtask_with_blocking_reservations(self):
-
- # create a reservation on station CS001
- with RARPC.create() as rarpc:
- ra_spec = { 'task_type': 'reservation',
- 'task_subtype': 'maintenance',
- 'status': 'prescheduled',
- 'starttime': datetime.utcnow()-timedelta(hours=1),
- 'endtime': datetime.utcnow() + timedelta(hours=1),
- 'cluster': None,
- 'specification': {} }
- inner_spec = { 'Observation.VirtualInstrument.stationList': ['CS001'],
- 'Observation.startTime': '2020-01-08 06:30:00',
- 'Observation.endTime': '2021-07-08 06:30:00' }
- ra_spec['specification'] = inner_spec
- assigned = rarpc.do_assignment(ra_spec)
- self.assertTrue(assigned)
+ def test_schedule_observation_subtask_with_one_blocking_reservation_failed(self):
+ """
+ Set (Resource Assigner) station CS001 to reserved
+ Schedule subtask with station CS001
+ Check if schedule of the subtask fail
+ """
+ self.assertTrue(create_reserved_stations_for_testing(['CS001']))
+
+ with tmss_test_env.create_tmss_client() as client:
+ task_blueprint_data = test_data_creator.TaskBlueprint(template_url=client.get_task_template(name="target observation")['url'])
+ task_blueprint = test_data_creator.post_data_and_get_response_as_json_object(task_blueprint_data, '/task_blueprint/')
+ subtask_template = client.get_subtask_template("observation control")
+ spec = get_default_json_object_for_schema(subtask_template['schema'])
+ spec['stations']['digital_pointings'][0]['subbands'] = [0]
+ cluster_url = client.get_path_as_json_object('/cluster/1')['url']
+
+ subtask_data = test_data_creator.Subtask(specifications_template_url=subtask_template['url'],
+ specifications_doc=spec,
+ cluster_url=cluster_url,
+ start_time=datetime.utcnow() + timedelta(minutes=5),
+ task_blueprint_url=task_blueprint['url'])
+ subtask = test_data_creator.post_data_and_get_response_as_json_object(subtask_data, '/subtask/')
+ subtask_id = subtask['id']
+ test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=subtask['url']), '/subtask_output/')
+
+ client.set_subtask_status(subtask_id, 'defined')
+
+ with self.assertRaises(Exception):
+ client.schedule_subtask(subtask_id)
+
+ subtask = client.get_subtask(subtask_id)
+ self.assertEqual('error', subtask['state_value'])
+ self.assertEqual('conflict', tmss_test_env.ra_test_environment.radb.getTask(tmss_id=subtask_id)['status'])
+
+ def test_schedule_observation_subtask_with_blocking_reservations_failed(self):
+ """
+ Set (Resource Assigner) station CS001, CS002, CS401, CS501 to reserved
+ Schedule subtask with stations CS001, CS002, CS401
+ Check if schedule of the subtask fail
+ """
+ self.assertTrue(create_reserved_stations_for_testing(['CS001','CS002','CS501','CS401' ]))
with tmss_test_env.create_tmss_client() as client:
task_blueprint_data = test_data_creator.TaskBlueprint(template_url=client.get_task_template(name="target observation")['url'])
@@ -131,11 +176,14 @@ class SchedulingTest(unittest.TestCase):
subtask_template = client.get_subtask_template("observation control")
spec = get_default_json_object_for_schema(subtask_template['schema'])
spec['stations']['digital_pointings'][0]['subbands'] = [0]
+ spec['stations']['station_list'] = ['CS001', 'CS002', 'CS401']
+
cluster_url = client.get_path_as_json_object('/cluster/1')['url']
subtask_data = test_data_creator.Subtask(specifications_template_url=subtask_template['url'],
specifications_doc=spec,
cluster_url=cluster_url,
+ start_time=datetime.utcnow() + timedelta(minutes=5),
task_blueprint_url=task_blueprint['url'])
subtask = test_data_creator.post_data_and_get_response_as_json_object(subtask_data, '/subtask/')
subtask_id = subtask['id']
@@ -152,6 +200,38 @@ class SchedulingTest(unittest.TestCase):
self.assertIsNotNone(ra_task)
self.assertEqual('conflict', ra_task['status'])
+ def test_schedule_observation_subtask_with_blocking_reservation_ok(self):
+ """
+ Set (Resource Assigner) station CS001 to reserved
+ Schedule subtask with station CS001, CS002, CS003
+ Check if schedule of the subtasks do not fail (it can schedule with station CS002 and CS003)
+ """
+ self.assertTrue(create_reserved_stations_for_testing(['CS001','CS003']))
+
+ with tmss_test_env.create_tmss_client() as client:
+ task_blueprint_data = test_data_creator.TaskBlueprint(template_url=client.get_task_template(name="target observation")['url'])
+ task_blueprint = test_data_creator.post_data_and_get_response_as_json_object(task_blueprint_data,'/task_blueprint/')
+ subtask_template = client.get_subtask_template("observation control")
+ spec = get_default_json_object_for_schema(subtask_template['schema'])
+ spec['stations']['digital_pointings'][0]['subbands'] = [0]
+ cluster_url = client.get_path_as_json_object('/cluster/1')['url']
+ spec['stations']['station_list'] = ['CS001', 'CS002', 'CS003']
+ subtask_data = test_data_creator.Subtask(specifications_template_url=subtask_template['url'],
+ specifications_doc=spec,
+ cluster_url=cluster_url,
+ start_time=datetime.utcnow()+timedelta(minutes=5),
+ task_blueprint_url=task_blueprint['url'])
+ subtask = test_data_creator.post_data_and_get_response_as_json_object(subtask_data, '/subtask/')
+ subtask_id = subtask['id']
+ test_data_creator.post_data_and_get_url(test_data_creator.SubtaskOutput(subtask_url=subtask['url']),
+ '/subtask_output/')
+
+ client.set_subtask_status(subtask_id, 'defined')
+
+ subtask = client.schedule_subtask(subtask_id)
+ self.assertEqual('scheduled', subtask['state_value'])
+ self.assertEqual('scheduled', tmss_test_env.ra_test_environment.radb.getTask(tmss_id=subtask_id)['status'])
+
def test_schedule_pipeline_subtask_with_enough_resources_available(self):
with tmss_test_env.create_tmss_client() as client:
cluster_url = client.get_path_as_json_object('/cluster/1')['url']
@@ -312,7 +392,7 @@ class SubtaskInputOutputTest(unittest.TestCase):
setting.value = True
setting.save()
- @mock.patch("lofar.sas.tmss.tmss.tmssapp.subtasks._assign_or_unassign_resources")
+ @mock.patch("lofar.sas.tmss.tmss.tmssapp.subtasks.assign_or_unassign_resources")
def test_schedule_pipeline_subtask_filters_predecessor_output_dataproducts_for_input(self, assign_resources_mock):
# setup:
# create observation subtask and outputs and dataproducts
@@ -388,7 +468,7 @@ class SAPTest(unittest.TestCase):
self.assertEqual(models.SAP.objects.first().specifications_doc['pointing']['angle1'], pointing['angle1'])
self.assertEqual(models.SAP.objects.first().specifications_doc['pointing']['angle2'], pointing['angle2'])
- @mock.patch("lofar.sas.tmss.tmss.tmssapp.subtasks._assign_or_unassign_resources")
+ @mock.patch("lofar.sas.tmss.tmss.tmssapp.subtasks.assign_or_unassign_resources")
def test_schedule_pipeline_subtask_copies_sap_from_input_to_output(self, assign_resources_mock):
# setup:
# create observation subtask and outputs and dataproducts
@@ -466,6 +546,31 @@ class TestWithUC1Specifications(unittest.TestCase):
cls.scheduling_unit_blueprints = scheduling_unit_draft.scheduling_unit_blueprints.all()
cls.scheduling_unit_blueprint = cls.scheduling_unit_blueprints[0]
cls.task_blueprints = cls.scheduling_unit_blueprint.task_blueprints.all()
+ # SubtaskId of the first observation subtask
+ observation_tbp = list(tb for tb in list(cls.task_blueprints) if tb.specifications_template.type.value == TaskType.Choices.OBSERVATION.value)
+ observation_tbp.sort(key=lambda tb: tb.relative_start_time)
+ cls.subtask_id_of_first_observation = list(st for st in observation_tbp[0].subtasks.all()
+ if st.specifications_template.type.value == SubtaskType.Choices.OBSERVATION.value)[0].id
+
+ def setUp(self):
+ # clean all specs/tasks/claims in RADB (cascading delete)
+ for spec in tmss_test_env.ra_test_environment.radb.getSpecifications():
+ tmss_test_env.ra_test_environment.radb.deleteSpecification(spec['id'])
+ # Set subtask back to 'defined', start_time to now (and no stoptime)
+ for tb in self.task_blueprints:
+ for subtask in tb.subtasks.all():
+ subtask.state = models.SubtaskState.objects.get(value="defined")
+ subtask.stop_time = None
+ subtask.start_time = datetime.utcnow()
+ subtask.save()
+
+ def _schedule_subtask_with_failure(self, station_reserved):
+ with tmss_test_env.create_tmss_client() as client:
+ with self.assertRaises(Exception) as context:
+ client.schedule_subtask(self.subtask_id_of_first_observation)
+ self.assertTrue("There are more stations in conflict than the specification is given" in str(context.exception).lower())
+ for station in station_reserved:
+ self.assertTrue(station in str(context.exception).lower())
def test_create_task_blueprints_and_subtasks_from_scheduling_unit_draft(self):
"""
@@ -537,6 +642,66 @@ class TestWithUC1Specifications(unittest.TestCase):
self.assertEqual(timedelta(0), task_blueprint.relative_start_time)
self.assertEqual(timedelta(0), task_blueprint.relative_stop_time)
+ def test_dutch_stations_conflicts_exception(self):
+ """
+ Test conflict of 'Dutch' station which are have a default of max_nr_missing=4,
+ Assign stations equal to max_nr_missing+1 before schedule it and check if it can NOT be scheduled
+ Check the context of the Exception
+ """
+ station_reserved = ['CS002', 'CS003', 'CS004', 'CS401', 'CS501']
+ self.assertTrue(create_reserved_stations_for_testing(station_reserved))
+ self._schedule_subtask_with_failure(station_reserved)
+
+ def test_dutch_stations_conflicts_ok(self):
+ """
+ Test conflict of 'Dutch' station which are have a default of max_nr_missing=4,
+ Assign stations equal to max_nr_missing before schedule it and check if it can be scheduled
+ """
+ station_reserved = ['CS002', 'CS003', 'CS004', 'CS401']
+ self.assertTrue(create_reserved_stations_for_testing(station_reserved))
+ with tmss_test_env.create_tmss_client() as client:
+ client.schedule_subtask(self.subtask_id_of_first_observation)
+
+ def test_international_stations_conflicts_failed(self):
+ """
+ Test conflict of 'International' stations which are have a default of max_nr_missing=2,
+ Assign stations equal to max_nr_missing+1 before schedule it and check if it can NOT be scheduled
+ Check the context of the Exception
+ """
+ station_reserved = ['SE607', 'PL610', 'PL612']
+ self.assertTrue(create_reserved_stations_for_testing(station_reserved))
+ self._schedule_subtask_with_failure(station_reserved)
+
+ def test_international_stations_conflicts_ok(self):
+ """
+ Test conflict of 'International' stations which are have a default of max_nr_missing=2,
+ Assign stations equal to max_nr_missing before schedule it and check if it can be scheduled
+ """
+ station_reserved = ['SE607', 'PL612']
+ self.assertTrue(create_reserved_stations_for_testing(station_reserved))
+ with tmss_test_env.create_tmss_client() as client:
+ client.schedule_subtask(self.subtask_id_of_first_observation)
+
+ def test_international_required_stations_conflicts_failed(self):
+ """
+ Test conflict of 'International Required' stations which are have a default of max_nr_missing=1,
+ Assign stations equal to max_nr_missing+1 before schedule it and check if it can NOT be scheduled
+ Check the context of the Exception
+ """
+ station_reserved = ['DE601', 'DE605']
+ self.assertTrue(create_reserved_stations_for_testing(station_reserved))
+ self._schedule_subtask_with_failure(station_reserved)
+
+ def test_international_required_stations_conflicts_ok(self):
+ """
+ Test conflict of 'International Required' stations which are have a default of max_nr_missing=1,
+ Assign stations equal to max_nr_missing before schedule it and check if it can be scheduled
+ """
+ station_reserved = ['DE605']
+ self.assertTrue(create_reserved_stations_for_testing(station_reserved))
+ with tmss_test_env.create_tmss_client() as client:
+ client.schedule_subtask(self.subtask_id_of_first_observation)
+
if __name__ == "__main__":
os.environ['TZ'] = 'UTC'
diff --git a/SAS/TMSS/test/t_tmssapp_specification_REST_API.py b/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
index 018c985f4b69f7b626564bda91f076dcc49591b9..e69a0a55f0bf1cbd466253a051ec5db88cd42392 100755
--- a/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
+++ b/SAS/TMSS/test/t_tmssapp_specification_REST_API.py
@@ -267,6 +267,70 @@ class SchedulingConstraintsTemplateTestCase(unittest.TestCase):
DELETE_and_assert_gone(self, url)
+class ReservationTemplateTestCase(unittest.TestCase):
+ def test_reservation_template_list_apiformat(self):
+ r = requests.get(BASE_URL + '/reservation_template/?format=api', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ self.assertTrue("Reservation Template List" in r.content.decode('utf8'))
+
+ def test_reservation_template_GET_nonexistant_raises_error(self):
+ GET_and_assert_equal_expected_code(self, BASE_URL + '/reservation_template/1234321/', 404)
+
+ def test_reservation_template_POST_and_GET(self):
+ # POST and GET a new item and assert correctness
+ test_data = test_data_creator.ReservationTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url+'?format=json', expected_data)
+
+ def test_reservation_template_PUT_invalid_raises_error(self):
+ test_data = test_data_creator.ReservationTemplate()
+ PUT_and_assert_expected_response(self, BASE_URL + '/reservation_template/9876789876/', test_data, 404, {})
+
+ def test_reservation_template_PUT(self):
+ # POST new item, verify
+ test_data = test_data_creator.ReservationTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+ # PUT new values, verify
+ test_data2 = test_data_creator.ReservationTemplate("reservationtemplate2")
+ expected_data2 = test_data_creator.update_schema_from_template("reservationtemplate", test_data2)
+ PUT_and_assert_expected_response(self, url, test_data2, 200, expected_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data2)
+
+ def test_reservation_template_PATCH(self):
+ # POST new item, verify
+ test_data = test_data_creator.ReservationTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ test_patch = {"name": "new_name",
+ "description": "better description",
+ "schema": minimal_json_schema(properties={"mykey": {"type":"string", "default":"my better value"}})}
+
+ # PATCH item and verify
+ expected_patch_data = test_data_creator.update_schema_from_template("reservationtemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(test_data)
+ expected_data.update(expected_patch_data)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+
+ def test_reservation_template_DELETE(self):
+ # POST new item, verify
+ test_data = test_data_creator.ReservationTemplate()
+ expected_data = test_data_creator.update_schema_from_template("reservationtemplate", test_data)
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation_template/', test_data, 201, expected_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, expected_data)
+ # DELETE and check it's gone
+ DELETE_and_assert_gone(self, url)
+
+
class TaskTemplateTestCase(unittest.TestCase):
def test_task_template_list_apiformat(self):
@@ -1099,7 +1163,6 @@ class ProjectQuotaTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, project_quota_url, project_quota_test_data)
-
class SchedulingSetTestCase(unittest.TestCase):
def test_scheduling_set_list_apiformat(self):
r = requests.get(BASE_URL + '/scheduling_set/?format=api', auth=AUTH)
@@ -2661,6 +2724,84 @@ class TaskSchedulingRelationDraftTestCase(unittest.TestCase):
GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/task_scheduling_relation_draft/%s/' % id2, test_data_2)
+class ReservationTestCase(unittest.TestCase):
+ def test_reservation_list_apiformat(self):
+ r = requests.get(BASE_URL + '/reservation/?format=api', auth=AUTH)
+ self.assertEqual(r.status_code, 200)
+ self.assertTrue("Reservation List" in r.content.decode('utf8'))
+
+ def test_reservation_GET_nonexistant_raises_error(self):
+ GET_and_assert_equal_expected_code(self, BASE_URL + '/reservation/1234321/', 404)
+
+ def test_reservation_POST_and_GET(self):
+ reservation_test_data = test_data_creator.Reservation(duration=60)
+
+ # POST and GET a new item and assert correctness
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation/', reservation_test_data, 201, reservation_test_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, reservation_test_data)
+
+ def test_reservation_PUT_invalid_raises_error(self):
+ reservation_test_data = test_data_creator.Reservation(duration=60)
+ PUT_and_assert_expected_response(self, BASE_URL + '/reservation/9876789876/', reservation_test_data, 404, {})
+
+ def test_reservation_PUT(self):
+ project_url = test_data_creator.post_data_and_get_url(test_data_creator.Project(), '/project/')
+ reservation_test_data = test_data_creator.Reservation(name="reservation 1", duration=50, project_url=project_url)
+
+ # POST new item, verify
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation/', reservation_test_data, 201, reservation_test_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, reservation_test_data)
+
+ reservation_test_data2 = test_data_creator.Reservation(name="reservation2", project_url=project_url)
+ # PUT new values, verify
+ PUT_and_assert_expected_response(self, url, reservation_test_data2, 200, reservation_test_data2)
+ GET_OK_and_assert_equal_expected_response(self, url, reservation_test_data2)
+
+ def test_reservation_PATCH(self):
+ reservation_test_data = test_data_creator.Reservation(duration=60)
+
+ # POST new item, verify
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation/', reservation_test_data, 201, reservation_test_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, reservation_test_data)
+
+ test_patch = {"description": "This is a new and improved description",
+ "duration": 90}
+
+ # PATCH item and verify
+ expected_patch_data = test_data_creator.update_schema_from_template("reservationtemplate", test_patch)
+ PATCH_and_assert_expected_response(self, url, test_patch, 200, expected_patch_data)
+ expected_data = dict(reservation_test_data)
+ expected_data.update(test_patch)
+ GET_OK_and_assert_equal_expected_response(self, url, expected_patch_data)
+
+ def test_reservation_DELETE(self):
+ reservation_test_data = test_data_creator.Reservation(duration=30, start_time=datetime.utcnow() + timedelta(days=1))
+ # POST new item, verify
+ r_dict = POST_and_assert_expected_response(self, BASE_URL + '/reservation/', reservation_test_data, 201, reservation_test_data)
+ url = r_dict['url']
+ GET_OK_and_assert_equal_expected_response(self, url, reservation_test_data)
+
+ # DELETE and check it's gone
+ DELETE_and_assert_gone(self, url)
+
+ def test_GET_Reservation_list_shows_entry(self):
+ test_data_1 = Reservation_test_data(duration=3600)
+ models.Reservation.objects.create(**test_data_1)
+ nbr_results = models.Reservation.objects.count()
+ GET_and_assert_in_expected_response_result_list(self, BASE_URL + '/reservation/', test_data_1, nbr_results)
+
+ def test_GET_Reservation_view_returns_correct_entry(self):
+ test_data_1 = Reservation_test_data(name="Reservation 1", duration=60, start_time=datetime.utcnow() + timedelta(days=1))
+ test_data_2 = Reservation_test_data(name="Reservation 2", duration=120, start_time=datetime.utcnow() + timedelta(days=2))
+ id1 = models.Reservation.objects.create(**test_data_1).id
+ id2 = models.Reservation.objects.create(**test_data_2).id
+ GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/reservation/' + str(id1) + '/', test_data_1)
+ GET_OK_and_assert_equal_expected_response(self, BASE_URL + '/reservation/' + str(id2) + '/', test_data_2)
+
+
if __name__ == "__main__":
logging.basicConfig(format='%(asctime)s %(levelname)s %(message)s',
level=logging.INFO)
diff --git a/SAS/TMSS/test/t_tmssapp_specification_django_API.py b/SAS/TMSS/test/t_tmssapp_specification_django_API.py
index c5c917319778ad16ef17e94331674b10af68309b..98bb7aad6b8dc43efef99e435ee2ebcb27cbb38b 100755
--- a/SAS/TMSS/test/t_tmssapp_specification_django_API.py
+++ b/SAS/TMSS/test/t_tmssapp_specification_django_API.py
@@ -446,6 +446,15 @@ class SchedulingUnitDraftTest(unittest.TestCase):
with self.assertRaises(IntegrityError):
models.SchedulingUnitDraft.objects.create(**test_data)
+ def test_SchedulingUnitDraft_gets_created_with_correct_default_ingest_permission_required(self):
+
+ # setup
+ entry = models.SchedulingUnitDraft.objects.create(**SchedulingUnitDraft_test_data())
+ #check the auto_ingest on project
+ self.assertEqual(False, entry.scheduling_set.project.auto_ingest)
+ #When auto_ingest=False (in project), the scheduling units should be created with ingest_permission_required = True
+ self.assertEqual(True, entry.ingest_permission_required)
+
class TaskDraftTest(unittest.TestCase):
@@ -650,6 +659,16 @@ class SchedulingUnitBlueprintTest(unittest.TestCase):
# assert
with self.assertRaises(IntegrityError):
models.SchedulingUnitBlueprint.objects.create(**test_data)
+
+
+ def test_SchedulingUnitBlueprint_gets_created_with_correct_default_ingest_permission_required(self):
+
+ # setup
+ entry = models.SchedulingUnitBlueprint.objects.create(**SchedulingUnitBlueprint_test_data())
+ #check the auto_ingest on project
+ self.assertEqual(False, entry.draft.scheduling_set.project.auto_ingest)
+ #When auto_ingest=False (in project), the scheduling units should be created with ingest_permission_required = True
+ self.assertEqual(True, entry.ingest_permission_required)
class TaskBlueprintTest(unittest.TestCase):
diff --git a/SAS/TMSS/test/test_utils.py b/SAS/TMSS/test/test_utils.py
index 1029deb3474ce830e83f3d8d0a26f07c9bf3620f..95bbec3a5cf6b123fd123be5501feaeaa6e4bf60 100644
--- a/SAS/TMSS/test/test_utils.py
+++ b/SAS/TMSS/test/test_utils.py
@@ -272,7 +272,8 @@ class TMSSTestEnvironment:
populate_schemas:bool=False, populate_test_data:bool=False,
start_ra_test_environment: bool=False, start_postgres_listener: bool=False,
start_subtask_scheduler: bool=False, start_dynamic_scheduler: bool=False,
- start_pipeline_control: bool=False, enable_viewflow: bool=False):
+ start_pipeline_control: bool=False,
+ start_workflow_service: bool=False, enable_viewflow: bool=False):
self._exchange = exchange
self._broker = broker
self._populate_schemas = populate_schemas
@@ -302,8 +303,10 @@ class TMSSTestEnvironment:
self._start_pipeline_control = start_pipeline_control
self.pipeline_control = None
- if enable_viewflow:
- os.environ['TMSS_ENABLE_VIEWFLOW'] = 'True'
+ self.enable_viewflow = enable_viewflow or start_workflow_service
+ self._start_workflow_service = start_workflow_service
+ self.workflow_service = None
+ os.environ['TMSS_ENABLE_VIEWFLOW'] = str(self.enable_viewflow)
# Check for correct Django version, should be at least 3.0
if django.VERSION[0] < 3:
@@ -366,6 +369,11 @@ class TMSSTestEnvironment:
self.pipeline_control = PipelineControlTMSS(exchange=self._exchange, broker=self._broker)
self.pipeline_control.start_listening()
+ if self._start_workflow_service:
+ from lofar.sas.tmss.services.workflow_service import create_workflow_service
+ self.workflow_service = create_workflow_service(exchange=self._exchange, broker=self._broker)
+ self.workflow_service.start_listening()
+
if self._populate_schemas or self._populate_test_data:
self.populate_schemas()
@@ -374,6 +382,10 @@ class TMSSTestEnvironment:
def stop(self):
+ if self.workflow_service is not None:
+ self.workflow_service.stop_listening()
+ self.workflow_service = None
+
if self.postgres_listener is not None:
self.postgres_listener.stop()
self.postgres_listener = None
@@ -488,7 +500,8 @@ def main_test_environment():
populate_schemas=options.schemas, populate_test_data=options.data,
start_ra_test_environment=options.services, start_postgres_listener=options.services,
start_subtask_scheduler=options.services, start_dynamic_scheduler=options.services,
- start_pipeline_control=options.services, enable_viewflow=options.viewflow) as tmss_test_env:
+ start_pipeline_control=options.services,
+ start_workflow_service=options.services and options.viewflow, enable_viewflow=options.viewflow) as tmss_test_env:
# print some nice info for the user to use the test servers...
# use print instead of log for clean lines.
diff --git a/SAS/TMSS/test/tmss_test_data_django_models.py b/SAS/TMSS/test/tmss_test_data_django_models.py
index 5edc2d0b9a87be9a108937fb0467fdff1476860d..8c59a5c8959d5252825dff208bb07bce1574cb65 100644
--- a/SAS/TMSS/test/tmss_test_data_django_models.py
+++ b/SAS/TMSS/test/tmss_test_data_django_models.py
@@ -68,7 +68,6 @@ def SchedulingConstraintsTemplate_test_data(name="my_SchedulingConstraintsTempla
"tags": ["TMSS", "TESTING"]}
-
def SchedulingUnitObservingStrategyTemplate_test_data(name="my_SchedulingUnitObservingStrategyTemplate",
scheduling_unit_template:models.SchedulingUnitTemplate=None,
template:dict=None) -> dict:
@@ -126,6 +125,7 @@ def Project_test_data(name: str=None, priority_rank: int = 1, archive_subdirecto
"name": name,
"description": 'my description ' + str(uuid.uuid4()),
"tags": [],
+ "auto_ingest": False,
"priority_rank": priority_rank,
"trigger_priority": 1000,
"can_trigger": False,
@@ -494,3 +494,32 @@ def SAPTemplate_test_data() -> dict:
"schema": minimal_json_schema(),
"tags": ["TMSS", "TESTING"]}
+
+def ReservationTemplate_test_data(name="my_ReservationTemplate", schema:dict=None) -> dict:
+ if schema is None:
+ schema = minimal_json_schema(properties={ "foo" : { "type": "string", "default": "bar" } }, required=["foo"])
+
+ return {"name": name,
+ "description": 'My ReservationTemplate description',
+ "schema": schema,
+ "tags": ["TMSS", "TESTING"]}
+
+
+def Reservation_test_data(name="MyReservation", duration=None, start_time=None, project: models.Project = None) -> dict:
+ if project is None:
+ project = models.Project.objects.create(**Project_test_data())
+
+ if start_time is None:
+ start_time = datetime.utcnow() + timedelta(hours=12)
+
+ specifications_template = models.ReservationTemplate.objects.create(**ReservationTemplate_test_data())
+ specifications_doc = get_default_json_object_for_schema(specifications_template.schema)
+
+ return {"name": name,
+ "project": project,
+ "description": "Test Reservation",
+ "tags": ["TMSS", "TESTING"],
+ "start_time": start_time,
+ "duration": duration, # can be None
+ "specifications_doc": specifications_doc,
+ "specifications_template": specifications_template}
diff --git a/SAS/TMSS/test/tmss_test_data_rest.py b/SAS/TMSS/test/tmss_test_data_rest.py
index 1a16d480f10c74cd783b3ea88d39fd363b1c2cfc..17f78eaf04f89360724f2c1896037f65c2029445 100644
--- a/SAS/TMSS/test/tmss_test_data_rest.py
+++ b/SAS/TMSS/test/tmss_test_data_rest.py
@@ -130,6 +130,16 @@ class TMSSRESTTestDataCreator():
"tags": ["TMSS", "TESTING"]}
+ def ReservationTemplate(self, name="reservationtemplate1", schema:dict=None) -> dict:
+ if schema is None:
+ schema = minimal_json_schema(properties={"foo": {"type": "string", "default": "bar"}})
+
+ return { "name": name,
+ "description": 'My description',
+ "schema": schema,
+ "tags": ["TMSS", "TESTING"]}
+
+
def SchedulingUnitObservingStrategyTemplate(self, name="my_SchedulingUnitObservingStrategyTemplate",
scheduling_unit_template_url=None,
template:dict=None) -> dict:
@@ -669,4 +679,30 @@ class TMSSRESTTestDataCreator():
return {"specifications_doc": specifications_doc,
"specifications_template": specifications_template_url,
- "tags": ['tmss', 'testing']}
\ No newline at end of file
+ "tags": ['tmss', 'testing']}
+
+ def Reservation(self, name="My Reservation", duration=None, start_time=None, project_url=None,
+ specifications_template_url=None, specifications_doc=None) -> dict:
+
+ if project_url is None:
+ project_url = self.post_data_and_get_url(self.Project(), '/project/')
+ if start_time is None:
+ start_time = datetime.utcnow() + timedelta(hours=12)
+
+ if specifications_template_url is None:
+ specifications_template_url = self.post_data_and_get_url(self.ReservationTemplate(), '/reservation_template/')
+
+ if specifications_doc is None:
+ specifications_doc = self.get_response_as_json_object(specifications_template_url + '/default')
+
+ if isinstance(start_time, datetime):
+ start_time = start_time.isoformat()
+
+ return {"name": name,
+ "project": project_url,
+ "description": "Test Reservation",
+ "tags": ["TMSS", "TESTING"],
+ "start_time": start_time,
+ "duration": duration, # can be None
+ "specifications_doc": specifications_doc,
+ "specifications_template": specifications_template_url}
\ No newline at end of file