Skip to content
GitLab
Explore
Sign in
Register
Primary navigation
Search or go to…
Project
P
Pipeline Dependency Analysis
Manage
Activity
Members
Labels
Plan
Issues
Issue boards
Milestones
Wiki
Requirements
Jira
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Locked files
Build
Pipelines
Jobs
Pipeline schedules
Test cases
Artifacts
Deploy
Releases
Package registry
Container registry
Model registry
Operate
Environments
Terraform modules
Monitor
Incidents
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Code review analytics
Issue analytics
Insights
Model experiments
Help
Help
Support
GitLab documentation
Compare GitLab plans
GitLab community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
Chiara Liotta
Pipeline Dependency Analysis
Commits
9fd40fa1
Commit
9fd40fa1
authored
4 months ago
by
Chiara Liotta
Browse files
Options
Downloads
Patches
Plain Diff
change data flow edge property from data_ids to data_id
parent
ac415119
No related branches found
No related tags found
No related merge requests found
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
graph_creation/cwl_processing.py
+1
-1
1 addition, 1 deletion
graph_creation/cwl_processing.py
graph_creation/utils.py
+1
-1
1 addition, 1 deletion
graph_creation/utils.py
neo4j_dependency_queries/create_edge_queries.py
+10
-43
10 additions, 43 deletions
neo4j_dependency_queries/create_edge_queries.py
with
12 additions
and
45 deletions
graph_creation/cwl_processing.py
+
1
−
1
View file @
9fd40fa1
...
...
@@ -68,7 +68,7 @@ def process_cwl_outputs(driver: Driver, cwl_entity: dict, step_lookup: dict) ->
# If it's not a workflow, create a relationship between the component and the output parameter
is_worflow
=
get_is_workflow
(
cwl_entity
)
if
not
is_worflow
:
create_out_param_relationship
(
driver
,
component_id
,
out_param_node_internal_id
)
create_out_param_relationship
(
driver
,
component_id
,
out_param_node_internal_id
,
output
[
'
id
'
]
)
# If the output has an 'outputSource', process the relationship(s) to the source(s)
if
'
outputSource
'
in
output
:
...
...
This diff is collapsed.
Click to expand it.
graph_creation/utils.py
+
1
−
1
View file @
9fd40fa1
...
...
@@ -48,7 +48,7 @@ def process_in_param(driver: Driver, param_id: str, component_id: str, param_typ
param_node
=
ensure_in_parameter_node
(
driver
,
param_id
,
component_id
,
param_type
,
component_type
)
if
component_type
!=
"
Workflow
"
:
ensure_component_node
(
driver
,
component_id
)
create_in_param_relationship
(
driver
,
component_id
,
param_node
[
0
])
create_in_param_relationship
(
driver
,
component_id
,
param_node
[
0
]
,
param_node
[
1
]
)
def
process_parameter_source
(
driver
:
Driver
,
param_node_internal_id
:
int
,
source_id
:
str
,
workflow_id
:
str
,
step_lookup
:
dict
,
step_id
:
str
=
""
)
->
None
:
"""
...
...
This diff is collapsed.
Click to expand it.
neo4j_dependency_queries/create_edge_queries.py
+
10
−
43
View file @
9fd40fa1
...
...
@@ -2,7 +2,7 @@ from neo4j import Driver
from
neo4j_dependency_queries.create_node_queries
import
ensure_component_node
from
neo4j_dependency_queries.utils
import
clean_component_id
def
create_in_param_relationship
(
driver
:
Driver
,
prefixed_component_id
:
str
,
parameter_internal_id
:
int
)
->
tuple
[
str
,
str
]:
def
create_in_param_relationship
(
driver
:
Driver
,
prefixed_component_id
:
str
,
parameter_internal_id
:
int
,
parameter_id
:
str
)
->
tuple
[
str
,
str
]:
"""
Creates a data dependency relationship in Neo4j between a component node with path prefixed_component_id
and an in-parameter node with Neo4j internal ID parameter_internal_id.
...
...
@@ -19,25 +19,11 @@ def create_in_param_relationship(driver: Driver, prefixed_component_id: str, par
tuple[str,str]: the component ID of the component, the parameter ID of the parameter
"""
component_id
=
clean_component_id
(
prefixed_component_id
)
ensure_component_node
(
driver
,
component_id
)
query
=
"""
MATCH (c:Component {component_id: $component_id}), (p:InParameter)
WHERE elementId(p) = $parameter_internal_id
MERGE (c)<-[r:DATA_FLOW {component_id: $component_id}]-(p)
SET r.data_ids =
CASE
WHEN r.data_ids IS NULL THEN [p.parameter_id]
WHEN NOT p.parameter_id IN r.data_ids THEN r.data_ids + [p.parameter_id]
ELSE r.data_ids
END
RETURN c.component_id AS component_id, p.parameter_id AS parameter_id
"""
with
driver
.
session
()
as
session
:
result
=
session
.
run
(
query
,
component_id
=
component_id
,
parameter_internal_id
=
parameter_internal_id
)
return
result
component_node_id
=
ensure_component_node
(
driver
,
component_id
)[
0
]
return
create_data_relationship
(
driver
,
parameter_internal_id
,
component_node_id
,
component_id
,
parameter_id
)
def
create_out_param_relationship
(
driver
:
Driver
,
prefixed_component_id
:
str
,
parameter_internal_id
:
int
)
->
tuple
[
str
,
str
]:
def
create_out_param_relationship
(
driver
:
Driver
,
prefixed_component_id
:
str
,
parameter_internal_id
:
int
,
parameter_id
:
str
)
->
tuple
[
str
,
str
]:
"""
Creates a data dependency relationship in Neo4j between a component node with path prefixed_component_id
and an out-parameter node with Neo4j internal ID parameter_internal_id.
...
...
@@ -54,23 +40,10 @@ def create_out_param_relationship(driver: Driver, prefixed_component_id: str, pa
tuple[str,str]: the component ID of the component, the parameter ID of the parameter
"""
component_id
=
clean_component_id
(
prefixed_component_id
)
ensure_component_node
(
driver
,
component_id
)
query
=
"""
MATCH (c:Component {component_id: $component_id}), (p: OutParameter)
WHERE elementId(p) = $parameter_internal_id
MERGE (c)-[r:DATA_FLOW {component_id: $component_id}]->(p)
SET r.data_ids =
CASE
WHEN r.data_ids IS NULL THEN [p.parameter_id]
WHEN NOT p.parameter_id IN r.data_ids THEN r.data_ids + [p.parameter_id]
ELSE r.data_ids
END
RETURN c.component_id AS component_id, p.parameter_id AS parameter_id
"""
with
driver
.
session
()
as
session
:
result
=
session
.
run
(
query
,
component_id
=
component_id
,
parameter_internal_id
=
parameter_internal_id
)
return
result
component_node_id
=
ensure_component_node
(
driver
,
component_id
)[
0
]
print
(
parameter_internal_id
)
print
(
component_node_id
)
return
create_data_relationship
(
driver
,
component_node_id
,
parameter_internal_id
,
component_id
,
parameter_id
)
def
create_data_relationship
(
driver
:
Driver
,
from_internal_node_id
:
int
,
to_internal_node_id
:
int
,
component_id
:
str
,
data_id
:
str
,
step_id
:
str
=
""
)
->
tuple
[
int
,
int
]:
...
...
@@ -91,13 +64,7 @@ def create_data_relationship(driver: Driver, from_internal_node_id: int, to_inte
query
=
"""
MATCH (a), (b)
WHERE elementId(a) = $from_internal_node_id AND elementId(b) = $to_internal_node_id
MERGE (a)-[r:DATA_FLOW {component_id: $component_id, step_id: $step_id}]->(b)
SET r.data_ids =
CASE
WHEN r.data_ids IS NULL THEN [$data_id]
WHEN NOT $data_id IN r.data_ids THEN r.data_ids + [$data_id]
ELSE r.data_ids
END
MERGE (a)-[r:DATA_FLOW {component_id: $component_id, step_id: $step_id, data_id: $data_id}]->(b)
RETURN elementId(a) AS id_1, elementId(b) AS id_2
"""
with
driver
.
session
()
as
session
:
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment