From 0c5662ae9b4518aa12a13c381ea0e88a2adf321f Mon Sep 17 00:00:00 2001
From: Giacomo Bernini <g.bernini@nextworks.it>
Date: Thu, 21 Oct 2021 17:11:05 +0200
Subject: [PATCH 1/4] updated SOL005 NS PM refs to v3.5.1

---
 .../IndividualPmJob.robot                     | 18 +++++------
 .../IndividualReport.robot                    | 12 +++----
 .../IndividualThreshold.robot                 | 18 +++++------
 .../NotificationEndpoint.robot                |  4 +--
 .../Notifications.robot                       |  4 +--
 .../NSPerformanceManagement-API/PMJobs.robot  | 32 +++++++++----------
 .../Thresholds.robot                          | 22 ++++++-------
 7 files changed, 55 insertions(+), 55 deletions(-)

diff --git a/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot b/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot
index a087adf08..5220e8583 100644
--- a/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot
+++ b/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot
@@ -11,7 +11,7 @@ GET individual NS Performance Job
     ...    Test title: Get individual NS Performance Job
     ...    Test objective: The objective is to test the retrieval of an individual NS Performance monitoring job and perform a JSON schema and content validation of the collected job data structure
     ...    Pre-conditions: A NS instance is instantiated. One or more NS Performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.3.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -25,7 +25,7 @@ GET individual NS Performance Job with invalid resource identifier
     ...    Test title: Get individual NS Performance Job with invalid resource identifier
     ...    Test objective: The objective is to test that the retrieval of an individual NS Performance monitoring job fails when using an invalid resource identifier, and perform the JSON schema validation of the failed operation HTTP response
     ...    Pre-conditions: A NS instance is instantiated. One or more NS Performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.3.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -38,7 +38,7 @@ DELETE Individual NS Performance Job with invalid resource identifier
     ...    Test title: Delete individual NS Performance Job with invalid resource identifier
     ...    Test objective: The objective is to test that the deletion of an individual NS Performance monitoring job fails when using an invalid resource identifier
     ...    Pre-conditions: A NS instance is instantiated. One or more NS Performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.3.3.5 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.5 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -50,7 +50,7 @@ POST Individual NS Performance Job - Method not implemented
     ...    Test title: POST Individual NS Performance Job - method not implemented
     ...    Test objective: The objective is to test that POST method is not allowed to create a new NS Performance Monitoring Job
     ...    Pre-conditions: none
-    ...    Reference: Clause 7.4.3.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -62,7 +62,7 @@ PUT Individual NS Performance Job - Method not implemented
     ...    Test title: PUT Individual NS Performance Job - method not implemented
     ...    Test objective: The objective is to test that PUT method is not allowed to update an existing NS Performance Monitoring Job
     ...    Pre-conditions: none
-    ...    Reference: Clause 7.4.3.3.3 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.3 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -74,7 +74,7 @@ PATCH Individual NS Performance Job
     ...    Test title: PATCH Individual NS Performance Job 
     ...    Test objective: The objective is to test that PATCH method modify an existing new NS Performance Monitoring Job
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 6.4.3.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 6.4.3.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: Resource modified
@@ -87,7 +87,7 @@ DELETE Individual NS Performance Job
     ...    Test title: Delete Individual NS Performance Job
     ...    Test objective: The objective is to test the deletion of an individual NS Performance monitoring job
     ...    Pre-conditions: A NS instance is instantiated. One or more NS Performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.3.3.5 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.5 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The NS Performance Job is no more available in the NFVO    
@@ -100,7 +100,7 @@ PATCH Individual NS Performance Job - Precondition failed
     ...    Test title: PATCH Individual NS Performance Job - Precondition failed
     ...    Test objective: The objective is to attempt to Modify an individual NS Performance job fails, where the precondition was not met
     ...    Pre-conditions: The related job already exist
-    ...    Reference: Clause 7.4.3.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The resource is not modified
@@ -113,7 +113,7 @@ PATCH Individual NS Performance Job - Unprocessable Entity
     ...    Test title: PATCH Individual NS Performance Job - Unprocessable Entity
     ...    Test objective: The objective is to test that PATCH method cannot modify an exsisting individual PM job resource when a request contains syntactically correct data but the data cannot be processed.
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.3.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.3.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
diff --git a/SOL005/NSPerformanceManagement-API/IndividualReport.robot b/SOL005/NSPerformanceManagement-API/IndividualReport.robot
index 0f2c4b5ab..44e8ef652 100644
--- a/SOL005/NSPerformanceManagement-API/IndividualReport.robot
+++ b/SOL005/NSPerformanceManagement-API/IndividualReport.robot
@@ -11,7 +11,7 @@ Get Individual Performance Report
     ...    Test title: Get Individual Performance Report
     ...    Test objective: The objective is to test the retrieval of an individual NS performance report associated to a monitoring job and perform a JSON schema validation of the collected report data structure
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance reports are set for a monitoring job in the NFVO.
-    ...    Reference: Clause 7.4.4.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.4.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -24,7 +24,7 @@ Get Individual Performance Report with invalid resource endpoint
     ...    Test title: Get Individual Performance Report with invalid resource endpoint
     ...    Test objective:  The objective is to test that the retrieval of an individual NS performance report associated to a monitoring job fails when using an invalid resource endpoint 
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance reports are set for a monitoring job in the NFVO.
-    ...    Reference: Clause 7.4.4.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.4.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -37,7 +37,7 @@ POST Individual Performance Report - Method not implemented
     ...    Test title: POST Individual Performance Report - Method not implemented
     ...    Test objective: The objective is to test that POST method is not allowed to create a new NS performance report within a monitoring job
     ...    Pre-conditions: none
-    ...    Reference: Clause 7.4.4.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.4.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -49,7 +49,7 @@ PUT Individual Performance Report - Method not implemented
     ...    Test title: PUT Individual Performance Report - Method not implemented
     ...    Test objective: The objective is to test that PUT method is not allowed to update an existing NS performance report within a monitoring job
     ...    Pre-conditions: none
-    ...    Reference: Clause 7.4.4.3.3 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.4.3.3 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -61,7 +61,7 @@ PATCH Individual Performance Report - Method not implemented
     ...    Test title: PATCH Individual Performance Report - Method not implemented
     ...    Test objective: The objective is to test that PATCH method is not allowed to modify an existing NS performance report within a monitoring job
     ...    Pre-conditions: none
-    ...    Reference: Clause 7.4.4.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.4.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -73,7 +73,7 @@ DELETE Individual Performance Report - Method not implemented
     ...    Test title: DELETE Individual Performance Report - Method not implemented
     ...    Test objective: The objective is to test that DELETE method is not allowed to delete an existing NS performance report within a monitoring job
     ...    Pre-conditions: none
-    ...    Reference: Clause 7.4.4.3.5 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.4.3.5 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
diff --git a/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot b/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot
index 67f24ec81..1a55bfb77 100644
--- a/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot
+++ b/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot
@@ -13,7 +13,7 @@ GET Individual Threshold
     ...    Test title: GET Individual Threshold
     ...    Test objective: The objective is to test the retrieval of an individual NS performance threshold and perform a JSON schema and content validation of the collected threshold data structure
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -27,7 +27,7 @@ GET Individual Threshold with invalid resource identifier
     ...    Test title: GET Individual Threshold with invalid resource identifier
     ...    Test objective: The objective is to test that the retrieval of an individual NS performance threshold fails when using an invalid resource identifier
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -40,7 +40,7 @@ DELETE Individual Threshold with invalid resource identifier
     ...    Test title: DELETE Individual Threshold with invalid resource identifier
     ...    Test objective: The objective is to test the deletion of an individual NS performance threshold
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.5 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.5 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none   
@@ -52,7 +52,7 @@ POST Individual Threshold - Method not implemented
     ...    Test title: POST Individual Threshold - Method not implemented
     ...    Test objective: The objective is to test that POST method is not allowed to create a new NS performance Threshold
     ...    Pre-conditions: A NS instance is instantiated
-    ...    Reference: Clause 7.4.6.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -64,7 +64,7 @@ PUT Individual Threshold - Method not implemented
     ...    Test title: PUT Individual Threshold - Method not implemented
     ...    Test objective: The objective is to test that PUT method is not allowed to update an existing NS performance threshold
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.3 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.3 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -76,7 +76,7 @@ PATCH Individual Threshold
     ...    Test title: PATCH Individual Threshold 
     ...    Test objective: The objective is to test that PATCH method modify an existing NS performance threshold
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The NS performance Threshold is modified by the operation
@@ -90,7 +90,7 @@ DELETE Individual Threshold
     ...    Test title: DELETE Individual Threshold
     ...    Test objective: The objective is to test the deletion of an individual NS performance threshold
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.5 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.5 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The NS performance Threshold is not available anymore in the NFVO    
@@ -103,7 +103,7 @@ PATCH Individual Threshold - Precondition failed
     ...    Test title: PATCH Individual Threshold - Precondition failed 
     ...    Test objective: The objective is to attempt to Modify an individual NS Performance threshold fails, where the precondition was not met
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The NS performance Threshold is not modified by the operation
@@ -117,7 +117,7 @@ PATCH Individual Threshold - Unprocessable Entity
     ...    Test title: PATCH Individual NS Performance Job - Unprocessable Entity
     ...    Test objective: The objective is to test that PATCH method cannot modify an exsisting individual PM job resource when a request contains syntactically correct data but the data cannot be processed.
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.6.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.6.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The NS performance Threshold is not modified by the operation
diff --git a/SOL005/NSPerformanceManagement-API/NotificationEndpoint.robot b/SOL005/NSPerformanceManagement-API/NotificationEndpoint.robot
index 5787d497a..94bea6251 100644
--- a/SOL005/NSPerformanceManagement-API/NotificationEndpoint.robot
+++ b/SOL005/NSPerformanceManagement-API/NotificationEndpoint.robot
@@ -13,7 +13,7 @@ NS Performance Information Availability Notification
     ...    Test title: NS Performance Information Availability Notification
     ...    Test objective: The objective is to test that NS Performance Information Availability Notification is delivered with success to the notification consumer
     ...    Pre-conditions: A NS performance job is created, and information availability notifications is available in the NFVO.
-    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_Notif_Endpoint
     ...    Applicability: none
     ...    Post-Conditions: none 
@@ -25,7 +25,7 @@ NS Threshold Crossed Notification
     ...    Test title: NS Threshold Crossed Notification
     ...    Test objective: The objective is to test that NS Threshold Crossed Notification is delivered with success to the notification consumer
     ...    Pre-conditions: A NS performance job is created, and threshold crossed notifications is available in the NFVO.
-    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_Notif_Endpoint
     ...    Applicability: none
     ...    Post-Conditions: none 
diff --git a/SOL005/NSPerformanceManagement-API/Notifications.robot b/SOL005/NSPerformanceManagement-API/Notifications.robot
index 82e77fc74..af3a71f63 100644
--- a/SOL005/NSPerformanceManagement-API/Notifications.robot
+++ b/SOL005/NSPerformanceManagement-API/Notifications.robot
@@ -16,7 +16,7 @@ NS Performance Information Availability Notification
     ...    Test title: NS Performance Information Availability Notification
     ...    Test objective: The objective is to test the dispatch of NS Performance Information Availability Notification when new NS performance information is available in the NFVO, and perform a JSON schema and content validation of the delivered notification. The action that triggers the notification under test is an explicit test step, but it is not performed by the test system.
     ...    Pre-conditions: A NS performance job is created, and a subscription for information availability notifications is available in the NFVO.
-    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none 
@@ -29,7 +29,7 @@ NS Threshold Crossed Notification
     ...    Test title: NS Threshold Crossed Notification
     ...    Test objective: The objective is to test the dispatch of NS Threshold Crossed Notification when a previously set NS performance metric threshold is crossed, and perform a JSON schema and content validation of the delivered notification. The action that triggers the notification under test is an explicit test step, but it is not performed by the test system.
     ...    Pre-conditions: A NS performance job is created, and a threshold subscription is available in the NFVO.
-    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.9.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none 
diff --git a/SOL005/NSPerformanceManagement-API/PMJobs.robot b/SOL005/NSPerformanceManagement-API/PMJobs.robot
index d36888cb4..404770817 100644
--- a/SOL005/NSPerformanceManagement-API/PMJobs.robot
+++ b/SOL005/NSPerformanceManagement-API/PMJobs.robot
@@ -13,7 +13,7 @@ GET all NS Performance Monitoring Jobs
     ...    Test title: GET all NS Performance Monitoring Jobs
     ...    Test objective: The objective is to test the retrieval of all the available NS performance monitoring jobs and perform a JSON schema and content validation of the collected jobs data structure
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -27,7 +27,7 @@ GET NS Performance Monitoring Jobs with attribute-based filter
     ...    Test title: GET all NS Performance Monitoring Jobs with attribute-based filter
     ...    Test objective: The objective is to test the retrieval of NS performance monitoring jobs using attribute-based filter, perform a JSON schema validation of the collected jobs data structure, and verify that the retrieved information matches the issued attribute-based filter
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -41,7 +41,7 @@ GET all NS Performance Monitoring Jobs with "all_fields" attribute selector
     ...    Test title: GET all NS Performance Monitoring Jobs with "all_fields" attribute selector
     ...    Test objective: The objective is to test the retrieval of all NS performance monitoring jobs "all_fields" attribute selector, perform a JSON schema validation of the collected jobs data structure, and verify that the retrieved information matches the issued "all_fileds" selector
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -55,7 +55,7 @@ GET all NS Performance Monitoring Jobs with "exclude_default" attribute selector
     ...    Test title: GET all NS Performance Monitoring Jobs with "exclude_default" attribute selector
     ...    Test objective: The objective is to test the retrieval of all NS performance monitoring jobs "exclude_default" attribute selector, perform a JSON schema validation of the collected jobs data structure, and verify that the retrieved information matches the issued "exclude_default" selector
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -69,7 +69,7 @@ GET all NS Performance Monitoring Jobs with "fields" attribute selector
     ...    Test title: GET all NS Performance Monitoring Jobs with "fields" attribute selector
     ...    Test objective: The objective is to test the retrieval of all NS performance monitoring jobs "fields" attribute selector, perform a JSON schema validation of the collected jobs data structure, and verify that the retrieved information matches the issued "include" selector
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: The NFVO supports the use of "include" attribute selector
     ...    Post-Conditions: none
@@ -83,7 +83,7 @@ GET all NS Performance Monitoring Jobs with "exclude" attribute selector
     ...    Test title: GET all NS Performance Monitoring Jobs with exclude_fields attribute selector
     ...    Test objective: The objective is to test the retrieval of all NS performance monitoring jobs "exclude" attribute selector, perform a JSON schema validation of the collected jobs data structure, and verify that the retrieved information matches the issued "exclude" selector
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: The NFVO supports the use of "exclude" attribute selector
     ...    Post-Conditions: none
@@ -97,7 +97,7 @@ GET NS Performance Monitoring Jobs with invalid attribute-based filter
     ...    Test title: GET NS Performance Monitoring Jobs with invalid attribute-based filter
     ...    Test objective: The objective is to test that the retrieval of NS performance monitoring jobs fails when using invalid attribute-based filter, and perform the JSON schema validation of the failed operation HTTP response
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -110,7 +110,7 @@ GET NS Performance Monitoring Jobs with invalid resource endpoint
     ...    Test title: GET NS Performance Monitoring Jobs with invalid resource endpoint
     ...    Test objective: The objective is to test that the retrieval of NS performance monitoring jobs fails when using invalid resource endpoint
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -123,7 +123,7 @@ Create new NS Performance Monitoring Job
     ...    Test title:  Create a new NS Performance Monitoring Job
     ...    Test objective: The objective is to test the creation of a new NS performance monitoring job and perform the JSON schema validation of the returned job data structure
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The NS Performance Job is successfully created on the NFVO
@@ -137,7 +137,7 @@ PUT all NS Performance Monitoring Jobs - Method not implemented
     ...    Test title: PUT all NS Performance Monitoring Jobs - Method not implemented
     ...    Test objective: The objective is to test that PUT method is not allowed to modify NS Performance Monitoring Jobs
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.3 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.3 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -149,7 +149,7 @@ PATCH all NS Performance Monitoring Jobs - Method not implemented
     ...    Test title: PATCH all NS Performance Monitoring Jobs - Method not implemented
     ...    Test objective: The objective is to test that PATCH method is not allowed to update NS Performance Monitoring Jobs
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -161,7 +161,7 @@ DELETE all NS Performance Monitoring Jobs - Method not implemented
     ...    Test title: DELETE all NS Performance Monitoring Jobs - Method not implemented
     ...    Test objective: The objective is to test that DELETE method is not allowed to delete NS Performance Monitoring Jobs
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.5 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.5 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -173,7 +173,7 @@ GET all NS Performance Monitoring Jobs as Paged Response
     ...    Test title: GET all NS Performance Monitoring Jobs as Paged Response
     ...    Test objective: The objective is to test the retrieval of all the available NS performance monitoring jobs as paged response.
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -186,7 +186,7 @@ GET NS Performance Monitoring Jobs - Bad Request Response too Big
     ...    Test title: GET NS Performance Monitoring Jobs - Bad Request Response too Big
     ...    Test objective: The objective is to test that the retrieval of NS performance monitoring jobs fails because reponse is too big, and perform the JSON schema validation of the failed operation HTTP response
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -199,7 +199,7 @@ GET all NS Performance Monitoring Jobs with "fields" and "exclude_default" attri
     ...    Test title: GET all NS Performance Monitoring Jobs with "fields" and "exclude_default" attribute selector
     ...    Test objective: The objective is to test the retrieval of all NS performance monitoring jobs "fields" and "exclude_default" attribute selector, perform a JSON schema validation of the collected jobs data structure, and verify that the retrieved information matches the issued selector
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 4.3.3.2.1, 7.4.2.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: The NFVO supports the use of "include" attribute selector
     ...    Post-Conditions: none
@@ -213,7 +213,7 @@ POST new NS Performance Monitoring Job - Unprocessable Entity
     ...    Test title: POST new NS Performance Monitoring Job - Unprocessable Entity
     ...    Test objective:  The objective is to test the creation of a new NS performance monitoring job fails when a request contains syntactically correct data but the data cannot be processed.
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance jobs are set in the NFVO.
-    ...    Reference: clause 7.4.2.3.1 - ETSI GS NFV-SOL 005 [3] V3.3.1
+    ...    Reference: clause 7.4.2.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
diff --git a/SOL005/NSPerformanceManagement-API/Thresholds.robot b/SOL005/NSPerformanceManagement-API/Thresholds.robot
index 4ef77d53b..6c662383a 100644
--- a/SOL005/NSPerformanceManagement-API/Thresholds.robot
+++ b/SOL005/NSPerformanceManagement-API/Thresholds.robot
@@ -13,7 +13,7 @@ GET All Performance Thresholds
     ...    Test title: GET All Performance Thresholds
     ...    Test objective: The objective is to test the retrieval of all the available NS performance thresholds and perform a JSON schema validation of the collected thresholds data structure
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -26,7 +26,7 @@ GET Performance Thresholds with attribute-based filter
     ...    Test title: GET Performance Thresholds with attribute-based filter
     ...    Test objective: The objective is to test the retrieval of all the available NS performance thresholds when using attribute-based filters, perform a JSON schema validation of the collected thresholds data structure, and verify that the retrieved information matches the issued attribute-based filter
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -40,7 +40,7 @@ GET Performance Thresholds with invalid attribute-based filter
     ...    Test title: GET Performance Thresholds with invalid attribute-based filter
     ...    Test objective: The objective is to test that the retrieval of NS performance thresholds fails when using invalid attribute-based filter, and perform the JSON schema validation of the failed operation HTTP response
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -53,7 +53,7 @@ GET Performance Thresholds with invalid resource endpoint
     ...    Test title: GET Performance Thresholds with invalid resource endpoint
     ...    Test objective: The objective is to test that the retrieval of NS performance thresholds fails when using invalid resource endpoint, and perform the JSON schema validation of the failed operation HTTP response
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -66,7 +66,7 @@ Create new Performance Threshold
     ...    Test title:  Create new Performance Threshold
     ...    Test objective: The objective is to test the creation of a new NS performance threshold and perform the JSON schema validation of the returned threshold data structure
     ...    Pre-conditions: A NS instance is instantiated.
-    ...    Reference: Clause 7.4.5.3.1 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: The NS performance Threshold is successfully created on the NFVO
@@ -81,7 +81,7 @@ PUT Performance Thresholds - Method not implemented
     ...    Test title: PUT Performance Thresholds - Method not implemented
     ...    Test objective: The objective is to test that PUT method is not allowed to modify NS performance Thresholds
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.3 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.3 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -93,7 +93,7 @@ PATCH Performance Thresholds - Method not implemented
     ...    Test title: PATCH Performance Thresholds - Method not implemented
     ...    Test objective: The objective is to test that PATCH method is not allowed to modify NS performance Thresholds
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.4 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.4 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -105,7 +105,7 @@ DELETE Performance Thresholds - Method not implemented
     ...    Test title: DELETE Performance Thresholds - Method not implemented
     ...    Test objective: The objective is to test that DELETE method is not allowed to update NS performance Thresholds
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.5 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.5 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -117,7 +117,7 @@ GET All Performance Thresholds as Paged Response
     ...    Test title: GET All Performance Thresholds as Paged Response
     ...    Test objective: The objective is to test the retrieval of all the available NS performance thresholds as paged response.
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -130,7 +130,7 @@ GET Performance Thresholds - Bad Request Response too Big
     ...    Test title: GET Performance Thresholds - Bad Request Response too Big
     ...    Test objective: The objective is to test that the retrieval of NS performance thresholds fails because reponse is too big, and perform the JSON schema validation of the failed operation HTTP response
     ...    Pre-conditions: A NS instance is instantiated. One or more NS performance thresholds are set in the NFVO.
-    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.3.1
+    ...    Reference: Clause 7.4.5.3.2 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
@@ -143,7 +143,7 @@ POST new Performance Threshold - Unprocessable Entity
     ...    Test title: POST new Performance Threshold - Unprocessable Entity
     ...    Test objective:  The objective is to test the creation of a new NS performance threshold fails when a request contains syntactically correct data but the data cannot be processed.
     ...    Pre-conditions: A NS instance is instantiated.
-    ...    Reference: clause 7.4.5.3.1 - ETSI GS NFV-SOL 005 [3] V3.3.1
+    ...    Reference: clause 7.4.5.3.1 - ETSI GS NFV-SOL 005 [3] v3.5.1
     ...    Config ID: Config_prod_NFVO
     ...    Applicability: none
     ...    Post-Conditions: none
-- 
GitLab


From f8f4621e6fe5fd6497545083685910f3eb4230c8 Mon Sep 17 00:00:00 2001
From: Giacomo Bernini <g.bernini@nextworks.it>
Date: Thu, 21 Oct 2021 17:35:08 +0200
Subject: [PATCH 2/4] updated SOL005 NS PM job creeation and update with
 mandatory test notification endpoint

---
 .../IndividualPmJob.robot                     |  5 ++++
 .../IndividualThreshold.robot                 |  5 ++++
 .../NSPerformanceManagementKeywords.robot     | 30 +++++++++++++++++--
 .../NSPerformanceManagement-API/PMJobs.robot  |  4 +++
 .../Thresholds.robot                          |  5 ++++
 .../jsons/CreatePmJobRequest.json             |  2 +-
 .../jsons/CreateThresholdRequest.json         |  2 +-
 .../jsons/PmJobModifications.json             |  2 +-
 .../jsons/ThresholdModifications.json         |  2 +-
 9 files changed, 51 insertions(+), 6 deletions(-)

diff --git a/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot b/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot
index 5220e8583..3db568934 100644
--- a/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot
+++ b/SOL005/NSPerformanceManagement-API/IndividualPmJob.robot
@@ -4,6 +4,11 @@ Resource          environment/variables.txt    # Generic Parameters
 Library           JSONLibrary
 Resource          NSPerformanceManagementKeywords.robot
 Library           REST    ${NFVO_SCHEMA}://${NFVO_HOST}:${NFVO_PORT}    ssl_verify=false
+Library           MockServerLibrary
+Library           Process
+
+Suite Setup       Create Sessions
+Suite Teardown    Terminate All Processes    kill=true
 
 *** Test Cases ***
 GET individual NS Performance Job
diff --git a/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot b/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot
index 1a55bfb77..2a61c5b14 100644
--- a/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot
+++ b/SOL005/NSPerformanceManagement-API/IndividualThreshold.robot
@@ -6,6 +6,11 @@ Resource          NSPerformanceManagementKeywords.robot
 Library           JSONLibrary
 Library           REST    ${NFVO_SCHEMA}://${NFVO_HOST}:${NFVO_PORT}    ssl_verify=false
 Library           OperatingSystem
+Library           MockServerLibrary
+Library           Process
+
+Suite Setup       Create Sessions
+Suite Teardown    Terminate All Processes    kill=true
 
 *** Test Cases ***
 GET Individual Threshold
diff --git a/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot b/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot
index 69f8826c5..29e0c5623 100644
--- a/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot
+++ b/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot
@@ -92,10 +92,19 @@ Send Post Request Create new NS Performance Monitoring Job
     Set Headers    {"Content-Type": "${CONTENT_TYPE_JSON}"}
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
     ${template}=    Get File    jsons/CreatePmJobRequest.json
-    ${body}=        Format String   ${template}     pmJobId=${pmJobId}
+    ${body}=        Format String   ${template}     pmJobId=${pmJobId}    callbackUri=${callback_uri}   callbackEndpoint=${callback_endpoint}   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    collectionPeriod=${collectionPeriod}    reportingPeriod=${reportingPeriod} 
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     POST    ${apiRoot}/${apiName}/${apiMajorVersion}/pm_jobs    ${body}
     ${output}=    Output    response
     Set Suite Variable    ${response}    ${output}
+    Log    Verify producer tested the notification endpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
+
     
 Send POST request for NS Performance Monitoring Job with unprocessable entity
     Log    trying to create a new NS PM Job
@@ -103,10 +112,18 @@ Send POST request for NS Performance Monitoring Job with unprocessable entity
     Set Headers    {"Content-Type": "${CONTENT_TYPE_JSON}"}
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
     ${template}=    Get File    jsons/CreatePmJobRequest.json
-    ${body}=        Format String   ${template}     objectInstanceIds=${objectInstanceIds}    callback_uri=${unreachable_callback_uri}:${callback_port}
+    ${body}=        Format String   ${template}     pmJobId=${pmJobId}    callbackUri=${callback_uri}   callbackEndpoint=${unreachable_callback_uri}   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    collectionPeriod=${collectionPeriod}    reportingPeriod=${reportingPeriod} 
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     POST    ${apiRoot}/${apiName}/${apiMajorVersion}/pm_jobs    ${body}
     ${output}=    Output    response
     Set Suite Variable    ${response}    ${output}
+    Log    Verify producer tested the notification endpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
 
 Send PUT Request for all NS Performance Monitoring Jobs 
     Log    Trying to perform a PUT. This method should not be implemented
@@ -251,9 +268,18 @@ Send Patch request for individual NS Performance Job
     Set Headers  {"Content-Type": "${CONTENT_TYPE_PATCH}"}
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
     ${body}=    Get File    jsons/PmJobModifications.json
+    ${body}=        Format String   ${body}     callbackUri=${callback_uri}   callbackEndpoint=${callback_endpoint}   callbackPort=${callback_port}
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     PATCH    ${apiRoot}/${apiName}/${apiMajorVersion}/pm_jobs/${pmJobId}    ${body}
     ${output}=    Output    response
     Set Suite Variable    ${response}    ${output}
+    Log    Verify producer tested the notification endpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
 
 Send Patch request for individual NS Performance Job - Etag mismatch 
     Log    Trying to perform a PATCH 
diff --git a/SOL005/NSPerformanceManagement-API/PMJobs.robot b/SOL005/NSPerformanceManagement-API/PMJobs.robot
index 404770817..bbb6ef7eb 100644
--- a/SOL005/NSPerformanceManagement-API/PMJobs.robot
+++ b/SOL005/NSPerformanceManagement-API/PMJobs.robot
@@ -6,6 +6,10 @@ Library           JSONLibrary
 Library           OperatingSystem
 Library           REST    ${NFVO_SCHEMA}://${NFVO_HOST}:${NFVO_PORT}    ssl_verify=false
 Library           MockServerLibrary
+Library           Process
+
+Suite Setup       Create Sessions
+Suite Teardown    Terminate All Processes    kill=true
 
 *** Test Cases ***
 GET all NS Performance Monitoring Jobs
diff --git a/SOL005/NSPerformanceManagement-API/Thresholds.robot b/SOL005/NSPerformanceManagement-API/Thresholds.robot
index 6c662383a..60f8f13ac 100644
--- a/SOL005/NSPerformanceManagement-API/Thresholds.robot
+++ b/SOL005/NSPerformanceManagement-API/Thresholds.robot
@@ -6,6 +6,11 @@ Resource          NSPerformanceManagementKeywords.robot
 Library           JSONLibrary
 Library           REST    ${NFVO_SCHEMA}://${NFVO_HOST}:${NFVO_PORT}    ssl_verify=false
 Library           OperatingSystem
+Library           MockServerLibrary
+Library           Process
+
+Suite Setup       Create Sessions
+Suite Teardown    Terminate All Processes    kill=true
 
 *** Test Cases ***
 GET All Performance Thresholds
diff --git a/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json b/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json
index f9f668265..2f55c065c 100644
--- a/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json
+++ b/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json
@@ -6,5 +6,5 @@
 		"collectionPeriod": {collectionPeriod},
 		"reportingPeriod": {reportingPeriod}
 	}},
-	"callbackUri": "{callback_uri}"
+	"callbackUri": "{callbackUri}:{callbackPort}{callbackEndpoint}"
 }}
\ No newline at end of file
diff --git a/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json b/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json
index 44a0b5981..145f2d681 100644
--- a/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json
+++ b/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json
@@ -8,5 +8,5 @@
 			"hysteresis": {hysteresis}
 		}}	
 	}},
-	"callbackUri": "{callback_uri}"
+	"callbackUri": "{callbackUri}:{callbackPort}{callbackEndpoint}"
 }}
\ No newline at end of file
diff --git a/SOL005/NSPerformanceManagement-API/jsons/PmJobModifications.json b/SOL005/NSPerformanceManagement-API/jsons/PmJobModifications.json
index f66ab98ec..3e630d3b0 100644
--- a/SOL005/NSPerformanceManagement-API/jsons/PmJobModifications.json
+++ b/SOL005/NSPerformanceManagement-API/jsons/PmJobModifications.json
@@ -1,3 +1,3 @@
 {
-	"callbackUri": ""
+	"callbackUri": "{callbackUri}:{callbackPort}{callbackEndpoint}"
 }
\ No newline at end of file
diff --git a/SOL005/NSPerformanceManagement-API/jsons/ThresholdModifications.json b/SOL005/NSPerformanceManagement-API/jsons/ThresholdModifications.json
index 60c7f184d..3e630d3b0 100644
--- a/SOL005/NSPerformanceManagement-API/jsons/ThresholdModifications.json
+++ b/SOL005/NSPerformanceManagement-API/jsons/ThresholdModifications.json
@@ -1,3 +1,3 @@
 {
-	"callbackUri":""
+	"callbackUri": "{callbackUri}:{callbackPort}{callbackEndpoint}"
 }
\ No newline at end of file
-- 
GitLab


From 147443441dd3b5dd503502d435dfb13b0368ad08 Mon Sep 17 00:00:00 2001
From: Giacomo Bernini <g.bernini@nextworks.it>
Date: Fri, 22 Oct 2021 11:49:23 +0200
Subject: [PATCH 3/4] updated SOL005 NS PM threshold creeation and update with
 mandatory test notification endpoint

---
 .../NSPerformanceManagementKeywords.robot     | 63 ++++++++++++++-----
 .../environment/variables.txt                 |  3 +
 .../jsons/CreatePmJobRequest.json             |  2 +-
 .../jsons/CreateThresholdRequest.json         |  7 ++-
 4 files changed, 56 insertions(+), 19 deletions(-)

diff --git a/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot b/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot
index 29e0c5623..15bd31168 100644
--- a/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot
+++ b/SOL005/NSPerformanceManagement-API/NSPerformanceManagementKeywords.robot
@@ -92,7 +92,7 @@ Send Post Request Create new NS Performance Monitoring Job
     Set Headers    {"Content-Type": "${CONTENT_TYPE_JSON}"}
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
     ${template}=    Get File    jsons/CreatePmJobRequest.json
-    ${body}=        Format String   ${template}     pmJobId=${pmJobId}    callbackUri=${callback_uri}   callbackEndpoint=${callback_endpoint}   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    collectionPeriod=${collectionPeriod}    reportingPeriod=${reportingPeriod} 
+    ${body}=        Format String   ${template}     pmJobObjectInstanceId=${pmJobObjectInstanceId}    callbackUri=${callback_uri}   callbackEndpoint=${callback_endpoint}   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    collectionPeriod=${collectionPeriod}    reportingPeriod=${reportingPeriod} 
     Log  Creating mock request and response to handle GET operation on notification endpoint
     &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
     &{notification_response}=  Create Mock Response	status_code=204
@@ -112,18 +112,10 @@ Send POST request for NS Performance Monitoring Job with unprocessable entity
     Set Headers    {"Content-Type": "${CONTENT_TYPE_JSON}"}
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
     ${template}=    Get File    jsons/CreatePmJobRequest.json
-    ${body}=        Format String   ${template}     pmJobId=${pmJobId}    callbackUri=${callback_uri}   callbackEndpoint=${unreachable_callback_uri}   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    collectionPeriod=${collectionPeriod}    reportingPeriod=${reportingPeriod} 
-    Log  Creating mock request and response to handle GET operation on notification endpoint
-    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
-    &{notification_response}=  Create Mock Response	status_code=204
-    Log    Issue the request
-    Create Mock Expectation  ${notification_request}  ${notification_response}
+    ${body}=        Format String   ${template}     pmJobObjectInstanceId=${pmJobObjectInstanceId}    callbackUri=${callback_uri}   callbackEndpoint=${unreachable_callback_uri}   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    collectionPeriod=${collectionPeriod}    reportingPeriod=${reportingPeriod} 
     POST    ${apiRoot}/${apiName}/${apiMajorVersion}/pm_jobs    ${body}
     ${output}=    Output    response
     Set Suite Variable    ${response}    ${output}
-    Log    Verify producer tested the notification endpoint
-	Verify Mock Expectation    ${notification_request}
-	Clear Requests    ${callback_endpoint}
 
 Send PUT Request for all NS Performance Monitoring Jobs 
     Log    Trying to perform a PUT. This method should not be implemented
@@ -289,9 +281,17 @@ Send Patch request for individual NS Performance Job - Etag mismatch
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
     ${template}=    Get File    jsons/PmJobModifications.json
     ${body}=        Format String   ${template}    callbackUri=${callback_uri}    callbackEndpoint=${callback_endpoint}    callbackPort=${callback_port}
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     PATCH    ${apiRoot}/${apiName}/${apiMajorVersion}/pm_jobs/${pmJobId}    ${body}
     ${output}=    Output    response
     Set Suite Variable    ${response}    ${output}
+    Log    Verify producer tested the notification endpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
 
 
 Send Patch request for individual NS Performance Job with unprocessable entity
@@ -420,10 +420,18 @@ Send Post Request Create new Performance Threshold
     Set Headers    {"Accept": "${ACCEPT_JSON}"}
     Set Headers    {"Content-Type": "${CONTENT_TYPE_JSON}"}
     ${template}=    Get File    jsons/CreateThresholdRequest.json
-    ${body}=        Format String   ${template}     pmJobId=${pmJobId}
+    ${body}=        Format String   ${template}     thresholdObjectInstanceId=${thresholdObjectInstanceId}    callbackUri=${callback_uri}   callbackEndpoint=${callback_endpoint}
+    ...   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    thresholdType=${thresholdType}    thresholdValue=${thresholdValue}    hysteresis=${hysteresis} 
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     POST    ${apiRoot}/${apiName}/${apiMajorVersion}/thresholds    ${body}
     ${output}=    Output    response
-    Set Suite Variable    ${response}    ${output}
+    Set Suite Variable    ${response}    ${output}ndpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
     
 Send POST request for Performance threshold with unprocessable entity
     Log    Creating a new Threshold
@@ -431,10 +439,18 @@ Send POST request for Performance threshold with unprocessable entity
     Set Headers    {"Accept": "${ACCEPT_JSON}"}
     Set Headers    {"Content-Type": "${CONTENT_TYPE_JSON}"}
     ${template}=    Get File    jsons/CreateThresholdRequest.json
-    ${body}=        Format String   ${template}    callback_uri=${unreachable_callback_uri}:${callback_port}
+    ${body}=        Format String   ${template}     thresholdObjectInstanceId=${thresholdObjectInstanceId}    callbackUri=${callback_uri}   callbackEndpoint=${unreachable_callback_uri}
+    ...   callbackPort=${callback_port}    performanceMetric=${performanceMetric}    thresholdType=${thresholdType}    thresholdValue=${thresholdValue}    hysteresis=${hysteresis}
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     POST    ${apiRoot}/${apiName}/${apiMajorVersion}/thresholds    ${body}
     ${output}=    Output    response
-    Set Suite Variable    ${response}    ${output}
+    Set Suite Variable    ${response}    ${output}ndpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
 
 Send PUT Request for all Performance Thresholds
     Log    PUT THresholds
@@ -530,10 +546,19 @@ Send Patch request for individual NS performance Threshold
     Set Headers    {"Accept": "${ACCEPT_JSON}"}
     Set Headers  {"Content-Type": "${CONTENT_TYPE_PATCH}"}
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
-    ${body}=    Get File    jsons/ThresholdModifications.json
+    ${body}=    Get File    jsons/ThresholdModifications.json  
+    ${body}=        Format String   ${body}     callbackUri=${callback_uri}   callbackEndpoint=${callback_endpoint}   callbackPort=${callback_port}
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     PATCH    ${apiRoot}/${apiName}/${apiMajorVersion}/thresholds/${thresholdId}    ${body}
     ${output}=    Output    response
     Set Suite Variable    ${response}    ${output}
+    Log    Verify producer tested the notification endpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
  
 Send Patch request for individual NS performance Threshold - Etag mismatch
     Log    Trying to PUT threshold
@@ -543,9 +568,17 @@ Send Patch request for individual NS performance Threshold - Etag mismatch
     Run Keyword If    ${AUTH_USAGE} == 1    Set Headers    {"${AUTHORIZATION_HEADER}":"${AUTHORIZATION_TOKEN}"}
     ${template}=    Get File    jsons/ThresholdModifications.json
     ${body}=        Format String   ${template}    callbackUri=${callback_uri}    callbackEndpoint=${callback_endpoint}    callbackPort=${callback_port}
+    Log  Creating mock request and response to handle GET operation on notification endpoint
+    &{notification_request}=  Create Mock Request Matcher	GET  ${callback_endpoint}
+    &{notification_response}=  Create Mock Response	status_code=204
+    Log    Issue the request
+    Create Mock Expectation  ${notification_request}  ${notification_response}
     PATCH    ${apiRoot}/${apiName}/${apiMajorVersion}/thresholds/${thresholdId}    ${body}
     ${output}=    Output    response
     Set Suite Variable    ${response}    ${output}
+    Log    Verify producer tested the notification endpoint
+	Verify Mock Expectation    ${notification_request}
+	Clear Requests    ${callback_endpoint}
   
 Send Patch request for Individual Threshold with unprocessable entity
     Set Headers  {"Accept":"${ACCEPT_JSON}"} 
diff --git a/SOL005/NSPerformanceManagement-API/environment/variables.txt b/SOL005/NSPerformanceManagement-API/environment/variables.txt
index 647fed250..e8b303c04 100644
--- a/SOL005/NSPerformanceManagement-API/environment/variables.txt
+++ b/SOL005/NSPerformanceManagement-API/environment/variables.txt
@@ -29,6 +29,9 @@ ${invalid_etag}    invalid etag
 ${total_polling_time}   2 min
 ${polling_interval}     10 sec
 
+${objectType}           my_type
+${pmJobObjectInstanceId}     my_id
+${thresholdObjectInstanceId}     my_id
 ${performanceMetric}    cpu_util
 ${thresholdType}        SIMPLE
 ${thresholdValue}       10
diff --git a/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json b/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json
index 2f55c065c..c0fb7de33 100644
--- a/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json
+++ b/SOL005/NSPerformanceManagement-API/jsons/CreatePmJobRequest.json
@@ -1,5 +1,5 @@
 {{
-	"objectInstanceIds": ["{pmJobId}"],
+	"objectInstanceIds": ["{pmJobObjectInstanceId} "],
 	"criteria": {{
 		"performanceMetric": "{performanceMetric}",
 		"performanceMetricGroup": [],
diff --git a/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json b/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json
index 145f2d681..5572d8469 100644
--- a/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json
+++ b/SOL005/NSPerformanceManagement-API/jsons/CreateThresholdRequest.json
@@ -1,11 +1,12 @@
 {{
-	"objectInstanceIds" :  "{pmJobId}",
+	"objectType": "{objectType}",
+	"objectInstanceId" :  "{thresholdObjectInstanceId}",
 	"criteria" : {{
 		"performanceMetric": "{performanceMetric}",
 		"thresholdType": "{thresholdType}",
 		"simpleThresholdDetails": {{
-			"thresholdValue": {thresholdValue},
-			"hysteresis": {hysteresis}
+			"thresholdValue": "{thresholdValue}",
+			"hysteresis": "{hysteresis}"
 		}}	
 	}},
 	"callbackUri": "{callbackUri}:{callbackPort}{callbackEndpoint}"
-- 
GitLab


From 6c9ed26be3e2453992095246e0cb0ac24fcf96f3 Mon Sep 17 00:00:00 2001
From: Elian Kraja <e.kraja@nextworks.it>
Date: Fri, 29 Oct 2021 18:14:02 +0200
Subject: [PATCH 4/4] Removing unused variables

---
 .../environment/variables.txt                        | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/SOL005/NSPerformanceManagement-API/environment/variables.txt b/SOL005/NSPerformanceManagement-API/environment/variables.txt
index e8b303c04..33dcf9430 100644
--- a/SOL005/NSPerformanceManagement-API/environment/variables.txt
+++ b/SOL005/NSPerformanceManagement-API/environment/variables.txt
@@ -15,21 +15,25 @@ ${NFVO_CHECKS_NOTIF_ENDPOINT}    1
 
 ${MOCK_SERVER_JAR}    ../../../bin/mockserver-netty-5.5.0-jar-with-dependencies.jar
 
-${testOptionalMethods}    0
+#${testOptionalMethods}    0
+
 
 ${callback_port}    9091
 ${callback_uri}    http://172.22.1.7
 ${unreachable_callback_uri}    http://not-reachable-uri
 ${callback_endpoint}    /nspm/subscriptions
-${callback_endpoint_error}    /subs_404
-${sleep_interval}    20s
+#${callback_endpoint_error}    /subs_404
+
+#${sleep_interval}    20s
+
 
 ${invalid_etag}    invalid etag
 
 ${total_polling_time}   2 min
 ${polling_interval}     10 sec
 
-${objectType}           my_type
+#${objectType}           my_type
+
 ${pmJobObjectInstanceId}     my_id
 ${thresholdObjectInstanceId}     my_id
 ${performanceMetric}    cpu_util
-- 
GitLab