Commits (147)
......@@ -3,4 +3,4 @@ results
*.pyc
*.http
.vscode/
.idea
.venv
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="All Unit Tests" type="tests" factoryName="Unittests">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="$PROJECT_DIR$/.venv/bin/python" />
<option name="SDK_NAME" value="Python 3.11 (ngsi-ld-test-suite)" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="false" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_pattern" value="&quot;&quot;" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="CheckTests Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_CheckTests.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Check Initial Setup" type="PythonConfigurationType" factoryName="Python">
<module name="ngsi-ld-test-suite" />
<option name="ENV_FILES" value="" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc/analysis" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/doc/analysis/initial_setup.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="CommonBehaviours Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_CommonBehaviours.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ContextInformation Consumption Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_ContextInformation_Consumption.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ContextInformation Provision Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_ContextInformation_Provision.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ContextInformation Subscription Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_ContextInformation_Subscription.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ContextSource Discovery Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_ContextSource_Discovery.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ContextSource RegistrationSubscription Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_ContextSource_RegistrationSubscription.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="ContextSource Registration Unit Tests" type="tests" factoryName="Autodetect">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="_new_additionalArguments" value="&quot;&quot;" />
<option name="_new_target" value="&quot;$PROJECT_DIR$/doc/tests/test_ContextSource_Registration.py&quot;" />
<option name="_new_targetType" value="&quot;PATH&quot;" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Generate Documentation Data" type="PythonConfigurationType" factoryName="Python">
<module name="ngsi-ld-test-suite" />
<option name="ENV_FILES" value="" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/doc/generateDocumentationData.py" />
<option name="PARAMETERS" value="047_01" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Run Selected Text" type="RobotFramework" factoryName="RobotFramework">
<option name="target" value="$FilePath$" />
<option name="args">
<list>
<option value="--outputdir" />
<option value="./results" />
<option value="--test" />
<option value="*$SelectedText$*" />
</list>
</option>
<option name="env">
<map />
</option>
<option name="workingDir" value="$ProjectFileDir$" />
<option name="makeSuite" value="true" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
<component name="ProjectRunConfigurationManager">
<configuration default="false" name="Statistics Documentation Data" type="PythonConfigurationType" factoryName="Python">
<module name="ngsi-ld-test-suite" />
<option name="INTERPRETER_OPTIONS" value="" />
<option name="PARENT_ENVS" value="true" />
<envs>
<env name="PYTHONUNBUFFERED" value="1" />
</envs>
<option name="SDK_HOME" value="" />
<option name="WORKING_DIRECTORY" value="$PROJECT_DIR$/doc" />
<option name="IS_MODULE_SDK" value="true" />
<option name="ADD_CONTENT_ROOTS" value="true" />
<option name="ADD_SOURCE_ROOTS" value="true" />
<EXTENSION ID="PythonCoverageRunConfigurationExtension" runner="coverage.py" />
<option name="SCRIPT_NAME" value="$PROJECT_DIR$/doc/statisticsDocumentationData.py" />
<option name="PARAMETERS" value="" />
<option name="SHOW_COMMAND_LINE" value="false" />
<option name="EMULATE_TERMINAL" value="false" />
<option name="MODULE_MODE" value="false" />
<option name="REDIRECT_INPUT" value="false" />
<option name="INPUT_FILE" value="" />
<method v="2" />
</configuration>
</component>
\ No newline at end of file
......@@ -30,11 +30,11 @@ In order to install the ETSI NGSI-LD Test Suite, download the configuration scri
- For MacOS and Ubuntu, download the following file:
```$ curl https://forge.etsi.org/rep/cim/ngsi-ld-test-suite/-/raw/windows11/scripts/configure.sh > configure.sh```
```$ curl https://forge.etsi.org/rep/cim/ngsi-ld-test-suite/-/raw/develop/scripts/configure.sh > configure.sh```
- For Windows, using Powershell download the following file (curl is an alias for Invoke-WebRequest in Powershell):
```> curl https://forge.etsi.org/rep/cim/ngsi-ld-test-suite/-/raw/windows11/scripts/configure.ps1 > configure.ps1```
```> curl https://forge.etsi.org/rep/cim/ngsi-ld-test-suite/-/raw/develop/scripts/configure.ps1 > configure.ps1```
- For MacOS and Ubuntu, be sure that you have the proper execution permissions of the file and the user is included
in the sudoers group, then execute the following script:
......@@ -63,7 +63,7 @@ In the `resources/variables.py` file, configure the following parameters:
- `url` : It is the url of the context broker which is to be tested (including the `/ngsi-ld/v1` path,
e.g., http://localhost:8080/ngsi-ld/v1).
-`temporal_api_url` : This is the url of the GET temporal operation API, in case that a Context Broker splits
- `temporal_api_url` : This is the url of the GET temporal operation API, in case that a Context Broker splits
this portion of the API (e.g., http://localhost:8080/ngsi-ld/v1).
- `ngsild_test_suite_context` : This is the url of the default context used in the ETSI NGSI-LD requests
(e.g., 'https://forge.etsi.org/rep/cim/ngsi-ld-test-suite/-/raw/develop/resources/jsonld-contexts/ngsi-ld-test-suite-compound.jsonld').
......@@ -84,6 +84,20 @@ Docker host.
``` ifconfig docker0 | grep inet | awk '{print $2}' ``` and that IP is the one that you need to use for
notification purposes.
Optionally, there are some extra variables that can be used during the generation of execution results of the Test
Case with a specific Robot Listener class to automatically generate a GitHub Issue in the GitHub repository of the
Context Broker of a failed Test Case. In case that you cannot or do not want to use this functionality, delete those
variables from the file.
As an explanation of the process, the GitHub Issue will be created if there is no other issue in the repository with
the same name or there is an issue with the same name, but it is not closed.
In order to create these issues, the [GitHub REST API](https://docs.github.com/en/rest) is used. For this purpose,
the authentication process is using a personal access token. The needed variables are the following:
* `github_owner`: Your GitHub user account.
* `github_broker_repo` : The corresponding URL of the Context Broker repository.
* `github_token` : Your personal access token. Please take a look to the GitHub documentation if you want to generate
your own [personal access token](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens).
## Execute the NGSI-LD Test Suite
......@@ -100,7 +114,7 @@ Now, you can launch the tests with the following command in MacOS or Linux:
```$ robot --outputdir ./results .```
For Windows system, you can lauch the tests with the following command:
For Windows system, you can launch the tests with the following command:
```> robot --outputdir .\results .\TP\NGSI-LD```
......@@ -131,10 +145,107 @@ test launch command followed by the file name.
> > .venv\scripts\deactivate.bat
> ```
## Test Suite Management (tsm)
The `tsm` script is designed to facilitate the selection and execution of the Test Suite, especially if not all the
endpoints of the API have been implemented for a specific Context Broker. This script provides a set of commands
to enable or disable Test Cases, Robot Test Suites or Collections (Robot Test Suite Groups), visualize the current
status of the different Test Cases, execute the selected Test Cases and perform other related operations as described
below.
The `tsm` script generates a pickle file named `tsm.pkl`, which stores the tuples list corresponding to each Test Case
with the following information:
(switch, running status, Test Case long name)
where the values and their meaning are the following:
* **switch**:
* `ON`: the Test Case is on, which means that it will be executed by the script.
* `OFF`: the Test Case is off, and therefore is not selected to be executed by the script.
* `MISSING`: the Test Case is not anymore in the Test Suite Structure. An update operation should be run to update
the `tsm.pkl` with the current set of available Test Cases in the filesystem.
* `NEW`: new Test Case discovered by the tsm after an update operation.
* **status**:
* `PASSED`: the robot framework executes the Test Case with result PASS.
* `FAILED`: the robot framework executes the Test Case with result FAIL.
* `PENDING`: the Test Case is pending to be executed by the robot framework.
* **test case long name**: the Test Case long name set by robot framework based on the Robot Test Suite number and
the Test Case name (e.g., NGSILD.032 02.032_02_01 Delete Unknown Subscription)
### Installation
The `tsm` script is integrated with arguments auto-completion for bash, therefore it is needed the installation of
the argcomplete python package on your system:
pip install argcomplete
and to enable the completion after the installation executing the following command:
activate-global-python-argcomplete
and then
eval "$(register-python-argcomplete tsm)"
Now, it is possible to autocomplete the commands and show possible options executing the script. Also –help argument
is available to obtain more information about the options.
### Execution
The tsm cases update command updates the `tsm.pkl` file with all the Robot Test Suite under the local path. If the
pickle file does not exist, it is created. After the creation of this file, it is possible to execute the script
to maintain and run the selected Test Cases from the pickle file. The list of commands is the following:
* **Test Cases (cases)**
* Switch ON Test Cases
tsm cases on [test_cases]
* Switch OFF Test Cases
tsm cases off [test_cases]
tsm cases off "NGSILD.032 01.032_01_02 InvalidId"
* List Test Cases based on the specific flag.
tsm cases list [on, off, missing, new, passed, failed, pending, all]
tsm cases list ./TP/NGSI-LD/CommonBehaviours
* Run Test Cases that are enabled
tsm cases run [on, off, missing, new, passed, failed, pending, [test_cases]]
tsm cases run NGSILD.048\ 01.048_01_06\ Endpoint\ post\ /temporal/entities/
tsm cases run pending
* Update the pickle file with the current Test Cases
tsm cases update
* Clean Test Cases, remove the Test Cases that were marked as MISSING
tsm cases clean
* **Robot Test Suites (suites)**
* Switch ON Robot Test Suites
tsm suites on [suites]
* Switch OFF Robot Test Suites
tsm suites off [suites]
* **Test Collections (collections)**
* Switch ON Test Collections
tsm collections on [collections]
tsm collections on ./TP/NGSI-LD/CommonBehaviours
* Switch OFF Test Collections
tsm collections off [collections]
## Contribute to the Test Suite
In order to contribute to the ETSI NGSI-LD Test Suite, you recommend the installation of an IDE with the corresponding
In order to contribute to the ETSI NGSI-LD Test Suite, it is recommended to install an IDE with the corresponding
Robot Framework. Our recommendations are:
- PyCharm
......@@ -146,10 +257,109 @@ Robot Framework. Our recommendations are:
- Install [Robot Framework Language Server](https://plugins.jetbrains.com/plugin/16086-robot-framework-language-server)
- Define as variable the path of the working directory. In Settings > Languages & Frameworks > Robot Framework (Project),
insert the following: `{"EXECDIR": "{path}/auth-test-suite"}`
- Define as variable the path of the working directory. In Settings > Languages & Frameworks > Robot Framework
(Project), insert the following: `{"EXECDIR": "{path}/ngsi-ld-test-suite"}`.
### Develop a new Test Case
In order to develop a new Test Case, some rules and conventions have to be followed.
#### Test Case Template
The following template shows the typical structure of a Test Case. It can be used as a template when creating a new
Test Case.
```
*** Settings ***
Documentation {Describe the behavior that is being checked by this Test Case}
Resource ${EXECDIR}/resources/any/necessary/resource/file
# For both setup and teardown steps, try as possible to use Keyword names already declared in doc/analysis/initial_setup.py
Suite Setup {An optional setup step to create data necessary for the Test Case}
Suite Teardown {An optional teardown step to delete data created in the setup step}
Test Template {If the Test Case uses permutations, the name of the Keyword that is called for each permutation}
*** Variables ***
${my_variable}= my_variable
*** Test Cases *** PARAM_1 PARAM_2
XXX_YY_01 Purpose of the first permutation
[Tags] {resource_request_reference} {section_reference_1} {section_reference_2}
param_1_value param_2_value
XXX_YY_02 Purpose of the second permutation
[Tags] {resource_request_reference} {section_reference_1} {section_reference_2}
param_1_value param_2_value
*** Keywords ***
{Keyword name describing what the Test Case is doing}
[Documentation] {Not sure this documentation brings something?}
[Arguments] ${param_1} ${param_2}
# Call operation that is being tested, passing any needed argument
# Perform checks on the response that has been received from the operation
# Optionally call another endpoint to do extra checks (e.g. check an entity has really been created)
# Add there keywords for setup and teardown steps
# Setup step typically creates data necessary for the Test Case and set some variables that will be used by the Test Case
# using Set Suite Variable or Set Test Variable keywords
# Teardown step must delete any data that has been created in setup step
```
Where :
- The possible values for `{resource_request_reference}` are defined in DGR/CIM-0015v211, section 6.1
- The meaning of the `{section_reference_x}` tags is defined in DGR/CIM-0015v211, section 6.2. A Test Case can reference
more than one section in the specification (for instance, a Test Case whose goal is to check that scopes are correctly
created when an entity is created should reference sections 4.18 NGSI-LD Scopes and 5.6.1 Create Entity of the
RGS/CIM-009v131 document)
#### Generate the documentation
The Conformance Tests include an option to automatically generate the documentation of them. If new tests are developed
but include the Check and Request operations already defined, it is not needed to modify the automatic generation system.
Nevertheless, if there are new operations to check and/or new request operations to develop in the Conformance Tests,
it is needed to describe them so that the automatic documentation generated covers these new operations.
Additionally, there is defined a Unit Test mechanism to identify if new Test Suites are included in the system
(See `doc/tests/test_CheckTests.py`) and if there were some changes in any Test Suite (See the `doc/tests/other test_*.py`).
In these cases, it is needed to provide the corresponding information in the Python code:
- When a new Test Case is created, it has to be declared in the corresponding `doc/tests/test_{group}.py` Python file
(where `{group}` represents the group of operations and how they are defined in the ETSI RGS/CIM-0012v211 document).
Complementary to this, updates have to be done:
- In `doc/analysis/initial_setup.py` if it uses a new keyword to set up the initial state of the Test Case
- In `doc/analysis/requests.py` if it updates an endpoint operation, for instance by adding support for a new request
parameter (in `self.op` with the list and position of parameters, in the method that pretty prints the operation in
the automated generation of the documentation associated with the Test Cases. If the positions array is empty, it
means that all the parameters will be taken into consideration)
- Run the documentation generation script (`python doc/generateDocumentationData.py {new_tc_id}`) for the new Test
Case and copy the generated JSON file in the folder containing all files for the given group and subgroup
(`cp doc/results/{new_tc_id}.json doc/files/{group}/{subgroup}`)
- When a new global Keyword is created, it has to be declared in the corresponding Python files:
- In `doc/analysis/checks.py` if it is an assertion type keyword (in `self.checks` to reference the method that pretty
prints the operation and `self.args` to identify the position of the arguments) and the corresponding method to
provide the description of this new operation
- In `doc/analysis/requests.py` if it is an endpoint operation (in `self.op` with the list and position of parameters,
in `self.description` to reference the method that pretty print the operation, and add the method that pretty prints
the operation)
- When a new permutation is added in an existing Test Case, run the documentation generation script
(`python doc/generateDocumentationData.py {tc_id}`) for the Test Case and copy the generated JSON file in the
folder containing all files for the given group and subgroup (`cp doc/results/{tc_id}.json doc/files/{group}/{subgroup}`)
- When a new directory containing Test Cases is created, it has to be declared in `doc/generaterobotdata.py` along with
its acronym
Finally, check that everything is OK by running the unit tests:
```shell
python -m unittest discover -s ./doc/tests -t ./doc
```
### Run configurations (PyCharm)
Two sample configurations have been created:
......@@ -225,7 +435,18 @@ And, if you want to generate a documentation for the Test Cases:
```$ python3 -m robot.testdoc TP/NGSI-LD api_docs/TestCases.html```
### Coding Style of Test Suites
## Generate output file details only for failed tests
It is possible to generate a report only for the failed tests through the use of a specific listener in the execution
of the robot framework. For example, if you want to execute the test suite number 043 and generate the report, you can
execute the following command:
```robot --listener libraries/ErrorListener.py --outputdir ./results ./TP/NGSI-LD/CommonBehaviours/043.robot```
It will generate a specific `errors.log` file into the `results` folder with the description of the different steps
developed and the mismatched observed on them.
## Coding Style of Test Suites
And if you want to tidy (code style) the Test Suites:
......
......@@ -29,26 +29,31 @@ ${content_type}= application/json
*** Test Cases ***
045_01_01 Endpoint /entities/{entityId}
[Documentation] Verify that on a GET HTTP request if nothing is specified on the Accept header, "application/json" is assumed (/entities/{entityId})
[Tags] e-query 6_3_4
[Tags] e-query cb-get 6_3_4
${id}= Generate Random Entity Id ${building_id_prefix}
${response}= Create Entity Selecting Content Type
... ${building_filename}
... ${id}
... ${CONTENT_TYPE_LD_JSON}
Check Response Status Code 201 ${response.status_code}
${response}= Query Entity ${id} context=${ngsild_test_suite_context} accept=*/*
${response}= Query Entity
... id=${id}
... context=${ngsild_test_suite_context}
... accept=*/*
Check Response Status Code 200 ${response.status_code}
Check Response Headers Containing Content-Type set to ${content_type} ${response.headers}
Check Response Headers Link Not Empty ${response.headers}
[Teardown] Delete Entity by Id Returning Response ${id}
[Teardown] Delete Entity by Id ${id}
045_01_02 Endpoint /subscriptions/{subscriptionId}
[Documentation] Verify that on a GET HTTP request if nothing is specified on the Accept header, "application/json" is assumed (/subscriptions/{subscriptionId})
[Tags] sub-retrieve 6_3_4
[Tags] sub-retrieve cb-get 6_3_4
${id}= Generate Random Entity Id ${subscription_id_prefix}
${response}= Create Subscription ${id} ${subscription_filename} ${CONTENT_TYPE_LD_JSON}
Check Response Status Code 201 ${response.status_code}
${response}= Retrieve Subscription ${id} accept=*/*
${response}= Retrieve Subscription
... id=${id}
... accept=*/*
Check Response Status Code 200 ${response.status_code}
Check Response Headers Containing Content-Type set to ${content_type} ${response.headers}
Check Response Headers Link Not Empty ${response.headers}
......@@ -56,7 +61,7 @@ ${content_type}= application/json
045_01_03 Endpoint /csourceRegistrations/
[Documentation] Verify that on a GET HTTP request if nothing is specified on the Accept header, "application/json" is assumed (/csourceRegistrations/)
[Tags] csr-query 6_3_4
[Tags] csr-query cb-get 6_3_4
${registration_id}= Generate Random Entity Id ${registration_id_prefix}
${payload}= Load JSON From File ${EXECDIR}/data/${registration_filename}
${updated_payload}= Update Value To JSON ${payload} $..id ${registration_id}
......@@ -74,12 +79,12 @@ ${content_type}= application/json
045_01_04 Endpoint /temporal/entities
[Documentation] Verify that on a GET HTTP request if nothing is specified on the Accept header, "application/json" is assumed (/temporal/entities)
[Tags] te-query 6_3_4
[Tags] te-query cb-get 6_3_4
${temporal_entity_representation_id}= Generate Random Entity Id ${tea_id_prefix}
${response}= Create Or Update Temporal Representation Of Entity Selecting Content Type
... ${temporal_entity_representation_id}
... ${tea_filename}
... ${CONTENT_TYPE_LD_JSON}
... temporal_entity_representation_id=${temporal_entity_representation_id}
... filename=${tea_filename}
... content_type=${CONTENT_TYPE_LD_JSON}
Check Response Status Code 201 ${response.status_code}
${response}= Query Temporal Representation Of Entities With Return
... entity_types=${teatype}
......
......@@ -24,7 +24,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration
*** Test Cases ***
043_01_01 Create entity
[Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create entity)
[Tags] e-create 5_2_2
[Tags] e-create cb-ldcontext 5_2_2
${entity_id}= Generate Random Entity Id ${building_id_prefix}
${response}= Create Entity Selecting Content Type
... ${building_filename}
......@@ -39,7 +39,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration
043_01_02 Create subscription
[Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create subscription)
[Tags] sub-create 5_2_2
[Tags] sub-create cb-ldcontext 5_2_2
${subscription_id}= Generate Random Entity Id ${subscription_id_prefix}
${response}= Create Subscription ${subscription_id} ${subscription_filename} ${CONTENT_TYPE_LD_JSON}
Check Response Status Code ${expected_status_code} ${response.status_code}
......@@ -51,12 +51,12 @@ ${registration_filename}= csourceRegistrations/context-source-registration
043_01_03 Create Temporal Representation of Entities
[Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create Temporal Representation of Entities)
[Tags] te-create 5_2_2
[Tags] te-create cb-ldcontext 5_2_2
${temporal_entity_representation_id}= Generate Random Entity Id ${tea_id_prefix}
${response}= Create Or Update Temporal Representation Of Entity Selecting Content Type
... ${temporal_entity_representation_id}
... ${tea_filename}
... ${CONTENT_TYPE_LD_JSON}
... temporal_entity_representation_id=${temporal_entity_representation_id}
... filename=${tea_filename}
... content_type=${CONTENT_TYPE_LD_JSON}
Check Response Status Code ${expected_status_code} ${response.status_code}
Check Response Body Containing ProblemDetails Element Containing Type Element set to
... ${response.json()}
......@@ -66,7 +66,7 @@ ${registration_filename}= csourceRegistrations/context-source-registration
043_01_04 Batch entity create
[Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Batch entity create)
[Tags] be-create 5_2_2
[Tags] be-create cb-ldcontext 5_2_2
${first_entity_id}= Generate Random Entity Id ${building_id_prefix}
${second_entity_id}= Generate Random Entity Id ${building_id_prefix}
${first_entity}= Load Entity ${building_filename} ${first_entity_id}
......@@ -79,11 +79,11 @@ ${registration_filename}= csourceRegistrations/context-source-registration
... ${response.json()}
... ${ERROR_TYPE_LD_CONTEXT_NOT_AVAILABLE}
Check Response Body Containing ProblemDetails Element Containing Title Element ${response.json()}
[Teardown] Batch Delete Entities @{entities_ids_to_be_created}
[Teardown] Batch Delete Entities entities_ids_to_be_deleted=@{entities_ids_to_be_created}
043_01_05 Create context source registration
[Documentation] Verify receiving 503 – LdContextNotAvailable error if remote JSON-LD @context cannot be retrieved (Create context source registration)
[Tags] csr-create 5_2_2
[Tags] csr-create cb-ldcontext 5_2_2
${registration_id}= Generate Random Entity Id ${registration_id_prefix}
${payload}= Load JSON From File ${EXECDIR}/data/${registration_filename}
${updated_payload}= Update Value To JSON ${payload} $..id ${registration_id}
......
......@@ -2,7 +2,6 @@
Documentation Verify that PATCH HTTP requests can be done with "application/merge-patch+json" as Content-Type
Resource ${EXECDIR}/resources/ApiUtils/ContextInformationProvision.resource
Resource ${EXECDIR}/resources/ApiUtils/ContextInformationSubscription.resource
Resource ${EXECDIR}/resources/AssertionUtils.resource
Resource ${EXECDIR}/resources/JsonUtils.resource
......@@ -18,14 +17,14 @@ ${attribute_id}= brandName
*** Test Cases ***
044_01_01 Endpoint /entities/{entityId}/attrs/{attrId}
044_01_01 endpoint /entities/{entityId}/attrs/{attrId}
[Documentation] Verify that PATCH HTTP requests can be done with "application/merge-patch+json" as Content-Type
[Tags] ea-partial-update 6_3_4
[Tags] ea-partial-update cb-mergepatch 6_3_4
${response}= Partial Update Entity Attributes
... ${entity_id}
... ${attribute_id}
... ${vehicle_fragment}
... ${CONTENT_TYPE_MERGE_PATCH_JSON}
... entityId=${entity_id}
... attributeId=${attribute_id}
... fragment_filename=${vehicle_fragment}
... content_type=${CONTENT_TYPE_MERGE_PATCH_JSON}
... context=${ngsild_test_suite_context}
Check Response Status Code 204 ${response.status_code}
......
......@@ -19,7 +19,7 @@ ${subscription_fragment}= subscriptions/fragments/subscription-update-samp
*** Test Cases ***
044_02_01 Endpoint /subscriptions/{subscriptionId}
[Documentation] Verify that PATCH HTTP requests can be done with "application/merge-patch+json" as Content-Type
[Tags] sub-update 6_3_4
[Tags] sub-update cb-mergepatch 6_3_4
${response}= Update Subscription
... ${subscription_id}
... ${subscription_fragment}
......
......@@ -21,35 +21,39 @@ ${subscription_id_prefix}= urn:ngsi-ld:Subscription:
*** Test Cases ***
049_01_01 Endpoint get /entities/{entityId}
[Documentation] Verify throwing 406 HTTP status code (Not Acceptable Media Type) if the "Accept" header does not imply "application/json" nor "application/ld+json" (get /entities/{entityId})
[Tags] e-query 6_3_4
[Tags] e-query cb-noacceptable-medtype 6_3_4
${entity_id}= Generate Random Entity Id ${building_id_prefix}
${response}= Query Entity ${entity_id} accept=${accept}
${response}= Query Entity
... id=${entity_id}
... accept=${accept}
Check Response Status Code ${status_code} ${response.status_code}
[Teardown] Delete Entity by Id Returning Response ${entity_id}
[Teardown] Delete Entity by Id ${entity_id}
049_01_02 Endpoint get /subscriptions/{subscriptionId}
[Documentation] Verify throwing 406 HTTP status code (Not Acceptable Media Type) if the "Accept" header does not imply "application/json" nor "application/ld+json" (get /subscriptions/{subscriptionId})
[Tags] sub-retrieve 6_3_4
[Tags] sub-retrieve cb-noacceptable-medtype 6_3_4
${id}= Generate Random Entity Id ${subscription_id_prefix}
${response}= Retrieve Subscription ${id} accept=${accept}
${response}= Retrieve Subscription
... id=${id}
... accept=${accept}
Check Response Status Code ${status_code} ${response.status_code}
[Teardown] Delete Subscription ${id}
049_01_03 Endpoint get /csourceRegistrations/
[Documentation] Verify throwing 406 HTTP status code (Not Acceptable Media Type) if the "Accept" header does not imply "application/json" nor "application/ld+json" (get /csourceRegistrations/)
[Tags] csr-query 6_3_4
[Tags] csr-query cb-noacceptable-medtype 6_3_4
${response}= Query Context Source Registrations With Return type=Building accept=${accept}
Check Response Status Code ${status_code} ${response.status_code}
049_01_04 Endpoint get /csourceSubscriptions/
[Documentation] Verify throwing 406 HTTP status code (Not Acceptable Media Type) if the "Accept" header does not imply "application/json" nor "application/ld+json" (get /csourceSubscriptions/)
[Tags] csrsub-query 6_3_4
[Tags] csrsub-query cb-noacceptable-medtype 6_3_4
${response}= Query Context Source Registration Subscriptions accept=${accept}
Check Response Status Code ${status_code} ${response.status_code}
049_01_05 Endpoint get /temporal/entities
[Documentation] Verify throwing 406 HTTP status code (Not Acceptable Media Type) if the "Accept" header does not imply "application/json" nor "application/ld+json" (get /temporal/entities)
[Tags] te-query 6_3_4
[Tags] te-query cb-noacceptable-medtype 6_3_4
${entity_types_to_be_retrieved}= Catenate SEPARATOR=, Vehicle
${response}= Query Temporal Representation Of Entities
... entity_types=${entity_types_to_be_retrieved}
......