align api to security spec & automated ci/cd test for json schema, bnf (#642)

* align api to security spec

* add ci workflow

* fix testcases

Author: Martin187187

.github/workflows/validate-spec-artifacts.yml

@@ -0,0 +1,30 @@
+name: Validate spec artifacts
+
+on:
+  pull_request:
+    types: [opened, synchronize, reopened]
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Fetch sources
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Install validation dependencies
+        run: python -m pip install "jsonschema[format]==4.25.1"
+
+      - name: Validate JSON schemas, examples, and BNF artifacts
+        run: python tools/validate_spec_artifacts.py
+
+      - name: Run query schema and regex tests
+        run: python -m unittest discover -s documentation/IDTA-01002-3/modules/ROOT/pages/http-rest-api/test/query

.gitignore

@@ -0,0 +1,2 @@
+__pycache__/
+*.pyc

Part2-API-Schemas/openapi.yaml

@@ -712,11 +712,11 @@ components:
     FieldIdentifier:
       type: string
       pattern: >-
-        ^(?:\$aas#(?:idShort|id|assetInformation\.assetKind|assetInformation\.assetType|assetInformation\.globalAssetId|assetInformation\.specificAssetIds\[(?:0|[1-9][0-9]*)\]\.(?:name|value|externalSubjectId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?)|submodels\[(?:0|[1-9][0-9]*)\]\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))|\$sm#(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|idShort|id)|\$sme(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[(?:0|[1-9][0-9]*)\])*(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[(?:0|[1-9][0-9]*)\])*)*)?#(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|idShort|value|valueType|language)|\$cd#(?:idShort|id)|\$aasdesc#(?:idShort|id|assetKind|assetType|globalAssetId|specificAssetIds\[(?:0|[1-9][0-9]*)\]\.(?:name|value|externalSubjectId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?)|endpoints\[(?:0|[1-9][0-9]*)\]\.(?:interface|protocolinformation\.href)|submodelDescriptors\[(?:0|[1-9][0-9]*)\]\.(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|idShort|id|endpoints\[(?:0|[1-9][0-9]*)\]\.(?:interface|protocolinformation\.href)))|\$smdesc#(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)\]\.(?:type|value)))?|idShort|id|endpoints\[(?:0|[1-9][0-9]*)\]\.(?:interface|protocolinformation\.href)))$
+        ^(?:\$aas#(?:idShort|id|assetInformation\.assetKind|assetInformation\.assetType|assetInformation\.globalAssetId|assetInformation\.specificAssetIds\[(?:0|[1-9][0-9]*)?\]\.(?:name|value|externalSubjectId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?)|submodels\[(?:0|[1-9][0-9]*)?\](?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?)|\$sm#(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|idShort|id)|\$sme(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[(?:0|[1-9][0-9]*)?\])*(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[(?:0|[1-9][0-9]*)?\])*)*)?#(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|idShort|value|valueType|language)|\$cd#(?:idShort|id)|\$aasdesc#(?:idShort|id|assetKind|assetType|globalAssetId|specificAssetIds\[(?:0|[1-9][0-9]*)?\]\.(?:name|value|externalSubjectId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?)|endpoints\[(?:0|[1-9][0-9]*)?\]\.(?:interface|protocolinformation\.href)|submodelDescriptors\[(?:0|[1-9][0-9]*)?\]\.(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|idShort|id|endpoints\[(?:0|[1-9][0-9]*)?\]\.(?:interface|protocolinformation\.href)))|\$smdesc#(?:semanticId(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\])?(?:\.(?:type|keys\[(?:0|[1-9][0-9]*)?\]\.(?:type|value)))?|idShort|id|endpoints\[(?:0|[1-9][0-9]*)?\]\.(?:interface|protocolinformation\.href)))$
     FragmentFieldIdentifier:
       type: string
       pattern: >-
-        ^(?:\$aas#(?:idShort|assetInformation\.assetType|assetInformation\.globalAssetId|assetInformation\.specificAssetIds(?:\[[0-9]*\](?:\.externalSubjectId(?:\.keys(?:\[[0-9]*\])?)?)?)?|submodels(?:\[[0-9]*\](?:\.keys(?:\[[0-9]*\])?)?)?)|\$sm#(?:semanticId(?:\.keys(?:\[[0-9]*\])?)?|supplementalSemanticIds(?:\[[0-9]*\](?:\.keys(?:\[[0-9]*\])?)?)?|idShort)|\$sme(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[[0-9]*\])*(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[[0-9]*\])*)*)?(?:#(?:semanticId(?:\.keys(?:\[[0-9]*\])?)?|supplementalSemanticIds(?:\[[0-9]*\](?:\.keys(?:\[[0-9]*\])?)?)?|idShort|value|valueType|language))?|\$cd#idShort|\$aasdesc#(?:idShort|description|displayName|extension|administration|assetKind|assetType|globalAssetId|specificAssetIds(?:\[[0-9]*\](?:\.externalSubjectId(?:\.keys(?:\[[0-9]*\])?)?)?)?|endpoints(?:\[[0-9]*\])?|submodelDescriptors(?:\[[0-9]*\](?:\.(?:semanticId(?:\.keys(?:\[[0-9]*\])?)?|supplementalSemanticIds(?:\[[0-9]*\](?:\.keys(?:\[[0-9]*\])?)?)?|idShort|endpoints(?:\[[0-9]*\])?))?)?)|\$smdesc#(?:semanticId(?:\.keys(?:\[[0-9]*\])?)?|supplementalSemanticIds(?:\[[0-9]*\](?:\.keys(?:\[[0-9]*\])?)?)?|idShort|endpoints(?:\[[0-9]*\])?))$
+        ^(?:\$aas#(?:idShort|assetInformation\.assetType|assetInformation\.globalAssetId|assetInformation\.specificAssetIds(?:\[(?:0|[1-9][0-9]*)?\](?:\.externalSubjectId(?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?)?)?|submodels(?:\[(?:0|[1-9][0-9]*)?\](?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?)?)|\$sm#(?:semanticId(?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\](?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?)?|idShort)|\$sme(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[(?:0|[1-9][0-9]*)?\])*(?:\.[A-Za-z](?:[A-Za-z0-9_-]*[A-Za-z0-9_])?(?:\[(?:0|[1-9][0-9]*)?\])*)*)?(?:#(?:semanticId(?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\](?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?)?|idShort|value|valueType|language))?|\$cd#idShort|\$aasdesc#(?:idShort|description|displayName|extension|administration|assetKind|assetType|globalAssetId|specificAssetIds(?:\[(?:0|[1-9][0-9]*)?\](?:\.externalSubjectId(?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?)?)?|endpoints(?:\[(?:0|[1-9][0-9]*)?\])?|submodelDescriptors(?:\[(?:0|[1-9][0-9]*)?\](?:\.(?:semanticId(?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\](?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?)?|idShort|endpoints(?:\[(?:0|[1-9][0-9]*)?\])?))?)?)|\$smdesc#(?:semanticId(?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?|supplementalSemanticIds(?:\[(?:0|[1-9][0-9]*)?\](?:\.keys(?:\[(?:0|[1-9][0-9]*)?\])?)?)?|idShort|endpoints(?:\[(?:0|[1-9][0-9]*)?\])?))$
     MultiLanguagePropertyMetadata: 
       allOf: 
         - $ref: "#/components/schemas/SubmodelElementAttributes"

documentation/IDTA-01002-3/modules/ROOT/pages/changelog.adoc

@@ -37,6 +37,7 @@ Minor Changes:
 * docs: Added cross-spec terminology matrix reference to align Metamodel classes, API path segments, and Query/Access-Rule prefixes across specifications.
 * docs: Added cross-specification alignment matrix to document API compatibility with IDTA-01001 (Metamodel v3.2 instead of v3.1), DTA-01003-a, IDTA-01003-b, IDTA-01004 (Security v3.1), and IDTA-01005 (from v3.1 to v3.2).
 * docs: Established IDTA-01002 as authoritative source for formula grammar and JSON Schema shared between API and Security specifications.
+* fix: Aligned Query Language BNF and JSON Schema with IDTA-01004 Security access-rule schema, including wildcard array indexes, stricter object/reference identifiers, and artifact validation tests.
 * docs: Added informative Conformance Test Corpus annex providing technology-neutral test cases for Query Language, HTTP/REST API, and error handling.
 * Added the `ServerNotImplemented` status code to the xref:specification/interfaces-payload.adoc#table-status-codes[Status Codes]. Note that the HTTP API mentioned this status code already, therefore, no change in OpenAPI or other related files. (https://github.com/admin-shell-io/aas-specs-api/issues/499[#499])
 * Removed the `GetAllAssetAdministrationShellDescriptorsByAssetType` operation from the OpenAPI files for the Asset Administration Shell Registry Service Specification, as it was not intended to be included in the first place and is not implemented by any known implementation. (https://github.com/admin-shell-io/aas-specs-api/issues/515[#515])
@@ -772,4 +773,3 @@ PostConceptDescription
 |PutConceptDescription |PutConceptDescriptionById |Changed |Naming pattern byId
 | |PostConceptDescription |New |
 |===
-

documentation/IDTA-01002-3/modules/ROOT/pages/http-rest-api/test/query/test_field_identifier_regex.py

@@ -60,6 +60,7 @@ def test_allowed_field_identifiers(self):
             "$aas#assetInformation.specificAssetIds[].externalSubjectId.type",
             "$aas#assetInformation.specificAssetIds[].externalSubjectId.keys[].type",
             "$aas#assetInformation.specificAssetIds[].externalSubjectId.keys[0].value",
+            "$aas#submodels[]",
             "$aas#submodels[].type",
             "$aas#submodels[].keys[].type",
             "$aas#submodels[0].keys[0].value",
@@ -141,14 +142,16 @@ def test_not_allowed_field_identifiers(self):
         not_allowed = [
             "$aas#assetInformation.specificAssetIds",
             "$aas#assetInformation.specificAssetIds[]",
+            "$aas#assetInformation.specificAssetIds[01].name",
             "$aas#assetInformation.specificAssetIds[].externalSubjectId.keys",
             "$aas#assetInformation.specificAssetIds[].externalSubjectId.keys[]",
             "$aas#submodels",
-            "$aas#submodels[]",
+            "$aas#submodels[01].type",
             "$aas#submodels[].keys",
             "$aas#submodels[].keys[]",
             "$sm#semanticId.keys",
             "$sm#semanticId.keys[]",
+            "$sm#supplementalSemanticIds[01]",
             "$sm#supplementalSemanticIds.keys",
             "$sm#supplementalSemanticIds.keys[]",
             "$sm#supplementalSemanticIds[].keys",
@@ -158,6 +161,7 @@ def test_not_allowed_field_identifiers(self):
             "$sme.AddressInformation[]",
             "$sme.AddressInformation#id",
             "$sme.AddressInformation#bogus",
+            "$sme.AddressInformation#supplementalSemanticIds[01]",
             "$sme.AddressInformation#supplementalSemanticIds.keys",
             "$sme.AddressInformation#supplementalSemanticIds[].keys",
             "$sme.1Invalid#value",
@@ -173,19 +177,23 @@ def test_not_allowed_field_identifiers(self):
             "$aasdesc#specificAssetIds[].externalSubjectId.keys[]",
             "$aasdesc#endpoints",
             "$aasdesc#endpoints[]",
+            "$aasdesc#endpoints[01].interface",
             "$aasdesc#endpoints[].protocolinformation",
             "$aasdesc#submodelDescriptors",
             "$aasdesc#submodelDescriptors[]",
+            "$aasdesc#submodelDescriptors[01].idShort",
             "$aasdesc#submodelDescriptors[].supplementalSemanticIds.keys",
             "$aasdesc#submodelDescriptors[].supplementalSemanticIds[].keys",
             "$aasdesc#submodelDescriptors[].endpoints",
             "$aasdesc#submodelDescriptors[].endpoints[]",
             "$smdesc#semanticId.keys",
             "$smdesc#semanticId.keys[]",
             "$smdesc#supplementalSemanticIds.keys",
+            "$smdesc#supplementalSemanticIds[01]",
             "$smdesc#supplementalSemanticIds[].keys",
             "$smdesc#endpoints",
             "$smdesc#endpoints[]",
+            "$smdesc#endpoints[01].interface",
             "$smdesc#endpoints[].protocolinformation",
         ]
 
@@ -198,5 +206,60 @@ def test_documented_field_identifier_patterns_are_in_sync(self):
         self.assertEqual(self.pattern, openapi_component_pattern("FieldIdentifier"))
 
 
+class ReferenceIdentifierRegexTest(unittest.TestCase):
+    @classmethod
+    def setUpClass(cls):
+        schema = json.loads(QUERY_SCHEMA.read_text(encoding="utf-8-sig"))
+        pattern = schema["definitions"]["ReferenceIdentifier"]["pattern"]
+        cls.pattern = pattern
+        cls.reference_identifier = re.compile(pattern)
+
+    def assert_allowed(self, value):
+        self.assertIsNotNone(
+            self.reference_identifier.fullmatch(value),
+            f"Expected ReferenceIdentifier to allow: {value}",
+        )
+
+    def assert_not_allowed(self, value):
+        self.assertIsNone(
+            self.reference_identifier.fullmatch(value),
+            f"Expected ReferenceIdentifier to reject: {value}",
+        )
+
+    def test_allowed_reference_identifiers(self):
+        allowed = [
+            '$aas("aas-id")#assetInformation.specificAssetIds[].externalSubjectId.keys[0].value',
+            '$sm("SubmodelID")#id',
+            '$sm("SubmodelID")#supplementalSemanticIds[].keys[].value',
+            '$cd("ConceptDescriptionID")#idShort',
+            '$sme("SubmodelID-OperationalData").machineState#value',
+            '$sme("SubmodelID").AddressInformation[0].Zipcode#semanticId.keys[].value',
+        ]
+
+        for value in allowed:
+            with self.subTest(value=value):
+                self.assert_allowed(value)
+
+    def test_not_allowed_reference_identifiers(self):
+        not_allowed = [
+            "$sm#id",
+            '$sm("SubmodelID")#bogus',
+            '$sm("SubmodelID")#supplementalSemanticIds[01]',
+            '$aas("aas-id")#assetInformation.specificAssetIds[]',
+            '$cd("ConceptDescriptionID")#description',
+            '$sme("SubmodelID").machineState',
+            '$sme("SubmodelID").machineState#id',
+            '$sme("SubmodelID").1Invalid#value',
+            '$aasdesc("aas-id")#idShort',
+        ]
+
+        for value in not_allowed:
+            with self.subTest(value=value):
+                self.assert_not_allowed(value)
+
+    def test_documented_reference_identifier_patterns_are_in_sync(self):
+        self.assertEqual(self.pattern, schema_page_pattern("ReferenceIdentifier"))
+
+
 if __name__ == "__main__":
     unittest.main()

documentation/IDTA-01002-3/modules/ROOT/pages/http-rest-api/test/query/test_fragment_field_identifier_regex.py

@@ -140,43 +140,54 @@ def test_not_allowed_fragment_field_identifiers(self):
             "$aas#id",
             "$aas#assetInformation.assetKind",
             "$aas#assetInformation.specificAssetIds.name",
+            "$aas#assetInformation.specificAssetIds[01]",
             "$aas#assetInformation.specificAssetIds[].name",
             "$aas#assetInformation.specificAssetIds[].value",
             "$aas#assetInformation.specificAssetIds.keys[]",
             "$aas#assetInformation.specificAssetIds.externalSubjectId.keys[]",
+            "$aas#submodels[01]",
             "$aas#submodels.keys[]",
             "$aas#submodels[].type",
             "$aas#submodels[].keys[].value",
             "$sm#semanticId.type",
             "$sm#semanticId.keys[].value",
             "$sm#id",
+            "$sm#supplementalSemanticIds[01]",
             "$sm#supplementalSemanticIds.type",
             "$sm#supplementalSemanticIds.keys",
             "$sm#supplementalSemanticIds[].type",
             "$sm#supplementalSemanticIds[].keys[].value",
             "$sme.1Invalid",
+            "$sme.AddressInformation[01]",
             "$sme.AddressInformation#id",
             "$sme.AddressInformation#bogus",
             "$sme.AddressInformation#semanticId.type",
             "$sme.AddressInformation#semanticId.keys[].value",
+            "$sme.AddressInformation#supplementalSemanticIds[01]",
             "$sme.AddressInformation#supplementalSemanticIds.type",
             "$sme.AddressInformation#supplementalSemanticIds.keys",
             "$sme.AddressInformation#supplementalSemanticIds[].keys[].value",
             "$cd#id",
             "$aasdesc#id",
             "$aasdesc#specificAssetIds.name",
+            "$aasdesc#specificAssetIds[01]",
             "$aasdesc#specificAssetIds[].name",
+            "$aasdesc#endpoints[01]",
             "$aasdesc#endpoints.protocolinformation.href",
+            "$aasdesc#submodelDescriptors[01]",
             "$aasdesc#submodelDescriptors.endpoints[]",
             "$aasdesc#submodelDescriptors[].id",
+            "$aasdesc#submodelDescriptors[].supplementalSemanticIds[01]",
             "$aasdesc#submodelDescriptors[].supplementalSemanticIds.type",
             "$aasdesc#submodelDescriptors[].supplementalSemanticIds.keys",
             "$aasdesc#submodelDescriptors[].supplementalSemanticIds[].keys[].value",
             "$aasdesc#submodelDescriptors[].endpoints.protocolinformation.href",
             "$smdesc#id",
+            "$smdesc#supplementalSemanticIds[01]",
             "$smdesc#supplementalSemanticIds.type",
             "$smdesc#supplementalSemanticIds.keys",
             "$smdesc#supplementalSemanticIds[].keys[].value",
+            "$smdesc#endpoints[01]",
             "$smdesc#endpoints.protocolinformation.href",
         ]