Merge branch 'msuhr1-replace-crendentials' into 'master'

Closes #5

See merge request !6

.gitignore

@@ -139,4 +139,7 @@ dmypy.json
 .pytype/
 
 # Cython debug symbols
-cython_debug/
\ No newline at end of file
+cython_debug/
+
+# PyCharm IDE
+.idea
\ No newline at end of file

.gitlab-ci.yml

@@ -7,11 +7,15 @@ stages:
   - test
   - publish
 
+services:
+  - couchdb
+
 variables:
   CDSTAR_URI: "http://vm18212.virt.gwdg.de:8080/v3"
   CDSTAR_VAULT: "medic"
-  COUCH_URI: "http://vm18212.virt.gwdg.de:8008"
-  COUCH_DB: "annotation_agent_test"
+  # variables for CouchDB service; refer to Gitlab project CI variables
+  COUCHDB_USER: $COUCH_USER
+  COUCHDB_PASSWORD: $COUCH_PASS
 
 build_image:
   stage: build
@@ -33,10 +37,16 @@ run_tests:
   tags:
     - docker
     - medic
-
+  
+  variables:
+    COUCH_URI: "http://couchdb:5984"
+    COUCH_DB: "annotation_agent_test"
   before_script:
     - curl --connect-timeout 5 $CDSTAR_URI
     - curl --connect-timeout 5 $COUCH_URI
+    - curl -X PUT http://$COUCHDB_USER:$COUCHDB_PASSWORD@couchdb:5984/_users
+    - curl -X PUT http://$COUCHDB_USER:$COUCHDB_PASSWORD@couchdb:5984/_replicator
+    - curl -X PUT http://$COUCHDB_USER:$COUCHDB_PASSWORD@couchdb:5984/$COUCH_DB
 
   script:
     - python3 -m pytest --junitxml=report.xml

annotator/__version__.py

@@ -2,6 +2,6 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-VERSION = (0, 4, 0)
+VERSION = (0, 5, 0)
 
 __version__ = ".".join(map(str, VERSION))

annotator/errors.py

@@ -18,4 +18,4 @@ class ConfigurationError(Exception):
 class UnauthorizedException(Exception):
     def __init__(self, message: str):
         self.name = "Unauthorized"
-        self.message = message
\ No newline at end of file
+        self.message = message

annotator/main.py

@@ -5,9 +5,12 @@
 import datetime
 import json
 import os
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, Optional, Union
 
-from liquid.context import Template
+import fastapi
+import fastapi.responses
+from fastapi import BackgroundTasks, FastAPI, Request
+from liquid import Environment
 
 from annotator import access_log, error_log
 from annotator import config
@@ -16,12 +19,6 @@ from annotator import models_activeworkflow as awmodels
 from annotator import stores
 from annotator import utils
 
-from fastapi import BackgroundTasks, FastAPI, Request
-import fastapi
-import fastapi.responses
-
-from liquid import Environment
-
 app = FastAPI()
 
 endpoint_name = "annotator"
@@ -39,11 +36,16 @@ def appinfo():
     response_model_exclude_none=True,
 )
 def aw_endpoint(payload: awmodels.RequestCommon, background_tasks: BackgroundTasks):
-    """Implements an remote agent API endpoint as specified by active_workflow,
+    """
+    Implements a remote agent API endpoint as specified by active_workflow,
     see [active_workflow docs](https://github.com/automaticmode/active_workflow/blob/master/docs/remote_agent_api.md)
     for more details. The call is forwarded to the correct endpoint.
     """
 
+    # Replace credentials in options
+    payload.params.options = utils.replace_credentials(data=payload.params.options,
+                                                       credentials=payload.params.credentials)
+
     if payload.method is not None and payload.method == "register":
         return register(payload=awmodels.RequestRegister.parse_obj(payload.dict()))
 
@@ -143,10 +145,10 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
     vault_id = settings["vault_id"]
 
     if not stores.is_valid_archive(
-        vault_id=vault_id,
-        archive_id=archive_id,
-        cdstar_uri=settings["cdstar_uri"],
-        cdstar_auth=(settings["cdstar_user"], settings["cdstar_pass"]),
+            vault_id=vault_id,
+            archive_id=archive_id,
+            cdstar_uri=settings["cdstar_uri"],
+            cdstar_auth=(settings["cdstar_user"], settings["cdstar_pass"]),
     ):
         response.result.errors.append(
             f"Archive {archive_id} is not available in CDSTAR vault {vault_id}."
@@ -189,11 +191,11 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
 
 
 def run_annotation(
-    archive_id: str,
-    annotations_archive: Dict[str, Any],
-    annotations_file: Dict[str, Any],
-    metafile: str,
-    settings: Dict[str, str],
+        archive_id: str,
+        annotations_archive: Dict[str, Any],
+        annotations_file: Dict[str, Any],
+        metafile: str,
+        settings: Dict[str, str],
 ) -> None:
     archive, filelist = stores.get_cdstar_metadata(
         vault_id=settings["vault_id"],
@@ -311,7 +313,7 @@ def check(payload: awmodels.RequestCheck):
 
 
 def get_setting_from_payload(
-    payload: Union[awmodels.RequestReceive, awmodels.RequestCheck], key: str
+        payload: Union[awmodels.RequestReceive, awmodels.RequestCheck], key: str
 ) -> Optional[str]:
     msg_payload = payload.params.message.payload.dict()
     if key in msg_payload.keys() and msg_payload[key]:
@@ -338,7 +340,7 @@ async def config_exception_handler(request: Request, exc: errors.ConfigurationEr
 
 
 @app.exception_handler(errors.ActiveWorkflowError)
-async def aw_exception_handler(request: Request, exc: errors.ConfigurationError):
+async def aw_exception_handler(request: Request, exc: errors.ActiveWorkflowError):
     error_log.error(f"{exc.name}: {exc.message}")
     resp = awmodels.ResponseCheck()
     resp.result.errors.append(f"{exc.name}: {exc.message}")

annotator/models_activeworkflow.py

@@ -187,16 +187,6 @@ class MemoryCommon(BaseModel):
         return {"archives": [("medic", "a1b2c3d4e5f6")]}
 
 
-class CredentialsCommon(BaseModel):
-    """
-    This model represents the expected credentials content to communicate credentials with the agent.
-    """
-
-    @staticmethod
-    def example() -> dict:
-        return {}
-
-
 #
 # Register Models
 #

annotator/test_utils.py

@@ -2,13 +2,68 @@
 #
 # SPDX-License-Identifier: GPL-3.0-or-later
 
-import tempfile
 import os
-
+import tempfile
 from typing import Any, Dict, Tuple
 
+import pytest
 import requests
 
+from annotator.errors import ActiveWorkflowError
+from annotator.models_activeworkflow import CredentialsCommon
+from annotator.utils import replace_credentials
+
+
+def test_replace_credentials():
+    """
+    Tests whether ActiveWorkflow credential syntax is replaced correctly
+    """
+    params = {
+        'option1': 'no credential reference',
+        'option2': '{% credential cred_ref_2 %}',
+        'option3': 'Just a normal {{ liquid.reference }}, should be ignored',
+        'option4': {'another': 'dict', 'with_ref': '{% credential cred1 %}'},
+        'option5': 'reference within {% credential cred1 %} normal text',
+        'option6': 'two references within {% credential cred1 %} normal {% credential cred_ref_2 %} text'
+    }
+    credentials = [CredentialsCommon(name="cred1", value="thisisasecret"),
+                   CredentialsCommon(name="cred_ref_2", value="thisisanothersecret")]
+    params_expected = {
+        'option1': 'no credential reference',
+        'option2': 'thisisanothersecret',
+        'option3': 'Just a normal {{ liquid.reference }}, should be ignored',
+        'option4': {'another': 'dict', 'with_ref': 'thisisasecret'},
+        'option5': 'reference within thisisasecret normal text',
+        'option6': 'two references within thisisasecret normal thisisanothersecret text'
+    }
+    params_replaced = replace_credentials(params, credentials)
+    assert params_replaced == params_expected
+
+    # we expect no changes, if the dictionary contains no references
+    params = {
+        'option1': 'no credential reference',
+        'option3': 'Just a normal {{ liquid.reference }}, should be ignored',
+        'option4': {'another': 'dict'}
+    }
+    params_replaced = replace_credentials(params, credentials)
+    assert params_replaced == params
+
+    # we expect no changes, if dictionary contains no references and no credentials are supplied
+    params_replaced = replace_credentials(params, [])
+    assert params_replaced == params
+
+    # MissingCredentialException is expected if a credential name is referenced that is not available
+    params = {
+        'option4': {'another': 'dict', 'with_ref': '{% credential cred1 %}'},
+        'option5': 'reference within {% credential doesntexist2 %} normal text'
+    }
+    with pytest.raises(ActiveWorkflowError):
+        replace_credentials(params, credentials)
+
+    # MissingCredentialException is expected, if no credentials are supplied
+    with pytest.raises(ActiveWorkflowError):
+        replace_credentials(params, [])
+
 
 def is_valid_response(json: Dict[str, Any]) -> bool:
     # see: https://github.com/automaticmode/active_workflow/blob/master/docs/remote_agent_api.md#receive-method
@@ -47,10 +102,10 @@ def is_schemaorg_jsonld(json: Dict[str, Any]) -> bool:
     """
     keys = json.keys()
     return (
-        "@context" in keys
-        and "@id" in keys
-        and "@type" in keys
-        and json["@context"].startswith("http://schema.org")
+            "@context" in keys
+            and "@id" in keys
+            and "@type" in keys
+            and json["@context"].startswith("http://schema.org")
     )
 
 
@@ -84,4 +139,4 @@ def has_memory_archive(json: Dict[str, Any]) -> bool:
     assert "archives" in json["result"]["memory"].keys()
     assert type(json["result"]["memory"]["archives"]) == list
     assert len(json["result"]["memory"]["archives"]) > 0
-    return True
\ No newline at end of file
+    return True

annotator/utils.py

@@ -6,15 +6,17 @@ import datetime
 import enum
 import json
 import os
+import re as regex
 import sys
 import tempfile
 import traceback
-
-from typing import Any, Dict, Optional, Union
+from typing import Any, Dict, Optional, Union, List
 
 from annotator import config
 from annotator import error_log
 from annotator import models_activeworkflow as awmodels
+from annotator.config import BasicSettings
+from annotator.errors import ActiveWorkflowError
 
 
 class AnnotationState(str, enum.Enum):
@@ -147,16 +149,14 @@ def remove_metafile(path: str) -> bool:
 
 
 def is_authorized(
-    params: Union[awmodels.ParamsRegister, awmodels.ParamsReceive, awmodels.ParamsCheck]
+        params: Union[awmodels.ParamsRegister, awmodels.ParamsReceive, awmodels.ParamsCheck]
 ) -> bool:
     """
     Check if a params section is authorized to use this Annotation agent instance.
 
     This method will check 1) if credentials are set in the configuration of the agent and 2)
-    whether the supplied params are authorized to use the agent. If not credentials are set
-    (which is the default), the method will return True. In any other case, the `api_key`
-    option has to contain the credential name in Liquid Templating notation which refers to
-    the credentials parameter.
+    whether the supplied params are authorized to use the agent. If no credentials are set
+    (which is the default), the method will return True.
 
     Parameters
     ----------
@@ -168,29 +168,63 @@ def is_authorized(
     bool
         True if parameters hold valid credentials, authorizing any request. False otherwise
     """
+    api_keys = BasicSettings().application_api_keys
 
-    if len(config.BasicSettings().application_api_keys) == 0:
+    if len(api_keys) == 0:
         # If there are no credentials set, no credentials are needed.
         return True
 
-    set_api_key = params.options.api_key
-    if set_api_key.startswith("{%") and set_api_key.endswith("%}"):
-        set_api_key = set_api_key[2:-2].strip()
-        if set_api_key.startswith("credential"):
-            search_credential_name = set_api_key[len("credential") :].strip()
-        else:
-            # If you end up here: an api key like `{% something %}` has been supplied. Either use
-            # the `credential` key word or use the plain API key, but do not mix.
-            return False
-    else:
-        if set_api_key.strip() in config.BasicSettings().application_api_keys:
-            return True
-        else:
-            return False
-
-    for credential in params.credentials:
-        if credential.name == search_credential_name:
-            if credential.value in config.BasicSettings().application_api_keys:
-                return True
+    if params.options.api_key in api_keys:
+        return True
 
     return False
+
+
+def replace_credentials(data: Dict, credentials: List[awmodels.CredentialsCommon]):
+    """
+    Replace occurrences of AW credential syntax in the items of a Dict
+
+    Searches for ActiveWorkflow credential references in Dict items' values, i.e. `{% credential ref_id %}` and
+    replaces the entire string with the value of `ref_id` if that is present as a key in the credentials Dict.
+
+    Parameters
+    ----------
+    data: Dict
+        Any dictionary that possibly contains ActiveWorkflow credential references
+    credentials: List[awmodels.CredentialsCommon]
+        A List of CredentialsCommon objects
+
+    Results
+    -------
+    data: Dict
+        The input Dict with replaced values (if applicable)
+    """
+    for key, value in data.items():
+        # Enter recursion if value itself is a dictionary
+        if type(value) is dict:
+            data[key] = replace_credentials(value, credentials)
+        # determine whether value contains an AW credential reference
+        # and look up the reference in the given list of credentials
+
+        if type(value) is str:
+            # Regular expression tests whether a reference to ActiveWorkflow credential is present,
+            # designated by double-curly-brackets: {% credential credential_name %}
+            regex_str = "({% credential )(.[^%}]*)( %})"
+            while (True):
+                matches = regex.search(regex_str, value)
+                if matches:
+                    # [0] is the whole string if the regex is matched, [1] is the first group, [2] the second group...
+                    # We need the 2nd group "(.*)" from the regex, which would be the name of the credential
+                    reference = matches[2].strip()
+                    replaced = False
+                    for c in credentials:
+                        if c.name == reference:
+                            value = value.replace(matches[0], c.value)
+                            data[key] = value
+                            replaced = True
+                    if not replaced:
+                        raise ActiveWorkflowError(name="Missing Credential",
+                                                  message=f"Credential '{reference}' is not supplied.")
+                else:
+                    break
+    return data