Commit 3f1ec591 authored by msuhr1's avatar msuhr1
Browse files

Replace AW credentials in options before applying liquid templating.

parent f537e417
......@@ -139,4 +139,7 @@ dmypy.json
.pytype/
# Cython debug symbols
cython_debug/
\ No newline at end of file
cython_debug/
# PyCharm IDE
.idea
\ No newline at end of file
......@@ -7,11 +7,15 @@ stages:
- test
- publish
services:
- couchdb
variables:
CDSTAR_URI: "http://vm18212.virt.gwdg.de:8080/v3"
CDSTAR_VAULT: "medic"
COUCH_URI: "http://vm18212.virt.gwdg.de:8008"
COUCH_DB: "annotation_agent_test"
# variables for CouchDB service; refer to Gitlab project CI variables
COUCHDB_USER: $COUCH_USER
COUCHDB_PASSWORD: $COUCH_PASS
build_image:
stage: build
......@@ -33,10 +37,16 @@ run_tests:
tags:
- docker
- medic
variables:
COUCH_URI: "http://couchdb:5984"
COUCH_DB: "annotation_agent_test"
before_script:
- curl --connect-timeout 5 $CDSTAR_URI
- curl --connect-timeout 5 $COUCH_URI
- curl -X PUT http://$COUCHDB_USER:$COUCHDB_PASSWORD@couchdb:5984/_users
- curl -X PUT http://$COUCHDB_USER:$COUCHDB_PASSWORD@couchdb:5984/_replicator
- curl -X PUT http://$COUCHDB_USER:$COUCHDB_PASSWORD@couchdb:5984/$COUCH_DB
script:
- python3 -m pytest --junitxml=report.xml
......
......@@ -2,6 +2,6 @@
#
# SPDX-License-Identifier: GPL-3.0-or-later
VERSION = (0, 4, 0)
VERSION = (0, 5, 0)
__version__ = ".".join(map(str, VERSION))
......@@ -18,4 +18,4 @@ class ConfigurationError(Exception):
class UnauthorizedException(Exception):
def __init__(self, message: str):
self.name = "Unauthorized"
self.message = message
\ No newline at end of file
self.message = message
......@@ -5,9 +5,12 @@
import datetime
import json
import os
from typing import Any, Dict, List, Optional, Union
from typing import Any, Dict, Optional, Union
from liquid.context import Template
import fastapi
import fastapi.responses
from fastapi import BackgroundTasks, FastAPI, Request
from liquid import Environment
from annotator import access_log, error_log
from annotator import config
......@@ -16,12 +19,6 @@ from annotator import models_activeworkflow as awmodels
from annotator import stores
from annotator import utils
from fastapi import BackgroundTasks, FastAPI, Request
import fastapi
import fastapi.responses
from liquid import Environment
app = FastAPI()
endpoint_name = "annotator"
......@@ -39,11 +36,16 @@ def appinfo():
response_model_exclude_none=True,
)
def aw_endpoint(payload: awmodels.RequestCommon, background_tasks: BackgroundTasks):
"""Implements an remote agent API endpoint as specified by active_workflow,
"""
Implements a remote agent API endpoint as specified by active_workflow,
see [active_workflow docs](https://github.com/automaticmode/active_workflow/blob/master/docs/remote_agent_api.md)
for more details. The call is forwarded to the correct endpoint.
"""
# Replace credentials in options
payload.params.options = utils.replace_credentials(data=payload.params.options,
credentials=payload.params.credentials)
if payload.method is not None and payload.method == "register":
return register(payload=awmodels.RequestRegister.parse_obj(payload.dict()))
......@@ -143,10 +145,10 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
vault_id = settings["vault_id"]
if not stores.is_valid_archive(
vault_id=vault_id,
archive_id=archive_id,
cdstar_uri=settings["cdstar_uri"],
cdstar_auth=(settings["cdstar_user"], settings["cdstar_pass"]),
vault_id=vault_id,
archive_id=archive_id,
cdstar_uri=settings["cdstar_uri"],
cdstar_auth=(settings["cdstar_user"], settings["cdstar_pass"]),
):
response.result.errors.append(
f"Archive {archive_id} is not available in CDSTAR vault {vault_id}."
......@@ -189,11 +191,11 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
def run_annotation(
archive_id: str,
annotations_archive: Dict[str, Any],
annotations_file: Dict[str, Any],
metafile: str,
settings: Dict[str, str],
archive_id: str,
annotations_archive: Dict[str, Any],
annotations_file: Dict[str, Any],
metafile: str,
settings: Dict[str, str],
) -> None:
archive, filelist = stores.get_cdstar_metadata(
vault_id=settings["vault_id"],
......@@ -311,7 +313,7 @@ def check(payload: awmodels.RequestCheck):
def get_setting_from_payload(
payload: Union[awmodels.RequestReceive, awmodels.RequestCheck], key: str
payload: Union[awmodels.RequestReceive, awmodels.RequestCheck], key: str
) -> Optional[str]:
msg_payload = payload.params.message.payload.dict()
if key in msg_payload.keys() and msg_payload[key]:
......@@ -338,7 +340,7 @@ async def config_exception_handler(request: Request, exc: errors.ConfigurationEr
@app.exception_handler(errors.ActiveWorkflowError)
async def aw_exception_handler(request: Request, exc: errors.ConfigurationError):
async def aw_exception_handler(request: Request, exc: errors.ActiveWorkflowError):
error_log.error(f"{exc.name}: {exc.message}")
resp = awmodels.ResponseCheck()
resp.result.errors.append(f"{exc.name}: {exc.message}")
......
......@@ -187,16 +187,6 @@ class MemoryCommon(BaseModel):
return {"archives": [("medic", "a1b2c3d4e5f6")]}
class CredentialsCommon(BaseModel):
"""
This model represents the expected credentials content to communicate credentials with the agent.
"""
@staticmethod
def example() -> dict:
return {}
#
# Register Models
#
......
......@@ -2,13 +2,68 @@
#
# SPDX-License-Identifier: GPL-3.0-or-later
import tempfile
import os
import tempfile
from typing import Any, Dict, Tuple
import pytest
import requests
from annotator.errors import ActiveWorkflowError
from annotator.models_activeworkflow import CredentialsCommon
from annotator.utils import replace_credentials
def test_replace_credentials():
"""
Tests whether ActiveWorkflow credential syntax is replaced correctly
"""
params = {
'option1': 'no credential reference',
'option2': '{% credential cred_ref_2 %}',
'option3': 'Just a normal {{ liquid.reference }}, should be ignored',
'option4': {'another': 'dict', 'with_ref': '{% credential cred1 %}'},
'option5': 'reference within {% credential cred1 %} normal text',
'option6': 'two references within {% credential cred1 %} normal {% credential cred_ref_2 %} text'
}
credentials = [CredentialsCommon(name="cred1", value="thisisasecret"),
CredentialsCommon(name="cred_ref_2", value="thisisanothersecret")]
params_expected = {
'option1': 'no credential reference',
'option2': 'thisisanothersecret',
'option3': 'Just a normal {{ liquid.reference }}, should be ignored',
'option4': {'another': 'dict', 'with_ref': 'thisisasecret'},
'option5': 'reference within thisisasecret normal text',
'option6': 'two references within thisisasecret normal thisisanothersecret text'
}
params_replaced = replace_credentials(params, credentials)
assert params_replaced == params_expected
# we expect no changes, if the dictionary contains no references
params = {
'option1': 'no credential reference',
'option3': 'Just a normal {{ liquid.reference }}, should be ignored',
'option4': {'another': 'dict'}
}
params_replaced = replace_credentials(params, credentials)
assert params_replaced == params
# we expect no changes, if dictionary contains no references and no credentials are supplied
params_replaced = replace_credentials(params, [])
assert params_replaced == params
# MissingCredentialException is expected if a credential name is referenced that is not available
params = {
'option4': {'another': 'dict', 'with_ref': '{% credential cred1 %}'},
'option5': 'reference within {% credential doesntexist2 %} normal text'
}
with pytest.raises(ActiveWorkflowError):
replace_credentials(params, credentials)
# MissingCredentialException is expected, if no credentials are supplied
with pytest.raises(ActiveWorkflowError):
replace_credentials(params, [])
def is_valid_response(json: Dict[str, Any]) -> bool:
# see: https://github.com/automaticmode/active_workflow/blob/master/docs/remote_agent_api.md#receive-method
......@@ -47,10 +102,10 @@ def is_schemaorg_jsonld(json: Dict[str, Any]) -> bool:
"""
keys = json.keys()
return (
"@context" in keys
and "@id" in keys
and "@type" in keys
and json["@context"].startswith("http://schema.org")
"@context" in keys
and "@id" in keys
and "@type" in keys
and json["@context"].startswith("http://schema.org")
)
......@@ -84,4 +139,4 @@ def has_memory_archive(json: Dict[str, Any]) -> bool:
assert "archives" in json["result"]["memory"].keys()
assert type(json["result"]["memory"]["archives"]) == list
assert len(json["result"]["memory"]["archives"]) > 0
return True
\ No newline at end of file
return True
......@@ -6,15 +6,17 @@ import datetime
import enum
import json
import os
import re as regex
import sys
import tempfile
import traceback
from typing import Any, Dict, Optional, Union
from typing import Any, Dict, Optional, Union, List
from annotator import config
from annotator import error_log
from annotator import models_activeworkflow as awmodels
from annotator.config import BasicSettings
from annotator.errors import ActiveWorkflowError
class AnnotationState(str, enum.Enum):
......@@ -147,16 +149,14 @@ def remove_metafile(path: str) -> bool:
def is_authorized(
params: Union[awmodels.ParamsRegister, awmodels.ParamsReceive, awmodels.ParamsCheck]
params: Union[awmodels.ParamsRegister, awmodels.ParamsReceive, awmodels.ParamsCheck]
) -> bool:
"""
Check if a params section is authorized to use this Annotation agent instance.
This method will check 1) if credentials are set in the configuration of the agent and 2)
whether the supplied params are authorized to use the agent. If not credentials are set
(which is the default), the method will return True. In any other case, the `api_key`
option has to contain the credential name in Liquid Templating notation which refers to
the credentials parameter.
whether the supplied params are authorized to use the agent. If no credentials are set
(which is the default), the method will return True.
Parameters
----------
......@@ -168,29 +168,63 @@ def is_authorized(
bool
True if parameters hold valid credentials, authorizing any request. False otherwise
"""
api_keys = BasicSettings().application_api_keys
if len(config.BasicSettings().application_api_keys) == 0:
if len(api_keys) == 0:
# If there are no credentials set, no credentials are needed.
return True
set_api_key = params.options.api_key
if set_api_key.startswith("{%") and set_api_key.endswith("%}"):
set_api_key = set_api_key[2:-2].strip()
if set_api_key.startswith("credential"):
search_credential_name = set_api_key[len("credential") :].strip()
else:
# If you end up here: an api key like `{% something %}` has been supplied. Either use
# the `credential` key word or use the plain API key, but do not mix.
return False
else:
if set_api_key.strip() in config.BasicSettings().application_api_keys:
return True
else:
return False
for credential in params.credentials:
if credential.name == search_credential_name:
if credential.value in config.BasicSettings().application_api_keys:
return True
if params.options.api_key in api_keys:
return True
return False
def replace_credentials(data: Dict, credentials: List[awmodels.CredentialsCommon]):
"""
Replace occurrences of AW credential syntax in the items of a Dict
Searches for ActiveWorkflow credential references in Dict items' values, i.e. `{% credential ref_id %}` and
replaces the entire string with the value of `ref_id` if that is present as a key in the credentials Dict.
Parameters
----------
data: Dict
Any dictionary that possibly contains ActiveWorkflow credential references
credentials: List[awmodels.CredentialsCommon]
A List of CredentialsCommon objects
Results
-------
data: Dict
The input Dict with replaced values (if applicable)
"""
for key, value in data.items():
# Enter recursion if value itself is a dictionary
if type(value) is dict:
data[key] = replace_credentials(value, credentials)
# determine whether value contains an AW credential reference
# and look up the reference in the given list of credentials
if type(value) is str:
# Regular expression tests whether a reference to ActiveWorkflow credential is present,
# designated by double-curly-brackets: {% credential credential_name %}
regex_str = "({% credential )(.[^%}]*)( %})"
while (True):
matches = regex.search(regex_str, value)
if matches:
# [0] is the whole string if the regex is matched, [1] is the first group, [2] the second group...
# We need the 2nd group "(.*)" from the regex, which would be the name of the credential
reference = matches[2].strip()
replaced = False
for c in credentials:
if c.name == reference:
value = value.replace(matches[0], c.value)
data[key] = value
replaced = True
if not replaced:
raise ActiveWorkflowError(name="Missing Credential",
message=f"Credential '{reference}' is not supplied.")
else:
break
return data
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment