Commit eddeeed2 authored by msuhr1's avatar msuhr1
Browse files

Refactored Liquid templating

Merge branch 'msuhr1-liquid-templating-refactored' into 'master'

See merge request !7
parents 7aed308b fad8311c
Pipeline #186947 passed with stages
in 1 minute and 34 seconds
...@@ -18,7 +18,7 @@ cloudant = "*" ...@@ -18,7 +18,7 @@ cloudant = "*"
pytest = "*" pytest = "*"
pycdstar3 = {git = "https://gitlab.gwdg.de/cdstar/pycdstar3"} pycdstar3 = {git = "https://gitlab.gwdg.de/cdstar/pycdstar3"}
jinja2 = "*" jinja2 = "*"
python-liquid = "*" liquidpy = "*"
[requires] [requires]
python_version = "3.8" python_version = "3.8"
......
This diff is collapsed.
# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> # SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
......
# SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
#
# SPDX-License-Identifier: GPL-3.0-or-later
from liquid import Tag, LiquidRenderError
class TagCredential(Tag):
"""Liquid engine extension to interpret and replace ActiveWorkflow credential syntax.
If a string contains ActiveWorkflow credential reference, e.g. `{% credential reference %}`,
the keyword "reference" shall be looked up in given dictionaries. If the dictionary "credentials"
contains a matching key, the whole Liquid sequence will be replaced with the respective value.
"""
# This is a void Tag extension, meaning that no closing counterpart is required but the
# `{% credential reference %}` sequence itself is complete (as opposed to Liquid constructions
# like `{% for ... %} ... {% endfor %}`)
VOID = True
# the start rule
START = "tag_credential"
# the grammar based on the base_grammar
GRAMMAR = "tag_credential: output"
def parse(self, force=False):
# Use the default parser to parse the output rule
return super().parse(force)
def _render(self, local_vars, global_vars):
# Look up the reference in the "credentials" dictionary (which has to be supplied when rendering
# is initiated by the Liquid engine.
if (
"credentials" in local_vars.keys()
and self.content in local_vars["credentials"].keys()
):
self.content = local_vars["credentials"][self.content]
else:
# Raise an exception if something is referenced but missing from the dictionary of replacement values
raise LiquidRenderError(msg=f"Credential '{self.content}' is not supplied.")
return self.content
# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> # SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import datetime import datetime
import json
import os import os
from typing import Any, Dict, Optional, Union from typing import Any, Dict, Optional, Union
import fastapi import fastapi
import fastapi.responses import fastapi.responses
from fastapi import BackgroundTasks, FastAPI, Request from fastapi import BackgroundTasks, FastAPI, Request
from liquid import Environment
from annotator import access_log, error_log from annotator import access_log, error_log
from annotator import config from annotator import config
...@@ -42,9 +40,13 @@ def aw_endpoint(payload: awmodels.RequestCommon, background_tasks: BackgroundTas ...@@ -42,9 +40,13 @@ def aw_endpoint(payload: awmodels.RequestCommon, background_tasks: BackgroundTas
for more details. The call is forwarded to the correct endpoint. for more details. The call is forwarded to the correct endpoint.
""" """
# Replace credentials in options # Replace Liquid Templating markup in provided options. Values from message and credentials may be
payload.params.options = utils.replace_credentials(data=payload.params.options, # referenced in Liquid statements.
credentials=payload.params.credentials) payload.params.options = utils.render_liquid(
options=payload.params.options,
message=payload.params.message,
credentials=payload.params.credentials,
)
if payload.method is not None and payload.method == "register": if payload.method is not None and payload.method == "register":
return register(payload=awmodels.RequestRegister.parse_obj(payload.dict())) return register(payload=awmodels.RequestRegister.parse_obj(payload.dict()))
...@@ -110,16 +112,10 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks) ...@@ -110,16 +112,10 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
response = awmodels.ResponseReceive() response = awmodels.ResponseReceive()
response.result.memory.archives += payload.params.memory.archives response.result.memory.archives += payload.params.memory.archives
# create a liquid templating environment and render the options as a JSON
env = Environment()
template = env.from_string(
payload.params.options.json(exclude_none=True, exclude_unset=True)
)
rendered_options = json.loads(template.render(payload.params.message.payload))
payload.params.message.payload = awmodels.PayloadInput.parse_obj( payload.params.message.payload = awmodels.PayloadInput.parse_obj(
payload.params.message.payload payload.params.message.payload
) )
for k, v in rendered_options.items(): for k, v in payload.params.options.dict().items():
if v: if v:
payload.params.message.payload.__setattr__(k, v) payload.params.message.payload.__setattr__(k, v)
...@@ -145,10 +141,10 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks) ...@@ -145,10 +141,10 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
vault_id = settings["vault_id"] vault_id = settings["vault_id"]
if not stores.is_valid_archive( if not stores.is_valid_archive(
vault_id=vault_id, vault_id=vault_id,
archive_id=archive_id, archive_id=archive_id,
cdstar_uri=settings["cdstar_uri"], cdstar_uri=settings["cdstar_uri"],
cdstar_auth=(settings["cdstar_user"], settings["cdstar_pass"]), cdstar_auth=(settings["cdstar_user"], settings["cdstar_pass"]),
): ):
response.result.errors.append( response.result.errors.append(
f"Archive {archive_id} is not available in CDSTAR vault {vault_id}." f"Archive {archive_id} is not available in CDSTAR vault {vault_id}."
...@@ -191,11 +187,11 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks) ...@@ -191,11 +187,11 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
def run_annotation( def run_annotation(
archive_id: str, archive_id: str,
annotations_archive: Dict[str, Any], annotations_archive: Dict[str, Any],
annotations_file: Dict[str, Any], annotations_file: Dict[str, Any],
metafile: str, metafile: str,
settings: Dict[str, str], settings: Dict[str, str],
) -> None: ) -> None:
archive, filelist = stores.get_cdstar_metadata( archive, filelist = stores.get_cdstar_metadata(
vault_id=settings["vault_id"], vault_id=settings["vault_id"],
...@@ -313,7 +309,7 @@ def check(payload: awmodels.RequestCheck): ...@@ -313,7 +309,7 @@ def check(payload: awmodels.RequestCheck):
def get_setting_from_payload( def get_setting_from_payload(
payload: Union[awmodels.RequestReceive, awmodels.RequestCheck], key: str payload: Union[awmodels.RequestReceive, awmodels.RequestCheck], key: str
) -> Optional[str]: ) -> Optional[str]:
msg_payload = payload.params.message.payload.dict() msg_payload = payload.params.message.payload.dict()
if key in msg_payload.keys() and msg_payload[key]: if key in msg_payload.keys() and msg_payload[key]:
......
...@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, validator ...@@ -9,6 +9,7 @@ from pydantic import BaseModel, Field, validator
from annotator import config from annotator import config
# TODO: for now, I use dict for annotations # TODO: for now, I use dict for annotations
# from annotator.models_metadata import MetadataBase # from annotator.models_metadata import MetadataBase
......
# SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
#
# SPDX-License-Identifier: GPL-3.0-or-later
# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> # SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
...@@ -13,9 +13,9 @@ import pycdstar3 ...@@ -13,9 +13,9 @@ import pycdstar3
import pytest import pytest
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from annotator.main import app, endpoint_name
from annotator import config from annotator import config
from annotator import test_utils from annotator.main import app, endpoint_name
from annotator.test import test_utils
client = TestClient(app) client = TestClient(app)
...@@ -70,7 +70,7 @@ def test_register_compliance_01(): ...@@ -70,7 +70,7 @@ def test_register_compliance_01():
""" """
register_request = {"method": "register", "params": {}} register_request = {"method": "register", "params": {}}
response = client.post(f"/{endpoint_name}", json=register_request) response = client.post(f"/{endpoint_name}", json=register_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
response_data = response.json() response_data = response.json()
assert response_data assert response_data
assert "result" in response_data.keys() assert "result" in response_data.keys()
...@@ -99,7 +99,7 @@ def disabled_test_receive_compliance_01(): ...@@ -99,7 +99,7 @@ def disabled_test_receive_compliance_01():
}, },
} }
response = client.post(f"/{endpoint_name}", json=receive_request) response = client.post(f"/{endpoint_name}", json=receive_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
response_data = response.json() response_data = response.json()
assert response_data assert response_data
assert test_utils.is_valid_response(response_data) assert test_utils.is_valid_response(response_data)
...@@ -119,7 +119,7 @@ def disabled_test_receive_compliance_02(): ...@@ -119,7 +119,7 @@ def disabled_test_receive_compliance_02():
}, },
} }
response = client.post(f"/{endpoint_name}", json=receive_request) response = client.post(f"/{endpoint_name}", json=receive_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
response_data = response.json() response_data = response.json()
assert response_data assert response_data
assert test_utils.is_valid_response(response_data) assert test_utils.is_valid_response(response_data)
...@@ -131,10 +131,15 @@ def disabled_test_check_compliance_01(): ...@@ -131,10 +131,15 @@ def disabled_test_check_compliance_01():
""" """
check_request = { check_request = {
"method": "check", "method": "check",
"params": {"message": None, "options": {}, "memory": {}, "credentials": [],}, "params": {
"message": None,
"options": {},
"memory": {},
"credentials": [],
},
} }
response = client.post(f"/{endpoint_name}", json=check_request) response = client.post(f"/{endpoint_name}", json=check_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
response_data = response.json() response_data = response.json()
assert response_data assert response_data
assert test_utils.is_valid_response(response_data) assert test_utils.is_valid_response(response_data)
...@@ -154,7 +159,7 @@ def disabled_test_check_compliance_02(): ...@@ -154,7 +159,7 @@ def disabled_test_check_compliance_02():
}, },
} }
response = client.post(f"/{endpoint_name}", json=check_request) response = client.post(f"/{endpoint_name}", json=check_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
response_data = response.json() response_data = response.json()
assert response_data assert response_data
assert test_utils.is_valid_response(response_data) assert test_utils.is_valid_response(response_data)
...@@ -177,7 +182,7 @@ def test_complete_workflow(cdstar_archive): ...@@ -177,7 +182,7 @@ def test_complete_workflow(cdstar_archive):
}, },
} }
response = client.post(f"/{endpoint_name}", json=receive_request) response = client.post(f"/{endpoint_name}", json=receive_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
json = response.json() json = response.json()
assert json assert json
...@@ -199,7 +204,7 @@ def test_complete_workflow(cdstar_archive): ...@@ -199,7 +204,7 @@ def test_complete_workflow(cdstar_archive):
}, },
} }
response = client.post(f"/{endpoint_name}", json=check_request) response = client.post(f"/{endpoint_name}", json=check_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
response_json = response.json() response_json = response.json()
assert response_json assert response_json
......
# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> # SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
...@@ -15,7 +15,7 @@ import pytest ...@@ -15,7 +15,7 @@ import pytest
from fastapi.testclient import TestClient from fastapi.testclient import TestClient
from annotator.main import app, endpoint_name from annotator.main import app, endpoint_name
from annotator import test_utils from annotator.test import test_utils
from annotator import utils from annotator import utils
client = TestClient(app) client = TestClient(app)
...@@ -146,4 +146,4 @@ def test_check_04(): ...@@ -146,4 +146,4 @@ def test_check_04():
response_data = response.json() response_data = response.json()
assert response_data assert response_data
if "archives" in response_data["result"]["memory"].keys(): if "archives" in response_data["result"]["memory"].keys():
assert len(response_data["result"]["memory"]["archives"]) == 0 assert len(response_data["result"]["memory"]["archives"]) == 0
\ No newline at end of file
# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> # SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
...@@ -21,7 +21,7 @@ from fastapi.testclient import TestClient ...@@ -21,7 +21,7 @@ from fastapi.testclient import TestClient
from annotator.main import app, endpoint_name from annotator.main import app, endpoint_name
from annotator import config from annotator import config
from annotator import test_utils from annotator.test import test_utils
client = TestClient(app) client = TestClient(app)
...@@ -90,7 +90,7 @@ def test_invalid_request_02(): ...@@ -90,7 +90,7 @@ def test_invalid_request_02():
""" """
payload = {"method": "invalid"} payload = {"method": "invalid"}
response = client.post(f"/{endpoint_name}", json=payload) response = client.post(f"/{endpoint_name}", json=payload)
assert response.status_code >= 400 and response.status_code <= 499 assert 400 <= response.status_code <= 499
def post_valid_receive_request( def post_valid_receive_request(
...@@ -117,7 +117,7 @@ def post_valid_receive_request( ...@@ -117,7 +117,7 @@ def post_valid_receive_request(
}, },
} }
response = client.post(f"/{endpoint_name}", json=register_request) response = client.post(f"/{endpoint_name}", json=register_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
json = response.json() json = response.json()
assert json assert json
assert test_utils.is_valid_response(json) assert test_utils.is_valid_response(json)
...@@ -174,7 +174,7 @@ def post_valid_liquid_request( ...@@ -174,7 +174,7 @@ def post_valid_liquid_request(
} }
response = client.post(f"/{endpoint_name}", json=register_request) response = client.post(f"/{endpoint_name}", json=register_request)
assert response.status_code >= 200 and response.status_code <= 299 assert 200 <= response.status_code <= 299
json = response.json() json = response.json()
assert json assert json
assert test_utils.is_valid_response(json) assert test_utils.is_valid_response(json)
...@@ -266,15 +266,16 @@ def test_receive_03(cdstar_archive): ...@@ -266,15 +266,16 @@ def test_receive_03(cdstar_archive):
vault_id = cdstar_archive[0] vault_id = cdstar_archive[0]
archive_id = cdstar_archive[1] archive_id = cdstar_archive[1]
post_valid_receive_request(archive_id) post_valid_receive_request(archive_id)
settings = config.BasicSettings()
# Check if metadata is available for each file of CDSTAR # Check if metadata is available for each file of CDSTAR
for file_offset in range(0, generated_files_count, file_list_limit): for file_offset in range(0, generated_files_count, file_list_limit):
# external request: use requests directly instead of the TestClient # external request: use requests directly instead of the TestClient
cdstar_json = test_utils.get_json_of_uri( cdstar_json = test_utils.get_json_of_uri(
f"{config.BasicSettings().cdstar_uri}/{vault_id}/{archive_id}?files&limit={file_list_limit}&offset={file_offset}", f"{settings.cdstar_uri}/{vault_id}/{archive_id}?files&limit={file_list_limit}&offset={file_offset}",
auth=( auth=(
config.BasicSettings().cdstar_user, settings.cdstar_user,
config.BasicSettings().cdstar_pass, settings.cdstar_pass,
), ),
) )
...@@ -283,10 +284,10 @@ def test_receive_03(cdstar_archive): ...@@ -283,10 +284,10 @@ def test_receive_03(cdstar_archive):
for file_info in cdstar_json["files"]: for file_info in cdstar_json["files"]:
# external request: use requests directly instead of the TestClient # external request: use requests directly instead of the TestClient
meta_json = test_utils.get_json_of_uri( meta_json = test_utils.get_json_of_uri(
f"{config.BasicSettings().couch_uri}/{config.BasicSettings().couch_db}/{file_info['id']}", f"{settings.couch_uri}/{settings.couch_db}/{file_info['id']}",
auth=( auth=(
config.BasicSettings().couch_user, settings.couch_user,
config.BasicSettings().couch_pass, settings.couch_pass,
), ),
) )
assert test_utils.is_schemaorg_jsonld(meta_json) assert test_utils.is_schemaorg_jsonld(meta_json)
...@@ -629,4 +630,4 @@ def test_receive_12(cdstar_archive): ...@@ -629,4 +630,4 @@ def test_receive_12(cdstar_archive):
) )
assert "name" in meta_file_json.keys() assert "name" in meta_file_json.keys()
assert meta_file_json["name"] == "AddedName" assert meta_file_json["name"] == "AddedName"
\ No newline at end of file
# SPDX-FileCopyrightText: 2020 UMG MeDIC <marcel.parciak@med.uni-goettingen.de> # SPDX-FileCopyrightText: 2020-2021 UMG MeDIC <medic.tech@med.uni-goettingen.de>
# #
# SPDX-License-Identifier: GPL-3.0-or-later # SPDX-License-Identifier: GPL-3.0-or-later
import os import os
import tempfile import tempfile
from typing import Any, Dict, Tuple from typing import Any, Dict, Tuple
import pytest import pytest
import requests import requests
from liquid import LiquidRenderError
from annotator.errors import ActiveWorkflowError
from annotator.models_activeworkflow import CredentialsCommon from annotator.models_activeworkflow import CredentialsCommon
from annotator.utils import replace_credentials from annotator.utils import render_liquid
def test_replace_credentials(): def test_render_liquid_credentials():
""" """Tests whether ActiveWorkflow credential syntax is replaced correctly."""
Tests whether ActiveWorkflow credential syntax is replaced correctly
"""
params = { params = {
'option1': 'no credential reference', "option1": "no credential reference",
'option2': '{% credential cred_ref_2 %}', "option2": "{% credential cred_ref_2 %}",
'option3': 'Just a normal {{ liquid.reference }}, should be ignored', "option3": "Just a normal {{ liquid.reference }}, should be ignored",
'option4': {'another': 'dict', 'with_ref': '{% credential cred1 %}'}, "option4": {"another": "dict", "with_ref": "{% credential cred1 %}"},
'option5': 'reference within {% credential cred1 %} normal text', "option5": "reference within {% credential cred1 %} normal text",
'option6': 'two references within {% credential cred1 %} normal {% credential cred_ref_2 %} text' "option6": "two references within {% credential cred1 %} normal {% credential cred_ref_2 %} text",
"option7": [
"lists",
"are",
"also possible with {% credential cred1 %}",
"{% credential cred_ref_2 %}",
],
} }
credentials = [CredentialsCommon(name="cred1", value="thisisasecret"), message = {"payload": {"liquid": {"reference": "replaced"}}}
CredentialsCommon(name="cred_ref_2", value="thisisanothersecret")] credentials = [
CredentialsCommon(name="cred1", value="thisisasecret"),
CredentialsCommon(name="cred_ref_2", value="thisisanothersecret"),
]
params_expected = { params_expected = {
'option1': 'no credential reference', "option1": "no credential reference",
'option2': 'thisisanothersecret', "option2": "thisisanothersecret",
'option3': 'Just a normal {{ liquid.reference }}, should be ignored', "option3": "Just a normal replaced, should be ignored",
'option4': {'another': 'dict', 'with_ref': 'thisisasecret'}, "option4": {"another": "dict", "with_ref": "thisisasecret"},
'option5': 'reference within thisisasecret normal text', "option5": "reference within thisisasecret normal text",
'option6': 'two references within thisisasecret normal thisisanothersecret text' "option6": "two references within thisisasecret normal thisisanothersecret text",
"option7": [
"lists",
"are",
"also possible with thisisasecret",
"thisisanothersecret",
],
} }
params_replaced = replace_credentials(params, credentials) params_replaced = render_liquid(
options=params, message=message, credentials=credentials
)
assert params_replaced == params_expected assert params_replaced == params_expected
# we expect no changes, if the dictionary contains no references # we expect no changes, if the dictionary contains no references
params = { params = {
'option1': 'no credential reference', "option1": "no credential reference",
'option3': 'Just a normal {{ liquid.reference }}, should be ignored', "option3": "Just a normal {{ liquid.reference }}, should be ignored",
'option4': {'another': 'dict'} "option4": {"another": "dict"},
} }
params_replaced = replace_credentials(params, credentials) params_expected = {
assert params_replaced == params "option1": "no credential reference",
"option3": "Just a normal replaced, should be ignored",
"option4": {"another": "dict"},
}
params_replaced = render_liquid(
options=params, message=message, credentials=credentials
)
assert params_replaced == params_expected