Commit 72e45435 authored by parciak's avatar parciak
Browse files

Added initial support for Liquid Templating via the options.


Signed-off-by: parciak's avatarMarcel Parciak <marcel.parciak@gmail.com>
parent 3766ee28
......@@ -20,6 +20,7 @@ alembic = "*"
pytest = "*"
pycdstar3 = {git = "https://gitlab.gwdg.de/cdstar/pycdstar3"}
jinja2 = "*"
python-liquid = "*"
[requires]
python_version = "3.8"
......
This diff is collapsed.
......@@ -3,8 +3,11 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import datetime
import json
import os
from typing import Any, Dict, Optional, Union
from typing import Any, Dict, List, Optional, Union
from liquid.context import Template
from annotator import access_log, error_log
from annotator import config
......@@ -17,6 +20,8 @@ from fastapi import BackgroundTasks, FastAPI, Request
import fastapi
import fastapi.responses
from liquid import Environment
app = FastAPI()
endpoint_name = "annotator"
......@@ -103,6 +108,19 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
response = awmodels.ResponseReceive()
response.result.memory.archives += payload.params.memory.archives
# create a liquid templating environment and render the options as a JSON
env = Environment()
template = env.from_string(
payload.params.options.json(exclude_none=True, exclude_unset=True)
)
rendered_options = json.loads(template.render(payload.params.message.payload))
payload.params.message.payload = awmodels.PayloadInput.parse_obj(
payload.params.message.payload
)
for k, v in rendered_options.items():
if v:
payload.params.message.payload.__setattr__(k, v)
archive_id = payload.params.message.payload.archive_id
settings = {}
for key in [
......
......@@ -3,7 +3,7 @@
# SPDX-License-Identifier: GPL-3.0-or-later
import abc
from typing import Any, Dict, List, Optional, Tuple
from typing import Any, Dict, List, Optional, Tuple, Union
from pydantic import BaseModel, Field, validator
......@@ -91,7 +91,7 @@ class PayloadInput(BaseModel):
This model represents the expected input payload when a message is received via `receive`.
"""
archive_id: str = Field(..., example="a1b2c3d4e5f6")
archive_id: str = Field(None, example="a1b2c3d4e5f6")
vault_id: str = Field(None, example="medic")
cdstar_uri: str = Field(None, example="http://localhost:8082/v3")
cdstar_user: str = Field(None, example="someuser")
......@@ -149,12 +149,13 @@ class OptionsCommon(BaseModel):
This model represents expected options to configure this agent (or rather a call to this agent).
"""
archive_id: str = Field(None, example="{{ archive_id_key }}")
vault_id: str = Field(None, example="medic")
cdstar_uri: str = Field(None, example="http://localhost:8082/v3")
cdstar_user: str = Field(None, example="someuser")
cdstar_pass: str = Field(None, example="somepass")
couch_db: str = Field(None, example="medic")
couch_uri: str = Field(None, example="http://lcoalhost:5984")
couch_uri: str = Field(None, example="http://localhost:5984")
couch_user: str = Field(None, example="someuser")
couch_pass: str = Field(None, example="somepass")
api_key: str = Field("", example="")
......@@ -179,9 +180,7 @@ class MemoryCommon(BaseModel):
This model represents the expected memory content to communicate state of the agent.
"""
archives: List[Tuple[str, str]] = Field([], example=[(
"medic", "a1b2c3d4e5f6"
)])
archives: List[Tuple[str, str]] = Field([], example=[("medic", "a1b2c3d4e5f6")])
@staticmethod
def example() -> dict:
......@@ -235,7 +234,7 @@ class ResponseRegister(ResponseCommon):
class MessageInputReceive(BaseModel):
payload: PayloadInput
payload: Union[Dict[str, Any], PayloadInput] # Union[PayloadInput, Dict[str, Any]]
class ParamsReceive(ParamsCommon):
......
......@@ -6,6 +6,8 @@ import datetime
import io
import json
import os
import random
import string
import tempfile
import time
from typing import Any, Dict
......@@ -23,7 +25,7 @@ from annotator import test_utils
client = TestClient(app)
generated_files_count = 250
generated_files_count = 25 # 0
file_list_limit = 50
......@@ -60,11 +62,13 @@ def delete_metadata(vault_id: str, archive_id: str):
user_basic_auth=True,
) as client:
db: CloudantDatabase.CouchDatabase = client[config.BasicSettings().couch_db]
wait_iterations = 3
wait_iterations = 1
while wait_iterations > 0 and archive_id not in db:
# wait for receive to finish in order to ensure no dead docs are created.
time.sleep(3)
wait_iterations -= 1
if archive_id not in db:
return
archive: CloudantDocument.Document = db[archive_id]
for part in archive["hasPart"]:
f: CloudantDocument.Document = db[part["@id"].split("/")[-1]]
......@@ -91,7 +95,6 @@ def test_invalid_request_02():
def post_valid_receive_request(
cdstar_archive: str,
annotation: Dict[str, Any] = {},
annot_archive: Dict[str, Any] = {},
annot_file: Dict[str, Any] = {},
) -> Dict[str, Any]:
......@@ -108,7 +111,7 @@ def post_valid_receive_request(
"annotations_file": annot_file,
}
},
"options": {"annotation": annotation},
"options": {},
"memory": {},
"credentials": [],
},
......@@ -121,6 +124,63 @@ def post_valid_receive_request(
return json
def post_valid_liquid_request(
cdstar_archive: str,
annot_archive: Dict[str, Any] = {},
annot_file: Dict[str, Any] = {},
) -> Dict[str, Any]:
"""
This method is used to create requests analogous to post_valid_receive_request while using
liquid templating.
"""
mock_message = {"archiv": {}, "datei": {}}
cdstar_key = "".join(random.choices(string.ascii_letters, k=8))
mock_message[cdstar_key] = cdstar_archive
# create a set of random keys for archive annotations
annot_archive_keys = {}
for k, v in annot_archive.items():
random_key = "".join(random.choices(string.ascii_letters, k=10))
while random_key in annot_archive_keys.keys():
random_key = "".join(random.choices(string.ascii_letters, k=10))
annot_archive_keys[random_key] = k
mock_message["archiv"][random_key] = v
# create a set of random keys for file annotations
annot_file_keys = {}
for k, v in annot_file.items():
random_key = "".join(random.choices(string.ascii_letters, k=12))
while random_key in annot_file_keys.keys():
random_key = "".join(random.choices(string.ascii_letters, k=12))
annot_file_keys[random_key] = k
mock_message["datei"][random_key] = v
register_request = {
"method": "receive",
"params": {
"message": {"payload": mock_message},
"options": {
"archive_id": "{{ " + cdstar_key + " }}",
"annotations_archive": {
v: "{{ archiv." + k + " }}" for k, v in annot_archive_keys.items()
},
"annotations_file": {
v: "{{ datei." + k + " }}" for k, v in annot_file_keys.items()
},
},
"memory": {},
"credentials": [],
},
}
response = client.post(f"/{endpoint_name}", json=register_request)
assert response.status_code >= 200 and response.status_code <= 299
json = response.json()
assert json
assert test_utils.is_valid_response(json)
return json
def test_receive_01(cdstar_archive):
"""
Tests if a JSON-LD using Schema.org is uploaded to CouchDB.
......@@ -329,7 +389,7 @@ def test_receive_06(cdstar_archive):
"used_job_version": "1.0",
}
response_data = post_valid_receive_request(
cdstar_archive[1], annotation={}, annot_archive=annot_archive, annot_file={}
cdstar_archive[1], annot_archive=annot_archive, annot_file={}
)
meta_json = test_utils.get_json_of_uri(
......@@ -365,7 +425,7 @@ def test_receive_07(cdstar_archive):
"abstract": "This is a dummy file, please do not do any science with it!",
}
response_data = post_valid_receive_request(
cdstar_archive[1], annotation={}, annot_archive={}, annot_file=annot_file
cdstar_archive[1], annot_archive={}, annot_file=annot_file
)
meta_json = test_utils.get_json_of_uri(
......@@ -409,3 +469,164 @@ def test_receive_invalid_archive_01():
assert len(response_data["result"]["errors"]) >= 1
assert len(response_data["result"]["logs"]) == 0
assert len(response_data["result"]["messages"]) == 0
def test_receive_08(cdstar_archive):
"""
Tests if a JSON-LD using Schema.org is uploaded to CouchDB using Liquid Templating.
"""
vault_id = cdstar_archive[0]
archive_id = cdstar_archive[1]
post_valid_liquid_request(archive_id)
assert os.path.exists(
os.path.join(
tempfile.gettempdir(),
f"meta_{vault_id}_{archive_id}.json",
)
)
# external request: use requests directly instead of the TestClient
meta_json = test_utils.get_json_of_uri(
f"{config.BasicSettings().couch_uri}/{config.BasicSettings().couch_db}/{archive_id}",
auth=(config.BasicSettings().couch_user, config.BasicSettings().couch_pass),
)
# assert that the response is a JSON-LD of schema.org/Dataset
assert test_utils.is_schemaorg_jsonld(meta_json)
assert meta_json["@type"] == "Dataset"
assert meta_json["identifier"] == archive_id
test_utils.cleanup_metafile(vault_id, archive_id)
def test_receive_09(cdstar_archive):
"""
Tests if the uploaded archive metadata matches the CDSTAR archive using liquid templating.
"""
vault_id = cdstar_archive[0]
archive_id = cdstar_archive[1]
post_valid_liquid_request(archive_id)
# external request: use requests directly instead of the TestClient
cdstar_json = test_utils.get_json_of_uri(
f"{config.BasicSettings().cdstar_uri}/{vault_id}/{archive_id}",
auth=(config.BasicSettings().cdstar_user, config.BasicSettings().cdstar_pass),
)
# external request: use requests directly instead of the TestClient
meta_json = test_utils.get_json_of_uri(
f"{config.BasicSettings().couch_uri}/{config.BasicSettings().couch_db}/{archive_id}",
auth=(config.BasicSettings().couch_user, config.BasicSettings().couch_pass),
)
# Compare both JSON replies and the data they contain
assert test_utils.is_schemaorg_jsonld(meta_json)
assert meta_json["identifier"] == archive_id
# across python versions, parsing datetimes does not work homogeneously (changed in Python 3.7). Fix this
cdstar_json["created"] = cdstar_json["created"][:-4] + cdstar_json["created"][
-4:
].replace(":", "")
cdstar_json["modified"] = cdstar_json["modified"][:-4] + cdstar_json["modified"][
-4:
].replace(":", "")
meta_json["dateCreated"] = meta_json["dateCreated"][:-4] + meta_json["dateCreated"][
-4:
].replace(":", "")
meta_json["dateModified"] = meta_json["dateModified"][:-4] + meta_json[
"dateModified"
][-4:].replace(":", "")
assert datetime.datetime.strptime(
cdstar_json["created"], "%Y-%m-%dT%H:%M:%S.%f%z"
) == datetime.datetime.strptime(meta_json["dateCreated"], "%Y-%m-%dT%H:%M:%S.%f%z")
assert datetime.datetime.strptime(
cdstar_json["modified"], "%Y-%m-%dT%H:%M:%S.%f%z"
) == datetime.datetime.strptime(meta_json["dateModified"], "%Y-%m-%dT%H:%M:%S.%f%z")
assert int(cdstar_json["file_count"]) == int(meta_json["size"]["value"])
assert int(cdstar_json["file_count"]) == len(meta_json["hasPart"])
test_utils.cleanup_metafile(vault_id, archive_id)
def test_receive_10(cdstar_archive):
"""
Tests if a receive request yields a memory for checking afterwards using liquid templating.
"""
vault_id = cdstar_archive[0]
archive_id = cdstar_archive[1]
response_data = post_valid_liquid_request(archive_id)
assert test_utils.has_memory_archive(response_data)
test_utils.cleanup_metafile(vault_id, archive_id)
def test_receive_11(cdstar_archive):
"""
Tests if supplied archive annotations are set to CouchDB accordingly using liquid templating.
"""
annot_archive = {
# TODO: adding whole subtrees does not work out of the box.
# "maintainer": {
# "@id": "../meta/marcel.parciak@med.uni-goettingen.de",
# "@type": "Person",
# },
"used_job": "DummyJob",
"used_job_version": "1.0",
}
response_data = post_valid_liquid_request(
cdstar_archive[1], annot_archive=annot_archive, annot_file={}
)
meta_json = test_utils.get_json_of_uri(
f"{config.BasicSettings().couch_uri}/{config.BasicSettings().couch_db}/{cdstar_archive[1]}",
auth=(config.BasicSettings().couch_user, config.BasicSettings().couch_pass),
)
assert "used_job" in meta_json.keys()
assert meta_json["used_job"] == "DummyJob"
assert "used_job_version" in meta_json.keys()
assert meta_json["used_job_version"] == "1.0"
def test_receive_12(cdstar_archive):
"""
Tests if supplied file annotations are set to CouchDB accordingly using liquid templating.
"""
annot_file = {
# TODO: addin whole subtrees does not work out of the box
# "creator": {
# "@id": "../meta/marcel.parciak@med.uni-goettingen.de",
# "@type": "Person",
# },
"abstract": "This is a dummy file, please do not do any science with it!",
"name": "AddedName",
}
response_data = post_valid_liquid_request(
cdstar_archive[1], annot_archive={}, annot_file=annot_file
)
meta_json = test_utils.get_json_of_uri(
f"{config.BasicSettings().couch_uri}/{config.BasicSettings().couch_db}/{cdstar_archive[1]}",
auth=(config.BasicSettings().couch_user, config.BasicSettings().couch_pass),
)
assert "hasPart" in meta_json.keys()
assert type(meta_json["hasPart"]) == list
for part in meta_json["hasPart"]:
meta_file_json = test_utils.get_json_of_uri(
f"{part['@id']}",
auth=(config.BasicSettings().couch_user, config.BasicSettings().couch_pass),
)
assert "abstract" in meta_file_json.keys()
assert (
meta_file_json["abstract"]
== "This is a dummy file, please do not do any science with it!"
)
assert "name" in meta_file_json.keys()
assert meta_file_json["name"] == "AddedName"
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment