Commit 13f161aa authored by parciak's avatar parciak
Browse files

Merge branch 'parciak-fix-missing-couchuri' into 'master'

Version 0.3.2

See merge request !4
parents 83cf44d1 21a303de
Pipeline #170943 failed with stages
in 3 minutes and 1 second
......@@ -7,10 +7,10 @@ stages:
- publish
variables:
CDSTAR_URI: "https://c109-199.cloud.gwdg.de/v3"
CDSTAR_URI: "http://vm18212.virt.gwdg.de:8080/v3"
CDSTAR_VAULT: "medic"
COUCH_URI: "https://c109-199.cloud.gwdg.de:5984"
COUCH_DB: "annotation_test"
COUCH_URI: "http://vm18212.virt.gwdg.de:8008"
COUCH_DB: "annotation_agent_test"
build_image:
stage: test
......@@ -78,7 +78,7 @@ dryrun_publish_to_harbor:
- master
before_script:
- docker login --username marcelparciak --password $HARBOR_PUSH_TOKEN https://harbor.umg.eu
- docker login --username $MEDIC_HARBOR_PUSH_USERNAME --password $MEDIC_HARBOR_PUSH_SECRET https://harbor.umg.eu
- AGENTVERSION=`python3 setup.py --version`
script:
......@@ -120,7 +120,7 @@ publish_to_harbor:
- master
before_script:
- docker login --username marcelparciak --password $HARBOR_PUSH_TOKEN https://harbor.umg.eu
- docker login --username $MEDIC_HARBOR_PUSH_USERNAME --password $MEDIC_HARBOR_PUSH_SECRET https://harbor.umg.eu
- AGENTVERSION=`python3 setup.py --version`
script:
......
......@@ -2,6 +2,6 @@
#
# SPDX-License-Identifier: GPL-3.0-or-later
VERSION = (0, 3, 1)
VERSION = (0, 3, 2)
__version__ = ".".join(map(str, VERSION))
......@@ -97,7 +97,7 @@ def receive(payload: awmodels.RequestReceive, background_tasks: BackgroundTasks)
if not utils.is_authorized(payload.params):
raise errors.UnauthorizedException(
message="Please supply valid credentials using the `api_key` option parameter referring to a stored credentials key. You may use Liquid templating: `\"api_key\": \"{% credential credential_name %}\"`.",
message='Please supply valid credentials using the `api_key` option parameter referring to a stored credentials key. You may use Liquid templating: `"api_key": "{% credential credential_name %}"`.',
)
response = awmodels.ResponseReceive()
......@@ -194,37 +194,48 @@ def run_annotation(
utils.merge_jsonld(f.to_jsonld(), annotations_file) for f in filelist
]
if "hasPart" not in jsonld_archive or type(jsonld_archive["hasPart"]) is not list:
jsonld_archive["hasPart"] = []
# files will be added after uploading them
jsonld_archive["hasPart"] = []
jsonld_archive = stores.upload_document(
jsonld_archive,
couch_db=settings["couch_db"],
couch_uri=settings["couch_uri"],
couch_auth=(settings["couch_user"], settings["couch_pass"]),
)
for meta_file in jsonld_files:
jsonld_archive["hasPart"].append(utils.get_jsonld_reference(meta_file))
meta_file["isPartOf"] = utils.get_jsonld_reference(jsonld_archive)
meta_file = stores.upload_document(
meta_file,
couch_db=settings["couch_db"],
couch_uri=settings["couch_uri"],
couch_auth=(settings["couch_user"], settings["couch_pass"]),
)
jsonld_archive["hasPart"].append(utils.get_jsonld_reference(meta_file))
if not stores.upload_archive(
jsonld_archive = stores.upload_document(
jsonld_archive,
couch_db=settings["couch_db"],
couch_uri=settings["couch_uri"],
couch_auth=(settings["couch_user"], settings["couch_pass"]),
):
)
# TODO: raise and catch an upload errors here!
"""
if not json_archive:
errmsg = (
f"Could not upload archive annotations for archive {archive_id} to CouchDB."
)
error_log.error(errmsg)
os.unlink(metafile)
return
"""
if not stores.upload_files(
jsonld_files,
couch_db=settings["couch_db"],
couch_uri=settings["couch_uri"],
couch_auth=(settings["couch_user"], settings["couch_pass"]),
):
errmsg = f"Could not upload file annotations for files of archive {archive_id} to CouchDB."
error_log.error(errmsg)
utils.write_state_to_metafile(utils.AnnotationState.incomplete, metafile)
return
utils.write_details_to_metafile(
{"link": jsonld_archive["@id"]},
metafile,
)
utils.write_state_to_metafile(utils.AnnotationState.completed, metafile)
......@@ -241,7 +252,7 @@ def check(payload: awmodels.RequestCheck):
if not utils.is_authorized(payload.params):
raise errors.UnauthorizedException(
message="Please supply valid credentials using the `api_key` option parameter referring to a stored credentials key. You may use Liquid templating: `\"api_key\": \"{% credential credential_name %}\"`.",
message='Please supply valid credentials using the `api_key` option parameter referring to a stored credentials key. You may use Liquid templating: `"api_key": "{% credential credential_name %}"`.',
)
resp = awmodels.ResponseCheck()
......@@ -262,7 +273,7 @@ def check(payload: awmodels.RequestCheck):
elif state["state"] == utils.AnnotationState.completed:
msg = awmodels.MessageOutput(
archive_id=archive_id, couch_uri=f"", stats=state
archive_id=archive_id, couch_uri=state["link"], stats=state
)
resp.result.messages.append(msg)
utils.remove_metafile(metafile)
......
......@@ -9,6 +9,7 @@ from typing import Any, Dict, List, Optional, Tuple
import cloudant
import cloudant.database as CloudantDatabase
import cloudant.document as CloudantDocument
import cloudant.error
from pycdstar3 import CDStar, CDStarVault
from annotator import error_log
......@@ -18,7 +19,10 @@ from annotator import models_cdstar as cdmodels
def is_valid_archive(
vault_id: str, archive_id: str, cdstar_uri: str, cdstar_auth: Tuple[str, str]
) -> bool:
cdstar = CDStar(cdstar_uri, auth=cdstar_auth,)
cdstar = CDStar(
cdstar_uri,
auth=cdstar_auth,
)
try:
vault = CDStarVault(cdstar, vault_id)
......@@ -33,7 +37,10 @@ def is_valid_archive(
def get_cdstar_metadata(
vault_id: str, archive_id: str, cdstar_uri: str, cdstar_auth: Tuple[str, str]
) -> Tuple[Optional[cdmodels.ArchiveInfo], List[cdmodels.FileInfo]]:
cdstar = CDStar(cdstar_uri, auth=cdstar_auth,)
cdstar = CDStar(
cdstar_uri,
auth=cdstar_auth,
)
try:
vault = CDStarVault(cdstar, vault_id)
......@@ -52,36 +59,29 @@ def get_cdstar_metadata(
return None, []
def upload_archive(
def upload_document(
archive_meta: Dict[str, Any],
couch_db: str,
couch_uri: str,
couch_auth: Tuple[str, str],
) -> bool:
return upload_jsonld(archive_meta, couch_db, couch_uri, couch_auth)
def upload_files(
files_metalist: List[Dict[str, Any]],
couch_db: str,
couch_uri: str,
couch_auth: Tuple[str, str],
) -> bool:
for file_meta in files_metalist:
if not upload_jsonld(file_meta, couch_db, couch_uri, couch_auth):
return False
return True
) -> Dict[str, Any]:
document_url = upload_jsonld(archive_meta, couch_db, couch_uri, couch_auth)
archive_meta["@id"] = document_url
return archive_meta
def upload_jsonld(
jsonld: Dict[str, Any], couch_db: str, couch_uri: str, couch_auth: Tuple[str, str]
) -> bool:
) -> Optional[str]:
if "identifier" not in jsonld.keys():
error_log.error(f"Supplied jsonld has no id for `upload_jsonld`: {jsonld}")
return False
return None
try:
with cloudant.couchdb(
couch_auth[0], couch_auth[1], url=couch_uri, use_basic_auth=True,
couch_auth[0],
couch_auth[1],
url=couch_uri,
use_basic_auth=True,
) as client:
database: CloudantDatabase.CouchDatabase = client[couch_db]
if not database.exists():
......@@ -90,10 +90,18 @@ def upload_jsonld(
)
return False
jsonld["_id"] = jsonld["identifier"]
created_doc: CloudantDocument.Document = database.create_document(jsonld)
return created_doc.exists()
try:
created_doc: CloudantDocument.Document = database.create_document(
jsonld, throw_on_exists=True
)
return created_doc.document_url
except cloudant.error.CloudantDatabaseException:
existing_doc = database[jsonld["_id"]]
existing_doc.update(jsonld)
existing_doc.save()
return existing_doc.document_url
except:
t, v, tb = sys.exc_info()
error_log.error("Could not connect to CouchDB.")
error_log.error(f"{t}: {v}\n{traceback.print_tb(tb)}")
return False
return None
......@@ -34,6 +34,7 @@ def cdstar_archive():
"state": utils.AnnotationState.completed,
"started_at": datetime.datetime.now().isoformat(),
"completed_at": datetime.datetime.now().isoformat(),
"link": "http://example.org/couch/1234abcd",
}
json.dump(mocked_state, metafile)
yield vault_id, archive_id
......
......@@ -64,6 +64,7 @@ def delete_metadata(vault_id: str, archive_id: str):
while wait_iterations > 0 and archive_id not in db:
# wait for receive to finish in order to ensure no dead docs are created.
time.sleep(3)
wait_iterations -= 1
archive: CloudantDocument.Document = db[archive_id]
for part in archive["hasPart"]:
f: CloudantDocument.Document = db[part["@id"].split("/")[-1]]
......@@ -128,7 +129,10 @@ def test_receive_01(cdstar_archive):
archive_id = cdstar_archive[1]
post_valid_receive_request(archive_id)
assert os.path.exists(
os.path.join(tempfile.gettempdir(), f"meta_{vault_id}_{archive_id}.json",)
os.path.join(
tempfile.gettempdir(),
f"meta_{vault_id}_{archive_id}.json",
)
)
# external request: use requests directly instead of the TestClient
......
......@@ -107,12 +107,30 @@ def get_state_from_metafile(path: str) -> Optional[Dict[str, Any]]:
try:
with open(path, "r") as m_f:
state = json.load(m_f)
state["state"] = state["state"]
return state
except FileNotFoundError:
return None
def write_details_to_metafile(details: Dict[str, str], path: str) -> bool:
try:
with open(path, "r") as m_f:
state = json.load(m_f)
for k, v in details.items():
if k != "state" and not k.endswith("_at"):
state[k] = v
with open(path, "w+") as m_f:
json.dump(state, m_f)
except:
error_log.error(f"Could not read metadfile: {path}.")
t, v, tb = sys.exc_info()
error_log.error("\n".join(traceback.format_exception(etype=t, value=v, tb=tb)))
return False
return True
def remove_metafile(path: str) -> bool:
try:
with open(path, "r") as m_f:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment