Commit 90f9636e authored by parciak's avatar parciak
Browse files

Initial commit based on agent barebone.


Signed-off-by: parciak's avatarMarcel Parciak <marcel.parciak@gmail.com>
parents
Pipeline #146250 failed with stage
in 15 seconds
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
stages:
- test
- publish
build_image:
stage: test
tags:
- shell
- medic
- umg
before_script:
- docker info
script:
- docker build -t $CI_REGISTRY_IMAGE:latest .
run_tests:
stage: test
tags:
- shell
- medic
- umg
before_script:
- pip3 install --user pytest
script:
- pytest --junitxml=report.xml
artifacts:
reports:
junit: report.xml
dryrun_publish_image:
stage: test
tags:
- shell
- medic
- umg
except:
refs:
- master
before_script:
- docker login --username $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY
- AGENTVERSION=`python3 setup.py --version`
script:
- docker build -t $CI_REGISTRY_IMAGE:latest .
- docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$AGENTVERSION
dryrun_publish_to_harbor:
stage: test
tags:
- shell
- medic
- wisslan
- umg
- harbor
except:
refs:
- master
before_script:
- docker login --username marcelparciak --password $HARBOR_PUSH_TOKEN https://harbor.umg.eu
- AGENTVERSION=`python3 setup.py --version`
script:
- docker build -t harbor.umg.eu/medic/$CI_PROJECT_NAME:latest .
- docker tag harbor.umg.eu/medic/$CI_PROJECT_NAME:latest harbor.umg.eu/medic/$CI_PROJECT_NAME:$AGENTVERSION
publish_image:
stage: publish
tags:
- shell
- medic
- umg
only:
refs:
- master
before_script:
- docker login --username $CI_REGISTRY_USER --password $CI_REGISTRY_PASSWORD $CI_REGISTRY
- AGENTVERSION=`python3 setup.py --version`
script:
- docker build -t $CI_REGISTRY_IMAGE:latest .
- docker tag $CI_REGISTRY_IMAGE:latest $CI_REGISTRY_IMAGE:$AGENTVERSION
- docker push $CI_REGISTRY_IMAGE:latest
- docker push $CI_REGISTRY_IMAGE:$AGENTVERSION
publish_to_harbor:
stage: publish
tags:
- shell
- medic
- wisslan
- umg
- harbor
only:
refs:
- master
before_script:
- docker login --username marcelparciak --password $HARBOR_PUSH_TOKEN https://harbor.umg.eu
- AGENTVERSION=`python3 setup.py --version`
script:
- docker build -t harbor.umg.eu/medic/$CI_PROJECT_NAME:latest .
- docker tag harbor.umg.eu/medic/$CI_PROJECT_NAME:latest harbor.umg.eu/medic/$CI_PROJECT_NAME:$AGENTVERSION
- docker push harbor.umg.eu/medic/$CI_PROJECT_NAME:latest
- docker push harbor.umg.eu/medic/$CI_PROJECT_NAME:$AGENTVERSION
{
"python.pythonPath": "/home/marcel/.local/share/virtualenvs/annotation-agent-1DIG_x-f/bin/python",
"editor.formatOnSave": true,
"python.linting.enabled": true,
"python.formatting.provider": "black"
}
\ No newline at end of file
FROM docker.io/tiangolo/uvicorn-gunicorn-fastapi:python3.8
ENV MODULE_NAME="annotator.main"
COPY ./requirements.txt /app/requirements.txt
RUN pip3.8 install -r requirements.txt
COPY ./prestart.sh /app/prestart.sh
COPY ./setup.py /app/setup.py
COPY ./annotator /app/annotator
COPY ./Readme.md /app/Readme.md
\ No newline at end of file
[[source]]
name = "pypi"
url = "https://pypi.org/simple"
verify_ssl = true
[dev-packages]
fastapi = "*"
uvicorn = "*"
black = "*"
[packages]
cloudant = "*"
sqlalchemy = "*"
alembic = "*"
pytest = "*"
pycdstar3 = {git = "https://gitlab.gwdg.de/cdstar/pycdstar3"}
[requires]
python_version = "3.8"
[pipenv]
allow_prereleases = true
This diff is collapsed.
An agent barebone for Active Workflow agents.
# Active Workflow Barebone Agent
This Agent complies to the [external REST-API](https://github.com/automaticmode/active_workflow/blob/master/docs/remote_agent_api.md) of [ActiveWorkflow](https://github.com/automaticmode/active_workflow).
# How to use
In the first step, you should rename the `app` directory to something meaningful (although this needs many modification in each file). Do not forget to change it in setup.py, too.
After renaming, you can start to work on the main.py, which is the main entrypoint for this agent. This barebone provides you with [pydantic](https://pydantic-docs.helpmanual.io/) models for Active Workflow requests and responses. In the models_activeworkflow.py file, the "Application specifics" models allow to tailor the models to your needs. Namely, you should modify `PayloadInput`, `MessageOutput`, `OptionsCommon`, `MemoryCommon` and `CredentialsCommon`.
`config.py` holds the configuration according to [FastAPI](https://fastapi.tiangolo.com/) (i.e. it can be set via environment variables when starting uvicorn or the provided Dockerfile). Use it to make your agent configurable.
Use the `__version__.py` for versioning of your module, setup.py will try to access this value to determine the version. Also, this will add versions to the .gitlab-ci.yml for automatic creation of tagged images. For now, this is a manual process, i.e. you have to change the `VERSION` tuple in `__version__.py` to assign a version.
Keep in mind you may need to add lines to the Dockerfile if you expand you agent, e.g. by implementing a SQL backend (using [sqlalchemy](https://www.sqlalchemy.org/) and [alembic](https://alembic.sqlalchemy.org/en/latest/) of course). Also, this repo uses pipenv for package management, the Dockerfile uses pip3 instead. Do not forget to freeze manually using pipenv or use a package like [pipenv_to_requirements](https://github.com/gsemet/pipenv-to-requirements).
Last but not least, change this Readme.md and add all necessary information to it. This Readme is used in the agent description part of the Active Workflow register response, meaning that a AW-user will see it as the description of this agent.
## API Caller Configurations
### Register
This is what happens when you call register:
#### Example
```
{
"some": "option"
}
```
### Receive
This is what happens when you call receive:
#### Example
Full request:
```
{
"method": "receive",
"params": {
"message": {
"payload": {
"context": {
"test_val": 12
}
}
},
"options": {
"job_name": "DummyJob",
"job_version": "1.0",
"context": {
"test_val": 8
}
},
"memory": {
"jobs": []
},
"credentials": {}
}
}
```
Full response:
```
{
"result": {
"errors": [],
"logs": ["Job DummyJob has been started with id 21."],
"memory": {
"jobs": [21]
},
"messages": [],
}
}
```
### Check
This is what happens when you call check:
#### Example
Full request:
```
{
"method": "check",
"params": {
"message": null
"options": {
"job_name": "DummyJob",
"job_version": "1.0",
"context": {
"test_val": 8
}
},
"memory": {
"jobs": [21]
},
"credentials": [],
}
}
```
Full response:
```
"result": {
"errors": [],
"logs": [
"25|2pdwaA|QvVW2g6IdzHS|key.compareTo(25) != 0 failed\n"
],
"memory": {
"jobs": []
},
"messages": [
{
"job_run_id": 21,
"job_id": 4,
"job_name": "DummyJob",
"job_version": "1.0",
"completed": true,
"returncode": 0,
"runtime": 1.036232,
"response": {
"data": [
{
"entity": 1,
"attribute": "wARNk5",
"value": "7riXpzHPFECz"
}
]
}
}
]
}
}
```
### Additional API Calls
Any non-active workflow based API endpoints can be explained here.
## Service Configuration
Configuration can be done using environment variables. It does not matter whether done locally or given to docker.
* `example_setting`: Some setting that configures your agent. Default: `"Unset"`
\ No newline at end of file
# Byte-compiled / optimized / DLL files
__pycache__/
*.py[cod]
*$py.class
# C extensions
*.so
# Distribution / packaging
.Python
build/
develop-eggs/
dist/
downloads/
eggs/
.eggs/
lib/
lib64/
parts/
sdist/
var/
wheels/
share/python-wheels/
*.egg-info/
.installed.cfg
*.egg
MANIFEST
# PyInstaller
# Usually these files are written by a python script from a template
# before PyInstaller builds the exe, so as to inject date/other infos into it.
*.manifest
*.spec
# Installer logs
pip-log.txt
pip-delete-this-directory.txt
# Unit test / coverage reports
htmlcov/
.tox/
.nox/
.coverage
.coverage.*
.cache
nosetests.xml
coverage.xml
*.cover
*.py,cover
.hypothesis/
.pytest_cache/
cover/
# Translations
*.mo
*.pot
# Django stuff:
*.log
local_settings.py
db.sqlite3
db.sqlite3-journal
# Flask stuff:
instance/
.webassets-cache
# Scrapy stuff:
.scrapy
# Sphinx documentation
docs/_build/
# PyBuilder
.pybuilder/
target/
# Jupyter Notebook
.ipynb_checkpoints
# IPython
profile_default/
ipython_config.py
# pyenv
# For a library or package, you might want to ignore these files since the code is
# intended to run in multiple environments; otherwise, check them in:
# .python-version
# pipenv
# According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
# However, in case of collaboration, if having platform-specific dependencies or dependencies
# having no cross-platform support, pipenv may install dependencies that don't work, or not
# install all needed dependencies.
#Pipfile.lock
# PEP 582; used by e.g. github.com/David-OConnor/pyflow
__pypackages__/
# Celery stuff
celerybeat-schedule
celerybeat.pid
# SageMath parsed files
*.sage.py
# Environments
.env
.venv
env/
venv/
ENV/
env.bak/
venv.bak/
# Spyder project settings
.spyderproject
.spyproject
# Rope project settings
.ropeproject
# mkdocs documentation
/site
# mypy
.mypy_cache/
.dmypy.json
dmypy.json
# Pyre type checker
.pyre/
# pytype static type analyzer
.pytype/
# Cython debug symbols
cython_debug/
import logging
access_log = logging.getLogger("gunicorn.access")
error_log = logging.getLogger("gunicorn.error")
VERSION = (0, 1, 0)
__version__ = ".".join(map(str, VERSION))
from pydantic import BaseSettings
import os
from annotator import __version__
class BasicSettings(BaseSettings):
application_name: str = "CouchDB Annotation Agent"
application_version: str = __version__.__version__
cdstar_uri: str = "http://localhost:8082/v3"
cdstar_user: str = "medic1"
cdstar_pass: str = "medIC!umG!9"
cdstar_vault: str = "medic"
couch_uri: str = "http://localhost:5984"
couch_user: str = "medic"
couch_pass: str = "medic2020"
couch_db: str = "medic"
def get_version():
return BasicSettings().application_version
agent_description = ""
with open(
os.path.join(os.path.abspath(os.path.dirname(__file__)), os.pardir, "Readme.md")
) as readme:
agent_description = readme.read()
class ActiveWorkflowException(Exception):
def __init__(self, name: str, message: str):
self.name = name
self.message = message
import datetime
from annotator import access_log, error_log
from annotator import stores
from annotator import models_activeworkflow as awmodels
from annotator.config import BasicSettings
from annotator.errors import ActiveWorkflowException