UMS-Agenten/Agenten-Plattform
3
0
Fork 0
Code Issues Actions Packages Releases Activity

Messages via Types with Validation etc.

Browse Source
This commit is contained in:
Magnus Bender
2024-10-06 15:09:55 +02:00
parent 4ea424e0d1
commit b889b581f2
36 changed files with 4274 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+5
View File
@@ -1,3 +1,8 @@
__pycache__
/data/*
/bin/
/lib/
/include/
/pyvenv.cfg
Readme.md
+12
View File
@@ -12,3 +12,15 @@
- `./docker-agent/`
- `./ums/agent/`
## Development
### Run via Docker
- `docker compose up`
### VS Code Autocomplete ...
- `python3 -m venv .` (only once)
- `source ./bin/activate`
- `pip install requests fastapi pdoc` (only once)
- Select Python from `./bin/python` in VS Code
build-mgmt.sh
+10
View File
@@ -1,5 +1,15 @@
#/bin/bash
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://stackoverflow.com/a/4774063
SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
docker-agent/Dockerfile
+10
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
ARG FROM_IMAGE=
FROM $FROM_IMAGE
docker-compose.yml
+11 -1
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
# This file is for development!!
# docker compose up
# See https://git.chai.uni-hamburg.de/UMS-Agenten/Agent-Template for production usage!
@@ -7,7 +17,7 @@ services:
image: git.chai.uni-hamburg.de/ums-agenten/management:arm64
#image: git.chai.uni-hamburg.de/ums-agenten/management:amd64
ports:
- 8080:80
- 8000:80
environment:
- AGENTS_PROCESS=http://agent_process_1:3001,http://agent_process_2:3001
- AGENTS_SOLVE=http://agent_solve_1:3001
docker-mgmt/Dockerfile
+10
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
FROM ubuntu:24.04
ARG H_GID
docker-mgmt/app.conf
+10
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
server {
listen 80 default_server;
docker-mgmt/nginx.conf
+10
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
user user;
worker_processes auto;
pid /run/nginx.pid;
docker-mgmt/requirements.txt
+10
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
# non frozen dependecies, to use latest
requests
tqdm
docker-mgmt/setup.py
+10
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
from setuptools import find_packages, setup
setup(
docker-mgmt/supervisor.conf
+10
View File
@@ -1,3 +1,13 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
[supervisord]
nodaemon=true
user=root
docs.sh
Executable
+17
View File
@@ -0,0 +1,17 @@
#!/bin/bash
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
pdoc ./ums/ \
--output-directory ./docs/ \
--no-browser \
--docformat google \
--template-directory ./utils/doc-template
docs/index.html
+7
View File
@@ -0,0 +1,7 @@
<!doctype html>
<html>
<head>
<meta charset="utf-8">
<meta http-equiv="refresh" content="0; url=./ums.html"/>
</head>
</html>
docs/search.js
+46
View File
File diff suppressed because one or more lines are too long
docs/ums.html
+249
View File
File diff suppressed because one or more lines are too long
docs/ums/__cli__.html
+248
View File
File diff suppressed because one or more lines are too long
docs/ums/agent.html
+248
View File
File diff suppressed because one or more lines are too long
docs/ums/management.html
+252
View File
File diff suppressed because one or more lines are too long
docs/ums/management/main.html
+402
View File
File diff suppressed because one or more lines are too long
docs/ums/utils.html
+265
View File
File diff suppressed because one or more lines are too long
docs/ums/utils/const.html
+295
View File
File diff suppressed because one or more lines are too long
docs/ums/utils/types.html
+1696
View File
File diff suppressed because one or more lines are too long
push-images.sh
+10
View File
@@ -1,5 +1,15 @@
#/bin/bash
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
# https://stackoverflow.com/a/4774063
SCRIPTPATH="$(cd -- "$(dirname "$0")" >/dev/null 2>&1; pwd -P)"
ums/__cli__.py
+9
View File
@@ -0,0 +1,9 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
ums/__init__.py
+9
View File
@@ -0,0 +1,9 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
ums/agent/__init__.py
+9
View File
@@ -0,0 +1,9 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
ums/management/__init__.py
+9
View File
@@ -0,0 +1,9 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
ums/management/main.py
+31 -7
View File
@@ -1,12 +1,20 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
# TEST ONLY
from ums.messages import Message, Riddle
from ums.utils import AgentMessage, RiddleData, RiddleDataType, RiddleSolution
from typing import Union
from fastapi import FastAPI
from fastapi.responses import PlainTextResponse
app = FastAPI()
@@ -14,13 +22,29 @@ app = FastAPI()
def read_root():
return {"Hello": "World"}
@app.get("/test", response_class=PlainTextResponse)
@app.get("/test")
def huhu():
a = Message(
id="hu", riddle="lala"
ex = AgentMessage(
id="ex1",
riddle={
"context":"Example 1",
"question":"Get the name of the person."
},
data=[
RiddleData(
type=RiddleDataType.TEXT,
file_plain="./cv.txt"
)
]
)
ex.status.extract.required = False
ex.solution = RiddleSolution(
solution="Otto",
explanation="Written in line 6 after 'Name:'"
)
a.status.steps["extract"] = False
return a.to_json()
return ex
@app.get("/items/{item_id}")
ums/messages/__init__.py
-4
View File
@@ -1,4 +0,0 @@
from ums.messages.message import Message
from ums.messages.specials import Riddle, RiddleData, RiddleStatus, Solution
ums/messages/message.py
-102
View File
@@ -1,102 +0,0 @@
import importlib, json
from typing import Any, List
class Message():
_USM_AGENT_CODE = "CZDygPSF2HJTLKVIqys1";
_ATTRIBUTES = { # * in name -> optional
"id" : str,
"sub_ids*" : List[str],
"riddle" : "ums.messages.specials.Riddle",
"solution*" : "ums.messages.specials.Solution",
"data" : "ums.messages.specials.RiddleData",
"status" : "ums.messages.specials.RiddleStatus"
}
_DEFAULTS = {
"data" : ("ums.messages.specials", "RiddleData"),
"status" : ("ums.messages.specials", "RiddleStatus")
}
def __init__(self, *args, **kwargs):
if len(args) > 0:
self.__dict__['items'] = list(args)
elif len(kwargs) > 0:
self.__dict__['items'] = kwargs
else:
self.__dict__['items'] = self._DEFAULTS
self._check_attr()
def __getattr__(self, name: str) -> Any:
if 'items' in self.__dict__ and isinstance(self.items, dict) and name in self.items:
return self.items[name]
def __setattr__(self, name: str, value: Any) -> None:
if hasattr(self, name):
setattr(self, name, value)
elif isinstance(self.items, dict):
self.items[name] = value
self._check_attr()
def _check_attr(self):
if isinstance(self._ATTRIBUTES, list):
assert isinstance(self.items, list), \
"{} content must be a list and not a dict!".format(self.__class__.__name__)
elif isinstance(self._ATTRIBUTES, dict):
assert isinstance(self.items, dict), \
"{} content must be a dict and not a list!".format(self.__class__.__name__)
for k,v in self._ATTRIBUTES.items():
if not k.endswith('*') and not k in self.items:
if k in self._DEFAULTS:
if isinstance(self._DEFAULTS[k], tuple):
def_class = getattr(
importlib.import_module(self._DEFAULTS[k][0]),
self._DEFAULTS[k][1]
)
self.items[k] = def_class()
else:
self.items[k] = self._DEFAULTS[k]
else:
raise ValueError("Message requires field {}, but not set (and no default)!".format(k))
if isinstance(v, str):
# a class name
pass
elif isinstance(v, dict):
# a sub structure
pass
else: # a type
pass
def _to_json(self):
return self.items
def to_json(self) -> str:
return json.dumps(
self._to_json(),
indent=2,
cls=JSONEncodeHelper,
sort_keys=True
)
def from_json(json:str):
# TODO
pass
class JSONEncodeHelper(json.JSONEncoder):
def default(self, o):
if isinstance(o, Message):
return o._to_json()
return super().default(o)
ums/messages/specials.py
-99
View File
@@ -1,99 +0,0 @@
import os
from typing import Generator, Tuple, IO
from ums.messages.message import Message
class Riddle(Message):
_ATTRIBUTES = { # * in name -> optional
"context" : str,
"question" : str,
"solution_before*" : str,
"review_before*" : str
}
_DEFAULTS = {}
class RiddleStatus(Message):
_ATTRIBUTES = { # * in name -> optional
"steps" : {
"extract" : bool,
"solve" : bool,
"validate" : bool
},
"trial" : int,
"solved" : bool
}
_DEFAULTS = {
"steps" : {
"extract" : True,
"solve" : True,
"validate" : True
},
"trial" : 0,
"solved" : False
}
class RiddleData(Message):
TYPE_TEXT = "text"
TYPE_IMAGE = "image"
TYPE_AUDIO = "audio"
DATA_TYPES = [
TYPE_IMAGE,
TYPE_TEXT,
TYPE_AUDIO
]
FILE_BASEPATH = '/ums-agenten/share'
_ATTRIBUTES = [
{
"type" : str,
"file_plain" : str,
"file_extracted*" : str
}
]
_DEFAULTS = []
def iterate(self,
yield_plain:bool=True, yield_extracted:bool=True,
file_pointer:bool=False
) -> Generator[Tuple[str, str|IO|None, str|IO|None], None, None]:
for item in self.items:
if item["type"] in self.DATA_TYPES:
f_e = None
if yield_extracted and "file_extracted" in item:
f_e_p = os.path.join(self.FILE_BASEPATH, item["file_extracted"])
if os.path.isfile(f_e_p):
f_e = open(f_e_p, 'r') if file_pointer else f_e_p
f_p = None
if yield_plain:
f_p_p = os.path.join(self.FILE_BASEPATH, item["file_plain"])
if os.path.isfile(f_p_p):
f_e = open(f_p_p, 'r') if file_pointer else f_p_p
yield item["type"], f_p, f_e
if hasattr(f_p, 'close'):
f_p.close()
if hasattr(f_e, 'close'):
f_e.close()
def __iter__(self):
yield from self.iterate()
class Solution(Message):
_ATTRIBUTES = {
"solution" : str,
"explanation" : str,
"used_data*" : "ums.messages.specials.RiddleData",
"review*" : str
}
_DEFAULTS = {}
ums/utils/__init__.py
+21
View File
@@ -0,0 +1,21 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
from ums.utils.types import (
RiddleInformation,
AgentMessage,
Riddle,
RiddleSolution,
RiddleData,
RiddleDataType,
RiddleStatus
)
from ums.utils.const import *
ums/utils/const.py
+19
View File
@@ -0,0 +1,19 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
"""
This file contains shared constants.
See the content ...
"""
import os
BASE_PATH = '/ums-agent'
SHARE_PATH = os.path.join(BASE_PATH, 'share')
ums/utils/types.py
+292
View File
@@ -0,0 +1,292 @@
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
"""
This represents the basic types used to interact with the management.
The types are implemented using [pydantic](https://docs.pydantic.dev/).
It provides validation, allow JSON serialization and works well with [FastAPI](https://fastapi.tiangolo.com/) which is used internally for the http request between the agents and the management.
### Example
```python
ex = AgentMessage(
id="ex1",
riddle={
"context":"Example 1",
"question":"Get the name of the person."
},
data=[
RiddleData(
type=RiddleDataType.TEXT,
file_plain="./cv.txt"
)
]
)
ex.status.extract.required = False
```
```json
{
"id": "ex1",
"sub_ids": [],
"riddle": {
"context": "Example 1",
"question": "Get the name of the person.",
"solutions_before": []
},
"solution": null,
"data": [
{
"type": "text",
"file_plain": "/ums-agent/share/cv.txt",
"file_extracted": null
}
],
"status": {
"extract": {
"required": false,
"finished": false
},
"solve": {
"required": true,
"finished": false
},
"validate": {
"required": true,
"finished": false
},
"trial": 0,
"solved": false
}
}
```
```python
ex.solution = RiddleSolution(
solution="Otto",
explanation="Written in line 6 after 'Name:'"
)
```
```json
{
...
"solution": {
"solution": "Otto",
"explanation": "Written in line 6 after 'Name:'",
"used_data": [],
"accepted": false,
"review": null
},
...
}
```
"""
import os
from enum import Enum
from typing import List
from typing_extensions import Annotated
from pydantic import BaseModel
from pydantic.functional_validators import AfterValidator
from ums.utils.const import SHARE_PATH
class RiddleInformation(BaseModel):
"""
This is the basic class used as superclass for all message and infos
about a riddle.
"""
class RiddleDataType(Enum):
"""
Enum for the three types of data used in a riddle.
"""
TEXT = "text"
IMAGE = "image"
AUDIO = "audio"
def _check_data_file(file_name:str) -> str:
if not file_name.startswith('/'):
file_name = os.path.join(SHARE_PATH, file_name)
assert file_name.startswith(SHARE_PATH), "The data file needs to be in {}!".format(SHARE_PATH)
file_name = os.path.realpath(file_name, strict=False)
assert os.path.isfile(file_name), "The data file {} does not exist!".format(file_name)
return file_name
class RiddleData(RiddleInformation):
"""
A data item to be used to solve the riddle
"""
type: RiddleDataType
"""
The type of the data item.
"""
file_plain: Annotated[str, AfterValidator(_check_data_file)]
"""
The plain file (as path to file system) without any processing.
The path will be validated and must start with `SHARE_PATH` (or be relative to `SHARE_PATH`).
The file must exist.
"""
file_extracted: Annotated[str, AfterValidator(_check_data_file)] | None = None
"""
The processed files (as path to file system), i.e., a schematic file containing all extracted informations.
The path will be validated and must start with `SHARE_PATH` (or be relative to `SHARE_PATH`).
The file must exist.
"""
class RiddleSolution(RiddleInformation):
"""
A solution of a riddle.
"""
solution: str
"""
The textual value of the solution.
"""
explanation: str
"""
An explanation of the solution.
"""
used_data: List[RiddleData] = []
"""
The data items used to create the solution (optional).
"""
accepted : bool = False
"""
If the solution is accepted by validator/ gatekeeper.
"""
review: str | None = None
"""
A review of the solution (if None: not tried to validate)
"""
class Riddle(RiddleInformation):
"""
The riddle (the task description and possibly a solution)
"""
context: str
"""
The context of the riddle (as textual string).
"""
question: str
"""
The actual main question of the riddle (as textual string).
"""
solutions_before: List[RiddleSolution] = []
"""
If already tried to solve this riddle before, the (not accepted) solutions are stored here
"""
class RiddleSubStatus(RiddleInformation):
"""
The sub status for each possible step a riddle may go though.
"""
required: bool = True
"""
Is this step required (i.e., requested)
"""
finished: bool = False
"""
Was this step already executed.
"""
class RiddleStatus(RiddleInformation):
"""
The status of a riddle, will be mostly changed by Management when the riddle is sent to different agents while solving it.
"""
extract: RiddleSubStatus = RiddleSubStatus()
"""
The first extract step (image, text, audio -> more sematic data)
The `RiddleData` items in `AgentMessage.data` shall have `file_extracted` afterwards.
"""
solve: RiddleSubStatus = RiddleSubStatus()
"""
The *main* solving step.
`AgentMessage.solution` shall be an `RiddleSolution` afterwards.
"""
validate: RiddleSubStatus = RiddleSubStatus()
"""
The validation step, i.e., does the gatekeeper accept the solution in `AgentMessage.solution`.
"""
trial: int = 0
"""
A counter for the number of trials.
Each time the gatekeeper does not accept a solution of this riddle, the value is incremented.
"""
solved: bool = False
"""
True, after the gatekeeper accepts the solution at `AgentMessage.solution`
"""
class AgentMessage(RiddleInformation):
"""
The basic message, which is sent be the agent and the management.
The objects will be JSON en- and decoded.
"""
id: str
"""
The riddle id, e.g., ``ex1``
This is a unique string and identifies the riddle.
"""
sub_ids: List[str] = []
"""
There might be cases, when an agent decided to split in riddle in multiple *smaller* steps.
Each *sub* riddle will then get its own id (i.e., ``ex1-sub1``) while the sub id is added here as reference.
"""
riddle: Riddle
"""
The riddle to solve.
"""
solution: RiddleSolution | None = None
"""
The solution of the riddle (or empty if no solution available)
"""
data: List[RiddleData] = []
"""
The data to get the solution from.
"""
status: RiddleStatus = RiddleStatus()
"""
The status of the riddle.
"""
utils/doc-template/module.html.jinja2
+20
View File
@@ -0,0 +1,20 @@
{# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
#}
{% extends "default/module.html.jinja2" %}
{% macro is_public(doc) %}
{% if doc.name in ("model_config", "model_fields", "model_computed_fields") %}
{% else %}
{{ default_is_public(doc) }}
{% endif %}
{% endmacro %}
vars.sh
+10
View File
@@ -1,5 +1,15 @@
#/bin/bash
# Agenten Plattform
#
# (c) 2024 Magnus Bender
# Institute of Humanities-Centered Artificial Intelligence (CHAI)
# Universitaet Hamburg
# https://www.chai.uni-hamburg.de/~bender
#
# source code released under the terms of GNU Public License Version 3
# https://www.gnu.org/licenses/gpl-3.0.txt
IMAGE_REGISTRY="git.chai.uni-hamburg.de"
IMAGE_OWNER="ums-agenten"
IMAGE_NAME_AGENT="base-agent"