v0.8.2 release

- Vision
- Code execution sessions
- Chat names

Author: frdel

README.md

@@ -4,9 +4,8 @@
 
 # `Agent Zero`
 
-[![Thanks to Sponsors](https://img.shields.io/badge/GitHub%20Sponsors-Thanks%20to%20Sponsors-FF69B4?style=for-the-badge&logo=githubsponsors&logoColor=white)](https://github.com/sponsors/frdel) [![Join our Skool Community](https://img.shields.io/badge/Skool-Join%20our%20Community-4A90E2?style=for-the-badge&logo=skool&logoColor=white)](https://www.skool.com/agent-zero) [![Join our Discord](https://img.shields.io/badge/Discord-Join%20our%20server-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/B8KZKNsPpj) [![Subscribe on YouTube](https://img.shields.io/badge/YouTube-Subscribe-red?style=for-the-badge&logo=youtube&logoColor=white)](https://www.youtube.com/@AgentZeroFW) [![Connect on LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=for-the-badge&logo=linkedin&logoColor=white)](https://www.linkedin.com/in/jan-tomasek/) [![Follow on Warpcast](https://img.shields.io/badge/Warpcast-Follow-5A32F3?style=for-the-badge)](https://warpcast.com/agent-zero)
+[![Thanks to Sponsors](https://img.shields.io/badge/GitHub%20Sponsors-Thanks%20to%20Sponsors-FF69B4?style=for-the-badge&logo=githubsponsors&logoColor=white)](https://github.com/sponsors/frdel) [![Follow on X](https://img.shields.io/badge/X-Follow-000000?style=for-the-badge&logo=x&logoColor=white)](https://x.com/Agent0_ai) [![Join our Skool Community](https://img.shields.io/badge/Skool-Join%20our%20Community-4A90E2?style=for-the-badge&logo=skool&logoColor=white)](https://www.skool.com/agent-zero) [![Join our Discord](https://img.shields.io/badge/Discord-Join%20our%20server-5865F2?style=for-the-badge&logo=discord&logoColor=white)](https://discord.gg/B8KZKNsPpj) [![Subscribe on YouTube](https://img.shields.io/badge/YouTube-Subscribe-red?style=for-the-badge&logo=youtube&logoColor=white)](https://www.youtube.com/@AgentZeroFW) [![Connect on LinkedIn](https://img.shields.io/badge/LinkedIn-Connect-blue?style=for-the-badge&logo=linkedin&logoColor=white)](https://www.linkedin.com/in/jan-tomasek/) [![Follow on Warpcast](https://img.shields.io/badge/Warpcast-Follow-5A32F3?style=for-the-badge)](https://warpcast.com/agent-zero)
 
-> **Note:** Agent Zero does not use Twitter/X. Any Twitter/X accounts claiming to represent this project are fake.
 
 [Installation](./docs/installation.md) •
 [How to update](./docs/installation.md#how-to-update-agent-zero) •

agent.py

@@ -10,15 +10,22 @@
 from langchain_core.prompt_values import ChatPromptValue
 from python.helpers import extract_tools, rate_limiter, files, errors, history, tokens
 from python.helpers.print_style import PrintStyle
-from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholder
-from langchain_core.messages import HumanMessage, SystemMessage, AIMessage
+from langchain_core.prompts import (
+    ChatPromptTemplate,
+    MessagesPlaceholder,
+    HumanMessagePromptTemplate,
+    StringPromptTemplate,
+)
+from langchain_core.prompts.image import ImagePromptTemplate
+from langchain_core.messages import HumanMessage, SystemMessage, AIMessage, BaseMessage
 from langchain_core.language_models.chat_models import BaseChatModel
 from langchain_core.language_models.llms import BaseLLM
 from langchain_core.embeddings import Embeddings
 import python.helpers.log as Log
 from python.helpers.dirty_json import DirtyJson
 from python.helpers.defer import DeferredTask
 from typing import Callable
+from python.helpers.history import OutputMessage
 
 
 class AgentContext:
@@ -89,7 +96,7 @@ def nudge(self):
         else:
             current_agent = self.agent0
 
-        self.task =self.run_task(current_agent.monologue)
+        self.task = self.run_task(current_agent.monologue)
         return self.task
 
     def communicate(self, msg: "UserMessage", broadcast_level: int = 1):
@@ -128,9 +135,9 @@ def run_task(
     async def _process_chain(self, agent: "Agent", msg: "UserMessage|str", user=True):
         try:
             msg_template = (
-                await agent.hist_add_user_message(msg)  # type: ignore
+                agent.hist_add_user_message(msg)  # type: ignore
                 if user
-                else await agent.hist_add_tool_result(
+                else agent.hist_add_tool_result(
                     tool_name="call_subordinate", tool_result=msg  # type: ignore
                 )
             )
@@ -187,7 +194,7 @@ class AgentConfig:
 @dataclass
 class UserMessage:
     message: str
-    attachments: list[str]
+    attachments: list[str] = field(default_factory=list[str])
 
 
 class LoopData:
@@ -281,9 +288,6 @@ async def stream_callback(chunk: str, full: str):
                                 printer.stream(chunk)
                                 self.log_from_stream(full, log)
 
-                        # store as last context window content
-                        self.set_data(Agent.DATA_NAME_CTX_WINDOW, prompt.format())
-
                         agent_response = await self.call_chat_model(
                             prompt, callback=stream_callback
                         )
@@ -294,18 +298,18 @@ async def stream_callback(chunk: str, full: str):
                             self.loop_data.last_response == agent_response
                         ):  # if assistant_response is the same as last message in history, let him know
                             # Append the assistant's response to the history
-                            await self.hist_add_ai_response(agent_response)
+                            self.hist_add_ai_response(agent_response)
                             # Append warning message to the history
                             warning_msg = self.read_prompt("fw.msg_repeat.md")
-                            await self.hist_add_warning(message=warning_msg)
+                            self.hist_add_warning(message=warning_msg)
                             PrintStyle(font_color="orange", padding=True).print(
                                 warning_msg
                             )
                             self.context.log.log(type="warning", content=warning_msg)
 
                         else:  # otherwise proceed with tool
                             # Append the assistant's response to the history
-                            await self.hist_add_ai_response(agent_response)
+                            self.hist_add_ai_response(agent_response)
                             # process tools requested in agent message
                             tools_result = await self.process_tools(agent_response)
                             if tools_result:  # final response of message loop available
@@ -317,7 +321,7 @@ async def stream_callback(chunk: str, full: str):
                     except RepairableException as e:
                         # Forward repairable errors to the LLM, maybe it can fix them
                         error_message = errors.format_error(e)
-                        await self.hist_add_warning(error_message)
+                        self.hist_add_warning(error_message)
                         PrintStyle(font_color="red", padding=True).print(error_message)
                         self.context.log.log(type="error", content=error_message)
                     except Exception as e:
@@ -356,19 +360,32 @@ async def prepare_prompt(self, loop_data: LoopData) -> ChatPromptTemplate:
             extras += history.Message(False, content=extra).output()
         loop_data.extras_temporary.clear()
 
-        # combine history and extras
-        history_combined = history.group_outputs_abab(loop_data.history_output + extras)
-
-        # convert history to LLM format
-        history_langchain = history.output_langchain(history_combined)
+        # convert history + extras to LLM format
+        history_langchain: list[BaseMessage] = history.output_langchain(
+            loop_data.history_output + extras
+        )
 
         # build chain from system prompt, message history and model
+        system_text = "\n\n".join(loop_data.system)
         prompt = ChatPromptTemplate.from_messages(
             [
-                SystemMessage(content="\n\n".join(loop_data.system)),
+                SystemMessage(content=system_text),
                 *history_langchain,
+                AIMessage(content="JSON:"), # force the LLM to start with json
             ]
         )
+
+        # store as last context window content
+        self.set_data(
+            Agent.DATA_NAME_CTX_WINDOW,
+            {
+                "text": prompt.format(),
+                "tokens": self.history.get_tokens()
+                + tokens.approximate_tokens(system_text)
+                + tokens.approximate_tokens(history.output_text(extras)),
+            },
+        )
+
         return prompt
 
     def handle_critical_exception(self, exception: Exception):
@@ -435,12 +452,12 @@ def get_data(self, field: str):
     def set_data(self, field: str, value):
         self.data[field] = value
 
-    def hist_add_message(self, ai: bool, content: history.MessageContent):
-        return self.history.add_message(ai=ai, content=content)
-
-    async def hist_add_user_message(
-        self, message: UserMessage, intervention: bool = False
+    def hist_add_message(
+        self, ai: bool, content: history.MessageContent, tokens: int = 0
     ):
+        return self.history.add_message(ai=ai, content=content, tokens=tokens)
+
+    def hist_add_user_message(self, message: UserMessage, intervention: bool = False):
         self.history.new_topic()  # user message starts a new topic in history
 
         # load message template based on intervention
@@ -470,16 +487,16 @@ async def hist_add_user_message(
         self.last_user_message = msg
         return msg
 
-    async def hist_add_ai_response(self, message: str):
+    def hist_add_ai_response(self, message: str):
         self.loop_data.last_response = message
         content = self.parse_prompt("fw.ai_response.md", message=message)
         return self.hist_add_message(True, content=content)
 
-    async def hist_add_warning(self, message: history.MessageContent):
+    def hist_add_warning(self, message: history.MessageContent):
         content = self.parse_prompt("fw.warning.md", message=message)
         return self.hist_add_message(False, content=content)
 
-    async def hist_add_tool_result(self, tool_name: str, tool_result: str):
+    def hist_add_tool_result(self, tool_name: str, tool_result: str):
         content = self.parse_prompt(
             "fw.tool_result.md", tool_name=tool_name, tool_result=tool_result
         )
@@ -613,9 +630,9 @@ async def handle_intervention(self, progress: str = ""):
             msg = self.intervention
             self.intervention = None  # reset the intervention message
             if progress.strip():
-                await self.hist_add_ai_response(progress)
+                self.hist_add_ai_response(progress)
             # append the intervention message
-            await self.hist_add_user_message(msg, intervention=True)
+            self.hist_add_user_message(msg, intervention=True)
             raise InterventionException(msg)
 
     async def wait_if_paused(self):
@@ -642,7 +659,7 @@ async def process_tools(self, msg: str):
                 return response.message
         else:
             msg = self.read_prompt("fw.msg_misformat.md")
-            await self.hist_add_warning(msg)
+            self.hist_add_warning(msg)
             PrintStyle(font_color="red", padding=True).print(msg)
             self.context.log.log(
                 type="error", content=f"{self.agent_name}: Message misformat"

docker/run/DockerfileKali

@@ -0,0 +1,33 @@
+# Use the latest slim version of Kali Linux
+FROM kalilinux/kali-rolling
+
+# Check if the argument is provided, else throw an error
+ARG BRANCH
+RUN if [ -z "$BRANCH" ]; then echo "ERROR: BRANCH is not set!" >&2; exit 1; fi
+ENV BRANCH=$BRANCH
+
+# Copy contents of the project to /a0
+COPY ./fs/ /
+
+# pre installation steps
+RUN bash /ins/pre_install.sh $BRANCH
+RUN bash /ins/pre_install_kali.sh $BRANCH
+
+# install additional software
+RUN bash /ins/install_additional.sh $BRANCH
+
+# install A0
+RUN bash /ins/install_A0.sh $BRANCH
+
+# cleanup repo and install A0 without caching, this speeds up builds
+ARG CACHE_DATE=none
+RUN echo "cache buster $CACHE_DATE" && bash /ins/install_A02.sh $BRANCH
+
+# post installation steps
+RUN bash /ins/post_install.sh $BRANCH
+
+# Expose ports
+EXPOSE 22 80
+
+# initialize runtime
+CMD ["/bin/bash", "-c", "/bin/bash /exe/initialize.sh $BRANCH"]

docker/run/build.txt

@@ -4,6 +4,9 @@ docker build -t agent-zero-run:local --build-arg BRANCH=development --build-arg
 # local image without cache
 docker build -t agent-zero-run:local --build-arg BRANCH=development --no-cache  .
 
+# local image from Kali
+docker build -f ./DockerfileKali -t agent-zero-run:hacking --build-arg BRANCH=main --build-arg CACHE_DATE=$(date +%Y-%m-%d:%H:%M:%S) .
+
 # dockerhub push:
 
 docker login

docker/run/fs/ins/install_additional.sh

@@ -1,4 +1,7 @@
 #!/bin/bash
 
+# install playwright
+bash /ins/install_playwright.sh "$@"
+
 # searxng
 bash /ins/install_searxng.sh "$@"

docker/run/fs/ins/install_playwright.sh

@@ -7,4 +7,12 @@
 pip install playwright
 
 # install chromium with dependencies
-playwright install --with-deps chromium-headless-shell
+# for kali-based
+if [ "$@" = "hacking" ]; then
+    apt-get install -y fonts-unifont libnss3 libnspr4
+    playwright install chromium-headless-shell
+else
+    # for debian based
+    playwright install --with-deps chromium-headless-shell
+fi
+

docker/run/fs/ins/post_install.sh

@@ -1,8 +1,5 @@
 #!/bin/bash
 
-# install playwright
-bash /ins/install_playwright.sh "$@"
-
 # Cleanup package list
 rm -rf /var/lib/apt/lists/*
 apt-get clean

docker/run/fs/ins/pre_install.sh

@@ -3,7 +3,6 @@
 # Update and install necessary packages
 apt-get update && apt-get install -y \
     python3 \
-    python3-pip \
     python3-venv \
     nodejs \
     npm \
@@ -14,5 +13,18 @@ apt-get update && apt-get install -y \
     git \
     ffmpeg 
 
-# prepare SSH daemon
-bash /ins/setup_ssh.sh "$@"
+# # Configure system alternatives so that /usr/bin/python3 points to Python 3.12
+# sudo update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.12 1
+# sudo update-alternatives --set python3 /usr/bin/python3.12
+
+# Update pip3 symlink: if pip3.12 exists, point pip3 to it;
+# otherwise, install pip using Python 3.12's ensurepip.
+# if [ -f /usr/bin/pip3.12 ]; then
+#   sudo ln -sf /usr/bin/pip3.12 /usr/bin/pip3
+# else
+  python3 -m ensurepip --upgrade
+  python3 -m pip install --upgrade pip
+# fi
+
+# Prepare SSH daemon
+bash /ins/setup_ssh.sh "$@"

docker/run/fs/ins/pre_install_kali.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+
+# ubuntu based dependencies for playwright
+# moved to install_playwright.sh
+# apt-get update && apt-get install -y fonts-unifont fonts-ubuntu
+

docker/run/fs/ins/setup_ssh.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
 # Set up SSH
-mkdir /var/run/sshd && \
+mkdir -p /var/run/sshd && \
     # echo 'root:toor' | chpasswd && \
     sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/' /etc/ssh/sshd_config

initialize.py

@@ -13,6 +13,7 @@ def initialize():
         provider=models.ModelProvider[current_settings["chat_model_provider"]],
         name=current_settings["chat_model_name"],
         ctx_length=current_settings["chat_model_ctx_length"],
+        vision=current_settings["chat_model_vision"],
         limit_requests=current_settings["chat_model_rl_requests"],
         limit_input=current_settings["chat_model_rl_input"],
         limit_output=current_settings["chat_model_rl_output"],

models.py

@@ -20,7 +20,7 @@
     HuggingFaceEndpoint,
 )
 from langchain_google_genai import (
-    GoogleGenerativeAI,
+    ChatGoogleGenerativeAI,
     HarmBlockThreshold,
     HarmCategory,
     embeddings as google_embeddings,
@@ -267,7 +267,7 @@ def get_google_chat(
 ):
     if not api_key:
         api_key = get_api_key("google")
-    return GoogleGenerativeAI(model=model_name, google_api_key=api_key, safety_settings={HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE}, **kwargs)  # type: ignore
+    return ChatGoogleGenerativeAI(model=model_name, google_api_key=api_key, safety_settings={HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE}, **kwargs)  # type: ignore
 
 
 def get_google_embedding(
@@ -277,7 +277,7 @@ def get_google_embedding(
 ):
     if not api_key:
         api_key = get_api_key("google")
-    return google_embeddings.GoogleGenerativeAIEmbeddings(model=model_name, api_key=api_key, **kwargs)  # type: ignore
+    return google_embeddings.GoogleGenerativeAIEmbeddings(model=model_name, google_api_key=api_key, **kwargs)  # type: ignore
 
 
 # Mistral models
@@ -332,7 +332,7 @@ def get_openrouter_chat(
             dotenv.get_dotenv_value("OPEN_ROUTER_BASE_URL")
             or "https://openrouter.ai/api/v1"
         )
-    return ChatOpenAI(api_key=api_key, model=model_name, base_url=base_url, **kwargs)  # type: ignore
+    return ChatOpenAI(api_key=api_key, model=model_name, base_url=base_url, stream_usage=True, **kwargs)  # type: ignore
 
 
 def get_openrouter_embedding(

prompts/default/agent.system.instruments.md

@@ -1,4 +1,5 @@
 # Instruments
-- following are instruments at disposal:
+- following are instruments at disposal
+- do not overly rely on them they might not be relevant
 
-{{instruments}}
+{{instruments}}

prompts/default/agent.system.memories.md

@@ -1,4 +1,5 @@
 # Memories on the topic
-- following are your memories about current topic:
+- following are memories about current topic
+- do not overly rely on them they might not be relevant
 
 {{memories}}

prompts/default/agent.system.solutions.md

@@ -1,4 +1,5 @@
 # Solutions from the past
-- following are your memories about successful solutions of related problems:
+- following are memories about successful solutions of related problems
+- do not overly rely on them they might not be relevant
 
 {{solutions}}