⚡️Improve AI translations formatting

CHANGELOG.md

@@ -26,10 +26,13 @@ and this project adheres to
 ## Added
 
 - ✨(frontend) synchronize language-choice #401
+- ✨(frontend) add Beautify action to AI transform #478
+- ✨(frontend) add Emojify action to AI transform #478
 
 ## Changed
 
 - Use sentry tags instead of extra scope
+- ♻️(frontend) Improve AI translations #478
 
 ## Fixed
 

src/backend/core/services/ai_services.py

@@ -1,8 +1,5 @@
 """AI services."""
 
-import json
-import re
-
 from django.conf import settings
 from django.core.exceptions import ImproperlyConfigured
 
@@ -12,32 +9,44 @@
 
 AI_ACTIONS = {
     "prompt": (
-        "Answer the prompt in markdown format. Return JSON: "
-        '{"answer": "Your markdown answer"}. '
-        "Do not provide any other information."
+        "Answer the prompt in markdown format. "
+        "Preserve the language and markdown formatting. "
+        "Do not provide any other information. "
+        "Preserve the language."
     ),
     "correct": (
         "Correct grammar and spelling of the markdown text, "
         "preserving language and markdown formatting. "
-        'Return JSON: {"answer": "your corrected markdown text"}. '
-        "Do not provide any other information."
+        "Do not provide any other information. "
+        "Preserve the language."
     ),
     "rephrase": (
         "Rephrase the given markdown text, "
         "preserving language and markdown formatting. "
-        'Return JSON: {"answer": "your rephrased markdown text"}. '
-        "Do not provide any other information."
+        "Do not provide any other information. "
+        "Preserve the language."
     ),
     "summarize": (
         "Summarize the markdown text, preserving language and markdown formatting. "
-        'Return JSON: {"answer": "your markdown summary"}. '
-        "Do not provide any other information."
+        "Do not provide any other information. "
+        "Preserve the language."
+    ),
+    "beautify": (
+        "Add formatting to the text to make it more readable. "
+        "Do not provide any other information. "
+        "Preserve the language."
+    ),
+    "emojify": (
+        "Add emojis to the important parts of the text. "
+        "Do not provide any other information. "
+        "Preserve the language."
     ),
 }
 
 AI_TRANSLATE = (
-    "Translate the markdown text to {language:s}, preserving markdown formatting. "
-    'Return JSON: {{"answer": "your translated markdown text in {language:s}"}}. '
+    "Keep the same html stucture and formatting. "
+    "Translate the content in the html to the specified language {language:s}. "
+    "Check the translation for accuracy and make any necessary corrections. "
     "Do not provide any other information."
 )
 
@@ -59,32 +68,18 @@ def call_ai_api(self, system_content, text):
         """Helper method to call the OpenAI API and process the response."""
         response = self.client.chat.completions.create(
             model=settings.AI_MODEL,
-            response_format={"type": "json_object"},
             messages=[
                 {"role": "system", "content": system_content},
-                {"role": "user", "content": json.dumps({"markdown_input": text})},
+                {"role": "user", "content": text},
             ],
         )
 
         content = response.choices[0].message.content
 
-        try:
-            sanitized_content = re.sub(r'\s*"answer"\s*:\s*', '"answer": ', content)
-            sanitized_content = re.sub(r"\s*\}", "}", sanitized_content)
-            sanitized_content = re.sub(r"(?<!\\)\n", "\\\\n", sanitized_content)
-            sanitized_content = re.sub(r"(?<!\\)\t", "\\\\t", sanitized_content)
-
-            json_response = json.loads(sanitized_content)
-        except (json.JSONDecodeError, IndexError):
-            try:
-                json_response = json.loads(content)
-            except json.JSONDecodeError as err:
-                raise RuntimeError("AI response is not valid JSON", content) from err
-
-        if "answer" not in json_response:
+        if not content:
             raise RuntimeError("AI response does not contain an answer")
 
-        return json_response
+        return {"answer": content}
 
     def transform(self, text, action):
         """Transform text based on specified action."""

src/backend/core/tests/documents/test_api_documents_ai_transform.py

@@ -71,9 +71,8 @@ def test_api_documents_ai_transform_anonymous_success(mock_create):
     """
     document = factories.DocumentFactory(link_reach="public", link_role="editor")
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-transform/"
@@ -83,17 +82,15 @@ def test_api_documents_ai_transform_anonymous_success(mock_create):
     assert response.json() == {"answer": "Salut"}
     mock_create.assert_called_once_with(
         model="llama",
-        response_format={"type": "json_object"},
         messages=[
             {
                 "role": "system",
                 "content": (
                     "Summarize the markdown text, preserving language and markdown formatting. "
-                    'Return JSON: {"answer": "your markdown summary"}. Do not provide any other '
-                    "information."
+                    "Do not provide any other information. Preserve the language."
                 ),
             },
-            {"role": "user", "content": '{"markdown_input": "Hello"}'},
+            {"role": "user", "content": "Hello"},
         ],
     )
 
@@ -170,9 +167,8 @@ def test_api_documents_ai_transform_authenticated_success(mock_create, reach, ro
 
     document = factories.DocumentFactory(link_reach=reach, link_role=role)
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-transform/"
@@ -182,16 +178,15 @@ def test_api_documents_ai_transform_authenticated_success(mock_create, reach, ro
     assert response.json() == {"answer": "Salut"}
     mock_create.assert_called_once_with(
         model="llama",
-        response_format={"type": "json_object"},
         messages=[
             {
                 "role": "system",
                 "content": (
-                    'Answer the prompt in markdown format. Return JSON: {"answer": '
-                    '"Your markdown answer"}. Do not provide any other information.'
+                    "Answer the prompt in markdown format. Preserve the language and markdown "
+                    "formatting. Do not provide any other information. Preserve the language."
                 ),
             },
-            {"role": "user", "content": '{"markdown_input": "Hello"}'},
+            {"role": "user", "content": "Hello"},
         ],
     )
 
@@ -246,9 +241,8 @@ def test_api_documents_ai_transform_success(mock_create, via, role, mock_user_te
             document=document, team="lasuite", role=role
         )
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-transform/"
@@ -258,16 +252,15 @@ def test_api_documents_ai_transform_success(mock_create, via, role, mock_user_te
     assert response.json() == {"answer": "Salut"}
     mock_create.assert_called_once_with(
         model="llama",
-        response_format={"type": "json_object"},
         messages=[
             {
                 "role": "system",
                 "content": (
-                    'Answer the prompt in markdown format. Return JSON: {"answer": '
-                    '"Your markdown answer"}. Do not provide any other information.'
+                    "Answer the prompt in markdown format. Preserve the language and markdown "
+                    "formatting. Do not provide any other information. Preserve the language."
                 ),
             },
-            {"role": "user", "content": '{"markdown_input": "Hello"}'},
+            {"role": "user", "content": "Hello"},
         ],
     )
 
@@ -315,9 +308,8 @@ def test_api_documents_ai_transform_throttling_document(mock_create):
     client = APIClient()
     document = factories.DocumentFactory(link_reach="public", link_role="editor")
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-transform/"
@@ -350,9 +342,8 @@ def test_api_documents_ai_transform_throttling_user(mock_create):
     client = APIClient()
     client.force_login(user)
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     for _ in range(3):

src/backend/core/tests/documents/test_api_documents_ai_translate.py

@@ -91,29 +91,28 @@ def test_api_documents_ai_translate_anonymous_success(mock_create):
     """
     document = factories.DocumentFactory(link_reach="public", link_role="editor")
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Ola"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-translate/"
     response = APIClient().post(url, {"text": "Hello", "language": "es"})
 
     assert response.status_code == 200
-    assert response.json() == {"answer": "Salut"}
+    assert response.json() == {"answer": "Ola"}
     mock_create.assert_called_once_with(
         model="llama",
-        response_format={"type": "json_object"},
         messages=[
             {
                 "role": "system",
                 "content": (
-                    "Translate the markdown text to Spanish, preserving markdown formatting. "
-                    'Return JSON: {"answer": "your translated markdown text in Spanish"}. '
+                    "Keep the same html stucture and formatting. "
+                    "Translate the content in the html to the specified language Spanish. "
+                    "Check the translation for accuracy and make any necessary corrections. "
                     "Do not provide any other information."
                 ),
             },
-            {"role": "user", "content": '{"markdown_input": "Hello"}'},
+            {"role": "user", "content": "Hello"},
         ],
     )
 
@@ -190,9 +189,8 @@ def test_api_documents_ai_translate_authenticated_success(mock_create, reach, ro
 
     document = factories.DocumentFactory(link_reach=reach, link_role=role)
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-translate/"
@@ -202,18 +200,18 @@ def test_api_documents_ai_translate_authenticated_success(mock_create, reach, ro
     assert response.json() == {"answer": "Salut"}
     mock_create.assert_called_once_with(
         model="llama",
-        response_format={"type": "json_object"},
         messages=[
             {
                 "role": "system",
                 "content": (
-                    "Translate the markdown text to Colombian Spanish, "
-                    "preserving markdown formatting. Return JSON: "
-                    '{"answer": "your translated markdown text in Colombian Spanish"}. '
+                    "Keep the same html stucture and formatting. "
+                    "Translate the content in the html to the "
+                    "specified language Colombian Spanish. "
+                    "Check the translation for accuracy and make any necessary corrections. "
                     "Do not provide any other information."
                 ),
             },
-            {"role": "user", "content": '{"markdown_input": "Hello"}'},
+            {"role": "user", "content": "Hello"},
         ],
     )
 
@@ -268,9 +266,8 @@ def test_api_documents_ai_translate_success(mock_create, via, role, mock_user_te
             document=document, team="lasuite", role=role
         )
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-translate/"
@@ -280,18 +277,18 @@ def test_api_documents_ai_translate_success(mock_create, via, role, mock_user_te
     assert response.json() == {"answer": "Salut"}
     mock_create.assert_called_once_with(
         model="llama",
-        response_format={"type": "json_object"},
         messages=[
             {
                 "role": "system",
                 "content": (
-                    "Translate the markdown text to Colombian Spanish, "
-                    "preserving markdown formatting. Return JSON: "
-                    '{"answer": "your translated markdown text in Colombian Spanish"}. '
+                    "Keep the same html stucture and formatting. "
+                    "Translate the content in the html to the "
+                    "specified language Colombian Spanish. "
+                    "Check the translation for accuracy and make any necessary corrections. "
                     "Do not provide any other information."
                 ),
             },
-            {"role": "user", "content": '{"markdown_input": "Hello"}'},
+            {"role": "user", "content": "Hello"},
         ],
     )
 
@@ -339,9 +336,8 @@ def test_api_documents_ai_translate_throttling_document(mock_create):
     client = APIClient()
     document = factories.DocumentFactory(link_reach="public", link_role="editor")
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     url = f"/api/v1.0/documents/{document.id!s}/ai-translate/"
@@ -374,9 +370,8 @@ def test_api_documents_ai_translate_throttling_user(mock_create):
     client = APIClient()
     client.force_login(user)
 
-    answer = '{"answer": "Salut"}'
     mock_create.return_value = MagicMock(
-        choices=[MagicMock(message=MagicMock(content=answer))]
+        choices=[MagicMock(message=MagicMock(content="Salut"))]
     )
 
     for _ in range(3):