examples: adapt to the new JSON format

Signed-off by: Yoan Salambashev <yoan.salambashev@broadcom.com>

Author: yonitoo

examples/embed-ingest-job-example/20_clean_and_embed_json_data.py

@@ -46,8 +46,8 @@ def load_and_clean_documents(json_file_path):
         documents = json.load(file)
 
     for doc in documents:
-        if "page_content" in doc:
-            cleaned_text = clean_text(doc["page_content"])
+        if "data" in doc:
+            cleaned_text = clean_text(doc["data"])
             cleaned_documents.append([cleaned_text])
 
     print(len(cleaned_documents))

examples/embed-ingest-job-example/30_create_schema.sql

@@ -13,6 +13,7 @@ CREATE TABLE IF NOT EXISTS public.vdk_confluence_doc_metadata_example
     id INTEGER PRIMARY KEY,
     title TEXT,
     source TEXT,
-    content TEXT,
+    data TEXT,
+    deleted BOOLEAN,
     CONSTRAINT fk_metadata_embeddings FOREIGN KEY (id) REFERENCES public.vdk_confluence_doc_embeddings_example(id)
 );

examples/embed-ingest-job-example/40_ingest_embeddings.py

@@ -44,8 +44,9 @@ def run(job_input: IJobInput):
         metadata_payload = {
             "id": document["metadata"]["id"],
             "title": document["metadata"]["title"],
-            "content": document["page_content"],
+            "data": document["data"],
             "source": document["metadata"]["source"],
+            "deleted": document["metadata"]["deleted"],
         }
         job_input.send_object_for_ingestion(
             payload=metadata_payload,

examples/embed-ingest-job-example/documents_example.json

@@ -3,45 +3,45 @@
     "metadata": {
       "title": "Getting Started",
       "id": "123213312",
-      "source": "https://github.com/vmware/versatile-data-kit/wiki/Getting-Started"
+      "source": "https://github.com/vmware/versatile-data-kit/wiki/Getting-Started",
+      "deleted": false
     },
-    "page_content": "VDK Getting Started guide",
-    "deleted": false
+    "data": "VDK Getting Started guide"
   },
   {
     "metadata": {
       "title": "VDK Wiki",
       "id": "747124724",
-      "source": "https://github.com/vmware/versatile-data-kit/wiki"
+      "source": "https://github.com/vmware/versatile-data-kit/wiki",
+      "deleted": false
     },
-    "page_content": "VDK Wiki",
-    "deleted": false
+    "data": "VDK Wiki"
   },
   {
     "metadata": {
       "title": "VDK Issues",
       "id": "721295269",
-      "source": "https://github.com/vmware/versatile-data-kit/issues"
+      "source": "https://github.com/vmware/versatile-data-kit/issues",
+      "deleted": false
     },
-    "page_content": "VDK Issues",
-    "deleted": false
+    "data": "VDK Issues"
   },
   {
     "metadata": {
       "title": "VDK PRs",
       "id": "1323122133",
-      "source": "https://github.com/vmware/versatile-data-kit/pulls"
+      "source": "https://github.com/vmware/versatile-data-kit/pulls",
+      "deleted": false
     },
-    "page_content": "VDK Pull Requests",
-    "deleted": false
+    "data": "VDK Pull Requests"
   },
   {
     "metadata": {
       "title": "VDK Main Page",
       "id": "312343243",
-      "source": "https://github.com/vmware/versatile-data-kit/tree/main"
+      "source": "https://github.com/vmware/versatile-data-kit/tree/main",
+      "deleted": false
     },
-    "page_content": "VDK: One framework to develop, deploy and operate data workflows with Python and SQL.",
-    "deleted": false
+    "data": "VDK: One framework to develop, deploy and operate data workflows with Python and SQL."
   }
 ]