feat: DIA-1924: "Import sample data" ability in the Data Import page (#7206)

Co-authored-by: Nick Skriabin <[email protected]>
Co-authored-by: bmartel <[email protected]>
Co-authored-by: robot-ci-heartex <[email protected]>
Co-authored-by: Nick Skriabin <[email protected]>
Co-authored-by: Marcel Canu <[email protected]>
Co-authored-by: mcanu <[email protected]>
Co-authored-by: pakelley <[email protected]>

Author: 6 people

web/apps/labelstudio/src/pages/CreateProject/CreateProject.jsx

@@ -89,6 +89,8 @@ export const CreateProject = ({ onClose, redirect = true }) => {
   const [name, setName] = React.useState("");
   const [error, setError] = React.useState();
   const [description, setDescription] = React.useState("");
+  const [sample, setSample] = React.useState(null);
+
   const setStep = React.useCallback((step) => {
     _setStep(step);
     const eventNameMap = {
@@ -103,7 +105,7 @@ export const CreateProject = ({ onClose, redirect = true }) => {
     setError(null);
   }, [name]);
 
-  const { columns, uploading, uploadDisabled, finishUpload, pageProps } = useImportPage(project);
+  const { columns, uploading, uploadDisabled, finishUpload, pageProps, uploadSample } = useImportPage(project, sample);
 
   const rootClass = cn("create-project");
   const tabClass = rootClass.elem("tab");
@@ -132,6 +134,10 @@ export const CreateProject = ({ onClose, redirect = true }) => {
     if (!imported) return;
 
     setWaitingStatus(true);
+
+    if (sample) {
+      await uploadSample(sample);
+    }
     const response = await api.callApi("updateProject", {
       params: {
         pk: project.id,
@@ -211,8 +217,10 @@ export const CreateProject = ({ onClose, redirect = true }) => {
         <ImportPage
           project={project}
           show={step === "import"}
-          {...pageProps}
+          sample={sample}
+          onSampleDatasetSelect={setSample}
           openLabelingConfig={() => setStep("config")}
+          {...pageProps}
         />
         <ConfigPage
           project={project}

web/apps/labelstudio/src/pages/CreateProject/Import/Import.jsx

@@ -1,39 +1,21 @@
-import { useCallback, useEffect, useReducer, useRef, useState } from "react";
-import { Modal } from "../../../components/Modal/Modal";
-import { cn } from "../../../utils/bem";
+import { ff } from "@humansignal/core";
+import { SampleDatasetSelect } from "@humansignal/core/blocks/SampleDatasetSelect/SampleDatasetSelect";
+import { IconError, IconFileUpload, IconInfo, IconTrash, IconUpload } from "@humansignal/icons";
+import { Badge } from "@humansignal/shad/components/ui/badge";
 import { cn as scn } from "@humansignal/shad/utils";
 import { CodeBlock, SimpleCard } from "@humansignal/ui";
-import { unique } from "../../../utils/helpers";
-import "./Import.scss";
-import { IconError, IconFileUpload, IconInfo, IconUpload } from "@humansignal/icons";
-import { useAPI } from "../../../providers/ApiProvider";
-import Input from "libs/datamanager/src/components/Common/Input/Input";
 import { Button } from "apps/labelstudio/src/components";
 import { useAtomValue } from "jotai";
+import Input from "libs/datamanager/src/components/Common/Input/Input";
+import { useCallback, useEffect, useReducer, useRef, useState } from "react";
+import { Modal } from "../../../components/Modal/Modal";
+import { useAPI } from "../../../providers/ApiProvider";
+import { cn } from "../../../utils/bem";
+import { unique } from "../../../utils/helpers";
 import { sampleDatasetAtom } from "../utils/atoms";
-import { ff } from "@humansignal/core";
-
-const testCode = `
-import { SimpleCard } from "../simple-card";
-
-export function CodeBlock({
-  code,
-  title,
-  description,
-  className,
-}: {
-  title?: string;
-  description?: string;
-  code: string;
-  className?: string;
-}) {
-  return (
-    <SimpleCard title={title} description={description} className={className}>
-      <div className="whitespace-pre-wrap font-mono mt-2 p-3 bg-gray-100 rounded-sm">{code}</div>
-    </SimpleCard>
-  );
-}
-`;
+import "./Import.scss";
+import samples from "./samples.json";
+import { importFiles } from "./utils";
 
 const importClass = cn("upload_page");
 const dropzoneClass = cn("dropzone");
@@ -175,9 +157,11 @@ const ErrorMessage = ({ error }) => {
 
 export const ImportPage = ({
   project,
+  sample,
   show = true,
   onWaiting,
   onFileListUpdate,
+  onSampleDatasetSelect,
   highlightCsvHandling,
   dontCommitToProject = false,
   csvHandling,
@@ -211,7 +195,7 @@ export const ImportPage = ({
   };
 
   const [files, dispatch] = useReducer(processFiles, { uploaded: [], uploading: [], ids: [] });
-  const showList = Boolean(files.uploaded?.length || files.uploading?.length);
+  const showList = Boolean(files.uploaded?.length || files.uploading?.length || sample);
 
   const loadFilesList = useCallback(
     async (file_upload_ids) => {
@@ -267,27 +251,18 @@ export const ImportPage = ({
     [addColumns, loadFilesList, setLoading],
   );
 
-  const importFiles = useCallback(
+  const importFilesImmediately = useCallback(
     async (files, body) => {
-      dispatch({ sending: files });
-
-      const query = dontCommitToProject ? { commit_to_project: "false" } : {};
-      // @todo use json for dataset uploads by URL
-      const contentType =
-        body instanceof FormData
-          ? "multipart/form-data" // usual multipart for usual files
-          : "application/x-www-form-urlencoded"; // chad urlencoded for URL uploads
-      const res = await api.callApi("importFiles", {
-        params: { pk: project.id, ...query },
-        headers: { "Content-Type": contentType },
+      importFiles({
+        files,
         body,
-        errorFilter: () => true,
+        project,
+        onError,
+        onFinish,
+        onUploadStart: (files) => dispatch({ sending: files }),
+        onUploadFinish: (files) => dispatch({ sent: files }),
+        dontCommitToProject,
       });
-
-      if (res && !res.error) onFinish?.(res);
-      else onError?.(res?.response);
-
-      dispatch({ sent: files });
     },
     [project, onFinish],
   );
@@ -306,9 +281,9 @@ export const ImportPage = ({
         }
         fd.append(f.name, f);
       }
-      return importFiles(files, fd);
+      return importFilesImmediately(files, fd);
     },
-    [importFiles, onStart],
+    [importFilesImmediately, onStart],
   );
 
   const onUpload = useCallback(
@@ -333,9 +308,9 @@ export const ImportPage = ({
       onWaiting?.(true);
       const body = new URLSearchParams({ url });
 
-      importFiles([{ name: url }], body);
+      importFilesImmediately([{ name: url }], body);
     },
-    [importFiles],
+    [importFilesImmediately],
   );
 
   const openConfig = useCallback(
@@ -375,7 +350,7 @@ export const ImportPage = ({
       {highlightCsvHandling && <div className={importClass.elem("csv-splash")} />}
       <input id="file-input" type="file" name="file" multiple onChange={onUpload} style={{ display: "none" }} />
 
-      <header>
+      <header className="flex gap-4">
         <form className={`${importClass.elem("url-form")} inline-flex`} method="POST" onSubmit={onLoadURL}>
           <Input placeholder="Dataset URL" name="url" ref={urlRef} style={{ height: 40 }} />
           <Button type="submit" look="primary">
@@ -391,6 +366,9 @@ export const ImportPage = ({
           <IconUpload width="16" height="16" className={importClass.elem("upload-icon")} />
           Upload {files.uploaded.length ? "More " : ""}Files
         </Button>
+        {ff.isActive(ff.FF_SAMPLE_DATASETS) && (
+          <SampleDatasetSelect samples={samples} sample={sample} onSampleApplied={onSampleDatasetSelect} />
+        )}
         <div
           className={importClass.elem("csv-handling").mod({ highlighted: highlightCsvHandling, hidden: !csvHandling })}
         >
@@ -475,10 +453,33 @@ export const ImportPage = ({
           {showList && (
             <table>
               <tbody>
+                {sample && (
+                  <tr key={sample.url}>
+                    <td>
+                      <div className="flex items-center gap-2">
+                        {sample.title}
+                        <Badge variant="info" className="h-5 text-xs rounded-sm">
+                          Sample
+                        </Badge>
+                      </div>
+                    </td>
+                    <td>{sample.description}</td>
+                    <td>
+                      <Button
+                        size="icon"
+                        look="destructive"
+                        style={{ height: 26, width: 26, padding: 0 }}
+                        onClick={() => onSampleDatasetSelect(undefined)}
+                      >
+                        <IconTrash style={{ width: 12, height: 12 }} />
+                      </Button>
+                    </td>
+                  </tr>
+                )}
                 {files.uploading.map((file, idx) => (
                   <tr key={`${idx}-${file.name}`}>
                     <td>{file.name}</td>
-                    <td>
+                    <td colSpan={2}>
                       <span className={importClass.elem("file-status").mod({ uploading: true })} />
                     </td>
                   </tr>

web/apps/labelstudio/src/pages/CreateProject/Import/Import.scss

@@ -44,7 +44,6 @@
     }
 
     & + span {
-      margin: 0 16px;
       color: var(--sand_600);
     }
   }

web/apps/labelstudio/src/pages/CreateProject/Import/ImportModal.jsx

@@ -18,9 +18,10 @@ export const Inner = () => {
   const refresh = useRefresh();
   const { project } = useProject();
   const [waiting, setWaitingStatus] = useState(false);
+  const [sample, setSample] = useState(null);
   const api = useAPI();
 
-  const { uploading, uploadDisabled, finishUpload, fileIds, pageProps } = useImportPage(project);
+  const { uploading, uploadDisabled, finishUpload, fileIds, pageProps, uploadSample } = useImportPage(project);
 
   const backToDM = useCallback(() => {
     const path = location.pathname.replace(ImportModal.path, "");
@@ -46,11 +47,17 @@ export const Inner = () => {
   }, [modal, project, fileIds, backToDM]);
 
   const onFinish = useCallback(async () => {
+    await uploadSample(
+      sample,
+      () => setWaitingStatus(true),
+      () => setWaitingStatus(false),
+    );
+
     const imported = await finishUpload();
 
     if (!imported) return;
     backToDM();
-  }, [backToDM, finishUpload]);
+  }, [backToDM, finishUpload, sample]);
 
   return (
     <Modal
@@ -78,11 +85,13 @@ export const Inner = () => {
       </Modal.Header>
       <ImportPage
         project={project}
-        {...pageProps}
+        sample={sample}
+        onSampleDatasetSelect={setSample}
         projectConfigured={Object.keys(project.parsed_label_config ?? {}).length > 0}
         openLabelingConfig={() => {
           history.push(`/projects/${project.id}/settings/labeling`);
         }}
+        {...pageProps}
       />
     </Modal>
   );

web/apps/labelstudio/src/pages/CreateProject/Import/samples.json

@@ -0,0 +1,12 @@
+[
+  {
+    "title": "COCO 2017 Images",
+    "description": "A 100-image subset of the COCO image segmentation benchmark dataset.",
+    "url": "https://hs-sandbox-pub.s3.us-east-1.amazonaws.com/sample_data/coco_2017_val_100tasks.json"
+  },
+  {
+    "title": "CoNLL 2012 NER",
+    "description": "The CoNLL dataset of 13k sentences for Named Entity Recognition.",
+    "url": "https://hs-sandbox-pub.s3.us-east-1.amazonaws.com/sample_data/conll_2012.json"
+  }
+]

web/apps/labelstudio/src/pages/CreateProject/Import/useImportPage.js

@@ -1,10 +1,11 @@
-import React from "react";
+import React, { useCallback } from "react";
 import { useAPI } from "../../../providers/ApiProvider";
 import { unique } from "../../../utils/helpers";
+import { importFiles } from "./utils";
 
 const DEFAULT_COLUMN = "$undefined$";
 
-export const useImportPage = (project) => {
+export const useImportPage = (project, sample) => {
   const [uploading, setUploadingStatus] = React.useState(false);
   const [fileIds, setFileIds] = React.useState([]);
   const [_columns, _setColumns] = React.useState([]);
@@ -35,6 +36,21 @@ export const useImportPage = (project) => {
     return imported;
   };
 
+  const uploadSample = useCallback(
+    async (sample, onStart, onFinish) => {
+      onStart?.();
+      const url = sample.url;
+      const body = new URLSearchParams({ url });
+      await importFiles({
+        files: [{ name: url }],
+        body,
+        project,
+      });
+      onFinish?.();
+    },
+    [project],
+  );
+
   const pageProps = {
     onWaiting: setUploadingStatus,
     // onDisableSubmit: onDisableSubmit,
@@ -46,5 +62,5 @@ export const useImportPage = (project) => {
     dontCommitToProject: true,
   };
 
-  return { columns, uploading, uploadDisabled, finishUpload, fileIds, pageProps };
+  return { columns, uploading, uploadDisabled, finishUpload, fileIds, pageProps, uploadSample };
 };

web/apps/labelstudio/src/pages/CreateProject/Import/utils.ts

@@ -0,0 +1,40 @@
+import { API } from "apps/labelstudio/src/providers/ApiProvider";
+
+export const importFiles = async ({
+  files,
+  body,
+  project,
+  onUploadStart,
+  onUploadFinish,
+  onFinish,
+  onError,
+  dontCommitToProject,
+}: {
+  files: { name: string }[];
+  body: Record<string, any> | FormData;
+  project: APIProject;
+  onUploadStart?: (files: { name: string }[]) => void;
+  onUploadFinish?: (files: { name: string }[]) => void;
+  onFinish?: (response: any) => void;
+  onError?: (response: any) => void;
+  dontCommitToProject?: boolean;
+}) => {
+  onUploadStart?.(files);
+
+  const query = dontCommitToProject ? { commit_to_project: "false" } : {};
+
+  const contentType =
+    body instanceof FormData
+      ? "multipart/form-data" // usual multipart for usual files
+      : "application/x-www-form-urlencoded"; // chad urlencoded for URL uploads
+  const res = await API.invoke(
+    "importFiles",
+    { pk: project.id, ...query },
+    { headers: { "Content-Type": contentType }, body },
+  );
+
+  if (res && !res.error) onFinish?.(res);
+  else onError?.(res?.response);
+
+  onUploadFinish?.(files);
+};

web/apps/labelstudio/src/utils/feature-flags.ts

@@ -1,3 +1,3 @@
 // TODO: migrate all usages to core feature flags instead of local ones
 // local ones used in LSO, Editor and DM
-export * from "@humansignal/core/lib/utils/feature-flags";
+export * from "@humansignal/core/lib/utils/feature-flags/ff";