feat: LSDV-4974: Add status and debug info to storages (HumanSignal#4059)

* docs: LSDV-4775: Small fixes in storage docs

* Revert

* feat: LDSV-4979: Add status and debug info to storages

* Fix ldclient debug level

* Fixes

* Fixes with storages

* Fix NGINX downloads for export

* Add migrations

* Fix bug with export annotations and expand_fields in BaseExportSerializer

* More feats

* Refactoring

* Update readme

* Add mermaid

* Add mermaid

* Some

* Fix last ping

* Fix duration

* Add validation on export storage add sync

* Add double validation in API

* Fix validation api

* Rename job_status

* Fixes

* Fix readme

* Add more info for statuses in readme

* Fix readme

* Add icon

* Add more help

* Fix texts

* Some fixes

* Fixes in storage validation

* ci: Build frontend

* Fix tests in LSE

* Fixes for azure export storage validation

* Fixed tests and added getLastTraceback

* ci: Build frontend

* Fixes

* Fix pytest

* Add migration

* Tests

* Fix wrong test

* Fix update progross in the correct place

* Fixes with time_in_progress

* Fix frontend

* ci: Build frontend

* Fixes

* ci: Build frontend

* fix: Presigned storage tests (HumanSignal#4184)

* fix: Presigned storage tests

* update all_urls

* fix presign url test

* Fix azure validation

* Update LSP

* ci: Build frontend

* Remove wrong placeholder in azure

* Add migrations

* Fix null in total annotations in LSP

* Update LSP build

* Use timezone.now() instead of datetime

* Fix migrations

---------

Co-authored-by: robot-ci-heartex <[email protected]>
Co-authored-by: bmartel <[email protected]>

Author: 3 people

label_studio/core/settings/base.py

@@ -65,7 +65,7 @@
         },
         'ldclient.util': {
             'handlers': ['console'],
-            'level': 'WARNING',
+            'level': 'ERROR',
         },
     },
 }
@@ -587,6 +587,9 @@ def collect_versions_dummy(**kwargs):
 
 FUTURE_SAVE_TASK_TO_STORAGE = get_bool_env('FUTURE_SAVE_TASK_TO_STORAGE', default=False)
 FUTURE_SAVE_TASK_TO_STORAGE_JSON_EXT = get_bool_env('FUTURE_SAVE_TASK_TO_STORAGE_JSON_EXT', default=True)
+STORAGE_IN_PROGRESS_TIMER = get_env('STORAGE_IN_PROGRESS_TIMER', 5.0)
+
+USE_NGINX_FOR_EXPORT_DOWNLOADS = get_bool_env('USE_NGINX_FOR_EXPORT_DOWNLOADS', False)
 
 if get_env('MINIO_STORAGE_ENDPOINT') and not get_bool_env('MINIO_SKIP', False):
     CLOUD_FILE_STORAGE_ENABLED = True

label_studio/data_export/api.py

@@ -552,14 +552,24 @@ def get(self, request, *args, **kwargs):
                 url = file.storage.url(file.name, storage_url=True)
             protocol = urlparse(url).scheme
 
-            # Let NGINX handle it
-            response = HttpResponse()
-            # The below header tells NGINX to catch it and serve, see docker-config/nginx-app.conf
-            redirect = '/file_download/' + protocol + '/' + url.replace(protocol + '://', '')
-
-            response['X-Accel-Redirect'] = redirect
-            response['Content-Disposition'] = 'attachment; filename="{}"'.format(file.name)
-            return response
+            # NGINX downloads are a solid way to make uwsgi workers free
+            if settings.USE_NGINX_FOR_EXPORT_DOWNLOADS:
+                # let NGINX handle it
+                response = HttpResponse()
+                # below header tells NGINX to catch it and serve, see docker-config/nginx-app.conf
+                redirect = '/file_download/' + protocol + '/' + url.replace(protocol + '://', '')
+                response['X-Accel-Redirect'] = redirect
+                response['Content-Disposition'] = 'attachment; filename="{}"'.format(file.name)
+                response['filename'] = os.path.basename(file.name)
+                return response
+
+            # No NGINX: standard way for export downloads in the community edition
+            else:
+                ext = file.name.split('.')[-1]
+                response = RangedFileResponse(request, file, content_type=f'application/{ext}')
+                response['Content-Disposition'] = f'attachment; filename="{file.name}"'
+                response['filename'] = os.path.basename(file.name)
+                return response
         else:
             if export_type is None:
                 file_ = snapshot.file

label_studio/data_export/migrations/0010_alter_convertedformat_export_type.py

@@ -0,0 +1,18 @@
+# Generated by Django 3.2.16 on 2023-04-19 11:36
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('data_export', '0009_alter_convertedformat_traceback'),
+    ]
+
+    operations = [
+        migrations.AlterField(
+            model_name='convertedformat',
+            name='export_type',
+            field=models.CharField(max_length=64),
+        ),
+    ]

label_studio/data_export/models.py

@@ -219,9 +219,7 @@ class Status(models.TextChoices):
         help_text='Traceback report in case of errors'
     )
     export_type = models.CharField(
-        max_length=64,
-        choices=Status.choices,
-        default=Status.CREATED,
+        max_length=64
     )
     created_at = models.DateTimeField(
         _('created at'),

label_studio/data_export/serializers.py

@@ -77,6 +77,7 @@ class Meta:
         expandable_fields = {
             'drafts': (AnnotationDraftSerializer, {'many': True}),
             'predictions': (PredictionSerializer, {'many': True}),
+            'annotations': (AnnotationSerializer, {'many': True})
         }
 
 

label_studio/frontend/package.json

@@ -93,7 +93,7 @@
     "truncate-middle": "^1.0.6",
     "typescript": "^4.2.2",
     "webpack": "^5.80.0",
-    "webpack-cli": "^5.0.2"
+    "webpack-cli": "^5.1.1"
   },
   "babel": {
     "presets": [

label_studio/frontend/src/assets/icons/index.js

@@ -24,4 +24,5 @@ export { default as IconTerminal } from './terminal.svg';
 export { default as LsThumbsDown } from './thumbs-down.svg';
 export { default as LsThumbsUp } from './thumbs-up.svg';
 export { default as IconUpload } from './upload.svg';
-export { default as LsPencil } from './pencil.svg';
+export { default as LsPencil } from './pencil.svg';
+export { default as IconInfoOutline } from './info-outline.svg';

label_studio/frontend/src/assets/icons/info-outline.svg

@@ -1,6 +1,7 @@
 .card
   border-radius 5px
   background-color rgba(#000, 0.03)
+  border 1px solid rgba(#000, 0.05)
 
   &__header
     display flex
@@ -9,9 +10,9 @@
     align-items center
     font-weight 500
     font-size 16px
-    line-height 22px
+    line-height 18px
     justify-content space-between
-    box-shadow 0 1px 0 0 rgba(#000, 0.15)
+    box-shadow 0 1px 0 0 rgba(#000, 0.1)
 
     &-content
       display flex

label_studio/frontend/src/components/Card/Card.styl

@@ -10,4 +10,4 @@
     font-weight 500
     font-size 16px
     line-height 22px
-    padding 0 16px
+    padding 0 16px 0 0

label_studio/frontend/src/components/Columns/Columns.styl

@@ -1,6 +1,8 @@
 import React from 'react';
 import { cn } from '../../utils/bem';
 import './DescriptionList.styl';
+import { IconInfoOutline } from '../../assets/icons';
+import { Tooltip } from '../../components/Tooltip/Tooltip';
 
 export const DescriptionList = ({style, className, children}) => {
   return (
@@ -10,10 +12,16 @@ export const DescriptionList = ({style, className, children}) => {
   );
 };
 
-DescriptionList.Item = ({ retmClassName, descriptionClassName, term, descriptionStyle, termStyle, children }) => {
+DescriptionList.Item = ({ retmClassName, descriptionClassName, term, descriptionStyle, termStyle, children, help }) => {
   return (
     <>
-      <dt className={cn('dl').elem('dt').mix(retmClassName)} style={descriptionStyle}>{term}</dt>
+      <dt className={cn('dl').elem('dt').mix(retmClassName)} style={descriptionStyle}>
+        {term} {help ? (
+          <Tooltip style={{ whiteSpace: "pre-wrap" }} title={help}>
+            <IconInfoOutline className={cn('help-icon')} width="14" height="14" />
+          </Tooltip>
+        ): "" }
+      </dt>
       <dd className={cn('dl').elem('dd').mix(descriptionClassName)} style={termStyle}>{children}</dd>
     </>
   );

label_studio/frontend/src/components/DescriptionList/DescriptionList.js

@@ -4,11 +4,17 @@
   font-size 16px
   line-height 22px
   color rgba(#000, 0.6)
-  grid-template-columns 30% 70%
+  grid-template-columns 40% 60%
   grid-row-gap 12px
 
   &__dt
     font-weight 500
+    min-width 300px
 
   &__dd
     margin 0
+
+.help-icon
+  opacity 0.5
+  top 1px
+  position relative

label_studio/frontend/src/components/DescriptionList/DescriptionList.styl

@@ -2,11 +2,12 @@ import { Children, cloneElement, forwardRef, useCallback, useEffect, useMemo, us
 import { createPortal } from "react-dom";
 import { Block, Elem } from "../../utils/bem";
 import { alignElements } from "../../utils/dom";
+import { isDefined } from "../../utils/helpers";
 import { aroundTransition } from "../../utils/transition";
 import "./Tooltip.styl";
 
 export const Tooltip = forwardRef(
-  ({ title, children, defaultVisible, disabled, style }, ref) => {
+  ({ title, children, alignment, defaultVisible, disabled, style }, ref) => {
     if (!children || Array.isArray(children)) {
       throw new Error("Tooltip does accept a single child only");
     }
@@ -18,18 +19,23 @@ export const Tooltip = forwardRef(
       defaultVisible ? "visible" : null,
     );
     const [injected, setInjected] = useState(false);
-    const [align, setAlign] = useState('top-center');
+    const [align, setAlign] = useState(alignment ?? 'top-center');
 
     const calculatePosition = useCallback(() => {
-      const { left, top, align: resultAlign } = alignElements(
-        triggerElement.current,
-        tooltipElement.current,
-        align,
-        10,
-      );
+      const parent = triggerElement.current;
+      const target = tooltipElement.current;
 
-      setOffset({ left, top });
-      setAlign(resultAlign);
+      if (isDefined(parent) && isDefined(target)) {
+        const { left, top, align: resultAlign } = alignElements(
+          parent,
+          target,
+          align,
+          10,
+        );
+
+        setOffset({ left, top });
+        setAlign(resultAlign);
+      }
     }, [triggerElement.current, tooltipElement.current]);
 
     const performAnimation = useCallback(

label_studio/frontend/src/components/Tooltip/Tooltip.js

@@ -43,9 +43,11 @@ export const StorageCard = ({
     setStorageData(storage);
   }, [storage]);
 
+  const notSyncedYet = synced !== null || ['in_progress', 'queued'].includes(storageData.status);
+
   return (
     <Card
-      header={storageData.title ?? `Untitled ${storageData.type}`}
+      header={storageData.title.slice(0, 70) ?? `Untitled ${storageData.type}`}
       extra={(
         <Dropdown.Trigger align="right" content={(
           <Menu size="compact" style={{ width: 110 }}>
@@ -58,21 +60,26 @@ export const StorageCard = ({
       )}
     >
       <StorageSummary
+        target={target}
         storage={storageData}
         className={rootClass.elem('summary')}
         storageTypes={storageTypes}
       />
       <div className={rootClass.elem('sync')}>
-        <Space size="small">
-          <Button waiting={syncing} onClick={startSync}>
+        <div>
+          <Button
+            waiting={syncing}
+            onClick={startSync}
+            disabled={notSyncedYet}
+          >
               Sync Storage
           </Button>
-          {synced !== null ? (
+          {notSyncedYet && (
             <div className={rootClass.elem('sync-count')}>
-                Synced {synced} task(s)
+              Syncing may take some time, please refresh the page to see the current status.
             </div>
-          ) : null}
-        </Space>
+          )}
+        </div>
       </div>
     </Card>
   );

label_studio/frontend/src/pages/Settings/StorageSettings/StorageCard.js

@@ -21,7 +21,7 @@ export const StorageSet = ({title, target, rootClass, buttonLabel}) => {
         target,
       },
     }).then(types => {
-      setStorageTypes(types);
+      setStorageTypes(types ?? []);
     });
   }, []);
 

label_studio/frontend/src/pages/Settings/StorageSettings/StorageSet.js

@@ -5,6 +5,7 @@ import { Block, cn } from '../../../utils/bem';
 import { StorageSet } from './StorageSet';
 import './StorageSettings.styl';
 
+
 export const StorageSettings = () => {
   const rootClass = cn("storage-settings");
 

label_studio/frontend/src/pages/Settings/StorageSettings/StorageSettings.js

@@ -1,5 +1,6 @@
 .storage-settings
   max-width 680px
+  padding 40px
 
   &__description
     font-size 16px
@@ -48,5 +49,7 @@
     align-items center
 
   &__sync-count
+    margin-top 14px
+    line-height 18px
     font-size 14px
     opacity 0.7