[feat] duckdb module updates (#2927)

- UI for SQL Query plugin
- DuckDb logic to ingest and generate datasets

Signed-off-by: Ihor Dykhta <[email protected]>

Author: igorDykhta

.eslintrc.js

@@ -38,7 +38,8 @@ module.exports = {
     'consistent-return': 0,
     'comma-dangle': 1,
     'enzyme-deprecation/no-shallow': 2,
-    'enzyme-deprecation/no-mount': 2
+    'enzyme-deprecation/no-mount': 2,
+    'no-constant-condition': ['error', {checkLoops: false}]
   },
   overrides: [
     {

src/duckdb/package.json

@@ -2,7 +2,7 @@
   "name": "@kepler.gl/duckdb",
   "author": "Shan He <[email protected]>",
   "version": "3.1.0-alpha.5",
-  "description": "kepler.gl duckDB plugin",
+  "description": "DuckDB plugin for Kepler.gl",
   "license": "MIT",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",
@@ -25,8 +25,17 @@
     "umd"
   ],
   "dependencies": {
+    "@duckdb/duckdb-wasm": "^1.28.0",
+    "@kepler.gl/common-utils": "3.1.0-alpha.5",
     "@kepler.gl/constants": "3.1.0-alpha.5",
-    "@kepler.gl/types": "3.1.0-alpha.5"
+    "@kepler.gl/processors": "3.1.0-alpha.5",
+    "@kepler.gl/table": "3.1.0-alpha.5",
+    "@kepler.gl/types": "3.1.0-alpha.5",
+    "@monaco-editor/react": "^4.6.0",
+    "@radix-ui/react-collapsible": "^1.1.0",
+    "apache-arrow": ">=15.0.0",
+    "monaco-editor": "^0.52.0",
+    "react-resizable-panels": "^2.1.7"
   },
   "nyc": {
     "sourceMap": false,

src/duckdb/src/components/index.tsx

@@ -0,0 +1 @@
+export * from './sql-panel';

src/duckdb/src/components/monaco-editor.tsx

@@ -0,0 +1,189 @@
+import React, {useCallback, useMemo, useRef} from 'react';
+import Editor, {OnChange, OnMount} from '@monaco-editor/react';
+import * as monaco from 'monaco-editor';
+// import {tableSchema as DEFAULT_SCHEMA} from './table-schema';
+import uniq from 'lodash.uniq';
+import uniqBy from 'lodash.uniqby';
+
+const MONACO_OPTIONS: monaco.editor.IStandaloneEditorConstructionOptions = {
+  minimap: {enabled: false},
+  language: 'sql',
+  contextmenu: false,
+  renderLineHighlight: 'none',
+  scrollBeyondLastLine: false,
+  scrollbar: {alwaysConsumeMouseWheel: false},
+  overviewRulerLanes: 0,
+  automaticLayout: true,
+  acceptSuggestionOnEnter: 'on',
+  quickSuggestionsDelay: 400,
+  matchOnWordStartOnly: false,
+  tabCompletion: 'off',
+  lineNumbers: 'off'
+};
+
+function parseSqlAndFindTableNameAndAliases(sql: string) {
+  const regex = /\b(?:FROM|JOIN)\s+([^\s.]+(?:\.[^\s.]+)?)\s*(?:AS)?\s*([^\s,]+)?/gi;
+  const tables: {table_name: string; alias: string}[] = [];
+
+  while (true) {
+    const match = regex.exec(sql);
+    if (!match) {
+      break;
+    }
+    const table_name = match[1];
+    if (!/\(/.test(table_name)) {
+      // exclude function calls
+      let alias = match[2] as string | null;
+      if (alias && /on|where|inner|left|right|join/.test(alias)) {
+        alias = null;
+      }
+      tables.push({
+        table_name,
+        alias: alias || table_name
+      });
+    }
+  }
+
+  return tables;
+}
+
+interface MonacoEditorProps {
+  code: string;
+  isReadOnly?: boolean;
+  onChange: OnChange;
+  onRunQuery: () => void;
+  tableSchema?: {table_name: string; column_name: string}[];
+}
+
+const MonacoEditor: React.FC<MonacoEditorProps> = ({
+  onRunQuery,
+  onChange,
+  code,
+  tableSchema,
+  isReadOnly
+}) => {
+  // private editor?: monaco.editor.IStandaloneCodeEditor;
+  const schemaTableNames = useMemo(
+    () => (tableSchema ? uniq(tableSchema.map(d => d.table_name)) : []),
+    [tableSchema]
+  );
+  const schemaTableNamesSet = useMemo(() => new Set(schemaTableNames), [schemaTableNames]);
+  const handleRunQueryRef = useRef(onRunQuery);
+  handleRunQueryRef.current = onRunQuery;
+
+  const handleEditorDidMount: OnMount = useCallback(
+    editor => {
+      // this.editor = editor;
+      editor.focus();
+
+      editor.addAction({
+        id: 'run-query',
+        label: 'Run Query',
+        keybindings: [
+          monaco.KeyMod.CtrlCmd | monaco.KeyCode.Enter,
+          monaco.KeyMod.Shift | monaco.KeyCode.Enter
+        ],
+        contextMenuGroupId: 'custom',
+        contextMenuOrder: 0,
+        run: () => handleRunQueryRef.current()
+      });
+
+      monaco.languages.registerCompletionItemProvider('*', {
+        provideCompletionItems: (model, position, context, cancelationToken) => {
+          const suggestions: monaco.languages.CompletionItem[] = [
+            {
+              label: 'myCustomSnippet',
+              kind: monaco.languages.CompletionItemKind.Snippet,
+              insertText: 'This is a piece of custom code',
+              insertTextRules: monaco.languages.CompletionItemInsertTextRule.InsertAsSnippet,
+              documentation: 'This is a piece of custom code'
+              // TODO: range is missing
+            } as monaco.languages.CompletionItem
+          ];
+
+          const fullQueryText = model.getValue();
+
+          const tableNamesAndAliases = new Map<string, string>(
+            parseSqlAndFindTableNameAndAliases(fullQueryText).map(({table_name, alias}) => [
+              alias,
+              table_name
+            ])
+          );
+
+          const thisLine = model.getValueInRange({
+            startLineNumber: position.lineNumber,
+            startColumn: 1,
+            endLineNumber: position.lineNumber,
+            endColumn: position.column
+          });
+          const thisToken = thisLine.trim().split(' ').slice(-1)?.[0] || '';
+
+          const lastTokenBeforeSpace = /\s?(\w+)\s+\w+$/.exec(thisLine.trim())?.[1];
+          const lastTokenBeforeDot = /(\w+)\.\w*$/.exec(thisToken)?.[1];
+
+          // console.log(tableNamesAndAliases, thisToken, lastTokenBeforeSpace, lastTokenBeforeDot);
+
+          if (lastTokenBeforeSpace && /from|join|update|into/.test(lastTokenBeforeSpace)) {
+            suggestions.push(
+              ...schemaTableNames.map(
+                table_name =>
+                  ({
+                    label: table_name,
+                    kind: monaco.languages.CompletionItemKind.Field,
+                    insertText: table_name
+                    // TODO: range is missing
+                  } as monaco.languages.CompletionItem)
+              )
+            );
+          }
+
+          if (lastTokenBeforeDot) {
+            let table_name = null as string | null;
+            if (schemaTableNamesSet.has(lastTokenBeforeDot)) {
+              table_name = lastTokenBeforeDot;
+            } else if (tableNamesAndAliases.get(lastTokenBeforeDot)) {
+              table_name = tableNamesAndAliases.get(lastTokenBeforeDot) as string;
+            }
+            if (table_name && tableSchema) {
+              suggestions.push(
+                ...tableSchema
+                  .filter(d => d.table_name === table_name)
+                  .map(
+                    ({table_name, column_name}) =>
+                      ({
+                        label: column_name,
+                        kind: monaco.languages.CompletionItemKind.Field,
+                        insertText: column_name
+                        // TODO: range is missing
+                      } as monaco.languages.CompletionItem)
+                  )
+              );
+            }
+          }
+
+          return {
+            suggestions: uniqBy(suggestions, s => s.insertText)
+          };
+        }
+      });
+    },
+    [tableSchema]
+  );
+
+  return (
+    <Editor
+      height="100%"
+      theme="vs-dark"
+      defaultLanguage="sql"
+      defaultValue={code}
+      onChange={onChange}
+      onMount={handleEditorDidMount}
+      options={{
+        ...MONACO_OPTIONS,
+        readOnly: isReadOnly
+      }}
+    />
+  );
+};
+
+export default MonacoEditor;

src/duckdb/src/components/preview-data-panel.tsx

@@ -0,0 +1,158 @@
+// Copyright 2022 Foursquare Labs, Inc. All Rights Reserved.
+
+import React, {useCallback, useMemo, useState, CSSProperties} from 'react';
+
+import {DataTable, renderedSize} from '@kepler.gl/components';
+import {parseFieldValue, createDataContainer} from '@kepler.gl/utils';
+import {arrowSchemaToFields} from '@kepler.gl/processors';
+import {DataForm} from '@kepler.gl/utils';
+import {withTheme} from 'styled-components';
+
+type BaseComponentProps = {
+  className?: string;
+  style?: CSSProperties;
+};
+
+const DEFAULT_ROWS_TO_CALCULATE_PREVIEW = 100;
+// min Cellsize should take into account option button and field token
+const minCellSize = 80;
+// option buttons and field token
+const optionButtonWidth = 20;
+const pinButton = 20;
+const cellPadding = 20;
+
+export type ColMeta = {
+  [key: string]: {
+    colIdx: number;
+    name: string;
+    displayName: string;
+    type: string;
+  };
+};
+
+export type DataTableStyle = {
+  minCellSize?: number;
+  cellPadding?: number;
+  fontSize?: number;
+  font?: string;
+  optionsButton?: number;
+};
+
+export type PreviewDataPanelProps = BaseComponentProps & {
+  result: any;
+  rowsToCalculatePreview?: number;
+  theme?: any;
+  setColumnDisplayFormat?: (formats: {[key: string]: string}) => void;
+  defaultPinnedColumns?: string[];
+  dataTableStyle: DataTableStyle;
+  onAddResultToMap: (result: any) => void;
+};
+
+const PreviewDataPanelWOTheme: React.FC<PreviewDataPanelProps> = ({
+  result,
+  rowsToCalculatePreview = DEFAULT_ROWS_TO_CALCULATE_PREVIEW,
+  setColumnDisplayFormat,
+  defaultPinnedColumns = [],
+  theme
+}) => {
+  const [pinnedColumns, setPinnedColumns] = useState<string[]>(defaultPinnedColumns);
+  const fields = useMemo(() => arrowSchemaToFields(result.schema), [result.schema]);
+  const dataContainer = useMemo(() => {
+    // const fields = arrowSchemaToFields(result.schema);
+
+    const cols = [...Array(result.numCols).keys()].map(i => result.getChildAt(i));
+
+    const dataContainer = createDataContainer(cols, {
+      fields,
+      inputDataFormat: DataForm.COLS_ARRAY
+    });
+    return dataContainer;
+  }, [result, fields]);
+
+  const columns = useMemo(() => fields.map(f => f.name), [fields]);
+  const colMeta = useMemo(
+    () =>
+      fields.reduce(
+        (acc, {name, displayName, type, displayFormat}, colIdx) => ({
+          ...acc,
+          [name]: {
+            // because '' || 'aaa' = 'aaa'
+            name: displayName !== undefined ? displayName : name,
+            displayName,
+            displayFormat,
+            type,
+            colIdx
+          }
+        }),
+        {}
+      ),
+    [fields]
+  );
+  const copyTableColumn = useCallback(
+    column => {
+      const {colIdx, type} = colMeta[column];
+      const text = dataContainer
+        .mapIndex(row => parseFieldValue(dataContainer.valueAt(row.index, colIdx), type))
+        .join('\n');
+      navigator?.clipboard.writeText(text);
+    },
+    [colMeta, dataContainer]
+  );
+  const pinTableColumn = useCallback(
+    column =>
+      pinnedColumns.includes(column)
+        ? setPinnedColumns(pinnedColumns.filter(c => c !== column))
+        : setPinnedColumns([...pinnedColumns, column]),
+    [pinnedColumns]
+  );
+
+  // TODO Potentially costly operation for non row based data containers. Revisit sorting below.
+  const dataTableStyle = useMemo(
+    () => ({
+      minCellSize,
+      cellPadding,
+      optionsButton:
+        theme.fieldTokenWidth + theme.fieldTokenRightMargin + optionButtonWidth + pinButton,
+      fontSize: theme.cellFontSize,
+      font: theme.fontFamily
+    }),
+    [theme]
+  );
+  const cellSizeCache = useMemo(() => {
+    return columns.reduce((acc, column) => {
+      const {colIdx, displayName, type} = colMeta[column];
+      return {
+        ...acc,
+        [column]: renderedSize({
+          text: {
+            dataContainer,
+            column: displayName
+          },
+          colIdx,
+          type,
+          numRowsToCalculate: rowsToCalculatePreview,
+          ...dataTableStyle
+        })
+      };
+    }, {});
+  }, [columns, colMeta, dataContainer, rowsToCalculatePreview, dataTableStyle]);
+
+  return (
+    <DataTable
+      colMeta={colMeta}
+      columns={columns}
+      cellSizeCache={cellSizeCache}
+      dataContainer={dataContainer}
+      pinnedColumns={pinnedColumns}
+      //   sortColumn={sortColumnConfig}
+      copyTableColumn={copyTableColumn}
+      pinTableColumn={pinTableColumn}
+      //   sortTableColumn={setTableSortColumn}
+      setColumnDisplayFormat={setColumnDisplayFormat ?? (() => null)}
+    />
+  );
+};
+
+export const PreviewDataPanel = withTheme(
+  PreviewDataPanelWOTheme
+) as React.FC<PreviewDataPanelProps>;