[Feat] Add an array field type

> fix keplergl#1637 and supersede keplergl#1666

Allow to import features with multiple values properties and convert them to an `array` type.
These fields can then be picked as filters or shown in the tooltip.

Author: lutangar

src/constants/src/default-settings.ts

@@ -329,6 +329,7 @@ export const ALL_FIELD_TYPES = keyMirror({
   boolean: null,
   date: null,
   geojson: null,
+  array: null,
   integer: null,
   real: null,
   string: null,
@@ -408,6 +409,10 @@ export const FIELD_TYPE_DISPLAY = {
     label: 'geo',
     color: BLUE2
   },
+  [ALL_FIELD_TYPES.array]: {
+    label: 'array',
+    color: BLUE2
+  },
   [ALL_FIELD_TYPES.integer]: {
     label: 'int',
     color: ORANGE
@@ -633,6 +638,17 @@ export const FIELD_OPTS = {
       legend: d => '...',
       tooltip: []
     }
+  },
+  array: {
+    type: 'array',
+    scale: {
+      ...notSupportedScaleOpts,
+      ...notSupportAggrOpts
+    },
+    format: {
+      legend: d => '...',
+      tooltip: []
+    }
   }
 };
 

src/processors/src/data-processor.ts

@@ -60,6 +60,16 @@ export const PARSE_FIELD_VALUE_FROM_STRING = {
     valid: (d: unknown): boolean => parseFloat(d) === d,
     // Note this will result in NaN for some string
     parse: parseFloat
+  },
+  [ALL_FIELD_TYPES.array]: {
+    valid: (d: unknown): boolean => Array.isArray(d),
+    parse: (d: unknown): any => {
+      try {
+        return JSON.parse(d as string);
+      } catch (e) {
+        return [];
+      }
+    }
   }
 };
 

src/utils/src/data-scale-utils.ts

@@ -47,7 +47,7 @@ export function getOrdinalDomain(
 ): string[] {
   const values = dataContainer.mapIndex(valueAccessor);
 
-  return unique(values)
+  return unique(values.flat())
     .filter(notNullorUndefined)
     .sort();
 }

src/utils/src/data-utils.ts

@@ -257,6 +257,8 @@ export function roundValToStep(minValue: number, step: number, val: number): num
  * Used in render tooltip value
  */
 export const defaultFormatter: FieldFormatter = v => (notNullorUndefined(v) ? String(v) : '');
+export const arrayFormatter: FieldFormatter = v =>
+  Array.isArray(v) ? `[${String(v)}]` : defaultFormatter(v);
 
 export const FIELD_DISPLAY_FORMAT: {
   [key: string]: FieldFormatter;
@@ -268,13 +270,8 @@ export const FIELD_DISPLAY_FORMAT: {
   [ALL_FIELD_TYPES.boolean]: defaultFormatter,
   [ALL_FIELD_TYPES.date]: defaultFormatter,
   [ALL_FIELD_TYPES.geojson]: d =>
-    typeof d === 'string'
-      ? d
-      : isPlainObject(d)
-      ? JSON.stringify(d)
-      : Array.isArray(d)
-      ? `[${String(d)}]`
-      : ''
+    typeof d === 'string' ? d : isPlainObject(d) ? JSON.stringify(d) : arrayFormatter(d),
+  [ALL_FIELD_TYPES.array]: arrayFormatter
 };
 
 /**

src/utils/src/dataset-utils.ts

@@ -418,6 +418,7 @@ export function getFieldsFromData(data: RowData, fieldOrder: string[]): Field[]
   const metadata = Analyzer.computeColMeta(
     data,
     [
+      {regex: /.*array/g, dataType: 'ARRAY'},
       {regex: /.*geojson|all_points/g, dataType: 'GEOMETRY'},
       {regex: /.*census/g, dataType: 'STRING'}
     ],
@@ -524,10 +525,11 @@ export function analyzerTypeToFieldType(aType: string): string {
     case GEOMETRY:
     case GEOMETRY_FROM_STRING:
     case PAIR_GEOMETRY_FROM_STRING:
-    case ARRAY:
     case OBJECT:
-      // TODO: create a new data type for objects and arrays
+      // TODO: create a new data type for objects
       return ALL_FIELD_TYPES.geojson;
+    case ARRAY:
+      return ALL_FIELD_TYPES.array;
     case NUMBER:
     case STRING:
     case ZIPCODE:

src/utils/src/filter-utils.ts

@@ -384,6 +384,7 @@ export function getFilterProps(
 
     case ALL_FIELD_TYPES.string:
     case ALL_FIELD_TYPES.date:
+    case ALL_FIELD_TYPES.array:
       // @ts-expect-error
       return {
         ...filterProps,
@@ -486,6 +487,9 @@ export function getFilterFunction<L extends {config: {dataId: string | null}; id
     case FILTER_TYPES.range:
       return data => isInRange(valueAccessor(data), filter.value);
     case FILTER_TYPES.multiSelect:
+      if (field?.type === ALL_FIELD_TYPES.array) {
+        return data => valueAccessor(data).some(v => filter.value.includes(v));
+      }
       return data => filter.value.includes(valueAccessor(data));
     case FILTER_TYPES.select:
       return data => valueAccessor(data) === filter.value;
@@ -1046,6 +1050,7 @@ export function mergeFilterDomainStep(
   switch (filterProps.fieldType) {
     case ALL_FIELD_TYPES.string:
     case ALL_FIELD_TYPES.date:
+    case ALL_FIELD_TYPES.array:
       return {
         ...newFilter,
         domain: unique(combinedDomain).sort()

src/utils/src/index.ts

@@ -55,6 +55,7 @@ export {
   normalizeSliderValue,
   roundValToStep,
   defaultFormatter,
+  arrayFormatter,
   FIELD_DISPLAY_FORMAT,
   parseFieldValue,
   arrayMove,
@@ -211,4 +212,4 @@ export {
 export {DataRow} from './data-row';
 
 export type {Centroid} from './h3-utils';
-export {getCentroid, idToPolygonGeo, h3IsValid, getHexFields} from './h3-utils';
+export {getCentroid, idToPolygonGeo, h3IsValid, getHexFields} from './h3-utils';

test/fixtures/geojson.js

@@ -559,7 +559,7 @@ export const geoStyleFields = [
     displayName: 'fillColor',
     format: '',
     fieldIdx: 1,
-    type: 'geojson',
+    type: 'array',
     analyzerType: 'ARRAY',
     valueAccessor: values => values[1]
   },
@@ -569,7 +569,7 @@ export const geoStyleFields = [
     displayName: 'lineColor',
     format: '',
     fieldIdx: 2,
-    type: 'geojson',
+    type: 'array',
     analyzerType: 'ARRAY',
     valueAccessor: values => values[2]
   },

test/node/utils/data-processor-test.js

@@ -426,6 +426,27 @@ test('Processor -> parseCsvRowsByFieldType -> boolean', t => {
   t.end();
 });
 
+test('Processor -> parseCsvRowsByFieldType -> array', t => {
+  const field = {
+    type: ALL_FIELD_TYPES.array
+  };
+
+  const rows = [
+    ['[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]'],
+    ['["tag1", "tag2", "tag3", "tag4", "tag5", "tag6", "tag7"]']
+  ];
+
+  const expected = [
+    [[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]],
+    [['tag1', 'tag2', 'tag3', 'tag4', 'tag5', 'tag6', 'tag7']]
+  ];
+
+  parseCsvRowsByFieldType(rows, -1, field, 0);
+
+  t.same(rows, expected, 'should parsed arrays properly');
+  t.end();
+});
+
 test('Processor -> getSampleForTypeAnalyze', t => {
   const fields = ['string', 'int', 'bool', 'time'];
 

test/node/utils/data-scale-utils-test.js

@@ -33,15 +33,15 @@ function numberSort(a, b) {
 }
 
 test('DataScaleUtils -> getOrdinalDomain', t => {
-  const data = [['a'], ['a'], ['b'], [undefined], [null], [0], null];
+  const data = [['a'], ['a'], ['b'], [undefined], [null], [0], null, [['c', 'd', null]]];
 
   function valueAccessor(d, dc) {
     return dc.valueAt(d.index, 0);
   }
 
   t.deepEqual(
     getOrdinalDomain(createDataContainer(data), valueAccessor),
-    [0, 'a', 'b'],
+    [0, 'a', 'b', 'c', 'd'],
     'should get correct ordinal domain'
   );
 

test/node/utils/data-utils-test.js

@@ -29,7 +29,8 @@ import {
   snapToMarks,
   arrayMove,
   getFormatter,
-  defaultFormatter
+  defaultFormatter,
+  arrayFormatter
 } from '@kepler.gl/utils';
 import {ALL_FIELD_TYPES} from '@kepler.gl/constants';
 
@@ -120,6 +121,16 @@ test('dataUtils -> defaultFormatter', t => {
   t.end();
 });
 
+test('dataUtils -> arrayFormatter', t => {
+  t.equal(arrayFormatter([]), '[]', 'arrayFormatter should be correct');
+  t.equal(arrayFormatter(['tag1']), '[tag1]', 'arrayFormatter should be correct');
+  t.equal(arrayFormatter(undefined), '', 'arrayFormatter should be correct');
+  t.equal(arrayFormatter(NaN), 'NaN', 'arrayFormatter should be correct');
+  t.equal(arrayFormatter(null), '', 'arrayFormatter should be correct');
+
+  t.end();
+});
+
 test('dataUtils -> getFormatter', t => {
   const TEST_CASES = [
     {