[feat] vector tile layer (#2839)

- This PR adds Vector Tile layer to Kepler with support initial support of mapbox vector tiles and remote .pmtiles (with mimeType application/vnd.mapbox-vector-tile)

Author: igorDykhta

docs/user-guides/c-types-of-layers/README.md

@@ -92,5 +92,13 @@ To use H3 layer, you need a `hex_id` in your dataset, which can be generated usi
 
 To use S2 layer, you need to assign a column containing S2 tokens.
 
+## Vector Tile Layer
+
+Vector Tile Layer makes it possible to visualize very large datasets through MVTs (Mapbox Vector Tiles). To optimize performance, the layer only loads and renders tiles containing features that are visible within the current viewport.
+
+Supported URL templates:
+
+- MVT (https://api.mapbox.com/v4/mapbox.mapbox-streets-v8/{z}/{x}/{y}.mvt?access_token=your-mapbox-acceess-token)
+- pmtiles (https://your-cdn/filename.pmtiles)
 
 [Back to table of contents](../README.md)

examples/demo-app/src/actions.js

@@ -5,6 +5,7 @@ import {push} from 'react-router-redux';
 import {fetch} from 'global';
 
 import {loadFiles, toggleModal} from '@kepler.gl/actions';
+import {parseUri} from '@kepler.gl/common-utils';
 import {load} from '@loaders.gl/core';
 import {CSVLoader} from '@loaders.gl/csv';
 import {GeoArrowLoader} from '@loaders.gl/arrow';
@@ -16,7 +17,6 @@ import {
   LOADING_SAMPLE_LIST_ERROR_MESSAGE,
   MAP_CONFIG_URL
 } from './constants/default-settings';
-import {parseUri} from './utils/url';
 
 // CONSTANTS
 export const INIT = 'INIT';

examples/demo-app/src/app.tsx

@@ -194,6 +194,76 @@ const App = props => {
     );
   }, [dispatch]);
 
+  const _loadVectorTileData = useCallback(() => {
+    dispatch(
+      addDataToMap({
+        datasets: [
+          {
+            info: {
+              label: 'Railroads',
+              id: 'railroads.pmtiles',
+              color: [255, 0, 0],
+              type: 'vectorTile'
+            },
+            data: {
+              rows: [],
+              fields: [
+                {
+                  name: 'continent',
+                  type: 'string',
+                  format: '',
+                  analyzerType: 'STRING'
+                }
+              ]
+            },
+            metadata: {
+              name: 'output.pmtiles',
+              description: 'output.pmtiles',
+              type: 'pmtiles',
+              tilesetDataUrl:
+                'https://4sq-studio-public.s3.us-west-2.amazonaws.com/pmtiles-test/161727fe-7952-4e57-aa05-850b3086b0b2.pmtiles',
+              tilesetMetadataUrl:
+                'https://4sq-studio-public.s3.us-west-2.amazonaws.com/pmtiles-test/161727fe-7952-4e57-aa05-850b3086b0b2.pmtiles',
+              id: 'sz6uy1xtj',
+              format: 'rows',
+              label: 'output.pmtiles',
+              metaJson: null,
+              bounds: [-150.1122219, -51.8952777, 179.3577783, 69.6043747],
+              center: [14.0625, 50.7026397, 6],
+              maxZoom: 6,
+              minZoom: 0,
+              fields: [
+                {
+                  name: 'continent',
+                  id: 'continent',
+                  format: '',
+                  filterProps: {
+                    domain: [
+                      'Africa',
+                      'Asia',
+                      'Europe',
+                      'North America',
+                      'Oceania',
+                      'South America'
+                    ],
+                    value: [],
+                    type: 'multiSelect',
+                    gpu: false
+                  },
+                  type: 'string',
+                  analyzerType: 'STRING'
+                }
+              ]
+            }
+          }
+        ],
+        options: {
+          autoCreateLayers: true
+        }
+      })
+    );
+  }, [dispatch]);
+
   const _loadPointData = useCallback(() => {
     dispatch(
       addDataToMap({
@@ -427,6 +497,7 @@ const App = props => {
     // _loadScenegraphLayer();
     // _loadGpsData();
     // _loadRowData();
+    // _loadVectorTileData();
     // eslint-disable-next-line react-hooks/exhaustive-deps
   }, [
     _loadPointData,
@@ -438,7 +509,8 @@ const App = props => {
     _loadScenegraphLayer,
     _loadGpsData,
     _loadRowData,
-    _replaceData
+    _replaceData,
+    _loadVectorTileData
   ]);
 
   return (

examples/demo-app/src/factories/load-data-modal.js

@@ -29,6 +29,7 @@ const CustomLoadDataModalFactory = (...deps) => {
   // add more loading methods
   const loadingMethods = [
     defaultLoadingMethods.find(lm => lm.id === 'upload'),
+    defaultLoadingMethods.find(lm => lm.id === 'tileset'),
     additionalMethods.remote,
     defaultLoadingMethods.find(lm => lm.id === 'storage'),
     additionalMethods.sample

examples/demo-app/src/utils/url.js

@@ -1,57 +1,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-// from http://blog.stevenlevithan.com/archives/parseuri
-/**
- * Allows to break down a url into multiple params
- * @param str
- */
-export function parseUri(str) {
-  const o = parseUri.options;
-  const m = o.parser[o.strictMode ? 'strict' : 'loose'].exec(str);
-  const uri = {};
-  let i = 14;
-
-  while (i--) uri[o.key[i]] = m[i] || '';
-
-  uri[o.q.name] = {};
-  uri[o.key[12]].replace(o.q.parser, ($0, $1, $2) => {
-    if ($1) uri[o.q.name][$1] = $2;
-  });
-
-  return uri;
-}
-
-parseUri.options = {
-  strictMode: false,
-  key: [
-    'source',
-    'protocol',
-    'authority',
-    'userInfo',
-    'user',
-    'password',
-    'host',
-    'port',
-    'relative',
-    'path',
-    'directory',
-    'file',
-    'query',
-    'anchor'
-  ],
-  q: {
-    name: 'queryKey',
-    parser: /(?:^|&)([^&=]*)=?([^&]*)/g
-  },
-  parser: {
-    strict:
-      /^(?:([^:/?#]+):)?(?:\/\/((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:/?#]*)(?::(\d*))?))?((((?:[^?#/]*\/)*)([^?#]*))(?:\?([^#]*))?(?:#(.*))?)/,
-    loose:
-      /^(?:(?![^:@]+:[^:@/]*@)([^:/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#/]*\.[^?#/.]+(?:[?#]|$)))*\/?)?([^?#/]*))(?:\?([^#]*))?(?:#(.*))?)/
-  }
-};
-
 /**
  * Validates an url
  * @param str

src/common-utils/src/data.ts

@@ -1,6 +1,10 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
+import {Analyzer, DATA_TYPES} from 'type-analyzer';
+
+import {Field} from '@kepler.gl/types';
+
 export function notNullorUndefined<T extends NonNullable<any>>(d: T | null | undefined): d is T {
   return d !== undefined && d !== null;
 }
@@ -45,3 +49,23 @@ export function toArray<T>(item: T | T[]): T[] {
 
   return [item];
 }
+
+/**
+ * Check whether geojson linestring's 4th coordinate is 1) not timestamp 2) unix time stamp 3) real date time
+ * @param timestamps array to be tested if its elements are timestamp
+ * @returns the type of timestamp: unix/datetime/invalid(not timestamp)
+ */
+export function containValidTime(timestamps: string[]): Field | null {
+  const formattedTimeStamps = timestamps.map(ts => ({ts}));
+  const ignoredDataTypes = Object.keys(DATA_TYPES).filter(
+    type => ![DATA_TYPES.TIME, DATA_TYPES.DATETIME, DATA_TYPES.DATE].includes(type)
+  );
+
+  // ignore all types but TIME to improve performance
+  const analyzedType = Analyzer.computeColMeta(formattedTimeStamps, [], {ignoredDataTypes})[0];
+
+  if (!analyzedType || analyzedType.category !== 'TIME') {
+    return null;
+  }
+  return analyzedType;
+}

src/common-utils/src/index.ts

@@ -4,6 +4,7 @@
 export * from './data';
 export * from './data-type';
 export * from './string';
+export * from './url';
 
 export {getCentroid, getHexFields, h3IsValid, idToPolygonGeo} from './h3-utils';
 export type {Centroid} from './h3-utils';

src/common-utils/src/url.ts

@@ -0,0 +1,54 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+/**
+ * Allows to break down a url into multiple params
+ * from http://blog.stevenlevithan.com/archives/parseuri
+ */
+export function parseUri(str: string): {[key: string]: any} {
+  const o = parseUri.options;
+  const m = o.parser[o.strictMode ? 'strict' : 'loose'].exec(str);
+  const uri = {};
+  let i = 14;
+
+  while (i--) uri[o.key[i]] = m?.[i] || '';
+
+  uri[o.q.name] = {};
+  uri[o.key[12]].replace(o.q.parser, ($0, $1, $2) => {
+    if ($1) uri[o.q.name][$1] = $2;
+  });
+
+  return uri;
+}
+
+parseUri.options = {
+  strictMode: false,
+  key: [
+    'source',
+    'protocol',
+    'authority',
+    'userInfo',
+    'user',
+    'password',
+    'host',
+    'port',
+    'relative',
+    'path',
+    'directory',
+    'file',
+    'query',
+    'anchor'
+  ],
+  q: {
+    name: 'queryKey',
+    parser: /(?:^|&)([^&=]*)=?([^&]*)/g
+  },
+  parser: {
+    strict:
+      // eslint-disable-next-line no-useless-escape
+      /^(?:([^:\/?#]+):)?(?:\/\/((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?))?((((?:[^?#\/]*\/)*)([^?#]*))(?:\?([^#]*))?(?:#(.*))?)/,
+    loose:
+      // eslint-disable-next-line no-useless-escape
+      /^(?:(?![^:@]+:[^:@\/]*@)([^:\/?#.]+):)?(?:\/\/)?((?:(([^:@]*)(?::([^:@]*))?)?@)?([^:\/?#]*)(?::(\d*))?)(((\/(?:[^?#](?![^?#\/]*\.[^?#\/.]+(?:[?#]|$)))*\/?)?([^?#\/]*))(?:\?([^#]*))?(?:#(.*))?)/
+  }
+};

src/components/src/hooks/use-fetch-vector-tile-metadata.ts

@@ -0,0 +1,90 @@
+// SPDX-License-Identifier: MIT
+// Copyright contributors to the kepler.gl project
+
+import {useCallback, useEffect, useState} from 'react';
+
+import {MVTSource, TileJSON} from '@loaders.gl/mvt';
+import {PMTilesSource, PMTilesMetadata} from '@loaders.gl/pmtiles';
+
+import {VectorTileMetadata, VectorTileType} from '@kepler.gl/layers';
+
+type FetchVectorTileMetadataProps = {
+  url: string | null;
+  type: VectorTileType;
+  process?: (json: PMTilesMetadata | TileJSON) => VectorTileMetadata | Error | null;
+};
+
+const DEFAULT_PROCESS_FUNCTION = (json: PMTilesMetadata | TileJSON): VectorTileMetadata => {
+  return {
+    metaJson: null,
+    bounds: null,
+    center: null,
+    maxZoom: null,
+    minZoom: null,
+    fields: [],
+    ...json
+  };
+};
+
+type FetchVectorTileMetadataReturn = {
+  data: VectorTileMetadata | null;
+  loading: boolean;
+  error: Error | null;
+};
+
+/** Hook to fetch and return mvt or pmtiles metadata. */
+export default function useFetchVectorTileMetadata({
+  type,
+  url,
+  process = DEFAULT_PROCESS_FUNCTION
+}: FetchVectorTileMetadataProps): FetchVectorTileMetadataReturn {
+  const [error, setError] = useState<Error | null>(null);
+  const [data, setData] = useState<VectorTileMetadata | null>(null);
+  const [loading, setLoading] = useState<boolean>(false);
+
+  const setProcessedData = useCallback(
+    (value: PMTilesMetadata | TileJSON | null) => {
+      if (!value) {
+        return setError(value);
+      }
+      const processedData = process(value);
+      if (processedData instanceof Error) {
+        setError(processedData);
+      } else {
+        setData(processedData);
+      }
+    },
+    [setError, setData, process]
+  );
+
+  useEffect(() => {
+    const getAndProcessMetadata = async () => {
+      setError(null);
+      setData(null);
+      if (url) {
+        setLoading(true);
+
+        try {
+          const tileSource =
+            type === VectorTileType.MVT
+              ? MVTSource.createDataSource('', {
+                  mvt: {
+                    metadataUrl: decodeURIComponent(url)
+                  }
+                })
+              : PMTilesSource.createDataSource(decodeURIComponent(url), {});
+
+          const metadata = await tileSource.metadata;
+          setProcessedData(metadata);
+        } catch (metadataError) {
+          setError(metadataError as any);
+        }
+        setLoading(false);
+      }
+    };
+
+    getAndProcessMetadata();
+  }, [url, type, setProcessedData]);
+
+  return {data, loading, error};
+}

src/components/src/hooks/use-on-click-outside.tsx

@@ -1,8 +1,6 @@
 // SPDX-License-Identifier: MIT
 // Copyright contributors to the kepler.gl project
 
-// Copyright 2022 Foursquare Labs, Inc. All Rights Reserved.
-
 import document from 'global/document';
 import {useCallback, useEffect, useRef, MutableRefObject} from 'react';
 export default function useOnClickOutside<T extends HTMLElement>(