feat(content): support useCache option (#772)

Author: kwiatkk1

docs/content/en/configuration.md

@@ -524,6 +524,13 @@ Your component should implement the following:
 
 You should be aware that you get the full markdown file content so this includes the front-matter. You can use `gray-matter` to split and join the markdown and the front-matter.
 
+### `useCache`
+
+- Type: `Boolean`
+- Default: `false`
+
+When `true`, the production server (`nuxt start`) will use cached version of the content (generated after running `nuxt build`) instead of parsing files. This improves app startup time, but makes app unaware of any content changes.
+
 ## Defaults
 
 ```js{}[nuxt.config.js]
@@ -535,6 +542,7 @@ export default {
     fullTextSearchFields: ['title', 'description', 'slug', 'text'],
     nestedProperties: [],
     liveEdit: true,
+    useCache: false,
     markdown: {
       remarkPlugins: [
         'remark-squeeze-paragraphs',

packages/content/lib/database.js

@@ -1,5 +1,6 @@
 const { join, extname } = require('path')
 const fs = require('graceful-fs').promises
+const mkdirp = require('mkdirp')
 const Hookable = require('hookable')
 const chokidar = require('chokidar')
 const JSON5 = require('json5')
@@ -18,18 +19,21 @@ class Database extends Hookable {
   constructor (options) {
     super()
     this.dir = options.dir || process.cwd()
-    this.cwd = options.cwd || process.cwd()
+    this.srcDir = options.srcDir || process.cwd()
+    this.buildDir = options.buildDir || process.cwd()
+    this.useCache = options.useCache || false
     this.markdown = new Markdown(options.markdown)
     this.yaml = new YAML(options.yaml)
     this.csv = new CSV(options.csv)
     this.xml = new XML(options.xml)
     // Create Loki database
     this.db = new Loki('content.db')
     // Init collection
-    this.items = this.db.addCollection('items', {
+    this.itemsCollectionOptions = {
       fullTextSearch: options.fullTextSearchFields.map(field => ({ field })),
       nestedProperties: options.nestedProperties
-    })
+    }
+    this.items = this.db.addCollection('items', this.itemsCollectionOptions)
     // User Parsers
     this.extendParser = options.extendParser || {}
     this.extendParserExtensions = Object.keys(this.extendParser)
@@ -58,19 +62,65 @@ class Database extends Hookable {
     }, this.options)
   }
 
+  async init () {
+    if (this.useCache) {
+      try {
+        return await this.initFromCache()
+      } catch (error) {}
+    }
+
+    await this.initFromFilesystem()
+  }
+
   /**
    * Clear items in database and load files into collection
    */
-  async init () {
+  async initFromFilesystem () {
+    const startTime = process.hrtime()
     this.dirs = ['/']
     this.items.clear()
-
-    const startTime = process.hrtime()
     await this.walk(this.dir)
     const [s, ns] = process.hrtime(startTime)
     logger.info(`Parsed ${this.items.count()} files in ${s}.${Math.round(ns / 1e8)} seconds`)
   }
 
+  async initFromCache () {
+    const startTime = process.hrtime()
+    const cacheFilePath = join(this.buildDir, this.db.filename)
+    const cacheFileData = await fs.readFile(cacheFilePath, 'utf-8')
+    const cacheFileJson = JSON.parse(cacheFileData)
+
+    this.db.loadJSONObject(cacheFileJson)
+
+    // recreate references
+    this.items = this.db.getCollection('items')
+    this.dirs = this.items.mapReduce(doc => doc.dir, dirs => [...new Set(dirs)])
+
+    const [s, ns] = process.hrtime(startTime)
+    logger.info(`Loaded ${this.items.count()} documents from cache in ${s},${Math.round(ns / 1e8)} seconds`)
+  }
+
+  /**
+   * Store database info file
+   * @param {string} [dir] - Directory containing database dump file.
+   * @param {string} [filename] - Database dump filename.
+   */
+  async save (dir, filename) {
+    dir = dir || this.buildDir
+    filename = filename || this.db.filename
+
+    await mkdirp(dir)
+    await fs.writeFile(join(dir, filename), this.db.serialize(), 'utf-8')
+  }
+
+  async rebuildCache () {
+    logger.info('Rebuilding content cache')
+    this.db = new Loki('content.db')
+    this.items = this.db.addCollection('items', this.itemsCollectionOptions)
+    await this.initFromFilesystem()
+    await this.save()
+  }
+
   /**
    * Walk dir tree recursively
    * @param {string} dir - Directory to browse.
@@ -145,7 +195,7 @@ class Database extends Hookable {
 
       const document = this.items.findOne({ path: item.path })
 
-      logger.info(`Updated ${path.replace(this.cwd, '.')}`)
+      logger.info(`Updated ${path.replace(this.srcDir, '.')}`)
       if (document) {
         this.items.update({ $loki: document.$loki, meta: document.meta, ...item })
         return
@@ -171,7 +221,7 @@ class Database extends Hookable {
    */
   async parseFile (path) {
     const extension = extname(path)
-    // If unkown extension, skip
+    // If unknown extension, skip
     if (!EXTENSIONS.includes(extension) && !this.extendParserExtensions.includes(extension)) {
       return
     }
@@ -204,7 +254,7 @@ class Database extends Hookable {
       // Force data to be an array
       data = Array.isArray(data) ? data : [data]
     } catch (err) {
-      logger.warn(`Could not parse ${path.replace(this.cwd, '.')}:`, err.message)
+      logger.warn(`Could not parse ${path.replace(this.srcDir, '.')}:`, err.message)
       return null
     }
 

packages/content/lib/index.js

@@ -1,6 +1,5 @@
 const { join, resolve } = require('path')
 const fs = require('graceful-fs').promises
-const mkdirp = require('mkdirp')
 const defu = require('defu')
 const logger = require('consola').withScope('@nuxt/content')
 const hash = require('hasha')
@@ -95,11 +94,21 @@ module.exports = async function (moduleOptions) {
     server.on('upgrade', (...args) => ws.callHook('upgrade', ...args))
   })
 
+  const useCache = options.useCache && !this.options.dev && this.options.ssr
+
   const database = new Database({
     ...options,
-    cwd: this.options.srcDir
+    srcDir: this.options.srcDir,
+    buildDir: resolve(this.options.buildDir, 'content'),
+    useCache
   })
 
+  if (useCache) {
+    this.nuxt.hook('builder:prepared', async () => {
+      await database.rebuildCache()
+    })
+  }
+
   // Database hooks
   database.hook('file:beforeInsert', item =>
     this.nuxt.callHook('content:file:beforeInsert', item, database)
@@ -187,12 +196,7 @@ module.exports = async function (moduleOptions) {
     this.nuxt.hook('generate:distRemoved', async () => {
       const dir = resolve(this.options.buildDir, 'dist', 'client', 'content')
 
-      await mkdirp(dir)
-      await fs.writeFile(
-        join(dir, `db-${dbHash}.json`),
-        database.db.serialize(),
-        'utf-8'
-      )
+      await database.save(dir, `db-${dbHash}.json`)
     })
 
     // Add client plugin

packages/content/lib/utils.js

@@ -4,6 +4,7 @@ const { camelCase } = require('change-case')
 const getDefaults = ({ dev = false } = {}) => ({
   editor: './editor.vue',
   watch: dev,
+  useCache: false,
   liveEdit: true,
   apiPrefix: '_content',
   dir: 'content',

packages/content/test/cache.test.js

@@ -0,0 +1,62 @@
+const path = require('path')
+const fs = require('graceful-fs').promises
+const { build, init, generatePort, loadConfig } = require('@nuxtjs/module-test-utils')
+
+describe('content cache', () => {
+  const config = {
+    ...loadConfig(__dirname),
+    buildDir: path.join(__dirname, 'fixture', '.nuxt-dev'),
+    content: {
+      useCache: true
+    }
+  }
+
+  const dbFilePath = path.join(config.buildDir, 'content', 'content.db')
+
+  describe('during build', () => {
+    let nuxt
+
+    beforeAll(async () => {
+      fs.unlink(dbFilePath).catch(() => {});
+      ({ nuxt } = (await build(config)))
+    }, 60000)
+
+    afterAll(async () => {
+      await nuxt.close()
+    })
+
+    test('should be generated', async () => {
+      await expect(fs.access(dbFilePath)).resolves.not.toThrow()
+    })
+
+    test('should be a valid json', async () => {
+      const fileTextContent = await fs.readFile(dbFilePath)
+      await expect(() => JSON.parse(fileTextContent)).not.toThrow()
+    })
+  })
+
+  describe('during start in production mode', () => {
+    const mockDbDump = '{"_env":"NODEJS","_serializationMethod":"normal","_autosave":false,"_autosaveInterval":5000,"_collections":[{"name":"items","unindexedSortComparator":"js","defaultLokiOperatorPackage":"js","_dynamicViews":[],"uniqueNames":[],"transforms":{},"rangedIndexes":{},"_data":[{"slug":"about","title":"Serialized test","toc":[],"body":{"type":"root","children":[{"type":"element","tag":"p","props":{},"children":[{"type":"text","value":"This is the serialized page!"}]}]},"text":"\\nThis is the serialized page!\\n","dir":"/","path":"/about","extension":".md","createdAt":"2021-02-11T22:10:21.655Z","updatedAt":"2021-02-12T20:13:23.079Z","meta":{"version":0,"revision":0,"created":1613160831274},"$loki":1}],"idIndex":[1],"maxId":1,"_dirty":true,"_nestedProperties":[],"transactional":false,"asyncListeners":false,"disableMeta":false,"disableChangesApi":true,"disableDeltaChangesApi":true,"cloneObjects":false,"cloneMethod":"deep","changes":[],"_fullTextSearch":{"ii":{"title":{"_store":true,"_optimizeChanges":true,"docCount":1,"docStore":[[0,{"fieldLength":2}]],"totalFieldLength":2,"root":{"k":[115,116],"v":[{"k":[101],"v":[{"k":[114],"v":[{"k":[105],"v":[{"k":[97],"v":[{"k":[108],"v":[{"k":[105],"v":[{"k":[122],"v":[{"k":[101],"v":[{"k":[100],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}]}]}]}]},{"k":[101],"v":[{"k":[115],"v":[{"k":[116],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}},"description":{"_store":true,"_optimizeChanges":true,"docCount":0,"docStore":[],"totalFieldLength":0,"root":{}},"slug":{"_store":true,"_optimizeChanges":true,"docCount":1,"docStore":[[0,{"fieldLength":1}]],"totalFieldLength":1,"root":{"k":[97],"v":[{"k":[98],"v":[{"k":[111],"v":[{"k":[117],"v":[{"k":[116],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}},"text":{"_store":true,"_optimizeChanges":true,"docCount":1,"docStore":[[0,{"fieldLength":5}]],"totalFieldLength":5,"root":{"k":[116,105,115,112],"v":[{"k":[104],"v":[{"k":[105,101],"v":[{"k":[115],"v":[{"d":{"df":1,"dc":[[0,1]]}}]},{"d":{"df":1,"dc":[[0,1]]}}]}]},{"k":[115],"v":[{"d":{"df":1,"dc":[[0,1]]}}]},{"k":[101],"v":[{"k":[114],"v":[{"k":[105],"v":[{"k":[97],"v":[{"k":[108],"v":[{"k":[105],"v":[{"k":[122],"v":[{"k":[101],"v":[{"k":[100],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}]}]}]}]},{"k":[97],"v":[{"k":[103],"v":[{"k":[101],"v":[{"k":[33],"v":[{"d":{"df":1,"dc":[[0,1]]}}]}]}]}]}]}}}}}],"databaseVersion":1.5,"engineVersion":1.5,"filename":"content.db","_persistenceAdapter":null,"_persistenceMethod":null,"_throttledSaves":true}'
+    let nuxt
+    let $content
+
+    beforeAll(async () => {
+      await fs.writeFile(dbFilePath, mockDbDump)
+      nuxt = await init(config)
+      await nuxt.listen(await generatePort())
+      $content = require('@nuxt/content').$content
+    }, 60000)
+
+    afterAll(async () => {
+      await nuxt.close()
+    })
+
+    test('should use cached db', async () => {
+      const item = await $content('about').fetch()
+
+      expect(item).toEqual(expect.objectContaining({
+        title: 'Serialized test'
+      }))
+    })
+  })
+})

packages/content/test/options.test.js

@@ -32,6 +32,7 @@ describe('options', () => {
       expect(options).toEqual(expect.objectContaining({
         apiPrefix: '_content',
         dir: 'content',
+        useCache: false,
         fullTextSearchFields: ['title', 'description', 'slug', 'text'],
         nestedProperties: [],
         csv: {},