Skip to content

Commit 650012d

Browse files
obeliskosobeliskos
committed
#657 : added syncronous LokiFsSyncAdapter for diagnostic purposes
To easily retrofit into synchronous code patterns (for node.js), I have created a synchronous version of the LokiFsAdapter. It is in an external module which needs to be required/imported. I do not recommend using this adapter unless you have requirement for totally syncronous database functionality. In that case, you would not use 'autoloadCallback' (which is forced async) but just initialize (if necessary) after loading synchronously or in the synchronous callback of loadDatabase.. Added 'benchmark/throttled-stress.js' stress test / benchmarks verifying validitity/effectiveness of throttled saves with overuse of throttleSaves, saving after each database operation. No problems are indicated with our current throttled save functionality and performance advantages with -async- adapter are significant and illustrated when user performs excessive saving without waiting for prior saves to complete.
1 parent 71139c9 commit 650012d

File tree

2 files changed

+295
-0
lines changed

2 files changed

+295
-0
lines changed

‎benchmark/throttled-stress.js

Lines changed: 190 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,190 @@
1+
/**
2+
* This stress test is designed to test effectiveness of throttled saves in
3+
* worst-case scenarios.
4+
*
5+
* In order to stress overlapping saves we will use an async adapter along
6+
* with synchronous logic with many calls to save in which we do not wait
7+
* for the save to complete before attempting to save again. This usage
8+
* pattern is not recommended but lokijs throttled saves is intended to
9+
* safeguard against it.
10+
*
11+
* The test will verify that nothing is lost when this usage pattern is
12+
* used by comparing the database when we are done with the copy in memory.
13+
*
14+
* We are forced to consider async adapter behavior on final save and reload,
15+
* since throttled saves protect only within a single loki object instance.
16+
* You must still wait after for the throttled queue to drain after finishing
17+
* before you can attempt to reload it and you must wait for the database
18+
* to finish loading before you can access its contents.
19+
*/
20+
21+
var crypto = require("crypto"); // for random string generation
22+
var loki = require('../src/lokijs.js');
23+
24+
const INITIALCOUNT = 2000;
25+
const ITERATIONS = 2000;
26+
const RANGE = 1000;
27+
28+
// synchronous adapter using LokiMemoryAdapter
29+
var memAdapterSync = new loki.LokiMemoryAdapter();
30+
31+
// simulate async adapter with 100ms save/load times
32+
var memAdapterAsync = new loki.LokiMemoryAdapter({
33+
asyncResponses: true,
34+
asyncTimeout: 100
35+
});
36+
37+
var db, db2;
38+
var maxThrottledCalls = 0;
39+
40+
// less memory 'leaky' way to generate random strings (node specific)
41+
function genRandomString() {
42+
return crypto.randomBytes(50).toString('hex');
43+
}
44+
45+
function genRandomObject() {
46+
var av = Math.floor(Math.random() * RANGE);
47+
var cv = Math.floor(Math.random() * RANGE);
48+
49+
return { "a": av, "b": genRandomString(), "c": cv };
50+
}
51+
52+
function setupDatabaseSync() {
53+
var newDatabase = new loki("throttle-test.db", { adapter: memAdapterSync });
54+
55+
// since our memory adapter is by default synchronous (unless simulating async),
56+
// we can assume any load will complete before our next statement executes.
57+
newDatabase.loadDatabase();
58+
59+
// initialize collections
60+
if (!newDatabase.getCollection("items")) {
61+
newDatabase.addCollection("items");
62+
}
63+
64+
return newDatabase;
65+
}
66+
67+
function setupDatabaseAsync(callback) {
68+
var newDatabase = new loki("throttle-test.db", { adapter: memAdapterAsync });
69+
70+
// database won't exist on first pass, but let's use forced
71+
// async syntax in case is did
72+
newDatabase.loadDatabase({}, function(err) {
73+
if (err) {
74+
callback(err);
75+
}
76+
77+
// initialize collections
78+
if (!newDatabase.getCollection("items")) {
79+
newDatabase.addCollection("items");
80+
81+
// bad practice, stress test
82+
newDatabase.saveDatabase();
83+
}
84+
85+
callback(err);
86+
});
87+
88+
return newDatabase;
89+
}
90+
91+
function performStressedOps() {
92+
var items = db.getCollection("items");
93+
var idx, op;
94+
95+
for(idx=0;idx<INITIALCOUNT;idx++) {
96+
items.insert(genRandomObject());
97+
98+
// bad practice, stress test
99+
db.saveDatabase();
100+
101+
if (db.throttledCallbacks.length > maxThrottledCalls) {
102+
maxThrottledCalls = db.throttledCallbacks.length;
103+
}
104+
}
105+
106+
for(idx=0;idx<ITERATIONS;idx++) {
107+
// randomly determine if this permutation will be insert/update/remove
108+
op = Math.floor(Math.random() * 3);
109+
switch(op) {
110+
// insert
111+
case 0: items.insert(genRandomObject());
112+
break;
113+
// update
114+
case 1: rnd = Math.floor(Math.random() * RANGE);
115+
items.chain().find({a:rnd}).update(function(obj) {
116+
obj.a = Math.floor(Math.random() * RANGE);
117+
obj.c = Math.floor(Math.random() * RANGE);
118+
obj.b = genRandomString();
119+
});
120+
break;
121+
// remove 2 matches of a single value in our range
122+
case 2: rnd = Math.floor(Math.random() * RANGE);
123+
items.chain().find({a:rnd}).limit(1).remove();
124+
break;
125+
}
126+
127+
db.saveDatabase();
128+
if (db.throttledCallbacks.length > maxThrottledCalls) {
129+
maxThrottledCalls = db.throttledCallbacks.length;
130+
}
131+
}
132+
}
133+
134+
function compareDatabases() {
135+
var c1 = db.getCollection("items");
136+
var c2 = db2.getCollection("items");
137+
var idx;
138+
139+
var count = c1.count();
140+
if (count !== c2.count()) return false;
141+
142+
for(idx=0; idx < count; idx++) {
143+
if (c1.data[idx].a !== c2.data[idx].a) return false;
144+
if (c1.data[idx].b !== c2.data[idx].b) return false;
145+
if (c1.data[idx].c !== c2.data[idx].c) return false;
146+
if (c1.data[idx].$loki !== c2.data[idx].$loki) return false;
147+
}
148+
149+
return true;
150+
}
151+
152+
console.log("");
153+
154+
// let's test in truly sync manner
155+
var start = process.hrtime();
156+
db = setupDatabaseSync();
157+
performStressedOps();
158+
db.saveDatabase();
159+
db2 = setupDatabaseSync();
160+
var end = process.hrtime(start);
161+
var result = compareDatabases();
162+
console.log("## Fully synchronous operations with excessive saving after each operation ##");
163+
console.log("Database are " + (result?"the same.":"NOT the same!"));
164+
console.log("Execution time (hr): %ds %dms", end[0], end[1]/1000000);
165+
console.log("maxThrottledCalls: " + maxThrottledCalls);
166+
167+
console.log("");
168+
169+
// now let's test with simulated async adpater
170+
// first pass setup will create in memory
171+
start = process.hrtime();
172+
db = setupDatabaseAsync(function() {
173+
performStressedOps();
174+
175+
// go ahead and do a final save (even though we save after every op)
176+
// and then wait for queue to drain before trying to reload it.
177+
db.saveDatabase();
178+
db.throttledSaveDrain(function () {
179+
db2 = setupDatabaseAsync(function () {
180+
var end = process.hrtime(start);
181+
var result = compareDatabases();
182+
183+
console.log("## Asynchronous operations with excessive saving after each operation ##");
184+
console.log("Async database are " + (result?"the same.":"NOT the same!"));
185+
console.log("Execution time (hr): %ds %dms", end[0], end[1]/1000000);
186+
console.log("maxThrottledCalls: " + maxThrottledCalls);
187+
});
188+
});
189+
});
190+

‎src/loki-fs-sync-adapter.js

Lines changed: 105 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,105 @@
1+
/*
2+
A synchronous version of the Loki Filesystem adapter for node.js
3+
4+
Intended for diagnostics or environments where synchronous i/o is required.
5+
6+
This adapter will perform worse than the default LokiFsAdapter but
7+
is provided for quick adaptation to synchronous code.
8+
*/
9+
10+
(function (root, factory) {
11+
if (typeof define === 'function' && define.amd) {
12+
// AMD
13+
define([], factory);
14+
} else if (typeof exports === 'object') {
15+
// Node, CommonJS-like
16+
module.exports = factory();
17+
} else {
18+
// Browser globals (root is window)
19+
root.LokiFsSyncAdapter = factory();
20+
}
21+
}(this, function () {
22+
return (function() {
23+
'use strict';
24+
25+
/**
26+
* A loki persistence adapter which persists using node fs module
27+
* @constructor LokiFsSyncAdapter
28+
*/
29+
function LokiFsSyncAdapter() {
30+
this.fs = require('fs');
31+
}
32+
33+
/**
34+
* loadDatabase() - Load data from file, will throw an error if the file does not exist
35+
* @param {string} dbname - the filename of the database to load
36+
* @param {function} callback - the callback to handle the result
37+
* @memberof LokiFsSyncAdapter
38+
*/
39+
LokiFsSyncAdapter.prototype.loadDatabase = function loadDatabase(dbname, callback) {
40+
var self = this;
41+
var contents;
42+
43+
try {
44+
var stats = this.fs.statSync(dbname);
45+
if (stats.isFile()) {
46+
contents = self.fs.readFileSync(dbname, {
47+
encoding: 'utf8'
48+
});
49+
50+
callback(contents);
51+
}
52+
else {
53+
callback(null);
54+
}
55+
}
56+
catch (err) {
57+
// first autoload when file doesn't exist yet
58+
// should not throw error but leave default
59+
// blank database.
60+
if (err.code === "ENOENT") {
61+
callback(null);
62+
}
63+
64+
callback(err);
65+
}
66+
};
67+
68+
/**
69+
* saveDatabase() - save data to file, will throw an error if the file can't be saved
70+
* might want to expand this to avoid dataloss on partial save
71+
* @param {string} dbname - the filename of the database to load
72+
* @param {function} callback - the callback to handle the result
73+
* @memberof LokiFsSyncAdapter
74+
*/
75+
LokiFsSyncAdapter.prototype.saveDatabase = function saveDatabase(dbname, dbstring, callback) {
76+
try {
77+
this.fs.writeFileSync(dbname, dbstring);
78+
callback();
79+
}
80+
catch (err) {
81+
callback(err);
82+
}
83+
};
84+
85+
/**
86+
* deleteDatabase() - delete the database file, will throw an error if the
87+
* file can't be deleted
88+
* @param {string} dbname - the filename of the database to delete
89+
* @param {function} callback - the callback to handle the result
90+
* @memberof LokiFsSyncAdapter
91+
*/
92+
LokiFsSyncAdapter.prototype.deleteDatabase = function deleteDatabase(dbname, callback) {
93+
try {
94+
this.fs.unlinkSync(dbname);
95+
callback();
96+
}
97+
catch (err) {
98+
callback(err);
99+
}
100+
};
101+
102+
return LokiFsSyncAdapter;
103+
104+
}());
105+
}));

0 commit comments

Comments
 (0)