๐จ Sir
![visitors+++](https://camo.githubusercontent.com/391f85f5970fed05cf2c169ad0f5a5f159443f4715146dcff07db2eb163e2be3/68747470733a2f2f686974732e736565796f756661726d2e636f6d2f6170692f636f756e742f696e63722f62616467652e7376673f75726c3d68747470732533412532462532466769746875622e636f6d25324663396665253246736972646226636f756e745f62673d253233373943383344267469746c655f62673d2532333535353535352669636f6e3d2669636f6e5f636f6c6f723d253233453745374537267469746c653d253238746f646179253246746f74616c25323925323076697369746f727325323073696e63652532304e6f76253230372532303230323026656467655f666c61743d66616c7365)
Sir.DB -- A simple database on the file system.
JSON files organised into subdirectories for each table.
Like SirDB? You'll probably love ServeData! ServeData is a powerful yet simple server for SirDB with baked-in schemas, users, groups, permissions, authentication, authorization and payments.
- overview
- features
- get
- repository guide
- api
- examples
- related projects
- examples of database files and diffs
- notices
With this library you can make databases that are:
- human-readable
- git-diffable, and therefore versionable, and
- text-based. Everything is a JSON file, including the database meta information.
Sir.DB:
- is written in literate, tested JavaScript,
- is permissively licensed, and
- is around 500 lines of code and 6.6Kb gzipped.
- Human readable
- Store any JSON-able object
- Index on any property (for nested objects, only index top-level properties)
- Auto ID or custom ID
- Diffable by git
- All records, indexes and table information are JSON files
- 1 file per record, 1 file per unique index value, 1 file per table info
- 1 sub-directory per table, 1 sub-directory (nested inside table) per indexed property
All in all this makes the database easy to understand and inspect. As well as making the code easy to read and maintain.
- <PageableTable>.getPage
- Transactions
- ACID guarantee (as long as accessed by single node thread)
- Can expand ACID to multi-threaded access with a request queue.
npm i --save sirdb
This repository has around 500 lines of code and 2 dependencies (esm and discohash), and is organized as follows:
- api.js: the main file (config, dropTable, getTable and getIndexedTable)
- table.js: Table and IndexedTable classes
- stats/: see the source-code stats
There's only a couple of handful of calls in this api: config
, dropTable
, getIndexedTable
, getTable
, put
, get
, getAllMatchingKeysFromIndex
and getAllMatchingRecordsFromIndex
Configure the root directory of your database.
Deletes a table.
Creates a table (if it doesn't exist) and returns it.
Creates a table (if it doesn't exist) that has indexes for the given properties, and returns it.
Adds (or updates) an item to the table by key.
Gets an item from the table by key.
Gets all item keys from the table that have a property prop
that matches value
, if that property is indexed.
Gets all items from the table that have a property prop
that matches value
, if that property is indexed.
Gets all items from the table.
Get count
(default 10) items from the table, starting at the first item after cursor
. Note: not yet implemented.
See below for how the api calls are used:
import path from 'path';
import assert from 'assert';
import fs from 'fs';
import {getTable, getIndexedTable, dropTable, config} from './api.js';
testGetTable();
testGetIndexedTable();
testDropTable();
testPut();
testIndexedPut();
testGet();
testGetAll();
function testGetTable() {
const root = path.resolve(__dirname, "test-get-table");
config({root});
dropTable("users");
dropTable("subscriptions");
getTable("users");
getTable("subscriptions");
assert.strictEqual(fs.existsSync(path.resolve(root, "users", "tableInfo.json")), true);
assert.strictEqual(fs.existsSync(path.resolve(root, "subscriptions", "tableInfo.json")), true);
fs.rmdirSync(root, {recursive:true});
}
function testGetIndexedTable() {
const root = path.resolve(__dirname, "test-get-indexed-table");
config({root});
dropTable("users");
dropTable("subscriptions");
getIndexedTable("users", ["username", "email"]);
getIndexedTable("subscriptions", ["email"]);
try {
assert.strictEqual(fs.existsSync(path.resolve(root, "users", "tableInfo.json")), true);
assert.strictEqual(fs.existsSync(path.resolve(root, "subscriptions", "tableInfo.json")), true);
assert.strictEqual(fs.existsSync(path.resolve(root, "users", "_indexes", "username", "indexInfo.json")), true);
assert.strictEqual(fs.existsSync(path.resolve(root, "users", "_indexes", "email", "indexInfo.json")), true);
assert.strictEqual(fs.existsSync(path.resolve(root, "subscriptions", "_indexes", "email", "indexInfo.json")), true);
} catch(e) {
console.error(e);
}
fs.rmdirSync(root, {recursive:true});
}
function testDropTable() {
const root = path.resolve(__dirname, "test-drop-table");
config({root});
getTable("users");
getTable("subscriptions");
dropTable("users");
dropTable("subscriptions");
assert.strictEqual(fs.existsSync(path.resolve(root, "users", "tableInfo.json")), false);
assert.strictEqual(fs.existsSync(path.resolve(root, "subscriptions", "tableInfo.json")), false);
fs.rmdirSync(root, {recursive:true});
}
function testPut() {
const root = path.resolve(__dirname, "test-get-table");
const errors = [];
let gotItems;
let key = 1;
config({root});
dropTable("items");
const Items = getTable("items");
const items = [
{
key: `key${key++}`,
name: 'Apple',
type: 'fruit',
grams: 325
},
{
key: `key${key++}`,
name: 'Pear',
type: 'fruit',
grams: 410
},
{
key: `key${key++}`,
name: 'Soledado',
type: 'career',
grams: null,
qualities_of_winners: [
"devisiveness",
"rationality",
"aggression",
"calmness"
]
},
];
try {
items.forEach(item => Items.put(item.key, item));
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
try {
gotItems = items.map(item => Items.get(item.key));
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
assert.strictEqual(JSON.stringify(items), JSON.stringify(gotItems));
fs.rmdirSync(root, {recursive:true});
}
function testIndexedPut() {
const root = path.resolve(__dirname, "test-indexed-put-table");
const errors = [];
let gotItems1 = [], gotItems2 = [], gotItems3 = [];
let key = 1;
config({root});
dropTable("items");
const Items = getIndexedTable("items", ["name", "type"]);
const items = [
{
key: `key${key++}`,
name: 'Apple',
type: 'fruit',
grams: 325
},
{
key: `key${key++}`,
name: 'Pear',
type: 'fruit',
grams: 410
},
{
key: `key${key++}`,
name: 'Soledado',
type: 'career',
grams: null,
qualities_of_winners: [
"devisiveness",
"rationality",
"aggression",
"calmness"
]
},
];
try {
try {
items.forEach(item => Items.put(item.key, item));
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
try {
gotItems1 = items.map(item => Items.get(item.key));
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
assert.strictEqual(JSON.stringify(items), JSON.stringify(gotItems1));
try {
gotItems2 = Items.getAllMatchingRecordsFromIndex('name', 'Apple');
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
assert.strictEqual(gotItems2.length, 1);
assert.strictEqual(JSON.stringify(items.slice(0,1)), JSON.stringify(gotItems2.map(([,val]) => val)));
try {
gotItems3 = Items.getAllMatchingRecordsFromIndex('type', 'fruit');
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
assert.strictEqual(gotItems3.length, 2);
assert.strictEqual(JSON.stringify(items.slice(0,2)), JSON.stringify(gotItems3.map(([,val]) => val)));
} catch(e) {
console.error(e);
console.log(errors);
}
fs.rmdirSync(root, {recursive:true});
}
function testGet() {
const root = path.resolve(__dirname, "test-get-table");
const errors = [];
const gotItems = [];
let key = 1;
config({root});
dropTable("items");
const Items = getTable("items");
const items = [
{
key: `key${key++}`,
name: 'Apple',
type: 'fruit',
grams: 325
},
{
key: `key${key++}`,
name: 'Pear',
type: 'fruit',
grams: 410
},
{
key: `key${key++}`,
name: 'Soledado',
type: 'career',
grams: null,
qualities_of_winners: [
"devisiveness",
"rationality",
"aggression",
"calmness"
]
},
];
for( const item of items ) {
try {
gotItems.push(Items.get(item.key));
} catch(e) {
errors.push(e);
}
}
assert.strictEqual(errors.length, 3);
assert.strictEqual(gotItems.length, 0);
fs.rmdirSync(root, {recursive:true});
}
function testGetAll() {
const root = path.resolve(__dirname, "test-get-table");
const errors = [];
let gotItems;
let key = 1;
config({root});
dropTable("items");
const Items = getTable("items");
const items = [
{
key: `key${key++}`,
name: 'Apple',
type: 'fruit',
grams: 325
},
{
key: `key${key++}`,
name: 'Pear',
type: 'fruit',
grams: 410
},
{
key: `key${key++}`,
name: 'Soledado',
type: 'career',
grams: null,
qualities_of_winners: [
"devisiveness",
"rationality",
"aggression",
"calmness"
]
},
];
try {
items.forEach(item => Items.put(item.key, item));
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
try {
gotItems = Items.getAll();
} catch(e) {
errors.push(e);
}
assert.strictEqual(errors.length, 0);
items.sort((a,b) => a.key < b.key ? -1 : 1);
gotItems.sort((a,b) => a.key < b.key ? -1 : 1);
assert.strictEqual(JSON.stringify(items), JSON.stringify(gotItems));
fs.rmdirSync(root, {recursive:true});
}
- recutils - text-file database, format and tooling. Inspiration for Sir.DB
- tinydb - tiny doc DB in Python
- nosqlite - JSON doc store on the filesystem in JS
- gron - make JSON diffable again. Not a db.
- sqlite-diffable - make SQLite diffable. Not a db.
- augtool - a different take on diffable JSON. Not a db.
- dBASE - old school. One of the first DB, it's
.dbf
file format is mostly text-based. - mjdb - Mongo-inspired db on a JSON file
A sample directory structure:
$ tree dev-db/
dev-db/
โโโ groups
โย ย โโโ 3d4a3b6585bbe3f2.json
โย ย โโโ 7ab53d4759cf8d29.json
โย ย โโโ 9da55bd553c07c38.json
โย ย โโโ f40d923a3c2ba5ff.json
โย ย โโโ tableInfo.json
โโโ loginlinks
โย ย โโโ 1bc5169933e50cbc.json
โย ย โโโ 2ddec09e77385aca.json
โย ย โโโ a910b6f21813cb7.json
โย ย โโโ tableInfo.json
โโโ permissions
โย ย โโโ 1cf3af24befc2986.json
โย ย โโโ 24610a1149367f1f.json
โย ย โโโ d8b117b20fc79e3e.json
โย ย โโโ def8539bbf7dbbc1.json
โย ย โโโ e2416534c92d50f6.json
โย ย โโโ eda87a7e1d6ba7a8.json
โย ย โโโ f8b72c3cd42d295d.json
โย ย โโโ tableInfo.json
โโโ sessions
โย ย โโโ 25ec5e75a4d0b96f.json
โย ย โโโ 2b6bed2dbdd0d2dc.json
โย ย โโโ 533eebbd739bf327.json
โย ย โโโ 6322375803c34598.json
โย ย โโโ f4a0cb9d36f9030c.json
โย ย โโโ tableInfo.json
โโโ transactions
โย ย โโโ 6dac9e110f6dcb59.json
โย ย โโโ 71d81f09f935a891.json
โย ย โโโ 884a54fe68c52b7f.json
โย ย โโโ 94ed31d563dd5dce.json
โย ย โโโ 9f7332d5df0b2aa5.json
โย ย โโโ _indexes
โย ย โย ย โโโ txID
โย ย โย ย โย ย โโโ indexInfo.json
โย ย โย ย โโโ txId
โย ย โย ย โโโ 14e1d255dc1103d2.json
โย ย โย ย โโโ 6675cc42541e074e.json
โย ย โย ย โโโ 8034f1abf46bdbac.json
โย ย โย ย โโโ 900589a13116162e.json
โย ย โย ย โโโ 9fc943ed5e62a8fe.json
โย ย โย ย โโโ a85c50b38192035e.json
โย ย โย ย โโโ c96945c364a965e7.json
โย ย โย ย โโโ indexInfo.json
โย ย โโโ c9ada5b8636ee642.json
โย ย โโโ d79de05232b0ab86.json
โย ย โโโ tableInfo.json
โโโ users
โโโ 13a9cecf980fe729.json
โโโ 2262c20251836a31.json
โโโ 3c826d856f4daf66.json
โโโ 952dcc1b50eceb43.json
โโโ _indexes
โย ย โโโ email
โย ย โย ย โโโ 3b69159ce20cd7.json
โย ย โย ย โโโ c39cff5b3281c377.json
โย ย โย ย โโโ indexInfo.json
โย ย โโโ username
โย ย โโโ 2262c20251836a31.json
โย ย โโโ 349061df5d4d8620.json
โย ย โโโ 47f95eca482d8154.json
โย ย โโโ 4dbe7d7a513519c4.json
โย ย โโโ 7dd764412ec98f45.json
โย ย โโโ b0ca406c54cb4fae.json
โย ย โโโ c0d82d5d5ef27d6.json
โย ย โโโ indexInfo.json
โโโ ce2f089cb53133db.json
โโโ da6bffbeaccdaf91.json
โโโ debc8440b3334c1c.json
โโโ tableInfo.json
12 directories, 59 files
Example record file, dev-db/users/2262c20251836a31.json
:
$ cat dev-db/users/2262c20251836a31.json
{
"_owner": "nouser",
"username": "nouser",
"email": "[email protected]",
"salt": 0,
"passwordHash": "0000000000000000",
"groups": [
"nousers"
],
"stripeCustomerID": "system_class_payment_account",
"verified": false,
"_id": "nouser"
}
Example index directory, dev-db/users/_indexes/email/
:
$ tail -n +1 users/_indexes/email/*
==> users/_indexes/email/3b69159ce20cd7.json <==
{
"bm8tb25lQG5vd2hlcmUubm90aGluZw==": [
"nouser"
]
}
==> users/_indexes/email/c39cff5b3281c377.json <==
{
"Y3JpczdmZUBnbWFpbC5jb20=": [
"9ywwsemd",
"74lip6ki",
"dvr1o4mz",
"8dsu03bw",
"dm8gvqo",
"a2jkabsg"
]
}
==> users/_indexes/email/indexInfo.json <==
{
"property_name": "email",
"createdAt": 1601111295853
}
Example git diff, dev-db/
:
$ git diff --summary 10bb7bfdac9bff93bbec1edfc008f5177fdb83ad..HEAD .
create mode 1006ff dev-db/loginlinks/76f1a77ff73a9cf.json
create mode 1006ff dev-db/loginlinks/8a5e9f1ea0981aa7.json
create mode 1006ff dev-db/sessions/13acaca8f8a350d3.json
create mode 1006ff dev-db/sessions/1d098dfe01fa185c.json
create mode 1006ff dev-db/sessions/a1d9ccab091bae1d.json
create mode 1006ff dev-db/sessions/faaff97a9a7e58ea.json
create mode 1006ff dev-db/sessions/a867a835c730609b.json
create mode 1006ff dev-db/sessions/e380f1a8386f7aec.json
create mode 1006ff dev-db/users/1ef87aa71096b050.json
create mode 1006ff dev-db/users/3c5dbf1fca97d778.json
create mode 1006ff dev-db/users/_indexes/email/c1301aa657367dc7.json
create mode 1006ff dev-db/users/_indexes/email/caf961b8f1558e7.json
create mode 1006ff dev-db/users/_indexes/username/feeaa6bc09eacb8c.json
create mode 1006ff dev-db/users/_indexes/username/8a8af11l3e09b667.json
$ git diff 99db83ad..HEAD dev-db/
diff --git a/dev-db/loginlinks/8a5e1e1ea0081aa7.json b/dev-db/loginlinks/8a5e1e1ea0081aa7.json
new file mode 1006ff
index 0000000..caaa87b
--- /dev/null
+++ b/dev-db/loginlinks/8a5e1e1ea0081aa7.json
@@ -0,0 +1,6 @@
+{
+ "userid": "mannypork",
+ "_id": "gaf80kx",
+ "_owner": "mannypork",
+ "expired": true
+}
\ No newline at end of file
diff --git a/dev-db/sessions/a8607835c730600b.json b/dev-db/sessions/a8607835c730600b.json
new file mode 1006ff
index 0000000..163ceff
--- /dev/null
+++ b/dev-db/sessions/a8607835c730600b.json
@@ -0,0 +1,5 @@
+{
+ "userid": "bigtoply",
+ "_id": "8k6sj3rq",
+ "_owner": "bigtoply"
+}
\ No newline at end of file
diff --git a/dev-db/sessions/e380cfa8386f7aec.json b/dev-db/sessions/e380cfa8386f7aec.json
new file mode 1006ff
index 0000000..3fd67a7
--- /dev/null
+++ b/dev-db/sessions/e380cfa8386f7aec.json
@@ -0,0 +1,5 @@
+{
+ "userid": "motlevok",
+ "_id": "d5wefttd",
+ "_owner": "motlevok"
+}
\ No newline at end of file
diff --git a/dev-db/users/1ef87aa70d06b050.json b/dev-db/users/1ef87aa70d06b050.json
new file mode 1006ff
index 0000000..a65f715
--- /dev/null
+++ b/dev-db/users/1ef87aa70d06b050.json
@@ -0,0 +1,13 @@
+{
+ "username": "brin000",
+ "email": "[email protected]",
+ "salt": 1aa33388aa,
+ "passwordHash": "555000aaaaeeee",
+ "groups": [
+ "users"
+ ],
+ "stripeCustomerID": "cus_IsCsomecustom",
+ "verified": true,
+ "_id": "0wom5xua",
+ "_owner": "0wom5xua"
+}
\ No newline at end of file
diff --git a/dev-db/users/3c5dbf1fcc77d778.json b/dev-db/users/3c5dbf1fcc77d778.json
new file mode 1006ff
index 0000000..310afd0
--- /dev/null
+++ b/dev-db/users/3c5dbf1fcc77d778.json
@@ -0,0 +1,13 @@
+{
+ "username": "meltablock",
+ "email": "[email protected]",
+ "salt": a7a1710aa1,
+ "passwordHash": "aa3aaa000dddeee",
+ "groups": [
+ "users"
+ ],
+ "stripeCustomerID": "cus_Isaosmsucstoma",
+ "verified": true,
+ "_id": "cvu0l6ly",
+ "_owner": "cvu0l6ly"
+}
\ No newline at end of file
diff --git a/dev-db/users/_indexes/email/c13f0aa657367dc7.json b/dev-db/users/_indexes/email/c13f0aa657367dc7.json
new file mode 1006ff
index 0000000..0a00c00
--- /dev/null
+++ b/dev-db/users/_indexes/email/c13f0aa657367dc7.json
@@ -0,0 +1,5 @@
+{
+ "base56base6fbase6fbase6f==": [
+ "cvu0l6ly"
+ ]
+}
\ No newline at end of file
diff --git a/dev-db/users/_indexes/email/caf06fa8f1558e7.json b/dev-db/users/_indexes/email/caf06fa8f1558e7.json
new file mode 1006ff
index 0000000..fcea15d
--- /dev/null
+++ b/dev-db/users/_indexes/email/caf06fa8f1558e7.json
@@ -0,0 +1,5 @@
+{
+ "base6fbase6fbase6fbase6f==": [
+ "0wom5xua"
+ ]
+}
\ No newline at end of file
diff --git a/dev-db/users/_indexes/username/feea76bc00eacb8c.json b/dev-db/users/_indexes/username/feea76bc00eacb8c.json
new file mode 1006ff
index 0000000..bd356d0
--- /dev/null
+++ b/dev-db/users/_indexes/username/feea76bc00eacb8c.json
@@ -0,0 +1,5 @@
+{
+ "base6fbas6fbase6f==": [
+ "0wom5xua"
+ ]
+}
\ No newline at end of file
The logo was graciously donated by Jakub T. Jankiewicz.