Coder Social home page Coder Social logo

nodejs-questdb-client's Introduction

QuestDB Node.js Client

Requirements

The client requires Node.js v16 or newer version.

Installation

npm i -s @questdb/nodejs-client

Configuration options

Detailed description of the client's configuration options can be found in the SenderOptions documentation.

Examples

The examples below demonstrate how to use the client.
For more details, please, check the Sender's documentation.

Basic API usage

const { Sender } = require('@questdb/nodejs-client');

async function run() {
    // create a sender using HTTP protocol
    const sender = Sender.fromConfig('http::addr=localhost:9000');

    // add rows to the buffer of the sender
    await sender.table('prices').symbol('instrument', 'EURUSD')
        .floatColumn('bid', 1.0195).floatColumn('ask', 1.0221)
        .at(Date.now(), 'ms');
    await sender.table('prices').symbol('instrument', 'GBPUSD')
        .floatColumn('bid', 1.2076).floatColumn('ask', 1.2082)
        .at(Date.now(), 'ms');

    // flush the buffer of the sender, sending the data to QuestDB
    // the buffer is cleared after the data is sent, and the sender is ready to accept new data
    await sender.flush();

    // add rows to the buffer again, and send it to the server
    await sender.table('prices').symbol('instrument', 'EURUSD')
        .floatColumn('bid', 1.0197).floatColumn('ask', 1.0224)
        .at(Date.now(), 'ms');
    await sender.flush();

    // close the connection after all rows ingested
    await sender.close();
}

run()
    .then(console.log)
    .catch(console.error);

Authentication and secure connection

const { Sender } = require('@questdb/nodejs-client');

async function run() {
    // create a sender using HTTPS protocol with username and password authentication
    const sender = Sender.fromConfig('https::addr=localhost:9000;username=user1;password=pwd');

    // send the data over the authenticated and secure connection
    await sender.table('prices').symbol('instrument', 'EURUSD')
        .floatColumn('bid', 1.0197).floatColumn('ask', 1.0224)
        .at(Date.now(), 'ms');
    await sender.flush();

    // close the connection after all rows ingested
    await sender.close();
}

run().catch(console.error);

TypeScript example

import { Sender } from '@questdb/nodejs-client';

async function run(): Promise<number> {
    // create a sender using HTTPS protocol with bearer token authentication
    const sender: Sender = Sender.fromConfig('https::addr=localhost:9000;token=Xyvd3er6GF87ysaHk');

    // send the data over the authenticated and secure connection
    sender.table('prices').symbol('instrument', 'EURUSD')
        .floatColumn('bid', 1.0197).floatColumn('ask', 1.0224).at(Date.now(), 'ms');
    await sender.flush();

    // close the connection after all rows ingested
    await sender.close();
}

run().catch(console.error);

Worker threads example

const { Sender } = require('@questdb/nodejs-client');
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');

// fake venue
// generates random prices for a ticker for max 5 seconds, then the feed closes
function* venue(ticker) {
    let end = false;
    setTimeout(() => { end = true; }, rndInt(5000));
    while (!end) {
        yield {'ticker': ticker, 'price': Math.random()};
    }
}

// market data feed simulator
// uses the fake venue to deliver price updates to the feed handler (onTick() callback)
async function subscribe(ticker, onTick) {
    const feed = venue(workerData.ticker);
    let tick;
    while (tick = feed.next().value) {
        await onTick(tick);
        await sleep(rndInt(30));
    }
}

async function run() {
    if (isMainThread) {
        const tickers = ['t1', 't2', 't3', 't4'];
        // main thread to start a worker thread for each ticker
        for (let ticker in tickers) {
            const worker = new Worker(__filename, { workerData: { ticker: ticker } })
                .on('error', (err) => { throw err; })
                .on('exit', () => { console.log(`${ticker} thread exiting...`); })
                .on('message', (msg) => {
                    console.log(`Ingested ${msg.count} prices for ticker ${msg.ticker}`);
                });
        }
    } else {
        // it is important that each worker has a dedicated sender object
        // threads cannot share the sender because they would write into the same buffer
        const sender = Sender.fromConfig('http::addr=localhost:9000');

        // subscribe for the market data of the ticker assigned to the worker
        // ingest each price update into the database using the sender
        let count = 0;
        await subscribe(workerData.ticker, async (tick) => {
            await sender
                .table('prices')
                .symbol('ticker', tick.ticker)
                .floatColumn('price', tick.price)
                .at(Date.now(), 'ms');
            await sender.flush();
            count++;
        });

        // let the main thread know how many prices were ingested
        parentPort.postMessage({'ticker': workerData.ticker, 'count': count});

        // close the connection to the database
        await sender.close();
    }
}

function sleep(ms) {
    return new Promise(resolve => setTimeout(resolve, ms));
}

function rndInt(limit) {
    return Math.floor((Math.random() * limit) + 1);
}

run()
    .then(console.log)
    .catch(console.error);

nodejs-questdb-client's People

Contributors

amunra avatar argshook avatar eugenels avatar glasstiger avatar jerrinot avatar juanarbol avatar puzpuzpuz avatar sklarsa avatar

Stargazers

 avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar  avatar

Watchers

 avatar  avatar

nodejs-questdb-client's Issues

Missing method to insert binary data

Hi,
in the QuestDB spec there's a column type "binary".
We want to use this column type to store base64 encoded data.
Unfortunately, there's no "binaryColumn" method in sender.js.
Is this planned or is there any other way to insert binary data using this library?
Thanks

Messages in console always visible

I want to use this module, but since it uses console.info to show some messages, it is a little bit slow due to the sync code with console.
The debug/info messages should be optional and defined as part of the options when the user create the sender

geohash support

would be great to have the geohash support, I am willing to contribute but I would need some guidence

Specify minimal supported Node.js version or support older ones like 14

Scenario:

  • Create a QuestDB Cloud instance
  • Run generated ILP ingestion script for Node.js with Node.js 14, e.g. v14.17.6
  • Observe the following error:
$ node insert-client.js 
Successfully connected to <host>:<port>
Authenticating with <host>:<port>
(node:392696) UnhandledPromiseRejectionWarning: TypeError [ERR_INVALID_ARG_TYPE]: The "key.key" property must be of type string or an instance of Buffer, TypedArray, or DataView. Received an instance of Object
    at prepareAsymmetricKey (internal/crypto/keys.js:288:13)
    at Object.createPrivateKey (internal/crypto/keys.js:349:5)
    at authenticate (/home/puzpuzpuz/projects/node-experiments/node_modules/@questdb/nodejs-client/src/sender.js:387:40)
    at TLSSocket.<anonymous> (/home/puzpuzpuz/projects/node-experiments/node_modules/@questdb/nodejs-client/src/sender.js:133:43)
    at TLSSocket.emit (events.js:400:28)
    at addChunk (internal/streams/readable.js:290:12)
    at readableAddChunk (internal/streams/readable.js:265:9)
    at TLSSocket.Readable.push (internal/streams/readable.js:204:10)
    at TLSWrap.onStreamRead (internal/stream_base_commons.js:188:23)
(Use `node --trace-warnings ...` to show where the warning was created)
(node:392696) UnhandledPromiseRejectionWarning: Unhandled promise rejection. This error originated either by throwing inside of an async function without a catch block, or by rejecting a promise which was not handled with .catch(). To terminate the node process on unhandled promise rejection, use the CLI flag `--unhandled-rejections=strict` (see https://nodejs.org/api/cli.html#cli_unhandled_rejections_mode). (rejection id: 2)
(node:392696) [DEP0018] DeprecationWarning: Unhandled promise rejections are deprecated. In the future, promise rejections that are not handled will terminate the Node.js process with a non-zero exit code.

The reason is that crypto.createPrivateKey(key) supports JWK object starting from v15.12.0: https://nodejs.org/api/crypto.html#cryptocreateprivatekeykey

Reduce the number of necessary parameters when using ILP auth

For connecting via ILP auth, only the key_id and the private_key/d parameter should be needed. However, this client also requests the public key pair, which is annoying as those are not really needed, and the developer need to treat them as secrets.

Some official clients like Go, JAVA, or C# don't need the extra parameters.

Once changed, the documentation should be updated to reflect the simplified connection params.

How to query?

I see there is sender implementation but where is reading?

Recommend Projects

  • React photo React

    A declarative, efficient, and flexible JavaScript library for building user interfaces.

  • Vue.js photo Vue.js

    ๐Ÿ–– Vue.js is a progressive, incrementally-adoptable JavaScript framework for building UI on the web.

  • Typescript photo Typescript

    TypeScript is a superset of JavaScript that compiles to clean JavaScript output.

  • TensorFlow photo TensorFlow

    An Open Source Machine Learning Framework for Everyone

  • Django photo Django

    The Web framework for perfectionists with deadlines.

  • D3 photo D3

    Bring data to life with SVG, Canvas and HTML. ๐Ÿ“Š๐Ÿ“ˆ๐ŸŽ‰

Recommend Topics

  • javascript

    JavaScript (JS) is a lightweight interpreted programming language with first-class functions.

  • web

    Some thing interesting about web. New door for the world.

  • server

    A server is a program made to process requests and deliver data to clients.

  • Machine learning

    Machine learning is a way of modeling and interpreting data that allows a piece of software to respond intelligently.

  • Game

    Some thing interesting about game, make everyone happy.

Recommend Org

  • Facebook photo Facebook

    We are working to build community through open source technology. NB: members must have two-factor auth.

  • Microsoft photo Microsoft

    Open source projects and samples from Microsoft.

  • Google photo Google

    Google โค๏ธ Open Source for everyone.

  • D3 photo D3

    Data-Driven Documents codes.