3/3 clients segfaulted when attempted to connect about aerospike-client-c

[BrianNichols]

inet_ntop() is getting called with a valid socket address "&a->sin_addr" (\300\250\001\001=192.168.1.1 or \300\250\001\002==192.168.1.2), so there must be some stack memory corruption that caused this. Were the clients running close to memory limits just before the crash?

Are the 3 clients running on the same machine?

Do you have a program that can reliably reproduce this crash?

[vytas-dauksa]

Were the clients running close to memory limits just before the crash?

Can't find any traces of it, each box got 12GB of ram.

Are the 3 clients running on the same machine?

No, they are running on 3 different boxes. Each box has Aerospike server running + a client for writing + a client for querying.

Do you have a program that can reliably reproduce this crash?

All Aerospike servers are now running v3.11.0.1 and all reading clients crashed again ( in the same sequence ):
Node 2 --> 2017-01-18 05:59:59
Node 1 --> 2017-01-18 06:00:07
Node 3 --> 2017-01-18 06:01:20

ran with query parameter ( ./scan_query query )

/*
 * Compile:
 * cc -std=gnu99 -g -Wall -fPIC -O3 -fno-common -fno-strict-aliasing -march=nocona -DMARCH_x86_64 -D_FILE_OFFSET_BITS=64 -D_REENTRANT -D_GNU_SOURCE -DAS_USE_LIBEV -I/usr/local/include -rdynamic -o scan_query scan_query.c -L/usr/local/lib -laerospike -lev -lssl -lcrypto -lpthread -lm -lz -lrt -ldl
 */

#include <aerospike/aerospike.h>
#include <aerospike/aerospike_key.h>
#include <aerospike/aerospike_scan.h>
#include <aerospike/as_monitor.h>
#include <aerospike/as_query.h>
#include <aerospike/aerospike_query.h>

#include <time.h>
#include <string.h>

/******************************************************************************
 *	Types
 *****************************************************************************/

typedef struct {
	int count;
} scan_counter;

/******************************************************************************
 *	Globals
 *****************************************************************************/

static const char* g_host = "127.0.0.1";
static int g_port = 3000;
static const char* g_namespace = "test";
static const char* g_set = "test";
static const char* g_bin = "auth";
static const char* g_index_bin = "authindex";

static aerospike as;
static as_monitor scan_monitor;

struct timespec tstart={0,0}, tend={0,0};
static const char* mode;

/******************************************************************************
 *	Forward Declarations
 *****************************************************************************/

static bool scan_query_listener(as_error* err, as_record* rec, void* udata, as_event_loop* event_loop);

/******************************************************************************
 *	Functions
 *****************************************************************************/

int main(int argc, char* argv[])
{
	mode = argv[1];
	if (! mode || ((strcmp(mode, "scan") != 0) && (strcmp(mode, "query") != 0))) {
		printf("Please supply mode ( 'scan' or 'query' )\n");
		return -1;
	}

	printf("Host=%s:%d\n", g_host, g_port);
	printf("Namespace=%s\n", g_namespace);
	printf("Set=%s\n", g_set);
	printf("Bin=%s\n", g_bin);
	printf("Index bin=%s\n", g_index_bin);
	printf("Mode=%s\n", mode);

	while( 1 ) {
		// Initialise
		as_config cfg;
		as_config_init(&cfg);
		as_config_add_host(&cfg, g_host, g_port);
		cfg.async_max_conns_per_node = 200;
		aerospike_init(&as, &cfg);

		// Create loop
		if (! as_event_create_loops(1)) {
			printf("Failed to create event loop\n");
			continue;
		}

		// Connect to cluster.
		as_error err;
		if (aerospike_connect(&as, &err) != AEROSPIKE_OK) {
			printf("Failed to connect to cluster\n");
			aerospike_destroy(&as);
			as_event_close_loops();
			continue;
		}

		as_monitor_init(&scan_monitor);

		clock_gettime(CLOCK_MONOTONIC, &tstart);

		scan_counter counter = {
			.count = 0,
		};

		if (strcmp(mode, "query") == 0) {
			as_query query;
			as_query_init(&query, g_namespace, g_set);

			// Generate an as_query.where condition. Note that as_query_destroy() takes
			// care of destroying all the query's member objects if necessary. However
			// using as_query_where_inita() does avoid internal heap usage.
			as_query_where_inita(&query, 1);
			as_query_where(&query, g_index_bin, as_integer_equals(1));

			// Execute the query.
			if (aerospike_query_async(&as, &err, NULL, &query, scan_query_listener, &counter, 0) != AEROSPIKE_OK) {
				as_event_loop* event_loop = as_event_loop_get();
				scan_query_listener(&err, NULL, NULL, event_loop);
			}

			as_query_destroy(&query);
		}

		if (strcmp(mode, "scan") == 0) {
			as_scan scan;
			as_scan_init(&scan, g_namespace, g_set);

			as_policy_scan local;
			as_policy_scan_init(&local);
			local.fail_on_cluster_change = true;

			as_scan_select_inita(&scan, 1);
			as_scan_select(&scan, g_bin);

			as_monitor_begin(&scan_monitor);

			if (aerospike_scan_async(&as, &err, &local, &scan, 0, scan_query_listener, &counter, 0) != AEROSPIKE_OK) {
				as_event_loop* event_loop = as_event_loop_get();
				scan_query_listener(&err, NULL, NULL, event_loop);
			}

			as_scan_destroy(&scan);
		}

		as_monitor_wait(&scan_monitor);

		// cleanup
		as_monitor_destroy(&scan_monitor);
		aerospike_close(&as, &err);
		aerospike_destroy(&as);

		as_event_close_loops();
	}
}

static bool scan_query_listener(as_error* err, as_record* rec, void* udata, as_event_loop* event_loop) {
	scan_counter* counter = (scan_counter*)udata;

	if (err) {
		printf("aerospike_%s_async() returned %d - %s\n", mode, err->code, err->message);
		as_monitor_notify(&scan_monitor);
		ev_sleep(0.2);
		return false;
	}

	if (! rec) {
		clock_gettime(CLOCK_MONOTONIC, &tend);

		float diff = ((double)tend.tv_sec + 1.0e-9*tend.tv_nsec) -
			((double)tstart.tv_sec + 1.0e-9*tstart.tv_nsec);

		// Scan has ended.
		printf("%s %u rec in %.5f sec\n", mode, counter->count,
				diff
				);

		if ( diff < 1 ) {
			// sleep for the remainder of the second
			ev_sleep( 1 - diff );
		} else {
//			printf( "===== Took longer than second -> %.5f\n", diff );
		}
		as_monitor_notify(&scan_monitor);

		return false;
	}

	counter->count++;

	return true;
}

[BrianNichols]

as_query_where_inita() uses alloca() which puts new variables on the stack on every iteration. These stack variable are not reclaimed until the function ends. Since the function runs in an infinite loop, your stack limits are exceeded and the program crashes.

If you put the query block in a separate function, as_query_where_inita() would work fine.

[vytas-dauksa]

I will reopen this issue if I find anything interesting once I get to it. Though, I wonder why this was not the case for as_scan_select_inita().

Thanks!

[BrianNichols]

as_scan_select_inita() allocates less stack memory than as_query_where_inita(), so it would take longer to fail.