-   Programming (
-   -   runtime loadable modules (

bigearsbilly 02-16-2012 05:33 PM

runtime loadable modules
I want to be able to load functions into a process while it is running, like kernel modules can.

I have a lot of real-time financial data coming in 24x7 from the outside world. These are saved to disk in many files.

I have a service reading the data files as they are
writing to disk with the requirement to format the data so they are graphable into new files, keeping the originals.

What I would like is to have run-time loadable filter modules, like kernel modules or shared libraries so that
I can apply different filters without stopping and recompiling. C/C++ code.

Any ideas how this can be done?

bigearsbilly 02-16-2012 06:27 PM


sundialsvcs 02-16-2012 06:30 PM

Dynamic loading and unloading of modules in applications occurs all the time. Search for, for instance, the term dyload.

Many modules, such as the ubiquitous glibc, are automatically loaded (that is to say, bound to the application ...) when any program that uses them is started, but programs can request and load or unload any other library at any time, particularly for situations such as the ones that you describe.

Nominal Animal 02-16-2012 07:28 PM

For C, see man 3 dlopen, man 3 dlsym and man 3 dlclose.

Designing good plugin API is more or less an art. You certainly want to implement only one filter per library. You probably should have a configuration or initialization function, and a matching release function, just in case the filter wants to set up some tables or something. Plus of course the filtering functions themselves.

If the filtering functions are fully independent, all the functions should have the same name in all filters. Otherwise you need to derive the function and symbol names somehow, e.g. based on the library name -- instead of just version, you'd look for prefix_version.

Whenever your application receives a SIGHUP signal, it should close all filters, and reload them. You can just scan a filter directory, for example. Each filter should obviously have some kind of an identifier. Then, you can build a linked list or a binary tree based on the identifiers, containing the function pointers. Essentially, you'll create a list or tree of "filter objects", each containing variable and function pointers you find out using dlsym().

For example, assume this is an example dynamic filter, which parses decimal integers:

/* Save as filter.c, then compile using
 *      gcc filter.c -o -shared -Wall
#include <stdlib.h>
#include <errno.h>

const int    version      = 1;
const char  identifier[] = "foo";

int initialize(void)
    return 0;

void finalize(void)

int parse(const char *string, void *valueptr)
    int value, negative;

    if (!string)
        return EINVAL;

    while (*string > 0 && *string <= 32)

    negative = 0;
    while (*string == '+' || *string == '-')
        if (*(string++) == '-')
            negative = !negative;

    if (*string >= '0' && *string <= '9')
        value = *(string++) - '0';
        return ENOENT;

    while (*string >= '0' && *string <= '9') {
        const int old = value;
        value = 10*value + (int)(*(string++) - '0');
        if (value < old)
            return EDOM;

    if (*string < 0 || *string > 32)
        return EDOM;

    if (negative)
        value = -value;

    if (valueptr)
        *((int *)valueptr) = value;

    return 0;

To load this filter dynamically, you could use

/* Save as loader.c, then compile using
 *      gcc loader.c -o loader -Wall -ldl
 * To test, run
 *      ./loader ./
 * (You need to specify the path to the library for dlopen(),
 *  or it will look for the library only in the standard locations.)
#include <stdlib.h>
#include <dlfcn.h>
#include <errno.h>
#include <string.h>
#include <stdio.h>

const char *const test[] = {
    " 1",
    " --43",

/* Acceptable version numbers */
#define MIN_VERSION  1
#define MAX_VERSION  255

struct filter {
    struct filter  *next;    /* Singly-linked list */
    void          *library;  /* dlopen()/dlsym()/dlclose() handle */
    int            version;
    const char    *filename;
    const char    *identifier;
    int          (*initialize)(void);
    void          (*finalize)(void);
    int          (*parse)(const char *, void *);

/* Singly-linked list of known filters. */
struct filter *filters = NULL;

const char *load_filter(const char *const filename)
    struct filter *current;
    void          *library, *ptr;
    int            result;

    if (!filename || !*filename)
        return strerror(EINVAL);

    errno = 0;
    library = dlopen(filename, RTLD_LAZY | RTLD_LOCAL);
    if (!library)
        return dlerror();

    /* Locate the version symbol. */
    ptr = dlsym(library, "version");
    if (!ptr) {
        return strerror(errno);
    if (*((int *)ptr) < MIN_VERSION || *((int *)ptr) > MAX_VERSION) {
        return strerror(ENOENT);

    /* Allocate a filter structure. */
    current = malloc(sizeof (struct filter));
    if (!current) {
        return strerror(ENOMEM);

    current->library = library;
    current->version = *((int *)ptr);
    current->filename = filename; /* This should really be strdup(filename) */

    ptr = dlsym(library, "identifier");
    if (ptr)
        current->identifier = (const char *)ptr;
        current->identifier = NULL;

    ptr = dlsym(library, "initialize");
    if (ptr)
        *(void **)&(current->initialize) = ptr;
        current->initialize = NULL; /* Or a stub function */

    ptr = dlsym(library, "finalize");
    if (ptr)
        *(void **)&(current->finalize) = ptr;
        current->finalize = NULL; /* Or a stub function */

    ptr = dlsym(library, "parse");
    if (ptr)
        *(void **)&(current->parse) = ptr;
        current->parse = NULL; /* Or a stub function */

    /* Cancel if missing symbols. */
    if (!current->identifier ||
        !current->initialize ||
        !current->finalize ||
        !current->parse) {
        return strerror(ENOENT);

    /* Initialize filter. */
    result = current->initialize();
    if (result) {
        return strerror(result);

    /* Note: dlclose(current->library) is called
    *      when/if this filter is unloaded;
    *      current->finalize() must be called first.

    /* Add to filter chain. */
    current->next = filters;
    filters      = current;

    return NULL;

int main(int argc, char *argv[])
    int                arg, result, value;
    const char        *failure;
    const char *const *proto;
    struct filter    *curr;

    for (arg = 1; arg < argc; arg++) {

        failure = load_filter(argv[arg]);
        if (failure)
            fprintf(stderr, "%s: Could not load filter: %s.\n", argv[arg], failure);
            fprintf(stderr, "%s: Filter loaded successfully.\n", argv[arg]);

    if (!filters) {
        fprintf(stderr, "Please supply the path(s) to dynamic filter(s).\n");
        return 1;

    curr = filters;
    while (curr) {
        printf("Filter %s:\n", curr->filename);
        printf("\tVersion: %d\n", curr->version);
        if (curr->identifier)
            printf("\tIdentifier: '%s'\n", curr->identifier);
        printf("\tinitialize(): %p\n", curr->initialize);
        printf("\tfinalize(): %p\n", curr->finalize);
        printf("\tparse(): %p\n", curr->parse);

        for (proto = test; *proto; proto++) {
            result = curr->parse(*proto, &value);
            if (result)
                printf("\tparse(\"%s\", &value) == %d\n", *proto, result);
                printf("\tparse(\"%s\", &value) == %d, value == %d\n", *proto, result, value);

        curr = curr->next;

    return 0;

Since you handle complex data formats, you should probably create a header file defining all the structures the filters can exchange data in -- basically, the filter API. You do not need to worry about endianness et cetera, because this is local to the process.

I'm pretty sure I could make better suggestions about what kind of structures to use, if you can describe the data and the operations in detail.

All times are GMT -5. The time now is 06:50 AM.