LinuxQuestions.org
Download your favorite Linux distribution at LQ ISO.
Go Back   LinuxQuestions.org > Forums > Linux Forums > Linux - Newbie
User Name
Password
Linux - Newbie This Linux forum is for members that are new to Linux.
Just starting out and have a question? If it is not in the man pages or the how-to's this is the place!

Notices


Reply
  Search this Thread
Old 04-15-2015, 06:22 AM   #1
lalitaddiwar
LQ Newbie
 
Registered: Apr 2015
Posts: 7

Rep: Reputation: Disabled
Help to disperse the data using bitsplitter


I am totally new to Linux. I am working on Nubisave project in TU Dresden. I am stuck in running the C program. Its bitsplitter program. Can anybody help me out? should I post the program here? Will it be helpful for me? I knnow little basics of C language and the code is totally new for me.
I have used the code of bitsplitter and it is running but I am not understanding the output. This is code from Tu Dresden. Can anybody explain me how it is working?

#include "bitsplit.h"

#include <stdio.h>
#include <sys/stat.h>
#include <sys/types.h>
#include <sys/errno.h>
#include <stdlib.h>
#include <string.h>

#define SPLITDEBUG 0

struct fillbyte_t {
char *bits;
int bitctr;
char bytes[1024];
int bytectr;
unsigned char numbits;
unsigned char shiftbits;
unsigned int pattern;
FILE *f;
char *buffer;
ssize_t bufferctr;
};
typedef struct fillbyte_t fillbyte;

struct bitsplit_t {
fillbyte *b;
int parts;
int redundant;
int totalbits;
};
typedef struct bitsplit_t bitsplit;

static void bitsplitfinish(bitsplit bs)
{
int j;

for(j = 0; j < (bs.parts + bs.redundant); j++)
{
if(bs.b[j].f)
fclose(bs.b[j].f);
free(bs.b[j].bits);
/*free(bs.b[j].buffer);*/ /* the caller clears this */
}
free(bs.b);
bs.b = NULL;
}

static bitsplit bitsplitprepare(const char *directory, int parts, int redundant, int bitwidth)
{
fillbyte *b;
bitsplit bs;
int ret;
char fname[1024];
int sumbits, totalbits;
int i, j;

bs.b = NULL;

if(directory)
{
ret = mkdir(directory, S_IRWXU);
if((ret != 0) && (errno != EEXIST)) return bs;
}

b = malloc((parts + redundant) * sizeof(fillbyte));
for(j = 0; j < (parts + redundant); j++)
{
b[j].bitctr = 0;
b[j].bytectr = 0;
b[j].buffer = NULL;
b[j].bufferctr = 0;
if(directory)
{
snprintf(fname, sizeof(fname), "%s/%i.stream", directory, j);
b[j].f = fopen(fname, "w");
}
else b[j].f = NULL;
}

sumbits = 0;
totalbits = (int)(((parts - 1) / bitwidth) + 1) * bitwidth;
#if SPLITDEBUG
printf("bitwidth: %i; totalbits: %i\n", bitwidth, totalbits);
#endif

if(redundant)
if(totalbits % parts != 0)
{
bitsplitfinish(bs);
return bs;
}

for(j = parts - 1; j >= 0; j--)
{
b[j].bits = (char*)malloc(totalbits * 8 * sizeof(char));

b[j].numbits = (int)((totalbits - sumbits) / (j + 1));
b[j].pattern = 0;
for(i = sumbits; i < sumbits + b[j].numbits; i++)
b[j].pattern |= (1 << i);
b[j].shiftbits = sumbits;
sumbits += b[j].numbits;
#if SPLITDEBUG
printf("bits: fragment %i=%i, pattern=%u\n", j, b[j].numbits, b[j].pattern);
#endif
}
for(j = parts; j < parts + redundant; j++)
{
b[j].bits = (char*)malloc(totalbits * 8 * sizeof(char));
}

bs.b = b;
bs.parts = parts;
bs.redundant = redundant;
bs.totalbits = totalbits;

return bs;
}

static int bitjoinwork(const char **inbuffers, ssize_t *insizes, bitsplit bs)
{
ssize_t i;
int j, k;
int target, revtarget;
int topk, topkcandidate;
int outbits;
int shifted;
int smallest;
int rels[16];
int inshift;

outbits = 8;
shifted = 0;

int xspecial = 0;

xspecial = 1;
if(bs.totalbits > 2 * bs.parts)
xspecial = 0;
if(xspecial)
for(j = 0; j < bs.parts; j++)
if(bs.b[j].numbits > 8) xspecial = 0;
if((bs.parts == 2) && (bs.totalbits == 11))
xspecial = 1;

if((!xspecial) && (bs.totalbits > 8))
outbits = 16;

topk = 0;
for(j = 0; j < bs.parts; j++)
{
topkcandidate = (int)((float)outbits / bs.b[j].numbits) - 1;
if(topkcandidate > topk)
topk = topkcandidate;
}
for(j = 0; j < bs.parts; j++)
{
rels[j] = (int)((topk + 1) / (outbits / bs.b[j].numbits));
// FIXME: mostly untested: =2, *=2 etc. yields 89 OKs
if((!xspecial) && (bs.totalbits > 8))
rels[j] = 2;
if(bs.b[j].numbits == 4)
{
if((bs.parts == 2) && (bs.totalbits == 9))
{
rels[j] = 1;
topk = 1;
}
}
#if SPLITDEBUG
printf("REL/%i=%i <<%i>>\n", j, rels[j], outbits);
#endif
}

smallest = (int)(insizes[0] / rels[0]);
if(((int)(float)insizes[0] * 2 / rels[0]) != smallest * 2)
smallest++;

inshift = (outbits - (topk + 1) * bs.b[0].numbits) % outbits;

int special = 0;
if((bs.parts == 2) && (bs.totalbits > 8))
special = 1;
int special2 = 0;
if((bs.parts == 2) && (bs.totalbits == 5))
special2 = 1;
if((bs.parts == 2) && (bs.totalbits == 7))
special2 = 1;
if((bs.parts == 3) && (bs.totalbits == 7))
special2 = 1;
int special3 = 0;
if((bs.parts == 5) && (bs.totalbits == 15))
special3 = 1;
if((bs.parts == 4) && (bs.totalbits == 12))
special3 = 1;
if((bs.parts == 3) && (bs.totalbits == 9))
special3 = 1;
int special4 = 0;
if((bs.parts == 3) && (bs.totalbits == 15))
special4 = 1;

target = 0;
for(i = 0; i < smallest; i++)
{
#if SPLITDEBUG
printf("- reverse source-pos:%li\n", i);
#endif
// FIXME: special; not for w=14||12, only for w=10!
if((special) && (bs.totalbits == 10) && (shifted >= 10))
{
shifted -= bs.totalbits;
if(shifted < 0)
shifted = 0;
}
#if SPLITDEBUG
printf(" remaining target:%i\n", target);
#endif

for(k = topk; k >= 0; k--)
{
for(j = bs.parts - 1; j >= 0; j--)
{
int packval = (int)(unsigned char)inbuffers[j][i * rels[j]];
// FIXME: read-ahead depends on the relative difference between numbits?
if(rels[j] == 2)
packval = (packval << 8) + (int)(unsigned char)inbuffers[j][i * rels[j] + 1];
// FIXME: special
if((!special) && (!special2) && (!special3) && (!special4))
packval >>= inshift;

int packshift = k * bs.b[j].numbits;
// FIXME: special; only for w=14||12, not for w=10!
if(special)
{
if(bs.b[j].numbits != 4)
packshift = k * 8 - bs.b[j].numbits;
if(bs.totalbits != 10)
packshift += 8;

// for 5+4 bits
if((packshift == 20) && (bs.b[j].numbits == 4))
packshift = 12;
else if((packshift == 16) && (bs.b[j].numbits == 4))
packshift = 8;
else if((packshift == 12) && (bs.b[j].numbits == 4))
packshift = 4;
else if((packshift == 8) && (bs.b[j].numbits == 4))
packshift = 0;
}
if(special2)
{
if((packshift == 9) && (bs.b[j].numbits == 3))
packshift = 13;
else if((packshift == 6) && (bs.b[j].numbits == 3))
packshift = 10;
else if((packshift == 3) && (bs.b[j].numbits == 3))
packshift = 5;
else if((packshift == 0) && (bs.b[j].numbits == 3))
packshift = 2;
}
if(special3)
{
if(packshift == 12)
packshift = 13;
else if(packshift == 9)
packshift = 10;
else if(packshift == 6)
packshift = 5;
else if(packshift == 3)
packshift = 2;
else if(packshift == 0)
continue;
}
if(special4)
{
if(packshift == 10)
packshift = 11;
else if(packshift == 5)
packshift = 3;
else if(packshift == 0)
continue;
}
#if SPLITDEBUG
printf("PACKSHIFT:%i (on val %i) [filtered:%i] [shifted:%i]\n", packshift, packval, (bs.b[j].pattern >> bs.b[j].shiftbits), shifted);
#endif
revtarget = (packval >> packshift) & (bs.b[j].pattern >> bs.b[j].shiftbits);
#if SPLITDEBUG
printf(" -> reverse target @%i[pos:%i]:%u\n", j, k, revtarget);
#endif
target += revtarget << shifted;
shifted += bs.b[j].numbits;
#if SPLITDEBUG
printf(" => target: %i (shifted:%i)\n", target, shifted);
#endif
if(shifted >= outbits)
{
while(shifted >= 8)
{
// FIXME: special; only for w>=10?
if((special) && (shifted == bs.totalbits))
break;
#if SPLITDEBUG
printf("@%li: %i [extra]\n", bs.b[0].bufferctr, target & 0xFF);
#endif
if(bs.b[0].bufferctr % 1024 == 0)
bs.b[0].buffer = (char*)realloc(bs.b[0].buffer, (bs.b[0].bufferctr + 1024) * sizeof(char));
bs.b[0].buffer[bs.b[0].bufferctr] = target & 0xFF;
shifted -= 8;
bs.b[0].bufferctr++;
target = (target - (target & 0xFF)) >> 8;
}
}
}
}
}

return 0;
}

static int bitsplitwork(const char *inbuffer, ssize_t insize, bitsplit bs)
{
ssize_t i;
int j, k, l;
int target, shifted, inshifted;
int precounted;

fillbyte *b;
int totalbits;
int parts;
int redundant;

int outbits;
int stuffingthreshold;

b = bs.b;
totalbits = bs.totalbits;
parts = bs.parts;
redundant = bs.redundant;

outbits = 8;
inshifted = 0;
stuffingthreshold = totalbits * 8;

for(i = 0; i < insize; i++)
{
if(inshifted > 8)
{
inshifted -= 8;
i += 1;
}
#if SPLITDEBUG
printf("- source-pos:%li\n", i);
#endif
for(j = 0; j < parts; j++)
{
//target = inbuffer[i] & b[j].pattern;
target = (*(int*)(inbuffer + i)) & (b[j].pattern << inshifted);
target = target >> (b[j].shiftbits + inshifted);
#if SPLITDEBUG
printf(" -> target @%i[pos:%i]:%u\n", j, b[j].bitctr, target);
#endif

b[j].bits[b[j].bitctr] = target;
b[j].bitctr++;
if(b[j].bitctr == stuffingthreshold)
{
target = 0;
shifted = outbits - b[j].numbits;
precounted = 0;
for(k = 0; k < b[j].bitctr; k++)
{
#if SPLITDEBUG
printf("@%i: (%i->%i) >> %i ; << %i\n", j, b[j].numbits, b[j].bits[k], b[j].shiftbits, shifted);
#endif
target |= (b[j].bits[k] << shifted);
shifted -= b[j].numbits;
if(shifted < 0)
{
b[j].bytes[b[j].bytectr] = target & 0xFF;
b[j].bytectr++;
shifted = outbits - b[j].numbits;
precounted++;
#if SPLITDEBUG
printf("@%i: [bytectr++]: %i {%i}\n", j, target & 0xFF, target);
#endif
target = target >> outbits;
}
}
//b[j].bytectr += (int)((b[j].numbits * totalbits * 8 - 1) / 8) + 1 - precounted;
b[j].bytectr += (int)((totalbits - 1) / 8);
#if SPLITDEBUG
printf("@%i: stuffed %i 'bits'/fragments; bytectr now %i\n", j, totalbits, b[j].bytectr);
#endif
b[j].bitctr = 0;
if(b[j].bytectr >= 1024 - stuffingthreshold)
{
if((j == 0) && (redundant))
{
#if SPLITDEBUG
printf(">> redundancy\n");
#endif
b[parts].bytectr = b[j].bytectr;
for(l = 0; l < b[parts].bytectr; l++)
{
b[parts].bytes[l] = 0;
for(k = 0; k < parts; k++)
b[parts].bytes[l] ^= b[k].bytes[l];
}
if(b[parts].f)
fwrite(b[parts].bytes, 1, b[parts].bytectr, b[parts].f);
else
{
b[parts].buffer = (char*)realloc(b[parts].buffer, (b[parts].bufferctr + b[parts].bytectr) * sizeof(char));
memcpy(b[parts].buffer + b[parts].bufferctr, b[parts].bytes, b[parts].bytectr);
b[parts].bufferctr += b[parts].bytectr;
}
}
if(b[j].f)
fwrite(b[j].bytes, 1, b[j].bytectr, b[j].f);
else
{
b[j].buffer = (char*)realloc(b[j].buffer, (b[j].bufferctr + b[j].bytectr) * sizeof(char));
memcpy(b[j].buffer + b[j].bufferctr, b[j].bytes, b[j].bytectr);
b[j].bufferctr += b[j].bytectr;
}
b[j].bytectr = 0;
#if SPLITDEBUG
printf("@%i: reset bytectr\n", j);
#endif
}
}
}

i += (int)((totalbits - 1) / 8);
if((totalbits % 8) != 0)
{
inshifted += totalbits % 8;
if(inshifted % 8 == 0)
{
i += (inshifted - 8) / 8;
inshifted = 0;
}
else
i--;
}
}

for(j = 0; j < parts; j++)
{
if(b[j].bitctr > 0)
{
target = 0;
shifted = outbits - b[j].numbits;
precounted = 0;
for(k = 0; k < b[j].bitctr; k++)
{
#if SPLITDEBUG
printf("@%i: (%i=>%u) >> %i ; << %i\n", j, b[j].numbits, b[j].bits[k], b[j].shiftbits, shifted);
#endif
target |= (b[j].bits[k] << shifted);
shifted -= b[j].numbits;
if(shifted < 0)
{
b[j].bytes[b[j].bytectr] = target & 0xFF;
b[j].bytectr++;
shifted = outbits - b[j].numbits;
precounted++;
#if SPLITDEBUG
printf("@%i: [final-bytectr++]: %i {%i}\n", j, target & 0xFF, target);
#endif
target = target >> 8;
}
}
if(shifted != outbits - b[j].numbits)
{
#if SPLITDEBUG
printf("@%i: final-partialbyte: %i\n", j, target & 0xFF);
#endif
b[j].bytes[b[j].bytectr] = target & 0xFF;
b[j].bytectr++;
precounted++;
}
//b[j].bytectr += (int)((b[j].numbits * b[j].bitctr - 1) / totalbits) + 1 - precounted;
// FIXME: this may need a more general treatment
if(totalbits != 16)
b[j].bytectr += (int)((totalbits - 1) / 8);
#if SPLITDEBUG
printf("@%i: final-stuffed %i 'bits'/fragments; bytectr now %i\n", j, b[j].bitctr, b[j].bytectr);
#endif
b[j].bitctr = 0;
}
if(b[j].bytectr > 0)
{
if((j == 0) && (redundant))
{
#if SPLITDEBUG
printf(">> final-redundancy\n");
#endif
b[parts].bytectr = b[j].bytectr;
for(l = 0; l < b[parts].bytectr; l++)
{
b[parts].bytes[l] = 0;
for(k = 0; k < parts; k++)
b[parts].bytes[l] ^= b[k].bytes[l];
}
if(b[parts].f)
fwrite(b[parts].bytes, 1, b[parts].bytectr, b[parts].f);
else
{
b[parts].buffer = (char*)realloc(b[parts].buffer, (b[parts].bufferctr + b[parts].bytectr) * sizeof(char));
memcpy(b[parts].buffer + b[parts].bufferctr, b[parts].bytes, b[parts].bytectr);
b[parts].bufferctr += b[parts].bytectr;
}
}
if(b[j].f)
fwrite(b[j].bytes, 1, b[j].bytectr, b[j].f);
else
{
b[j].buffer = (char*)realloc(b[j].buffer, (b[j].bufferctr + b[j].bytectr) * sizeof(char));
memcpy(b[j].buffer + b[j].bufferctr, b[j].bytes, b[j].bytectr);
b[j].bufferctr += b[j].bytectr;
}
}
}

return 0;
}

int bitsplitbuffer(const char *inbuffer, ssize_t insize, char ***outbuffers, ssize_t **outsizes, int parts, int redundant, int bitwidth)
{
int ret;
int j;
bitsplit bs;

if((!inbuffer) || (parts <= 0)) return -1;
if(bitwidth < 1) return -1;

bs = bitsplitprepare(NULL, parts, redundant, bitwidth);
if(!bs.b) return -1;

*outbuffers = (char**)malloc((parts + redundant) * sizeof(char*));
*outsizes = (ssize_t*)malloc((parts + redundant) * sizeof(ssize_t));
for(j = 0; j < (parts + redundant); j++)
{
(*outbuffers)[j] = NULL;
(*outsizes)[j] = 0;
}

ret = bitsplitwork(inbuffer, insize, bs);

for(j = 0; j < (parts + redundant); j++)
{
(*outbuffers)[j] = bs.b[j].buffer;
(*outsizes)[j] = bs.b[j].bufferctr;
}

bitsplitfinish(bs);
return ret;
}

int bitjoinbuffer(const char **inbuffers, ssize_t *insizes, char **outbuffer, ssize_t *outsize, int parts, int bitwidth)
{
int ret;
bitsplit bs;

if((!inbuffers) || (parts <= 0)) return -1;

bs = bitsplitprepare(NULL, parts, 0, bitwidth);
if(!bs.b) return -1;

ret = bitjoinwork(inbuffers, insizes, bs);

*outbuffer = bs.b[0].buffer;
*outsize = bs.b[0].bufferctr;

bitsplitfinish(bs);
return ret;
}

int bitsplitfile(const char *filename, const char *directory, int parts, int redundant, int bitwidth)
{
FILE *f;
char buf[1024];
ssize_t size;
int ret;
bitsplit bs;

if((!filename) || (parts <= 0)) return -1;

f = fopen(filename, "r");
if(!f) return -1;

bs = bitsplitprepare(directory, parts, redundant, bitwidth);
if(!bs.b) return -1;

while(!feof(f))
{
size = fread(buf, 1, sizeof(buf), f);
#if SPLITDEBUG
printf("read %li bytes\n", size);
#endif

ret = bitsplitwork(buf, size, bs);
if(ret != 0)
{
fclose(f);
bitsplitfinish(bs);
return ret;
}
}

fclose(f);
bitsplitfinish(bs);
return 0;
}

int bitjoinfile(const char *filename, const char *directory, int parts, int bitwidth)
{
FILE *f, **dfs;
char **dbufs;
ssize_t *dsizes;
int ret;
//bitsplit bs;
int j;
char fname[1024];

if((!filename) || (parts <= 0)) return -1;

f = fopen(filename, "w");
if(!f) return -1;

dfs = (FILE**)malloc(parts * sizeof(FILE*));
dbufs = (char**)malloc(parts * sizeof(char**));
dsizes = (ssize_t*)malloc(parts * sizeof(ssize_t));

for(j = 0; j < parts; j++)
{
snprintf(fname, sizeof(fname), "%s/%i.stream", directory, j);
dfs[j] = fopen(fname, "r");
if(!dfs[j]) return -1;

dbufs[j] = (char*)malloc(1024 * sizeof(char));
}

//bs = bitsplitprepare(directory, parts, redundant, bitwidth);
//if(!bs.b) return -1;

while(!feof(dfs[0]))
{
for(j = 0; j < parts; j++)
{
dsizes[j] = fread(dbufs[j], 1, sizeof(dbufs[j]), dfs[j]);
#if SPLITDEBUG
printf("read %li bytes\n", dsizes[j]);
#endif
}

ret = -1;
//ret = bitsplitwork(buf, size, bs);
if(ret != 0)
{
for(j = 0; j < parts; j++)
fclose(dfs[j]);
fclose(f);
//bitsplitfinish(bs);
return ret;
}
}

for(j = 0; j < parts; j++)
fclose(dfs[j]);
fclose(f);
//bitsplitfinish(bs);
return 0;
}

please respond as soon as possible.
 
  


Reply


Thread Tools Search this Thread
Search this Thread:

Advanced Search

Posting Rules
You may not post new threads
You may not post replies
You may not post attachments
You may not edit your posts

BB code is On
Smilies are On
[IMG] code is Off
HTML code is Off



Similar Threads
Thread Thread Starter Forum Replies Last Post
slackware 14.1 copying data to usb drives results data loss corruption TuxDork Slackware 7 02-17-2015 12:10 PM
Adding data upload and data download time to Squid access.log sanjeetkumar086 Linux - Server 0 08-19-2014 12:44 PM
Adding data upload and data download time to Squid access.log sanjeetkumar086 Linux - Enterprise 1 08-18-2014 10:59 AM
upgrade system with lvm on software raid-1 data volume w/o losing data BinWondrin Linux - General 1 01-13-2009 04:25 PM
Processing data from a 'foreign' database with mysql, or tools to pre-process data. linker3000 Linux - Software 1 08-14-2007 09:36 PM


All times are GMT -5. The time now is 06:34 PM.

Main Menu
Advertisement
My LQ
Write for LQ
LinuxQuestions.org is looking for people interested in writing Editorials, Articles, Reviews, and more. If you'd like to contribute content, let us know.
Main Menu
Syndicate
RSS1  Latest Threads
RSS1  LQ News
Twitter: @linuxquestions
Facebook: linuxquestions Google+: linuxquestions
Open Source Consulting | Domain Registration