Jerasure  1.2
A Library in C/C++ Facilitating Erasure Coding for Storage Applications
Defines | Functions
jerasure.c File Reference
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "galois.h"
#include "jerasure.h"

Defines

#define talloc(type, num)   (type *) malloc(sizeof(type)*(num))

Functions

void jerasure_print_matrix (int *m, int rows, int cols, int w)
void jerasure_print_bitmatrix (int *m, int rows, int cols, int w)
int jerasure_make_decoding_matrix (int k, int m, int w, int *matrix, int *erased, int *decoding_matrix, int *dm_ids)
int jerasure_make_decoding_bitmatrix (int k, int m, int w, int *matrix, int *erased, int *decoding_matrix, int *dm_ids)
int jerasure_matrix_decode (int k, int m, int w, int *matrix, int row_k_ones, int *erasures, char **data_ptrs, char **coding_ptrs, int size)
int * jerasure_matrix_to_bitmatrix (int k, int m, int w, int *matrix)
void jerasure_matrix_encode (int k, int m, int w, int *matrix, char **data_ptrs, char **coding_ptrs, int size)
void jerasure_bitmatrix_dotprod (int k, int w, int *bitmatrix_row, int *src_ids, int dest_id, char **data_ptrs, char **coding_ptrs, int size, int packetsize)
void jerasure_do_parity (int k, char **data_ptrs, char *parity_ptr, int size)
int jerasure_invert_matrix (int *mat, int *inv, int rows, int w)
int jerasure_invertible_matrix (int *mat, int rows, int w)
int * jerasure_erasures_to_erased (int k, int m, int *erasures)
void jerasure_free_schedule (int **schedule)
void jerasure_free_schedule_cache (int k, int m, int ***cache)
void jerasure_matrix_dotprod (int k, int w, int *matrix_row, int *src_ids, int dest_id, char **data_ptrs, char **coding_ptrs, int size)
int jerasure_bitmatrix_decode (int k, int m, int w, int *bitmatrix, int row_k_ones, int *erasures, char **data_ptrs, char **coding_ptrs, int size, int packetsize)
int jerasure_schedule_decode_lazy (int k, int m, int w, int *bitmatrix, int *erasures, char **data_ptrs, char **coding_ptrs, int size, int packetsize, int smart)
int jerasure_schedule_decode_cache (int k, int m, int w, int ***scache, int *erasures, char **data_ptrs, char **coding_ptrs, int size, int packetsize)
int *** jerasure_generate_schedule_cache (int k, int m, int w, int *bitmatrix, int smart)
int jerasure_invert_bitmatrix (int *mat, int *inv, int rows)
int jerasure_invertible_bitmatrix (int *mat, int rows)
int * jerasure_matrix_multiply (int *m1, int *m2, int r1, int c1, int r2, int c2, int w)
void jerasure_get_stats (double *fill_in)
void jerasure_do_scheduled_operations (char **ptrs, int **operations, int packetsize)
void jerasure_schedule_encode (int k, int m, int w, int **schedule, char **data_ptrs, char **coding_ptrs, int size, int packetsize)
int ** jerasure_dumb_bitmatrix_to_schedule (int k, int m, int w, int *bitmatrix)
int ** jerasure_smart_bitmatrix_to_schedule (int k, int m, int w, int *bitmatrix)
void jerasure_bitmatrix_encode (int k, int m, int w, int *bitmatrix, char **data_ptrs, char **coding_ptrs, int size, int packetsize)

Define Documentation

#define talloc (   type,
  num 
)    (type *) malloc(sizeof(type)*(num))

Function Documentation

int jerasure_bitmatrix_decode ( int  k,
int  m,
int  w,
int *  bitmatrix,
int  row_k_ones,
int *  erasures,
char **  data_ptrs,
char **  coding_ptrs,
int  size,
int  packetsize 
)

This function

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
bitmatrixArray of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j]
erasuresArray of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1.
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by coding_ptrs/data_ptrs in bytes.
packetsizeThe size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize.
Todo:

return data

formula

fix

example code

description

references

See also:
jerasure_matrix_decode(int k, int m, int w, int *matrix, int row_k_ones, int *erasures, char **data_ptrs, char **coding_ptrs, int size)
void jerasure_bitmatrix_dotprod ( int  k,
int  w,
int *  bitmatrix_row,
int *  src_ids,
int  dest_id,
char **  data_ptrs,
char **  coding_ptrs,
int  size,
int  packetsize 
)

This function

Parameters:
kNumber of data devices
wWord size
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by coding_ptrs/data_ptrs in bytes.
packetsizeThe size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize.
Todo:

fix

example code

description

void jerasure_bitmatrix_encode ( int  k,
int  m,
int  w,
int *  bitmatrix,
char **  data_ptrs,
char **  coding_ptrs,
int  size,
int  packetsize 
)

This function encodes a matrix with a bit-matrix in $GF(2^w)$. $w$ my be any number between 1 and 32.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
bitmatrixArray of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j]
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by data_ptrs in bytes.
packetsizeThe size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize.
Todo:
example code
See also:
jerasure_matrix_encode(int k, int m, int w, int *matrix, char **data_ptrs, char **coding_ptrs, int size)
void jerasure_do_parity ( int  k,
char **  data_ptrs,
char *  parity_ptr,
int  size 
)

This function calculates the parity of size bytes of data from each of k regions of memory accessed by data_ptrs. It put the result into the size pointed to by parity_ptr.

Parameters:
kNumber of data devices
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
sizeSize of memory allocated by data_ptrs in bytes.
Todo:

fix

example code

void jerasure_do_scheduled_operations ( char **  ptrs,
int **  schedule,
int  packetsize 
)

This function executes the schedule on w*packetsize worth of bytes from each device. ptrs is an array of pointers which should have as many elements as the highest referenced device in the schedule.

Parameters:
scheduleArray of schedule operations. If there are m operations, then schedule[m][0] = -1.
packetsizeThe size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize.
Todo:

fix

example code

int** jerasure_dumb_bitmatrix_to_schedule ( int  k,
int  m,
int  w,
int *  bitmatrix 
)

This function turns a bitmatrix into a schedule using the straightforward algorithm -- just schedule the dot products defined by each row of the matrix.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
bitmatrixArray of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j]
Todo:

return (int**)

example code

See also:
jerasure_smart_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix)
jerasure_free_schedule_cache(int k, int m, int ***cache)
int* jerasure_erasures_to_erased ( int  k,
int  m,
int *  erasures 
)

This function allocates and returns erased from erasures.

Parameters:
kNumber of data devices
mNumber of coding devices
erasuresArray of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1.
Todo:

return data

usage example

void jerasure_free_schedule ( int **  schedule)

This function frees a schedule that was allocated with jerasure_XXX_bitmatrix_to_schedule.

Parameters:
scheduleArray of schedule operations. If there are m operations, then schedule[m][0] = -1.
Todo:
example code
See also:
jerasure_dumb_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix)
jerasure_smart_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix)
void jerasure_free_schedule_cache ( int  k,
int  m,
int ***  cache 
)

This function reads a schedule cache that was created with jerasure_generate_schedule_cache.

Parameters:
kNumber of data devices
mNumber of coding devices
See also:
jerasure_generate_schedule_cache(int k, int m, int w, int *bitmatrix, int smart)
jerasure_smart_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix)
Todo:

fix

example code

int*** jerasure_generate_schedule_cache ( int  k,
int  m,
int  w,
int *  bitmatrix,
int  smart 
)

This function precalcalculate all the schedule for the given distribution bitmatrix. M must equal 2.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
bitmatrixArray of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j]
See also:
jerasure_free_schedule_cache(int k, int m, int ***cache)
Todo:

return data

fix

example code

void jerasure_get_stats ( double *  fill_in)

This function fills in a vector of three doubles: fill_in[0] is the number of bytes that have been XOR'd, fill_in[1] is the number of bytes that have been copied, fill_in[2] is the number of bytes that have been multiplied by a constant in $GF(2^w)$. When jerasure_get_stats() is called, it resets its values.

Parameters:
fill_invector of three doubles to be filled
 jerasure_get_stats(); // reset all values
int jerasure_invert_bitmatrix ( int *  mat,
int *  inv,
int  rows 
)

This function

Todo:

fix

return data

description

int jerasure_invert_matrix ( int *  mat,
int *  inv,
int  rows,
int  w 
)

This function

Parameters:
wWord size
Todo:

fix

return data

example code

description

int jerasure_invertible_bitmatrix ( int *  mat,
int  rows 
)

This function

Todo:

return data

fix

description

int jerasure_invertible_matrix ( int *  mat,
int  rows,
int  w 
)

This function

Parameters:
wWord size
Todo:

fix

return data

description

int jerasure_make_decoding_bitmatrix ( int  k,
int  m,
int  w,
int *  matrix,
int *  erased,
int *  decoding_matrix,
int *  dm_ids 
)

This function makes the k*k decoding matrix (or wk*wk bitmatrix) by taking the rows corresponding to k non-erased devices of the distribution matrix, and then inverting that matrix. You should already have allocated the decoding matrix and dm_ids, which is a vector of k integers. These will be filled in appropriately. dm_ids[i] is the id of element i of the survivors vector. I.e. row i of the decoding matrix times dm_ids equals data drive i. Both of these routines take "erased" instead of "erasures". Erased is a vector with k+m elements, which has 0 or 1 for each device's id, according to whether the device is erased.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
matrixArray of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j]
Todo:

fix

example code

int jerasure_make_decoding_matrix ( int  k,
int  m,
int  w,
int *  matrix,
int *  erased,
int *  decoding_matrix,
int *  dm_ids 
)

This function makes the k*k decoding matrix (or wk*wk bitmatrix) by taking the rows corresponding to k non-erased devices of the distribution matrix, and then inverting that matrix. You should already have allocated the decoding matrix and dm_ids, which is a vector of k integers. These will be filled in appropriately. dm_ids[i] is the id of element i of the survivors vector. I.e. row i of the decoding matrix times dm_ids equals data drive i. Both of these routines take "erased" instead of "erasures". Erased is a vector with k+m elements, which has 0 or 1 for each device's id, according to whether the device is erased.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
Todo:

fix

return data

example code

int jerasure_matrix_decode ( int  k,
int  m,
int  w,
int *  matrix,
int  row_k_ones,
int *  erasures,
char **  data_ptrs,
char **  coding_ptrs,
int  size 
)

This function decodes unsing a matrix in $GF(2^w)$, only when $w$ = 8|16|32.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
matrixArray of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j]
erasuresArray of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1.
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by data_ptrs/coding_ptrs in bytes.
Returns:
0 if it worked, -1 if it failed
Todo:

crossreferences

example code

See also:
jerasure_bitmatrix_decode(int k, int m, int w, int *bitmatrix, int row_k_ones, int *erasures, char **data_ptrs, char **coding_ptrs, int size, int packetsize)
jerasure_matrix_encode(int k, int m, int w, int *matrix, char **data_ptrs, char **coding_ptrs, int size)
void jerasure_matrix_dotprod ( int  k,
int  w,
int *  matrix_row,
int *  src_ids,
int  dest_id,
char **  data_ptrs,
char **  coding_ptrs,
int  size 
)

This function only works when w = 8|16|32.

Parameters:
kNumber of data devices
wWord size
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
Todo:

fix

sample code

description

void jerasure_matrix_encode ( int  k,
int  m,
int  w,
int *  matrix,
char **  data_ptrs,
char **  coding_ptrs,
int  size 
)

This function encodes a matrix in $GF(2^w)$. $w$ must be either 8, 16 or 32.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
matrixArray of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j]
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by coding_ptrs in bytes.
See also:
jerasure_matrix_decode(int k, int m, int w, int *matrix, int row_k_ones, int *erasures, char **data_ptrs, char **coding_ptrs, int size)
void jerasure_bitmatrix_encode(int k, int m, int w, int *bitmatrix, char **data_ptrs, char **coding_ptrs, int size, int packetsize)
int* jerasure_matrix_multiply ( int *  m1,
int *  m2,
int  r1,
int  c1,
int  r2,
int  c2,
int  w 
)

This function

Parameters:
wWord size
Todo:

fix

return data

description

int* jerasure_matrix_to_bitmatrix ( int  k,
int  m,
int  w,
int *  matrix 
)

This function turns a $m \times k$ matrix in $GF(2^w)$ into a $wm \times wk$ bitmatrix (in $GF(2)$). For a detailed explanation see: J. Blomer, M. Kalfane, M. Karpinski, R. Karp, M. Luby and D. Zuckerman: An XOR-based erasure-resilinet coding scheme. Technical Report TR-95-048, International Computer Science Institute, August 1995

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
matrixArray of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j]
Returns:
bit-matrix (int*)
Todo:
example code
void jerasure_print_bitmatrix ( int *  matrix,
int  rows,
int  cols,
int  w 
)

This function

Parameters:
matrixArray of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j]
wWord size
Todo:

fix

description

void jerasure_print_matrix ( int *  matrix,
int  rows,
int  cols,
int  w 
)

This function

Parameters:
matrixArray of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j]
wWord size
Todo:

fix

description

int jerasure_schedule_decode_cache ( int  k,
int  m,
int  w,
int ***  scache,
int *  erasures,
char **  data_ptrs,
char **  coding_ptrs,
int  size,
int  packetsize 
)

This function

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
erasuresArray of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1.
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by coding_ptrs/data_ptrs in bytes.
packetsizeThe size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize.
Todo:

return data

fix

example code

int jerasure_schedule_decode_lazy ( int  k,
int  m,
int  w,
int *  bitmatrix,
int *  erasures,
char **  data_ptrs,
char **  coding_ptrs,
int  size,
int  packetsize,
int  smart 
)

This function generates the schedule on the fly.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
bitmatrixArray of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j]
erasuresArray of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1.
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by coding_ptrs/data_ptrs in bytes.
packetsizeThe size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize.
Todo:

return data

formula

fix

example code

void jerasure_schedule_encode ( int  k,
int  m,
int  w,
int **  schedule,
char **  data_ptrs,
char **  coding_ptrs,
int  size,
int  packetsize 
)

This function encodes with a previously created schedule.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
scheduleArray of schedule operations. If there are m operations, then schedule[m][0] = -1.
data_ptrsArray of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned.
coding_ptrsArray of m pointers to coding data which is size bytes
sizeSize of memory allocated by data_ptrs in bytes.
packetsizeThe size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize.
See also:
jerasure_dumb_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix)
jerasure_smart_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix)
Todo:
example code
int** jerasure_smart_bitmatrix_to_schedule ( int  k,
int  m,
int  w,
int *  bitmatrix 
)

This function turns a bitmatrix into a schedule, but tries to use previous dot products to calculate new ones. This is the optimization explained in the original Liberation code paper.

Parameters:
kNumber of data devices
mNumber of coding devices
wWord size
bitmatrixArray of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j]
Todo:

return data

example code

See also:
jerasure_free_schedule_cache(int k, int m, int ***cache)
jerasure_dumb_bitmatrix_to_schedule(int k, int m, int w, int *bitmatrix)
 All Files Functions Variables Defines