Jerasure
1.2
A Library in C/C++ Facilitating Erasure Coding for Storage Applications
|
#include "galois.h"
Go to the source code of this file.
Functions | |
int * | jerasure_matrix_to_bitmatrix (int k, int m, int w, int *matrix) |
int ** | jerasure_dumb_bitmatrix_to_schedule (int k, int m, int w, int *bitmatrix) |
int ** | jerasure_smart_bitmatrix_to_schedule (int k, int m, int w, int *bitmatrix) |
int *** | jerasure_generate_schedule_cache (int k, int m, int w, int *bitmatrix, int smart) |
void | jerasure_free_schedule (int **schedule) |
void | jerasure_free_schedule_cache (int k, int m, int ***cache) |
void | jerasure_do_parity (int k, char **data_ptrs, char *parity_ptr, int size) |
void | jerasure_matrix_encode (int k, int m, int w, int *matrix, char **data_ptrs, char **coding_ptrs, int size) |
void | jerasure_bitmatrix_encode (int k, int m, int w, int *bitmatrix, char **data_ptrs, char **coding_ptrs, int size, int packetsize) |
void | jerasure_schedule_encode (int k, int m, int w, int **schedule, char **data_ptrs, char **coding_ptrs, int size, int packetsize) |
int | jerasure_matrix_decode (int k, int m, int w, int *matrix, int row_k_ones, int *erasures, char **data_ptrs, char **coding_ptrs, int size) |
int | jerasure_bitmatrix_decode (int k, int m, int w, int *bitmatrix, int row_k_ones, int *erasures, char **data_ptrs, char **coding_ptrs, int size, int packetsize) |
int | jerasure_schedule_decode_lazy (int k, int m, int w, int *bitmatrix, int *erasures, char **data_ptrs, char **coding_ptrs, int size, int packetsize, int smart) |
int | jerasure_schedule_decode_cache (int k, int m, int w, int ***scache, int *erasures, char **data_ptrs, char **coding_ptrs, int size, int packetsize) |
int | jerasure_make_decoding_matrix (int k, int m, int w, int *matrix, int *erased, int *decoding_matrix, int *dm_ids) |
int | jerasure_make_decoding_bitmatrix (int k, int m, int w, int *matrix, int *erased, int *decoding_matrix, int *dm_ids) |
int * | jerasure_erasures_to_erased (int k, int m, int *erasures) |
void | jerasure_matrix_dotprod (int k, int w, int *matrix_row, int *src_ids, int dest_id, char **data_ptrs, char **coding_ptrs, int size) |
void | jerasure_bitmatrix_dotprod (int k, int w, int *bitmatrix_row, int *src_ids, int dest_id, char **data_ptrs, char **coding_ptrs, int size, int packetsize) |
void | jerasure_do_scheduled_operations (char **ptrs, int **schedule, int packetsize) |
int | jerasure_invert_matrix (int *mat, int *inv, int rows, int w) |
int | jerasure_invert_bitmatrix (int *mat, int *inv, int rows) |
int | jerasure_invertible_matrix (int *mat, int rows, int w) |
int | jerasure_invertible_bitmatrix (int *mat, int rows) |
void | jerasure_print_matrix (int *matrix, int rows, int cols, int w) |
void | jerasure_print_bitmatrix (int *matrix, int rows, int cols, int w) |
int * | jerasure_matrix_multiply (int *m1, int *m2, int r1, int c1, int r2, int c2, int w) |
void | jerasure_get_stats (double *fill_in) |
int jerasure_bitmatrix_decode | ( | int | k, |
int | m, | ||
int | w, | ||
int * | bitmatrix, | ||
int | row_k_ones, | ||
int * | erasures, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size, | ||
int | packetsize | ||
) |
This function
k | Number of data devices |
m | Number of coding devices |
w | Word size |
bitmatrix | Array of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j] |
erasures | Array of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1. |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by coding_ptrs/data_ptrs in bytes. |
packetsize | The size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize. |
return data
formula
fix
example code
description
references
void jerasure_bitmatrix_dotprod | ( | int | k, |
int | w, | ||
int * | bitmatrix_row, | ||
int * | src_ids, | ||
int | dest_id, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size, | ||
int | packetsize | ||
) |
This function
k | Number of data devices |
w | Word size |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by coding_ptrs/data_ptrs in bytes. |
packetsize | The size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize. |
fix
example code
description
void jerasure_bitmatrix_encode | ( | int | k, |
int | m, | ||
int | w, | ||
int * | bitmatrix, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size, | ||
int | packetsize | ||
) |
This function encodes a matrix with a bit-matrix in .
my be any number between 1 and 32.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
bitmatrix | Array of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j] |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by data_ptrs in bytes. |
packetsize | The size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize. |
void jerasure_do_parity | ( | int | k, |
char ** | data_ptrs, | ||
char * | parity_ptr, | ||
int | size | ||
) |
This function calculates the parity of size bytes of data from each of k regions of memory accessed by data_ptrs. It put the result into the size pointed to by parity_ptr.
k | Number of data devices |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
size | Size of memory allocated by data_ptrs in bytes. |
fix
example code
void jerasure_do_scheduled_operations | ( | char ** | ptrs, |
int ** | schedule, | ||
int | packetsize | ||
) |
This function executes the schedule on w*packetsize worth of bytes from each device. ptrs is an array of pointers which should have as many elements as the highest referenced device in the schedule.
schedule | Array of schedule operations. If there are m operations, then schedule[m][0] = -1. |
packetsize | The size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize. |
fix
example code
int** jerasure_dumb_bitmatrix_to_schedule | ( | int | k, |
int | m, | ||
int | w, | ||
int * | bitmatrix | ||
) |
This function turns a bitmatrix into a schedule using the straightforward algorithm -- just schedule the dot products defined by each row of the matrix.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
bitmatrix | Array of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j] |
return (int**)
example code
int* jerasure_erasures_to_erased | ( | int | k, |
int | m, | ||
int * | erasures | ||
) |
This function allocates and returns erased from erasures.
k | Number of data devices |
m | Number of coding devices |
erasures | Array of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1. |
return data
usage example
void jerasure_free_schedule | ( | int ** | schedule | ) |
This function frees a schedule that was allocated with jerasure_XXX_bitmatrix_to_schedule.
schedule | Array of schedule operations. If there are m operations, then schedule[m][0] = -1. |
void jerasure_free_schedule_cache | ( | int | k, |
int | m, | ||
int *** | cache | ||
) |
This function reads a schedule cache that was created with jerasure_generate_schedule_cache.
k | Number of data devices |
m | Number of coding devices |
fix
example code
int*** jerasure_generate_schedule_cache | ( | int | k, |
int | m, | ||
int | w, | ||
int * | bitmatrix, | ||
int | smart | ||
) |
This function precalcalculate all the schedule for the given distribution bitmatrix. M must equal 2.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
bitmatrix | Array of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j] |
return data
fix
example code
void jerasure_get_stats | ( | double * | fill_in | ) |
This function fills in a vector of three doubles: fill_in[0] is the number of bytes that have been XOR'd, fill_in[1] is the number of bytes that have been copied, fill_in[2] is the number of bytes that have been multiplied by a constant in . When jerasure_get_stats() is called, it resets its values.
fill_in | vector of three doubles to be filled jerasure_get_stats(); // reset all values |
int jerasure_invert_bitmatrix | ( | int * | mat, |
int * | inv, | ||
int | rows | ||
) |
This function
fix
return data
description
int jerasure_invert_matrix | ( | int * | mat, |
int * | inv, | ||
int | rows, | ||
int | w | ||
) |
int jerasure_invertible_bitmatrix | ( | int * | mat, |
int | rows | ||
) |
This function
return data
fix
description
int jerasure_invertible_matrix | ( | int * | mat, |
int | rows, | ||
int | w | ||
) |
int jerasure_make_decoding_bitmatrix | ( | int | k, |
int | m, | ||
int | w, | ||
int * | matrix, | ||
int * | erased, | ||
int * | decoding_matrix, | ||
int * | dm_ids | ||
) |
This function makes the k*k decoding matrix (or wk*wk bitmatrix) by taking the rows corresponding to k non-erased devices of the distribution matrix, and then inverting that matrix. You should already have allocated the decoding matrix and dm_ids, which is a vector of k integers. These will be filled in appropriately. dm_ids[i] is the id of element i of the survivors vector. I.e. row i of the decoding matrix times dm_ids equals data drive i. Both of these routines take "erased" instead of "erasures". Erased is a vector with k+m elements, which has 0 or 1 for each device's id, according to whether the device is erased.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
matrix | Array of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j] |
fix
example code
int jerasure_make_decoding_matrix | ( | int | k, |
int | m, | ||
int | w, | ||
int * | matrix, | ||
int * | erased, | ||
int * | decoding_matrix, | ||
int * | dm_ids | ||
) |
This function makes the k*k decoding matrix (or wk*wk bitmatrix) by taking the rows corresponding to k non-erased devices of the distribution matrix, and then inverting that matrix. You should already have allocated the decoding matrix and dm_ids, which is a vector of k integers. These will be filled in appropriately. dm_ids[i] is the id of element i of the survivors vector. I.e. row i of the decoding matrix times dm_ids equals data drive i. Both of these routines take "erased" instead of "erasures". Erased is a vector with k+m elements, which has 0 or 1 for each device's id, according to whether the device is erased.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
fix
return data
example code
int jerasure_matrix_decode | ( | int | k, |
int | m, | ||
int | w, | ||
int * | matrix, | ||
int | row_k_ones, | ||
int * | erasures, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size | ||
) |
This function decodes unsing a matrix in , only when
= 8|16|32.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
matrix | Array of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j] |
erasures | Array of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1. |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by data_ptrs/coding_ptrs in bytes. |
crossreferences
example code
void jerasure_matrix_dotprod | ( | int | k, |
int | w, | ||
int * | matrix_row, | ||
int * | src_ids, | ||
int | dest_id, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size | ||
) |
This function only works when w = 8|16|32.
k | Number of data devices |
w | Word size |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
fix
sample code
description
void jerasure_matrix_encode | ( | int | k, |
int | m, | ||
int | w, | ||
int * | matrix, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size | ||
) |
This function encodes a matrix in .
must be either 8, 16 or 32.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
matrix | Array of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j] |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by coding_ptrs in bytes. |
int* jerasure_matrix_multiply | ( | int * | m1, |
int * | m2, | ||
int | r1, | ||
int | c1, | ||
int | r2, | ||
int | c2, | ||
int | w | ||
) |
int* jerasure_matrix_to_bitmatrix | ( | int | k, |
int | m, | ||
int | w, | ||
int * | matrix | ||
) |
This function turns a matrix in
into a
bitmatrix (in
). For a detailed explanation see: J. Blomer, M. Kalfane, M. Karpinski, R. Karp, M. Luby and D. Zuckerman: An XOR-based erasure-resilinet coding scheme. Technical Report TR-95-048, International Computer Science Institute, August 1995
k | Number of data devices |
m | Number of coding devices |
w | Word size |
matrix | Array of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j] |
void jerasure_print_bitmatrix | ( | int * | matrix, |
int | rows, | ||
int | cols, | ||
int | w | ||
) |
This function
matrix | Array of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j] |
w | Word size |
fix
description
void jerasure_print_matrix | ( | int * | matrix, |
int | rows, | ||
int | cols, | ||
int | w | ||
) |
This function
matrix | Array of k*m integers. It represents an m by k matrix. Element i,j is in matrix[i*k+j] |
w | Word size |
fix
description
int jerasure_schedule_decode_cache | ( | int | k, |
int | m, | ||
int | w, | ||
int *** | scache, | ||
int * | erasures, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size, | ||
int | packetsize | ||
) |
This function
k | Number of data devices |
m | Number of coding devices |
w | Word size |
erasures | Array of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1. |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by coding_ptrs/data_ptrs in bytes. |
packetsize | The size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize. |
return data
fix
example code
int jerasure_schedule_decode_lazy | ( | int | k, |
int | m, | ||
int | w, | ||
int * | bitmatrix, | ||
int * | erasures, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size, | ||
int | packetsize, | ||
int | smart | ||
) |
This function generates the schedule on the fly.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
bitmatrix | Array of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j] |
erasures | Array of id's of erased devices. Id's are integers between 0 and k+m-1. Id's 0 to k-1 are id's of data devices. Id's k to k+m-1 are id's of coding devices: Coding device id = id-k. If there are e erasures, erasures[e] = -1. |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by coding_ptrs/data_ptrs in bytes. |
packetsize | The size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize. |
return data
formula
fix
example code
void jerasure_schedule_encode | ( | int | k, |
int | m, | ||
int | w, | ||
int ** | schedule, | ||
char ** | data_ptrs, | ||
char ** | coding_ptrs, | ||
int | size, | ||
int | packetsize | ||
) |
This function encodes with a previously created schedule.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
schedule | Array of schedule operations. If there are m operations, then schedule[m][0] = -1. |
data_ptrs | Array of k pointers to data which is size bytes. Size must be a multiple of sizeof(long). Pointers must also be longword aligned. |
coding_ptrs | Array of m pointers to coding data which is size bytes |
size | Size of memory allocated by data_ptrs in bytes. |
packetsize | The size of a coding block with bitmatrix coding. When you code with a bitmatrix, you will use w packets of size packetsize. |
int** jerasure_smart_bitmatrix_to_schedule | ( | int | k, |
int | m, | ||
int | w, | ||
int * | bitmatrix | ||
) |
This function turns a bitmatrix into a schedule, but tries to use previous dot products to calculate new ones. This is the optimization explained in the original Liberation code paper.
k | Number of data devices |
m | Number of coding devices |
w | Word size |
bitmatrix | Array of k*m*w*w integers. It represents an mw by kw matrix. Element i,j is in matrix[i*k*w+j] |
return data
example code