//------------------------------------------------------------------------------------
// DSt_cube constructor. Example creation: DSt_cube<int> *integer_cube = new DSt_cube<int>(nwZ, nwY, nwX);
//
// Description: Creates cube with dimensions (nwZ, nwY, nwX). Can use ->data[#][#][#], or datac(#,#,#) to access spots.
//
// Parameters:
// nwZ, nwY, nwX - dimensions of cube to create
// max_block_size - max contiguous bytes allowed, useful for large cubes that could run into fragmentation issues
//------------------------------------------------------------------------------------
template <class T>
DSt_cube<T>::DSt_cube(unsigned short nwZ, unsigned short nwY, unsigned short nwX, mem_block max_block_size_bytes/* = DST_AUTO_SIZE_BLOCK*/)
{
// Fill in sample size information
wZ = nwZ;
wY = nwY;
wX = nwX;
dimensions.from_zyx(wZ, wY, wX);
max_block_size = max_block_size_bytes;
cache_state = CACHE_DISABLED;
// Allocate cube
allocateCube(nwZ, nwY, nwX);
};
//------------------------------------------------------------------------------------
// DSt_cube using file cache constructor.
//
// Description: Creates cube with dimensions (nwZ, nwY, nwX), but uses disc to store most of the data.
// A cache in RAM of size (cache_wZ, cache_wY, cache_wX) will be automatically used and kept up to date.
// MUST use datac(#,#,#) to access spots. Slower, but allows much larger cubes.
//
// NOTE: There are three levels of performance:
// 1) cache_wY and cache_wX are size of nwY and nwX. Fastest in file access, but uses most memory.
// 2) cache_wX is same size as nwX. Middle ground for performance.
// 3) None of the sizes match cache, this can result in any size cache cube, but slow disc access.
// Cache type should be based on algorithm to be used. If cube will be traveled in a linear fashion, option (1) is
// probably best. If cube is traveled in random directions, or equally in all, then (3) is probably best.
//
// Parameters:
// nwZ, nwY, nwX - dimensions of cube to create
// max_block_size - max contiguous bytes allowed, useful for large cubes that could run into fragmentation issues
//------------------------------------------------------------------------------------
template <class T>
DSt_cube<T>::DSt_cube( unsigned short nwZ, unsigned short nwY, unsigned short nwX,
cache_type cache_config, UINT64 desired_cache_size_bytes /*= 100000000*/, unsigned char sub_row_size_2pow/*=7*/ )
{
// Fill in sample size and cache size information
wZ = nwZ;
wY = nwY;
wX = nwX;
dimensions.from_zyx(wZ, wY, wX);
data = NULL; // Not used, set NULL to prevent deallocation at destruction
// Disable multiple blocks for simplicity (might want to change this in the future)
max_block_size = DST_SINGLE_BLOCK;
cache_is_full = false;
cache_sub_row_shift_val = sub_row_size_2pow; // Hard code for now at 128 size subrows
#if CACHING_IS_ENABLED
if ( ((UINT64)nwZ*nwY*nwX) < (desired_cache_size_bytes/sizeof(T)) )
{
cache_state = CACHE_DISABLED;
}
else
{
cache_state = cache_config;
// If subrow size is bigger than wX, might as well ust FULL_ROW mode
if ( (cache_state == CACHE_SUB_ROW) && (wX < (1 << cache_sub_row_shift_val)) )
{
cache_state = CACHE_FULL_ROW;
}
}
#else
cache_state = CACHE_DISABLED;
#endif
// Determine what type of cache to use, depending on how well sample sizes match
// requested cache size.
switch ( cache_state )
{
case CACHE_DISABLED:
max_block_size = DST_AUTO_SIZE_BLOCK;
// Allocate cube normally and RETURN
allocateCube(nwZ, nwY, nwX);
return;
case CACHE_FULL_SLICE:
cache_validity_table_dim.from_zyx( wZ, 1, 1 );
cache_entry_size = wX*wY;
cache_entry_dimensions.from_zyx( 1, wY, wX );
if ( allocateCacheMemory(desired_cache_size_bytes) != DST_NO_ERROR )
{
write_to_log("DSt_cube<T>::DSt_cube - FATAL ERROR - Out of memory while creating cache validity matrix CACHE_FULL_SLICE: %d, %d, %d.",
wZ, 1, 1);
exit(1);
}
break;
case CACHE_FULL_ROW:
cache_validity_table_dim.from_zyx( wZ, wY, 1 );
cache_entry_size = wX;
cache_entry_dimensions.from_zyx( 1, 1, wX );
if ( allocateCacheMemory(desired_cache_size_bytes) != DST_NO_ERROR )
{
write_to_log("DSt_cube<T>::DSt_cube - FATAL ERROR - Out of memory while creating cache validity matrix CACHE_FULL_ROW: %d, %d, %d.",
wZ, wY, 1);
exit(1);
}
break;
case CACHE_SUB_ROW:
cache_sub_row_last_seg_indx = (unsigned short)(wX >> cache_sub_row_shift_val);
cache_validity_table_dim.from_zyx( wZ, wY, cache_sub_row_last_seg_indx + 1 );
cache_entry_size = (unsigned short)(1 << cache_sub_row_shift_val);
cache_entry_dimensions.from_zyx( 1, 1, cache_entry_size );
cache_sub_row_last_entry_size = (unsigned short)( (~(0xFFFF << cache_sub_row_shift_val)) & wX );
if ( allocateCacheMemory(desired_cache_size_bytes) != DST_NO_ERROR )
{
write_to_log("DSt_cube<T>::DSt_cube - FATAL ERROR - Out of memory while creating cache validity matrix CACHE_SUB_ROW: %d, %d, %d.",
wZ, wY, cache_validity_table_dim.x);
exit(1);
}
break;
default:
write_to_log("DSt_cube<T>::DSt_cube - FATAL ERROR - Invalid Cache type requested %d.", cache_state);
exit(1);
}
// Create and open the cache file
++dst_cache_file_num;
sprintf( cache_file_name, "poromedia_cache_file_%d.cache", dst_cache_file_num );
// Create the file with temporary and random access flags, system will delete on close
cache_file = CreateFile( cache_file_name, GENERIC_READ | GENERIC_WRITE, FILE_SHARE_READ, NULL, CREATE_NEW,
/*FILE_FLAG_SEQUENTIAL_SCAN |*/ FILE_FLAG_DELETE_ON_CLOSE/* | FILE_FLAG_RANDOM_ACCESS*/, NULL );
if( cache_file == INVALID_HANDLE_VALUE )
{
write_to_log("DSt_cube<T>::DSt_cube - FATAL ERROR - Cannot create a cache file, %s", cache_file_name);
exit(1); // Fatal error
}
// Allocate entire file size in advance
LARGE_INTEGER file_size;
DWORD bytes_written;
file_size.QuadPart = (LONGLONG) wZ*wY*wX*sizeof(T);
SetFilePointerEx( cache_file, file_size, NULL, FILE_BEGIN );
T single_item = (T) 0;
WriteFile( cache_file, &single_item, sizeof(T), &bytes_written, NULL );
FlushFileBuffers( cache_file );
};
//------------------------------------------------------------------------------------
// allocateCube
//
// Description: Allocates memory for 'data' cube of desired size
//------------------------------------------------------------------------------------
template <class T>
dst_status DSt_cube<T>::allocateCube(unsigned short nwZ, unsigned short nwY, unsigned short nwX)
{
register unsigned short int y, z;
// Check if we need to break up into multiple smaller block sizes. If auto_size is specified, and total cube
// size is less than 512 MB, then we will not break up into multiple blocks.
if ( (max_block_size == DST_AUTO_SIZE_BLOCK) && (((unsigned int)nwZ*nwY*nwX*sizeof(T)) < 512000000) )
{
memory_num_of_blocks = 1;
slices_per_block = nwZ;
slices_per_block_last = nwZ;
}
else
{
// Determine how many slices can fit within block memory limit (max block size divided by memory required per slice)
unsigned int block_size_in_bytes = (max_block_size == DST_AUTO_SIZE_BLOCK) ? (DST_SMALL_BLOCK) : (max_block_size);
slices_per_block = (unsigned short)(block_size_in_bytes / (sizeof(T)*nwX*nwY) );
// Last block needs to be initialized to same size as original, in case of perfect fit
slices_per_block_last = slices_per_block;
// Must be at least one slice per block
if ( slices_per_block == 0 )
{
slices_per_block = 1;
}
// Number of blocks needed will be Ceiling of total number of slices divided by slices per block
memory_num_of_blocks = nwZ/slices_per_block;
if ( (nwZ%slices_per_block) > 0 )
{
memory_num_of_blocks = memory_num_of_blocks + 1;
}
if ( memory_num_of_blocks == 1 )
{
slices_per_block = wZ;
}
}
// Create array of pointers for start of each block, set to null
memory_block_ptrs = new T * [memory_num_of_blocks];
memset( memory_block_ptrs, 0, memory_num_of_blocks*sizeof(T) );
data = new T ** [nwZ];
rows = new T * [nwZ*nwY];
// Allocate memory for each block
for ( z = 0, y = 0; z < memory_num_of_blocks; z++)
{
if ( (y + slices_per_block) > nwZ )
{ // Allocate fewer slices if we are at last block of imperfect block divisions
slices_per_block_last = nwZ - y;
memory_block_ptrs[z] = new T [slices_per_block_last * nwX * nwY];
break;
}
else
{
memory_block_ptrs[z] = new T [slices_per_block * nwX * nwY];
y = y + slices_per_block;
}
}
// Make sure memory allocation is successful
if (!data || !rows || !memory_block_ptrs)
{
write_to_log("DSt_cube<T>::allocateCube - Out of memory while creating cube of dimensions %dx%dx%d.", nwX, nwY, nwZ);
data = NULL;
return DST_ERROR_MEMORY_ALLOCATION;
}
unsigned short cur_block = 0;
unsigned short cur_block_slice = 0;
// Make pointers point into correct places
for (z = 0; z < nwZ; z++)
{
for (y = 0; y < nwY; y++)
{ // Link row point into memory rows
rows[z*nwY + y] = &memory_block_ptrs[cur_block][cur_block_slice*nwY*nwX + y*nwX];
}
data[z] = &rows[z*nwY]; // Link final data pointers into rows
++cur_block_slice;
// Check to see if we should move on to the next block
if ( cur_block_slice == slices_per_block )
{
++cur_block;
cur_block_slice = 0;
}
}
return DST_NO_ERROR;
};
//------------------------------------------------------------------------------------
// allocateCube
//
// Description: Allocates all memory needed for cache'ing
//------------------------------------------------------------------------------------
template <class T>
dst_status DSt_cube<T>::allocateCacheMemory( UINT64 desired_size )
{
// Allocate row validity 2d matrix.
cache_validity_table_mem = new cache_entry_info* [cache_validity_table_dim.z * cache_validity_table_dim.y * cache_validity_table_dim.x];
cache_validity_table_rows = new cache_entry_info** [cache_validity_table_dim.z*cache_validity_table_dim.y];
cache_validity_table = new cache_entry_info*** [cache_validity_table_dim.z];
// Make sure memory allocation is successful
if ( (cache_validity_table_mem == NULL) || (cache_validity_table == NULL) )
{
return DST_ERROR_MEMORY_ALLOCATION;
}
// Make pointers point into correct places
for (int z = 0; z < cache_validity_table_dim.z; z++)
{
for (int y = 0; y < cache_validity_table_dim.y; y++)
{ // Link row point into memory rows
cache_validity_table_rows[z*cache_validity_table_dim.y + y] =
&cache_validity_table_mem[z*cache_validity_table_dim.y*cache_validity_table_dim.x + y*cache_validity_table_dim.x];
}
cache_validity_table[z] = &cache_validity_table_rows[ z*cache_validity_table_dim.y ];
}
// Now allocate buffer memory
return allocateBufferMemory( desired_size );
}
//------------------------------------------------------------------------------------
// allocateCube
//
// Description: Allocates actual buffer memory needed for cache'ing
//------------------------------------------------------------------------------------
template <class T>
dst_status DSt_cube<T>::allocateBufferMemory( UINT64 desired_size )
{
// Need to make sure validity table is completely invalid if we are messing with buffer
memset( cache_validity_table_mem, 0,
sizeof(cache_entry_info*) * cache_validity_table_dim.z * cache_validity_table_dim.y * cache_validity_table_dim.x );
// Determine number of cache entries we can hold at once
cache_number_of_entries = (int)( desired_size / (sizeof(T) * cache_entry_size) );
if ( cache_number_of_entries == 0 )
{
cache_number_of_entries = 1;
}
// --- Allocate memory for each entry --- //
memory_num_of_blocks = 1;
memory_block_ptrs = new T* [memory_num_of_blocks];
memory_total_used = cache_number_of_entries * cache_entry_size;
memory_block_ptrs[0] = new T [memory_total_used];
rows = new T * [cache_number_of_entries*cache_entry_dimensions.y];
// Make sure memory allocation is successful
if ( (memory_block_ptrs == NULL) || (memory_block_ptrs == NULL) || (rows == NULL) )
{
return DST_ERROR_MEMORY_ALLOCATION;
}
#if CACHING_USE_LINKED_LIST
cache_available_indx = 0;
cache_age_queue = new LinkedList<cache_entry_info>( cache_number_of_entries );
// Make sure memory allocation is successful
if ( cache_age_queue == NULL )
{
return DST_ERROR_MEMORY_ALLOCATION;
}
// Link up the queue
for (int i = 0; i < cache_number_of_entries; i++)
{
oldest_seg_ptr = cache_age_queue->getNewNode();
oldest_seg_ptr->item.needs_sync = false;
oldest_seg_ptr->item.queue_node_ptr = oldest_seg_ptr;
for ( int j = 0; j < cache_entry_dimensions.y; j++ )
{ // Link row point into memory rows
rows[ i*cache_entry_dimensions.y + j ] = &memory_block_ptrs[0][i*cache_entry_size + j*cache_entry_dimensions.x];
}
oldest_seg_ptr->item.data = &( rows[i * cache_entry_dimensions.y] );
cache_age_queue->insertAtStart( oldest_seg_ptr );
}
#else
// Now allocate and initialize the circular array
cache_oldest_indx = 0;
cache_available_indx = 0;
cache_age_list = new cache_entry_info [cache_number_of_entries];
// Make sure memory allocation is successful
if ( cache_age_list == NULL )
{
return DST_ERROR_MEMORY_ALLOCATION;
}
for (int i = 0; i < cache_number_of_entries; i++)
{
cache_age_list[i].needs_sync = false;
for ( int j = 0; j < cache_entry_dimensions.y; j++ )
{ // Link row point into memory rows
rows[ i*cache_entry_dimensions.y + j ] =
&memory_block_ptrs[0][i*cache_entry_size + j*cache_entry_dimensions.x];
}
cache_age_list[i].data = &( rows[i * cache_entry_dimensions.y] );
}
#endif
return DST_NO_ERROR;
}
//------------------------------------------------------------------------------------
// DSt_cube<T>::~DSt_cube
// deletes all memory belonging to object
//------------------------------------------------------------------------------------
template <class T>
DSt_cube<T>::~DSt_cube(void)
{
if (data)
delete[] data;
if (rows)
delete[] rows;
if (memory_block_ptrs)
{
// Delete each individual block
for (unsigned short i = 0; i < memory_num_of_blocks; i++)
{
if ( memory_block_ptrs[i] )
delete[] memory_block_ptrs[i];
}
// Delete array
delete[] memory_block_ptrs;
}
// Close and delete cache related stuff if used
if ( cache_state != CACHE_DISABLED )
{
// Close the cache file, system should delete it automatically
CloseHandle( cache_file );
// Cache memory
delete[] cache_validity_table;
delete[] cache_validity_table_rows;
delete[] cache_validity_table_mem;
#if CACHING_USE_LINKED_LIST
delete cache_age_queue;
#else
delete[] cache_age_list;
#endif
}
};