CUBRID Engine  latest
external_sort.c File Reference
#include "config.h"
#include <stdio.h>
#include <string.h>
#include <stddef.h>
#include <stdlib.h>
#include <math.h>
#include "error_manager.h"
#include "system_parameter.h"
#include "memory_alloc.h"
#include "external_sort.h"
#include "file_manager.h"
#include "page_buffer.h"
#include "log_manager.h"
#include "disk_manager.h"
#include "slotted_page.h"
#include "overflow_file.h"
#include "boot_sr.h"
#include "server_support.h"
#include "thread_entry_task.hpp"
#include "thread_manager.hpp"
#include <functional>

Go to the source code of this file.

Classes

struct  file_contents
 
struct  vol_info
 
struct  vol_list
 
struct  px_tree_node
 
struct  sort_param
 
struct  sort_rec_list
 
struct  slotted_pheader
 
struct  slot
 
struct  run_struct
 
struct  srun
 
struct  sort_stack
 

Macros

#define SORT_MULTIPAGE_FILE_SIZE_ESTIMATE   20
 
#define SORT_MAX_HALF_FILES   4
 
#define SORT_MIN_HALF_FILES   2
 
#define SORT_INITIAL_DYN_ARRAY_SIZE   30
 
#define SORT_EXPAND_DYN_ARRAY_RATIO   1.5
 
#define SORT_MAXREC_LENGTH   ((ssize_t)(DB_PAGESIZE - sizeof(SLOTTED_PAGE_HEADER) - sizeof(SLOT)))
 
#define SORT_SWAP_PTR(a, b)   { char **temp; temp = a; a = b; b = temp; }
 
#define SORT_CHECK_DUPLICATE(a, b)
 
#define SORT_PARTITION_RUN_SIZE_MIN   (ONE_M)
 

Typedefs

typedef struct file_contents FILE_CONTENTS
 
typedef struct vol_info VOL_INFO
 
typedef struct vol_list VOL_LIST
 
typedef struct px_tree_node PX_TREE_NODE
 
typedef struct sort_param SORT_PARAM
 
typedef struct sort_rec_list SORT_REC_LIST
 
typedef struct slotted_pheader SLOTTED_PAGE_HEADER
 
typedef struct slot SLOT
 
typedef struct run_struct RUN
 
typedef struct srun SRUN
 
typedef struct sort_stack SORT_STACK
 
typedef void FIND_RUN_FN(char **, long *, SORT_STACK *, long, SORT_CMP_FUNC *, void *)
 
typedef void MERGE_RUN_FN(char **, char **, SORT_STACK *, SORT_CMP_FUNC *, void *)
 

Functions

static int sort_validate (char **vector, long size, SORT_CMP_FUNC *compare, void *comp_arg)
 
static PX_TREE_NODEpx_sort_assign (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param, int px_id, char **px_buff, char **px_vector, long px_vector_size, int px_height, int px_myself)
 
static int px_sort_myself (THREAD_ENTRY *thread_p, PX_TREE_NODE *px_node)
 
static int sort_inphase_sort (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param, SORT_GET_FUNC *get_next, void *arguments, unsigned int *total_numrecs)
 
static int sort_exphase_merge_elim_dup (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param)
 
static int sort_exphase_merge (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param)
 
static int sort_get_avg_numpages_of_nonempty_tmpfile (SORT_PARAM *sort_param)
 
static void sort_return_used_resources (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param)
 
static int sort_add_new_file (THREAD_ENTRY *thread_p, VFID *vfid, int file_pg_cnt_est, bool force_alloc, bool tde_encrypted)
 
static int sort_write_area (THREAD_ENTRY *thread_p, VFID *vfid, int first_page, INT32 num_pages, char *area_start)
 
static int sort_read_area (THREAD_ENTRY *thread_p, VFID *vfid, int first_page, INT32 num_pages, char *area_start)
 
static int sort_get_num_half_tmpfiles (int tot_buffers, int input_pages)
 
static int sort_checkalloc_numpages_of_outfiles (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param)
 
static int sort_get_numpages_of_active_infiles (const SORT_PARAM *sort_param)
 
static int sort_find_inbuf_size (int tot_buffers, int in_sections)
 
static char * sort_retrieve_longrec (THREAD_ENTRY *thread_p, RECDES *address, RECDES *memory)
 
static char ** sort_run_sort (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param, char **base, long limit, long sort_numrecs, char **otherbase, long *srun_limit)
 
static int sort_run_add_new (FILE_CONTENTS *file_contents, int num_pages)
 
static void sort_run_remove_first (FILE_CONTENTS *file_contents)
 
static void sort_run_flip (char **start, char **stop)
 
static void sort_run_find (char **source, long *top, SORT_STACK *st_p, long limit, SORT_CMP_FUNC *compare, void *comp_arg, SORT_DUP_OPTION option)
 
static void sort_run_merge (char **low, char **high, SORT_STACK *st_p, SORT_CMP_FUNC *compare, void *comp_arg, SORT_DUP_OPTION option)
 
static int sort_run_flush (THREAD_ENTRY *thread_p, SORT_PARAM *sort_param, int out_curfile, int *cur_page, char *output_buffer, char **index_area, int numrecs, int rec_type)
 
static int sort_get_num_file_contents (FILE_CONTENTS *file_contents)
 
static void sort_spage_initialize (PAGE_PTR pgptr, INT16 slots_type, INT16 alignment)
 
static INT16 sort_spage_get_numrecs (PAGE_PTR pgptr)
 
static INT16 sort_spage_insert (PAGE_PTR pgptr, RECDES *recdes)
 
static SCAN_CODE sort_spage_get_record (PAGE_PTR pgptr, INT16 slotid, RECDES *recdes, bool peek_p)
 
static int sort_spage_offsetcmp (const void *s1, const void *s2)
 
static int sort_spage_compact (PAGE_PTR pgptr)
 
static INT16 sort_spage_find_free (PAGE_PTR pgptr, SLOT **sptr, INT16 length, INT16 type, INT16 *space)
 
static void sort_append (const void *pk0, const void *pk1)
 
int sort_listfile (THREAD_ENTRY *thread_p, INT16 volid, int est_inp_pg_cnt, SORT_GET_FUNC *get_fn, void *get_arg, SORT_PUT_FUNC *put_fn, void *put_arg, SORT_CMP_FUNC *cmp_fn, void *cmp_arg, SORT_DUP_OPTION option, int limit, bool includes_tde_class)
 

Macro Definition Documentation

#define SORT_CHECK_DUPLICATE (   a,
 
)
Value:
do { \
if (cmp == 0) { \
sort_append(a, b); \
*(a) = NULL; \
dup_num++; \
} \
} while (0)
#define NULL
Definition: freelistheap.h:34
if(extra_options)
Definition: dynamic_load.c:958
#define cmp
Definition: mprec.h:351
static void sort_append(const void *pk0, const void *pk1)

Definition at line 84 of file external_sort.c.

Referenced by sort_run_find().

#define SORT_EXPAND_DYN_ARRAY_RATIO   1.5

Definition at line 77 of file external_sort.c.

Referenced by sort_run_add_new().

#define SORT_INITIAL_DYN_ARRAY_SIZE   30

Definition at line 74 of file external_sort.c.

Referenced by sort_listfile().

#define SORT_MAXREC_LENGTH   ((ssize_t)(DB_PAGESIZE - sizeof(SLOTTED_PAGE_HEADER) - sizeof(SLOT)))

Definition at line 79 of file external_sort.c.

Referenced by sort_inphase_sort(), and sort_spage_insert().

#define SORT_MIN_HALF_FILES   2

Definition at line 71 of file external_sort.c.

Referenced by sort_get_num_half_tmpfiles().

#define SORT_MULTIPAGE_FILE_SIZE_ESTIMATE   20

Definition at line 57 of file external_sort.c.

#define SORT_PARTITION_RUN_SIZE_MIN   (ONE_M)

Referenced by px_sort_myself().

#define SORT_SWAP_PTR (   a,
 
)    { char **temp; temp = a; a = b; b = temp; }

Definition at line 82 of file external_sort.c.

Typedef Documentation

typedef struct file_contents FILE_CONTENTS

Definition at line 94 of file external_sort.c.

typedef void FIND_RUN_FN(char **, long *, SORT_STACK *, long, SORT_CMP_FUNC *, void *)

Definition at line 241 of file external_sort.c.

typedef void MERGE_RUN_FN(char **, char **, SORT_STACK *, SORT_CMP_FUNC *, void *)

Definition at line 242 of file external_sort.c.

typedef struct px_tree_node PX_TREE_NODE

Definition at line 121 of file external_sort.c.

typedef struct run_struct RUN

Definition at line 218 of file external_sort.c.

typedef struct slot SLOT

Definition at line 210 of file external_sort.c.

Definition at line 196 of file external_sort.c.

typedef struct sort_param SORT_PARAM

Definition at line 144 of file external_sort.c.

typedef struct sort_rec_list SORT_REC_LIST

Definition at line 188 of file external_sort.c.

typedef struct sort_stack SORT_STACK

Definition at line 234 of file external_sort.c.

typedef struct srun SRUN

Definition at line 225 of file external_sort.c.

typedef struct vol_info VOL_INFO

Definition at line 104 of file external_sort.c.

typedef struct vol_list VOL_LIST

Definition at line 111 of file external_sort.c.

Function Documentation

static int sort_add_new_file ( THREAD_ENTRY thread_p,
VFID vfid,
int  file_pg_cnt_est,
bool  force_alloc,
bool  tde_encrypted 
)
static
static void sort_append ( const void *  pk0,
const void *  pk1 
)
static

Definition at line 856 of file external_sort.c.

References SORT_REC::next.

Referenced by px_sort_myself(), and sort_run_merge().

Here is the caller graph for this function:

static int sort_find_inbuf_size ( int  tot_buffers,
int  in_sections 
)
static

Definition at line 4850 of file external_sort.c.

Referenced by sort_exphase_merge(), and sort_exphase_merge_elim_dup().

Here is the caller graph for this function:

static int sort_get_avg_numpages_of_nonempty_tmpfile ( SORT_PARAM sort_param)
static

Definition at line 4380 of file external_sort.c.

References sort_param::file_contents, file_contents::first_run, i, file_contents::last_run, file_contents::num_pages, and sort_param::tot_tempfiles.

Referenced by sort_listfile().

Here is the caller graph for this function:

static int sort_get_num_file_contents ( FILE_CONTENTS file_contents)
static

Definition at line 4935 of file external_sort.c.

References file_contents::first_run, file_contents::last_run, and file_contents::num_pages.

Referenced by sort_exphase_merge(), and sort_exphase_merge_elim_dup().

Here is the caller graph for this function:

static int sort_get_num_half_tmpfiles ( int  tot_buffers,
int  input_pages 
)
static

Definition at line 4649 of file external_sort.c.

References CEIL_PTVDIV, SORT_MAX_HALF_FILES, and SORT_MIN_HALF_FILES.

Referenced by sort_listfile().

Here is the caller graph for this function:

static int sort_get_numpages_of_active_infiles ( const SORT_PARAM sort_param)
static

Definition at line 4811 of file external_sort.c.

References sort_param::file_contents, file_contents::first_run, sort_param::half_files, i, and sort_param::in_half.

Referenced by sort_exphase_merge(), and sort_exphase_merge_elim_dup().

Here is the caller graph for this function:

static int sort_inphase_sort ( THREAD_ENTRY thread_p,
SORT_PARAM sort_param,
SORT_GET_FUNC get_next,
void *  arguments,
unsigned int *  total_numrecs 
)
static
int sort_listfile ( THREAD_ENTRY thread_p,
INT16  volid,
int  est_inp_pg_cnt,
SORT_GET_FUNC get_fn,
void *  get_arg,
SORT_PUT_FUNC put_fn,
void *  put_arg,
SORT_CMP_FUNC cmp_fn,
void *  cmp_arg,
SORT_DUP_OPTION  option,
int  limit,
bool  includes_tde_class 
)

Definition at line 1345 of file external_sort.c.

References ARG_FILE_LINE, assert_release, CEIL_PTVDIV, cleanup(), sort_param::cmp_arg, sort_param::cmp_fn, DB_PAGESIZE, db_private_alloc, ER_CSS_PTHREAD_MUTEX_INIT, ER_ERROR_SEVERITY, er_log_debug, ER_OUT_OF_VIRTUAL_MEMORY, er_set(), error(), sort_param::file_contents, vfid::fileid, file_contents::first_run, free_and_init, sort_param::half_files, i, sort_param::in_half, sort_param::internal_memory, file_contents::last_run, sort_param::limit, sort_param::multipage_file, NO_ERROR, NULL, NULL_FILEID, NULL_VOLID, file_contents::num_pages, file_contents::num_slots, sort_param::option, prm_get_integer_value(), PRM_ID_SR_NBUFFERS, pthread_mutex_init, sort_param::put_arg, sort_param::put_fn, sort_param::px_array, sort_param::px_array_size, sort_param::px_height_max, rv, sort_add_new_file(), SORT_ELIM_DUP, sort_exphase_merge(), sort_exphase_merge_elim_dup(), sort_get_avg_numpages_of_nonempty_tmpfile(), sort_get_num_half_tmpfiles(), SORT_INITIAL_DYN_ARRAY_SIZE, sort_inphase_sort(), SORT_MAX_HALF_FILES, sort_return_used_resources(), sort_param::tde_encrypted, sort_param::temp, thread_set_sort_stats_active(), sort_param::tmp_file_pgs, sort_param::tot_buffers, sort_param::tot_runs, sort_param::tot_tempfiles, vol_list::vol_cnt, vol_list::vol_ent_cnt, vol_list::vol_info, sort_param::vol_list, vol_list::volid, and vfid::volid.

Referenced by btree_index_sort(), qexec_execute_analytic(), qexec_groupby(), and qfile_sort_list_with_func().

Here is the caller graph for this function:

static int sort_read_area ( THREAD_ENTRY thread_p,
VFID vfid,
int  first_page,
INT32  num_pages,
char *  area_start 
)
static
static char * sort_retrieve_longrec ( THREAD_ENTRY thread_p,
RECDES address,
RECDES memory 
)
static

Definition at line 2802 of file external_sort.c.

References recdes::area_size, recdes::data, free_and_init, NULL, overflow_get(), overflow_get_length(), and S_SUCCESS.

Referenced by sort_exphase_merge(), sort_exphase_merge_elim_dup(), and sort_inphase_sort().

Here is the caller graph for this function:

static int sort_run_add_new ( FILE_CONTENTS file_contents,
int  num_pages 
)
static
static void sort_run_find ( char **  source,
long *  top,
SORT_STACK st_p,
long  limit,
SORT_CMP_FUNC compare,
void *  comp_arg,
SORT_DUP_OPTION  option 
)
static

Definition at line 886 of file external_sort.c.

References CAST_BUFLEN, cmp, srun::low_high, NULL, SORT_CHECK_DUPLICATE, sort_run_flip(), sort_stack::srun, srun::start, srun::stop, sort_stack::top, and srun::tree_depth.

Referenced by sort_run_sort().

Here is the caller graph for this function:

static void sort_run_flip ( char **  start,
char **  stop 
)
static

Definition at line 833 of file external_sort.c.

Referenced by sort_run_find().

Here is the caller graph for this function:

static void sort_run_merge ( char **  low,
char **  high,
SORT_STACK st_p,
SORT_CMP_FUNC compare,
void *  comp_arg,
SORT_DUP_OPTION  option 
)
static

Definition at line 1007 of file external_sort.c.

References cmp, srun::low_high, sort_append(), SORT_DUP, sort_stack::srun, srun::start, srun::stop, sort_stack::top, and srun::tree_depth.

Referenced by sort_run_sort().

Here is the caller graph for this function:

static void sort_run_remove_first ( FILE_CONTENTS file_contents)
static

Definition at line 4914 of file external_sort.c.

References file_contents::first_run, and file_contents::last_run.

Referenced by sort_exphase_merge(), and sort_exphase_merge_elim_dup().

Here is the caller graph for this function:

static char ** sort_run_sort ( THREAD_ENTRY thread_p,
SORT_PARAM sort_param,
char **  base,
long  limit,
long  sort_numrecs,
char **  otherbase,
long *  srun_limit 
)
static
static INT16 sort_spage_find_free ( PAGE_PTR  pgptr,
SLOT **  sptr,
INT16  length,
INT16  type,
INT16 *  space 
)
static
static INT16 sort_spage_get_numrecs ( PAGE_PTR  pgptr)
static

Definition at line 366 of file external_sort.c.

References slotted_pheader::nrecs.

Referenced by sort_exphase_merge(), sort_exphase_merge_elim_dup(), and sort_run_flush().

Here is the caller graph for this function:

static INT16 sort_spage_insert ( PAGE_PTR  pgptr,
RECDES recdes 
)
static
static int sort_spage_offsetcmp ( const void *  s1,
const void *  s2 
)
static

Definition at line 382 of file external_sort.c.

References slot::roffset.

Referenced by sort_spage_compact().

Here is the caller graph for this function:

static int sort_validate ( char **  vector,
long  size,
SORT_CMP_FUNC compare,
void *  comp_arg 
)
static

Definition at line 1577 of file external_sort.c.

References assert, cmp, DB_LT, ER_FAILED, NO_ERROR, and NULL.

Referenced by px_sort_myself(), and sort_run_sort().

Here is the caller graph for this function:

static int sort_write_area ( THREAD_ENTRY thread_p,
VFID vfid,
int  first_page,
INT32  num_pages,
char *  area_start 
)
static