12 #ifndef __PROTOBUFFILE_H__
13 #define __PROTOBUFFILE_H__
19 #include <google/protobuf/message.h>
59 CProtobufFile(
const char* fname,
char rw=
'r',
const char* name=NULL);
71 virtual void get_vector(int8_t*& vector, int32_t& len);
72 virtual void get_vector(uint8_t*& vector, int32_t& len);
73 virtual void get_vector(
char*& vector, int32_t& len);
74 virtual void get_vector(int32_t*& vector, int32_t& len);
75 virtual void get_vector(uint32_t*& vector, int32_t& len);
79 virtual void get_vector(int16_t*& vector, int32_t& len);
80 virtual void get_vector(uint16_t*& vector, int32_t& len);
81 virtual void get_vector(int64_t*& vector, int32_t& len);
82 virtual void get_vector(uint64_t*& vector, int32_t& len);
94 uint8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
96 int8_t*& matrix, int32_t& num_feat, int32_t& num_vec);
98 char*& matrix, int32_t& num_feat, int32_t& num_vec);
100 int32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
102 uint32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
104 int64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
106 uint64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
108 float32_t*& matrix, int32_t& num_feat, int32_t& num_vec);
110 float64_t*& matrix, int32_t& num_feat, int32_t& num_vec);
112 floatmax_t*& matrix, int32_t& num_feat, int32_t& num_vec);
114 int16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
116 uint16_t*& matrix, int32_t& num_feat, int32_t& num_vec);
128 uint8_t*& array, int32_t*& dims, int32_t& num_dims);
130 char*& array, int32_t*& dims, int32_t& num_dims);
132 int32_t*& array, int32_t*& dims, int32_t& num_dims);
134 float32_t*& array, int32_t*& dims, int32_t& num_dims);
136 float64_t*& array, int32_t*& dims, int32_t& num_dims);
138 int16_t*& array, int32_t*& dims, int32_t& num_dims);
140 uint16_t*& array, int32_t*& dims, int32_t& num_dims);
189 int32_t& max_string_len);
192 int32_t& max_string_len);
195 int32_t& max_string_len);
198 int32_t& max_string_len);
201 int32_t& max_string_len);
204 int32_t& max_string_len);
207 int32_t& max_string_len);
210 int32_t& max_string_len);
213 int32_t& max_string_len);
216 int32_t& max_string_len);
219 int32_t& max_string_len);
222 int32_t& max_string_len);
235 virtual void set_vector(
const int8_t* vector, int32_t len);
236 virtual void set_vector(
const uint8_t* vector, int32_t len);
237 virtual void set_vector(
const char* vector, int32_t len);
238 virtual void set_vector(
const int32_t* vector, int32_t len);
239 virtual void set_vector(
const uint32_t* vector, int32_t len);
243 virtual void set_vector(
const int16_t* vector, int32_t len);
244 virtual void set_vector(
const uint16_t* vector, int32_t len);
245 virtual void set_vector(
const int64_t* vector, int32_t len);
246 virtual void set_vector(
const uint64_t* vector, int32_t len);
257 const uint8_t* matrix, int32_t num_feat, int32_t num_vec);
259 const int8_t* matrix, int32_t num_feat, int32_t num_vec);
261 const char* matrix, int32_t num_feat, int32_t num_vec);
263 const int32_t* matrix, int32_t num_feat, int32_t num_vec);
265 const uint32_t* matrix, int32_t num_feat, int32_t num_vec);
267 const int64_t* matrix, int32_t num_feat, int32_t num_vec);
269 const uint64_t* matrix, int32_t num_feat, int32_t num_vec);
271 const float32_t* matrix, int32_t num_feat, int32_t num_vec);
273 const float64_t* matrix, int32_t num_feat, int32_t num_vec);
275 const floatmax_t* matrix, int32_t num_feat, int32_t num_vec);
277 const int16_t* matrix, int32_t num_feat, int32_t num_vec);
279 const uint16_t* matrix, int32_t num_feat, int32_t num_vec);
351 virtual const char*
get_name()
const {
return "ProtobufFile"; }
358 void write_big_endian_uint(uint32_t number, uint8_t* array, uint32_t size);
361 uint32_t read_big_endian_uint(uint8_t* array, uint32_t size);
364 int32_t compute_num_messages(uint64_t len, int32_t sizeof_type)
const;
382 void write_vector_header(int32_t len, int32_t num_messages);
383 void write_matrix_header(int32_t num_feat, int32_t num_vec, int32_t num_messages);
390 void write_sparse_matrix_header(
392 void write_sparse_matrix_header(
394 void write_sparse_matrix_header(
396 void write_sparse_matrix_header(
398 void write_sparse_matrix_header(
400 void write_sparse_matrix_header(
402 void write_sparse_matrix_header(
404 void write_sparse_matrix_header(
406 void write_sparse_matrix_header(
408 void write_sparse_matrix_header(
410 void write_sparse_matrix_header(
412 void write_sparse_matrix_header(
414 void write_sparse_matrix_header(
422 void write_string_list_header(
424 void write_string_list_header(
426 void write_string_list_header(
428 void write_string_list_header(
430 void write_string_list_header(
432 void write_string_list_header(
434 void write_string_list_header(
436 void write_string_list_header(
438 void write_string_list_header(
440 void write_string_list_header(
442 void write_string_list_header(
444 void write_string_list_header(
449 void read_message(google::protobuf::Message& message);
452 void write_message(
const google::protobuf::Message& message);
459 void read_memory_block(uint8_t*& vector, uint64_t len, int32_t num_messages);
460 void read_memory_block(int8_t*& vector, uint64_t len, int32_t num_messages);
461 void read_memory_block(
char*& vector, uint64_t len, int32_t num_messages);
462 void read_memory_block(int32_t*& vector, uint64_t len, int32_t num_messages);
463 void read_memory_block(uint32_t*& vector, uint64_t len, int32_t num_messages);
464 void read_memory_block(
float64_t*& vector, uint64_t len, int32_t num_messages);
465 void read_memory_block(
float32_t*& vector, uint64_t len, int32_t num_messages);
466 void read_memory_block(
floatmax_t*& vector, uint64_t len, int32_t num_messages);
467 void read_memory_block(int16_t*& vector, uint64_t len, int32_t num_messages);
468 void read_memory_block(uint16_t*& vector, uint64_t len, int32_t num_messages);
469 void read_memory_block(int64_t*& vector, uint64_t len, int32_t num_messages);
470 void read_memory_block(uint64_t*& vector, uint64_t len, int32_t num_messages);
478 void write_memory_block(
const int8_t* vector, uint64_t len, int32_t num_messages);
479 void write_memory_block(
const uint8_t* vector, uint64_t len, int32_t num_messages);
480 void write_memory_block(
const char* vector, uint64_t len, int32_t num_messages);
481 void write_memory_block(
const int32_t* vector, uint64_t len, int32_t num_messages);
482 void write_memory_block(
const uint32_t* vector, uint64_t len, int32_t num_messages);
483 void write_memory_block(
const float32_t* vector, uint64_t len, int32_t num_messages);
484 void write_memory_block(
const float64_t* vector, uint64_t len, int32_t num_messages);
485 void write_memory_block(
const floatmax_t* vector, uint64_t len, int32_t num_messages);
486 void write_memory_block(
const int16_t* vector, uint64_t len, int32_t num_messages);
487 void write_memory_block(
const uint16_t* vector, uint64_t len, int32_t num_messages);
488 void write_memory_block(
const int64_t* vector, uint64_t len, int32_t num_messages);
489 void write_memory_block(
const uint64_t* vector, uint64_t len, int32_t num_messages);
524 void write_sparse_matrix(
526 void write_sparse_matrix(
528 void write_sparse_matrix(
530 void write_sparse_matrix(
532 void write_sparse_matrix(
534 void write_sparse_matrix(
536 void write_sparse_matrix(
538 void write_sparse_matrix(
540 void write_sparse_matrix(
542 void write_sparse_matrix(
544 void write_sparse_matrix(
546 void write_sparse_matrix(
548 void write_sparse_matrix(
582 void write_string_list(
584 void write_string_list(
586 void write_string_list(
588 void write_string_list(
590 void write_string_list(
592 void write_string_list(
594 void write_string_list(
596 void write_string_list(
598 void write_string_list(
600 void write_string_list(
602 void write_string_list(
604 void write_string_list(
613 int32_t message_size;
619 uint8_t uint_buffer[4];
virtual void set_string_list(const SGString< uint8_t > *strings, int32_t num_str)
virtual void get_string_list(SGString< uint8_t > *&strings, int32_t &num_str, int32_t &max_string_len)
virtual void get_ndarray(uint8_t *&array, int32_t *&dims, int32_t &num_dims)
template class SGSparseVector The assumtion is that the stored SGSparseVectorEntry* vector is orde...
virtual void get_vector(int8_t *&vector, int32_t &len)
virtual void get_matrix(uint8_t *&matrix, int32_t &num_feat, int32_t &num_vec)
virtual void set_sparse_matrix(const SGSparseVector< bool > *matrix, int32_t num_feat, int32_t num_vec)
A File access base class.
virtual void set_vector(const int8_t *vector, int32_t len)
virtual void get_sparse_matrix(SGSparseVector< bool > *&matrix, int32_t &num_feat, int32_t &num_vec)
virtual void set_matrix(const uint8_t *matrix, int32_t num_feat, int32_t num_vec)
Class for work with binary file in protobuf format.
virtual const char * get_name() const