如何在 HDF5 中编写固定长度的字符串？
Posted
技术标签:
【中文标题】如何在 HDF5 中编写固定长度的字符串？【英文标题】：How to write fixed length strings in HDF5? 【发布时间】：2015-06-14 05:44:05 【问题描述】：我有一些 C++ 代码调用 HDF5 C API 来写出固定长度的字符串。由于某种原因，结果是完全垃圾。以下是 h5dump 的示例：
DATASET "simple"
DATATYPE H5T_STRING
STRSIZE 10;
STRPAD H5T_STR_NULLTERM;
CSET H5T_CSET_ASCII;
CTYPE H5T_C_S1;
DATASPACE SIMPLE ( 100 ) / ( 100 )
DATA
(0): "\001\026", "", "\37777777635\177", "", "y\026", "",
(6): "\37777777635\177", "", "\37777777761\026", "",
(10): "\37777777635\177", "", "i\027", "", "\37777777635\177", "",
(16): "\37777777741\027", "", "\37777777635\177", "", "Y\030", "",
(22): "\37777777635\177", "", "\37777777721\030", "",
(26): "\37777777635\177", "", "I\031", "", "\37777777635\177", "",
(32): "\37777777701\031", "", "\37777777635\177", "", "9\032", "",
但如果我将大小更改为H5Tset_size (datatype_id, H5T_VARIABLE);，输出看起来与预期一样。
--- 编辑 ----
这是一个较小的例子，只有 C 语言有同样的问题。可以用h5cc构建：
#include <string.h>
#include <stdio.h>
#include "hdf5.h"
#define FILE "chard.h5"
int main()
hid_t file_id, dataset_id, dataspace_id; /* identifiers */
herr_t status;
hid_t dtype;
size_t size;
file_id = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
hsize_t dims[1] = 4;
dataspace_id = H5Screate_simple(1, dims, NULL);
dtype = H5Tcopy (H5T_C_S1);
size = 5;
status = H5Tset_size (dtype, size);
char *strs[4] =
"this",
"this",
"this",
"this";
dataset_id = H5Dcreate(file_id, "simp", dtype, dataspace_id, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, strs);
status = H5Dclose(dataset_id);
status = H5Sclose(dataspace_id);
status = H5Fclose(file_id);
--- 结束编辑 ---
这是我的代码：
#include <iostream>
#include <assert.h>
#include <cstring>
#include <memory>
#include <string>
#include <vector>
#include "hdf5.h"
const char** vec_to_ptr(const std::vector<std::string>& v)
const char** ret;
ret = new const char*[v.size()];
for (size_t i = 0; i < v.size(); ++i)
ret[i] = v[i].c_str();
return ret;
hid_t get_datatype_id(const std::vector<std::string>& v)
hid_t datatype_id = H5Tcopy (H5T_C_S1);
/* H5Tset_size (datatype_id, H5T_VARIABLE); */
// If I replace the H5Tset_size line below with the commented one above, this
// code works well... but compression isn't nearly as good since almost all
// my strings are of the same length
herr_t status = H5Tset_size (datatype_id, 10);
assert( status >= 0 );
v.size(); // shutup, compiler
return datatype_id;
// str_vec: a vector of string to be written out
// group_id: a group_id which has already been opened
// dataset_name: the to write out to
// release_type: if 'true', release the datatype
// compression_level: the level of compression (6 seems reasonable)
//
// return: the status of H5Dwrite (last H5 operation)
template <typename T>
herr_t data_to_h5(
const std::vector<T>& str_vec,
hid_t group_id,
const std::string& dataset_name,
bool release_type,
uint compression_level = 6
)
herr_t status;
hsize_t dims[1] = str_vec.size();
// create the propery which allows for compression
hid_t prop_id = H5Pcreate(H5P_DATASET_CREATE);
// chunk size is same size as vector
status = H5Pset_chunk(prop_id, 1, dims);
assert( status >= 0 );
// compress using gzip
status = H5Pset_deflate(prop_id, compression_level);
assert( status >= 0 );
// create the data type
hid_t datatype_id = get_datatype_id(str_vec);
// create the dataspace
hid_t dataspace_id = H5Screate_simple(1, dims, NULL);
// create the dataset
hid_t dataset_id = H5Dcreate(group_id, dataset_name.c_str(), datatype_id,
dataspace_id, H5P_DEFAULT, prop_id, H5P_DEFAULT);
// get the ptrs from the string and write out
auto ptr = vec_to_ptr(str_vec);
status = H5Dwrite(dataset_id, datatype_id, H5S_ALL, H5S_ALL, H5P_DEFAULT,
ptr);
assert( status >= 0 );
status = H5Pclose(prop_id);
assert( status >= 0 );
status = H5Dclose(dataset_id);
assert( status >= 0 );
status = H5Sclose(dataspace_id);
assert( status >= 0 );
if (release_type)
status = H5Tclose(datatype_id);
assert( status >= 0 );
delete [] ptr;
return status;
int main(int argc, char *argv[])
if (argc != 2)
std::cerr << "Usage: h5_string output.h5" << std::endl;
return 1;
std::string fname(argv[1]);
std::cout << "Opening file " << fname << std::endl;
std::vector<std::string> str_vec;
// make a bunch of strings
const auto NSTR = 100;
for (auto i = 0; i < NSTR; ++i)
std::string cur_str = std::to_string( i );
str_vec.push_back( cur_str );
hid_t file_id;
file_id = H5Fcreate(fname.c_str(), H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
herr_t status;
hid_t root;
root = H5Gopen(file_id, "/", H5P_DEFAULT);
assert(status >= 0);
status = data_to_h5(str_vec, root, "simple", true, 6);
assert(status >= 0);
status = H5Gclose(root);
assert(status >= 0);
status = H5Fclose(file_id);
assert(status >= 0);
return 0;
以及对应的CMakeLists.txt：
find_package(HDF5)
add_executable(h5_string h5_string.cpp)
if(HDF5_FOUND)
include_directories( $HDF5_INCLUDE_DIR )
target_link_libraries( h5_string $HDF5_LIBRARIES )
else()
message(FATAL_ERROR "HDF5 not found. Required to output files")
endif()
非常感谢任何帮助。提前致谢！
【问题讨论】：
【参考方案1】：您的代码在设置大小方面工作正常（但我认为您偏离了一个）。真正的问题在于如何定义字符串。
char *strs[4] =
"this",
"this",
"this",
"this";
您将它们定义为指针数组与二维数组：
char strs[4][4] =
"this",
"this",
"this",
"this";
并且还修正了一个：
dtype = H5Tcopy (H5T_C_S1);
size = 4;
status = H5Tset_size (dtype, size);
我也倾向于关闭数据类型。
status = H5Tclose(dtype);
这是您的完整测试程序：
#include <string.h>
#include <stdio.h>
#include "hdf5.h"
#define FILE "chard.h5"
int main()
hid_t file_id, dataset_id, dataspace_id; /* identifiers */
herr_t status;
hid_t dtype;
size_t size;
file_id = H5Fcreate(FILE, H5F_ACC_TRUNC, H5P_DEFAULT, H5P_DEFAULT);
hsize_t dims[1] = 4;
dataspace_id = H5Screate_simple(1, dims, NULL);
dtype = H5Tcopy (H5T_C_S1);
size = 4 * sizeof(char);
status = H5Tset_size (dtype, size);
char strs[4][4] =
"this",
"this",
"this",
"this";
dataset_id = H5Dcreate(file_id, "simp", dtype, dataspace_id, H5P_DEFAULT,
H5P_DEFAULT, H5P_DEFAULT);
status = H5Dwrite (dataset_id, dtype, H5S_ALL, H5S_ALL, H5P_DEFAULT, strs);
status = H5Dclose(dataset_id);
status = H5Sclose(dataspace_id);
status = H5Tclose(dtype);
status = H5Fclose(file_id);
如果编译、运行并转储输出：
~$ h5pcc -g -O0 -Wall -o foo foo.c
~$ ./foo
~$ h5dump chard.h5
HDF5 "chard.h5"
GROUP "/"
DATASET "simp"
DATATYPE H5T_STRING
STRSIZE 4;
STRPAD H5T_STR_NULLTERM;
CSET H5T_CSET_ASCII;
CTYPE H5T_C_S1;
DATASPACE SIMPLE ( 4 ) / ( 4 )
DATA
(0): "this", "this", "this", "this"
【讨论】：
关键是对H5Dwrite的调用需要strs指向的连续内存。该函数假设您只是传输一维字节数组。
char strs[4][4] 应该是 char strs[4][5] 以说明终止 \0。
@RedX - 不是真的。 HDF5 不会终止字符串——它是 FORTRAN 的遗产（尽管如果你愿意，你可以这样做）。但是，我很惊讶编译时没有警告。
在这里完全同意@MarkLakata。正如我在人们编写多维数组时看到的那样，它们分别 malloc 行。它需要一个连续的块。【参考方案2】：
另一种（更容易）在 C++ 中的 HDF5 中编写固定长度字符串的方法是使用 HDFql。使用 HDFql，可以这样做：
// declare variable 'data'
char data[4][4];
// populate variable 'data' with values
memcpy(data[0], "this", 4);
memcpy(data[1], "that", 4);
memcpy(data[2], "what", 4);
memcpy(data[3], "when", 4);
// create an HDF5 file named 'chard.h5' and use (i.e. open) it
HDFql::execute("create and use file chard.h5");
// register variable 'data' for subsequent usage (by HDFql)
HDFql::variableRegister(data);
// create a dataset named 'simp' of fixed-length string and write content of 'data' into it
HDFql::execute("create dataset simp as char(4, 4) values from memory 0");
【讨论】：
以上是关于如何在 HDF5 中编写固定长度的字符串？的主要内容，如果未能解决你的问题，请参考以下文章