15 #include "nc4internal.h"
16 #include "nc4dispatch.h"
18 #include <H5DSpublic.h>
31 extern int num_plists;
32 extern int num_spaces;
35 #define MIN_DEFLATE_LEVEL 0
36 #define MAX_DEFLATE_LEVEL 9
40 #define REFERENCE_LIST "REFERENCE_LIST"
42 #define DIMENSION_LIST "DIMENSION_LIST"
46 static int NC4_enddef(
int ncid);
47 static int nc4_rec_read_types(NC_GRP_INFO_T *grp);
48 static int nc4_rec_read_vars(NC_GRP_INFO_T *grp);
49 static int close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5,
int abort);
59 size_t nc4_chunk_cache_size = CHUNK_CACHE_SIZE;
60 size_t nc4_chunk_cache_nelems = CHUNK_CACHE_NELEMS;
61 float nc4_chunk_cache_preemption = CHUNK_CACHE_PREEMPTION;
65 static int virgin = 1;
70 static hid_t native_type_constant[NUM_TYPES];
72 static char nc_type_name[NUM_TYPES][
NC_MAX_NAME + 1] = {
"char",
"byte",
"short",
"int",
"float",
73 "double",
"ubyte",
"ushort",
"uint",
74 "int64",
"uint64",
"string"};
75 int nc4_free_global_hdf_string_typeid();
80 nc_set_chunk_cache(
size_t size,
size_t nelems,
float preemption)
82 if (preemption < 0 || preemption > 1)
84 nc4_chunk_cache_size = size;
85 nc4_chunk_cache_nelems = nelems;
86 nc4_chunk_cache_preemption = preemption;
93 nc_get_chunk_cache(
size_t *sizep,
size_t *nelemsp,
float *preemptionp)
96 *sizep = nc4_chunk_cache_size;
99 *nelemsp = nc4_chunk_cache_nelems;
102 *preemptionp = nc4_chunk_cache_preemption;
108 nc_set_chunk_cache_ints(
int size,
int nelems,
int preemption)
110 if (size <= 0 || nelems <= 0 || preemption < 0 || preemption > 100)
112 nc4_chunk_cache_size = size;
113 nc4_chunk_cache_nelems = nelems;
114 nc4_chunk_cache_preemption = (float)preemption / 100;
119 nc_get_chunk_cache_ints(
int *sizep,
int *nelemsp,
int *preemptionp)
122 *sizep = (int)nc4_chunk_cache_size;
124 *nelemsp = (int)nc4_chunk_cache_nelems;
126 *preemptionp = (int)(nc4_chunk_cache_preemption * 100);
156 #define MAGIC_NUMBER_LEN 4
157 #define NC_HDF5_FILE 1
158 #define NC_HDF4_FILE 2
160 nc_check_for_hdf(
const char *path,
int use_parallel, MPI_Comm comm, MPI_Info info,
163 char blob[MAGIC_NUMBER_LEN];
165 assert(hdf_file && path);
166 LOG((3,
"nc_check_for_hdf: path %s", path));
171 *hdf_file = NC_HDF5_FILE;
182 if ((retval = MPI_File_open(comm, (
char *)path, MPI_MODE_RDONLY,
183 info, &fh)) != MPI_SUCCESS)
185 if ((retval = MPI_File_read(fh, blob, MAGIC_NUMBER_LEN, MPI_CHAR,
186 &status)) != MPI_SUCCESS)
188 if ((retval = MPI_File_close(&fh)) != MPI_SUCCESS)
195 if (!(fp = fopen(path,
"r")) ||
196 fread(blob, MAGIC_NUMBER_LEN, 1, fp) != 1) {
205 if (!strncmp(blob,
"\016\003\023\001", MAGIC_NUMBER_LEN))
206 *hdf_file = NC_HDF4_FILE;
216 nc4_create_file(
const char *path,
int cmode, MPI_Comm comm, MPI_Info info,
219 hid_t fcpl_id, fapl_id;
223 NC_HDF5_FILE_INFO_T* nc4_info = NULL;
231 flags = H5F_ACC_TRUNC;
233 flags = H5F_ACC_EXCL;
235 flags = H5F_ACC_TRUNC;
237 LOG((3,
"nc4_create_file: path %s mode 0x%x", path, cmode));
242 if (cmode & NC_DISKLESS) {
247 }
else if ((cmode & NC_NOCLOBBER) && (fp = fopen(path,
"r"))) {
253 if ((retval = nc4_nc4f_list_add(nc, path, (
NC_WRITE | cmode))))
255 nc4_info = NC4_DATA(nc);
256 assert(nc4_info && nc4_info->root_grp);
266 if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
272 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
275 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG))
284 nc4_info->parallel++;
285 if (cmode & NC_MPIIO)
287 LOG((4,
"creating parallel file with MPI/IO"));
288 if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
293 LOG((4,
"creating parallel file with MPI/posix"));
294 if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0)
299 if(cmode & NC_DISKLESS) {
300 if (H5Pset_fapl_core(fapl_id, 4096, persist))
303 if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
304 nc4_chunk_cache_preemption) < 0)
306 LOG((4,
"nc4_create_file: set HDF raw chunk cache to size %d nelems %d preemption %f",
307 nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption));
310 if (H5Pset_libver_bounds(fapl_id, H5F_LIBVER_LATEST, H5F_LIBVER_LATEST) < 0)
314 if ((fcpl_id = H5Pcreate(H5P_FILE_CREATE)) < 0)
323 if (H5Pset_link_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
324 H5P_CRT_ORDER_INDEXED)) < 0)
326 if (H5Pset_attr_creation_order(fcpl_id, (H5P_CRT_ORDER_TRACKED |
327 H5P_CRT_ORDER_INDEXED)) < 0)
331 if ((nc4_info->hdfid = H5Fcreate(path, flags, fcpl_id, fapl_id)) < 0)
337 if ((nc4_info->root_grp->hdf_grpid = H5Gopen2(nc4_info->hdfid,
"/",
342 if (H5Pclose(fapl_id) < 0 ||
343 H5Pclose(fcpl_id) < 0)
351 nc4_info->flags |= NC_INDEF;
359 if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
360 if(!nc4_info)
return retval;
361 close_netcdf4_file(nc4_info,1);
363 if (nc4_info->hdfid > 0) H5Fclose(nc4_info->hdfid);
364 if (nc4_info->root_grp) {
365 free(nc4_info->root_grp->name);
366 free(nc4_info->root_grp);
389 NC4_create(
const char* path,
int cmode,
size_t initialsz,
int basepe,
390 size_t *chunksizehintp,
int use_parallel,
void *mpidata,
391 NC_Dispatch *dispatch, NC* nc_file)
397 int comm = 0, info = 0;
401 assert(nc_file && path);
403 LOG((1,
"nc4_create_file: path %s cmode 0x%x comm %d info %d",
404 path, cmode, comm, info));
409 comm = ((NC_MPI_INFO *)mpidata)->comm;
410 info = ((NC_MPI_INFO *)mpidata)->info;
417 if (H5Eset_auto(NULL, NULL) < 0)
418 LOG((0,
"Couldn't turn off HDF5 error messages!"));
419 LOG((1,
"HDF5 error messages have been turned off."));
430 || (cmode & NC_MPIIO && cmode & NC_MPIPOSIX)
432 || (cmode & (NC_MPIIO | NC_MPIPOSIX) && cmode & NC_DISKLESS)
444 LOG((2,
"cmode after applying default format: 0x%x", cmode));
446 nc_file->int_ncid = nc_file->ext_ncid;
447 res = nc4_create_file(path, cmode, comm, info, nc_file);
450 if (cmode & NC_PNETCDF)
452 NC_HDF5_FILE_INFO_T* nc4_info;
453 nc4_info = NC4_DATA(nc_file);
456 nc4_info->pnetcdf_file++;
457 res = ncmpi_create(comm, path, cmode, info, &(nc_file->int_ncid));
470 read_scale(NC_GRP_INFO_T *grp, hid_t datasetid,
const char *obj_name,
471 hsize_t scale_size, hsize_t max_scale_size,
472 int *dim_without_var)
481 if ((retval = nc4_dim_list_add(&grp->dim)))
485 grp->dim->dimid = grp->nc4_info->next_dimid++;
491 if ((attid = H5Aopen_by_name(datasetid,
".", NC_DIMID_ATT_NAME,
492 H5P_DEFAULT, H5P_DEFAULT)) > 0)
494 if (H5Aread(attid, H5T_NATIVE_INT, &grp->dim->dimid) < 0)
496 if (H5Aclose(attid) < 0)
502 if (!(grp->dim->name = malloc((max_len + 1) *
sizeof(
char))))
504 strncpy(grp->dim->name, obj_name, max_len + 1);
508 grp->dim->too_long = 1;
511 grp->dim->len = scale_size;
512 grp->dim->hdf_dimscaleid = datasetid;
516 if (max_scale_size == H5S_UNLIMITED)
517 grp->dim->unlimited++;
522 if (H5DSget_scale_name(datasetid, dimscale_name_att,
NC_MAX_NAME) >= 0)
524 if (!strncmp(dimscale_name_att, DIM_WITHOUT_VARIABLE,
525 strlen(DIM_WITHOUT_VARIABLE)))
527 if (grp->dim->unlimited)
529 size_t len = 0, *lenp = &len;
530 if ((retval = nc4_find_dim_len(grp, grp->dim->dimid, &lenp)))
532 grp->dim->len = *lenp;
534 (*dim_without_var)++;
544 read_coord_dimids(NC_VAR_INFO_T *var)
546 hid_t coord_att_typeid = -1, coord_attid = -1, spaceid = -1;
552 if ((coord_attid = H5Aopen_name(var->hdf_datasetid, COORDINATES)) < 0) ret++;
553 if (!ret && (coord_att_typeid = H5Aget_type(coord_attid)) < 0) ret++;
554 if (!ret && H5Aread(coord_attid, coord_att_typeid, var->dimids) < 0) ret++;
555 LOG((4,
"dimscale %s is multidimensional and has coords", var->name));
558 if ((spaceid = H5Aget_space(coord_attid)) < 0) ret++;
562 if (H5Sget_simple_extent_npoints(spaceid) < 0) ret++;
565 if (spaceid >= 0 && H5Sclose(spaceid) < 0) ret++;
569 if (coord_att_typeid >= 0 && H5Tclose(coord_att_typeid) < 0) ret++;
570 if (coord_attid >= 0 && H5Aclose(coord_attid) < 0) ret++;
577 dimscale_visitor(hid_t did,
unsigned dim, hid_t dsid,
578 void *dimscale_hdf5_objids)
583 if (H5Gget_objinfo(dsid,
".", 1, &statbuf) < 0)
589 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[0] = statbuf.fileno[0];
590 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).fileno[1] = statbuf.fileno[1];
591 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[0] = statbuf.objno[0];
592 (*(HDF5_OBJID_T *)dimscale_hdf5_objids).objno[1] = statbuf.objno[1];
598 get_netcdf_type(NC_HDF5_FILE_INFO_T *h5, hid_t native_typeid,
601 NC_TYPE_INFO_T *type;
603 htri_t is_str, equal = 0;
607 if ((
class = H5Tget_class(native_typeid)) < 0)
612 if (
class == H5T_STRING)
614 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
622 else if (
class == H5T_INTEGER ||
class == H5T_FLOAT)
626 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SCHAR)) < 0)
633 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_SHORT)) < 0)
640 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_INT)) < 0)
647 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_FLOAT)) < 0)
654 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_DOUBLE)) < 0)
661 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UCHAR)) < 0)
668 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_USHORT)) < 0)
675 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_UINT)) < 0)
682 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_LLONG)) < 0)
689 if ((equal = H5Tequal(native_typeid, H5T_NATIVE_ULLONG)) < 0)
700 if((type = nc4_rec_find_hdf_type(h5->root_grp, native_typeid)))
702 *xtype = type->nc_typeid;
714 get_type_info2(NC_HDF5_FILE_INFO_T *h5, hid_t datasetid,
715 nc_type *xtype, NC_TYPE_INFO_T **type_info)
717 NC_TYPE_INFO_T *type;
718 htri_t is_str, equal = 0;
719 hid_t
class, native_typeid, hdf_typeid;
728 int type_size[NUM_TYPES] = {
sizeof(char),
sizeof(
char),
sizeof(short),
729 sizeof(
int),
sizeof(float),
sizeof(
double),
730 sizeof(
unsigned char),
sizeof(
unsigned short),
731 sizeof(
unsigned int),
sizeof(
long long),
732 sizeof(
unsigned long long), 0};
735 assert(h5 && xtype && type_info);
743 if (!native_type_constant[1])
745 native_type_constant[1] = H5T_NATIVE_SCHAR;
746 native_type_constant[2] = H5T_NATIVE_SHORT;
747 native_type_constant[3] = H5T_NATIVE_INT;
748 native_type_constant[4] = H5T_NATIVE_FLOAT;
749 native_type_constant[5] = H5T_NATIVE_DOUBLE;
750 native_type_constant[6] = H5T_NATIVE_UCHAR;
751 native_type_constant[7] = H5T_NATIVE_USHORT;
752 native_type_constant[8] = H5T_NATIVE_UINT;
753 native_type_constant[9] = H5T_NATIVE_LLONG;
754 native_type_constant[10] = H5T_NATIVE_ULLONG;
758 if ((hdf_typeid = H5Dget_type(datasetid)) < 0)
763 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
767 if ((
class = H5Tget_class(native_typeid)) < 0)
771 if (
class == H5T_STRING ||
class == H5T_INTEGER ||
class == H5T_FLOAT)
774 if (!(*type_info = calloc(1,
sizeof(NC_TYPE_INFO_T))))
776 (*type_info)->class =
class;
780 if (
class == H5T_STRING)
782 if ((is_str = H5Tis_variable_str(native_typeid)) < 0)
785 if (is_str || H5Tget_size(hdf_typeid) > 1)
790 else if (
class == H5T_INTEGER ||
class == H5T_FLOAT)
792 for (t = 1; t < NUM_TYPES - 1; t++)
794 if ((equal = H5Tequal(native_typeid, native_type_constant[t])) < 0)
799 my_nc_type = nc_type_constant[t];
806 if (
class == H5T_INTEGER)
808 if ((order = H5Tget_order(hdf_typeid)) < 0)
810 if (order == H5T_ORDER_LE)
812 else if (order == H5T_ORDER_BE)
817 (*type_info)->endianness = endianness;
820 *xtype = nc_type_constant[t];
821 (*type_info)->nc_typeid = nc_type_constant[t];
822 (*type_info)->size = type_size[t];
823 if (!((*type_info)->name = malloc((strlen(nc_type_name[t]) + 1) *
sizeof(
char))))
825 strcpy((*type_info)->name, nc_type_name[t]);
826 (*type_info)->class =
class;
827 (*type_info)->hdf_typeid = hdf_typeid;
828 (*type_info)->native_typeid = native_typeid;
829 (*type_info)->close_hdf_typeid = 1;
835 if((type = nc4_rec_find_hdf_type(h5->root_grp, native_typeid)))
837 *xtype = type->nc_typeid;
844 if (H5Tclose(native_typeid) < 0)
846 if (H5Tclose(hdf_typeid) < 0)
859 read_hdf5_att(NC_GRP_INFO_T *grp, hid_t attid, NC_ATT_INFO_T *att)
861 hid_t spaceid = 0, file_typeid = 0;
862 hsize_t dims[1] = {0};
866 hssize_t att_npoints;
867 H5T_class_t att_class;
868 int fixed_len_string = 0;
869 size_t fixed_size = 0;
872 LOG((5,
"read_hdf5_att: att->attnum %d att->name %s "
873 "att->xtype %d att->len %d", att->attnum, att->name,
874 att->xtype, att->len));
877 if ((file_typeid = H5Aget_type(attid)) < 0)
879 if ((att->native_typeid = H5Tget_native_type(file_typeid, H5T_DIR_DEFAULT)) < 0)
881 if ((att_class = H5Tget_class(att->native_typeid)) < 0)
883 if (att_class == H5T_STRING && !H5Tis_variable_str(att->native_typeid))
886 if (!(fixed_size = H5Tget_size(att->native_typeid)))
889 if ((retval = get_netcdf_type(grp->nc4_info, att->native_typeid, &(att->xtype))))
894 if ((spaceid = H5Aget_space(attid)) < 0)
899 if ((att_ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
901 if ((att_npoints = H5Sget_simple_extent_npoints(spaceid)) < 0)
906 if (att_ndims == 0 && att_npoints == 0)
911 dims[0] = att_npoints;
913 else if (att->xtype ==
NC_CHAR)
919 if (!(dims[0] = H5Tget_size(file_typeid)))
926 dims[0] = att_npoints;
936 if (H5Sget_simple_extent_dims(spaceid, dims, NULL) < 0)
947 if ((retval = nc4_get_typelen_mem(grp->nc4_info, att->xtype, 0,
950 if (att_class == H5T_VLEN)
952 if (!(att->vldata = malloc((
unsigned int)(att->len *
sizeof(hvl_t)))))
954 if (H5Aread(attid, att->native_typeid, att->vldata) < 0)
959 if (!(att->stdata = calloc(att->len,
sizeof(
char *))))
972 if (fixed_len_string)
975 char *contig_buf, *cur;
978 if (!(contig_buf = malloc(att->len * fixed_size *
sizeof(
char))))
982 if (H5Aread(attid, att->native_typeid, contig_buf) < 0)
989 for (i = 0; i < att->len; i++)
991 if (!(att->stdata[i] = malloc(fixed_size)))
993 strncpy(att->stdata[i], cur, fixed_size);
1003 if (H5Aread(attid, att->native_typeid, att->stdata) < 0)
1009 if (!(att->data = malloc((
unsigned int)(att->len * type_size))))
1011 if (H5Aread(attid, att->native_typeid, att->data) < 0)
1016 if (H5Tclose(file_typeid) < 0)
1018 if (H5Sclose(spaceid) < 0)
1027 if (H5Tclose(file_typeid) < 0)
1029 if (spaceid > 0 && H5Sclose(spaceid) < 0)
1041 read_type(NC_GRP_INFO_T *grp,
char *type_name)
1043 NC_TYPE_INFO_T *type;
1045 hid_t hdf_typeid, native_typeid = 0;
1047 hid_t member_hdf_typeid, base_hdf_typeid = 0;
1048 char *member_name = NULL;
1049 size_t type_size = 0, member_offset;
1057 assert(grp && type_name);
1062 LOG((4,
"read_type: type_name %s grp->name %s", type_name, grp->name));
1064 if ((hdf_typeid = H5Topen2(grp->hdf_grpid, type_name, H5P_DEFAULT)) < 0)
1068 if ((native_typeid = H5Tget_native_type(hdf_typeid, H5T_DIR_DEFAULT)) < 0)
1072 if (!(type_size = H5Tget_size(native_typeid)))
1074 LOG((5,
"type_size %d", type_size));
1077 if ((
class = H5Tget_class(hdf_typeid)) < 0)
1092 if ((ret = H5Tis_variable_str(hdf_typeid)) < 0)
1102 if (!(base_hdf_typeid = H5Tget_super(native_typeid)))
1106 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1110 if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid,
1113 LOG((5,
"base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1114 base_hdf_typeid, type_size, base_nc_type));
1120 if (!(type_size = H5Tget_size(hdf_typeid)))
1122 LOG((5,
"type_size %d", type_size));
1129 if (!(base_hdf_typeid = H5Tget_super(hdf_typeid)))
1132 if (!(type_size = H5Tget_size(base_hdf_typeid)))
1135 if ((retval = get_netcdf_type(grp->nc4_info, base_hdf_typeid,
1138 LOG((5,
"base_hdf_typeid 0x%x type_size %d base_nc_type %d",
1139 base_hdf_typeid, type_size, base_nc_type));
1142 LOG((0,
"unknown class"));
1147 if ((retval = nc4_type_list_add(&grp->type, &type)))
1152 type->nc_typeid = grp->nc4_info->next_typeid++;
1153 type->size = type_size;
1154 if (!(type->name = malloc((strlen(type_name) + 1) *
sizeof(
char))))
1156 strcpy(type->name, type_name);
1157 type->class = ud_type_type;
1158 type->base_nc_type = base_nc_type;
1160 type->hdf_typeid = hdf_typeid;
1161 type->native_typeid = native_typeid;
1166 if ((nmembers = H5Tget_nmembers(hdf_typeid)) < 0)
1168 LOG((5,
"compound type has %d members", nmembers));
1169 for (m = 0; m < nmembers; m++)
1171 H5T_class_t mem_class;
1172 hid_t member_native_typeid;
1179 if ((member_hdf_typeid = H5Tget_member_type(type->native_typeid, m)) < 0)
1181 if ((member_native_typeid = H5Tget_native_type(member_hdf_typeid, H5T_DIR_DEFAULT)) < 0)
1185 member_name = H5Tget_member_name(type->native_typeid, m);
1186 if (!member_name || strlen(member_name) >
NC_MAX_NAME)
1190 member_offset = H5Tget_member_offset(type->native_typeid, m);
1193 if ((mem_class = H5Tget_class(member_hdf_typeid)) < 0)
1195 if (mem_class == H5T_ARRAY)
1197 if ((ndims = H5Tget_array_ndims(member_hdf_typeid)) < 0)
1199 if (H5Tget_array_dims(member_hdf_typeid, dims, NULL) != ndims)
1201 for (d = 0; d < ndims; d++)
1202 dim_size[d] = dims[d];
1204 if ((retval = get_netcdf_type(grp->nc4_info, H5Tget_super(member_hdf_typeid),
1211 if ((retval = get_netcdf_type(grp->nc4_info, member_native_typeid,
1219 if ((retval = nc4_field_list_add(&type->field, type->num_fields++, member_name,
1220 member_offset, H5Tget_super(member_hdf_typeid),
1221 H5Tget_super(member_native_typeid),
1222 member_xtype, ndims, dim_size)))
1227 if ((retval = nc4_field_list_add(&type->field, type->num_fields++, member_name,
1228 member_offset, member_hdf_typeid, member_native_typeid,
1229 member_xtype, 0, NULL)))
1237 else if (ud_type_type ==
NC_VLEN)
1239 type->base_hdf_typeid = base_hdf_typeid;
1241 else if (ud_type_type ==
NC_ENUM)
1244 type->base_hdf_typeid = base_hdf_typeid;
1247 if ((type->num_enum_members = H5Tget_nmembers(hdf_typeid)) < 0)
1251 if (!(value = calloc(1, type_size)))
1255 for (i = 0; i < type->num_enum_members; i++)
1258 if (!(member_name = H5Tget_member_name(hdf_typeid, i))) {
1259 if(value) free(value);
1262 if (!member_name || strlen(member_name) >
NC_MAX_NAME) {
1263 if(value) free(value);
1266 if (H5Tget_member_value(hdf_typeid, i, value) < 0)
1270 if ((retval = nc4_enum_member_add(&type->enum_member, type->size,
1271 member_name, value)))
1290 read_var(NC_GRP_INFO_T *grp, hid_t datasetid,
const char *obj_name,
1291 size_t ndims,
int is_scale,
int num_scales, hid_t access_pid)
1293 NC_VAR_INFO_T *var = NULL;
1298 char att_name[NC_MAX_HDF5_NAME + 1];
1300 #define CD_NELEMS_ZLIB 1
1301 #define CD_NELEMS_SZIP 4
1302 H5Z_filter_t filter;
1304 unsigned int cd_values[CD_NELEMS_SZIP];
1305 size_t cd_nelems = CD_NELEMS_SZIP;
1307 H5D_fill_value_t fill_status;
1308 H5D_layout_t layout;
1314 assert(obj_name && grp);
1315 LOG((4,
"read_var: obj_name %s", obj_name));
1318 if ((retval = nc4_var_list_add(&grp->var, &var)))
1325 var->hdf_datasetid = datasetid;
1326 var->varid = grp->nvars++;
1334 if (!(var->dim = calloc(var->ndims,
sizeof(NC_DIM_INFO_T *))))
1336 if (!(var->dimids = calloc(var->ndims,
sizeof(
int))))
1341 if ((H5Pget_chunk_cache(access_pid, &(var->chunk_cache_nelems),
1342 &(var->chunk_cache_size), &rdcc_w0)) < 0)
1344 var->chunk_cache_preemption = rdcc_w0;
1347 if (!(var->name = malloc((strlen(obj_name) + 1) *
sizeof(
char))))
1353 if (!strncmp(obj_name, NON_COORD_PREPEND, strlen(NON_COORD_PREPEND)))
1357 strcpy(var->name, &obj_name[strlen(NON_COORD_PREPEND)]);
1360 strcpy(var->name, obj_name);
1365 if ((propid = H5Dget_create_plist(datasetid)) < 0)
1372 if ((layout = H5Pget_layout(propid)) < -1)
1374 if (layout == H5D_CHUNKED)
1378 if (!(var->chunksizes = malloc(var->ndims *
sizeof(
size_t))))
1380 for (d = 0; d < var->ndims; d++)
1381 var->chunksizes[d] = chunksize[d];
1383 else if (layout == H5D_CONTIGUOUS)
1388 if ((num_filters = H5Pget_nfilters(propid)) < 0)
1390 for (f = 0; f < num_filters; f++)
1392 if ((filter = H5Pget_filter2(propid, f, NULL, &cd_nelems,
1393 cd_values, 0, NULL, NULL)) < 0)
1397 case H5Z_FILTER_SHUFFLE:
1400 case H5Z_FILTER_FLETCHER32:
1401 var->fletcher32 = 1;
1403 case H5Z_FILTER_DEFLATE:
1405 if (cd_nelems != CD_NELEMS_ZLIB ||
1406 cd_values[0] > MAX_DEFLATE_LEVEL)
1408 var->deflate_level = cd_values[0];
1410 case H5Z_FILTER_SZIP:
1412 if (cd_nelems != CD_NELEMS_SZIP)
1414 var->options_mask = cd_values[0];
1415 var->pixels_per_block = cd_values[1];
1418 LOG((1,
"Yikes! Unknown filter type found on dataset!"));
1424 if ((retval = get_type_info2(grp->nc4_info, datasetid,
1425 &var->xtype, &var->type_info)))
1429 if (H5Pfill_value_defined(propid, &fill_status) < 0)
1433 if (fill_status == H5D_FILL_VALUE_USER_DEFINED)
1436 if (!var->fill_value)
1438 if (var->type_info->class ==
NC_VLEN)
1440 if (!(var->fill_value = malloc(
sizeof(
nc_vlen_t))))
1443 else if (var->type_info->size)
1445 if (!(var->fill_value = malloc(var->type_info->size)))
1450 if (!(var->fill_value = malloc(
sizeof(
char *))))
1456 if (H5Pget_fill_value(propid, var->type_info->native_typeid,
1457 var->fill_value) < 0)
1472 if ((retval = read_coord_dimids(var)))
1477 var->dimids[0] = grp->dim->dimid;
1478 var->dim[0] = grp->dim;
1482 if (num_scales && ndims &&
1483 !(var->dimscale_attached = calloc(ndims,
sizeof(
int))))
1489 if (!is_scale && num_scales)
1493 if (!(var->dimscale_hdf5_objids = malloc(ndims *
sizeof(
struct hdf5_objid))))
1495 for (d = 0; d < var->ndims; d++)
1497 LOG((5,
"read_var: about to iterate over scales for dim %d", d));
1498 if (H5DSiterate_scales(var->hdf_datasetid, d, NULL, dimscale_visitor,
1499 &(var->dimscale_hdf5_objids[d])) < 0)
1507 var->dimscale_attached[d]++;
1513 if ((natts = H5Aget_num_attrs(var->hdf_datasetid)) < 0)
1515 for (a = 0; a < natts; a++)
1520 if (attid && H5Aclose(attid) < 0)
1524 if ((attid = H5Aopen_idx(var->hdf_datasetid, (
unsigned int)a)) < 0)
1526 if (H5Aget_name(attid, NC_MAX_HDF5_NAME, att_name) < 0)
1528 LOG((4,
"read_var: a %d att_name %s", a, att_name));
1531 if (strcmp(att_name, REFERENCE_LIST) &&
1532 strcmp(att_name, CLASS) &&
1533 strcmp(att_name, DIMENSION_LIST) &&
1534 strcmp(att_name, NAME) &&
1535 strcmp(att_name, COORDINATES))
1539 if (!strcmp(att_name, NC_DIMID_ATT_NAME))
1546 if ((retval = nc4_att_list_add(&var->att)))
1548 for (att = var->att; att->next; att = att->next)
1552 att->attnum = var->natts++;
1553 if (!(att->name = malloc((strlen(att_name) + 1) *
sizeof(
char))))
1555 strcpy(att->name, att_name);
1559 if ((retval = read_hdf5_att(grp, attid, att)))
1563 if ((retval = nc4_att_list_del(&var->att, att)))
1578 if ((retval = nc4_adjust_var_cache(grp, var)))
1584 if(nc4_var_list_del(&grp->var, var))
1587 if (propid > 0 && H5Pclose(propid) < 0)
1592 if (attid > 0 && H5Aclose(attid) < 0)
1600 read_grp_atts(NC_GRP_INFO_T *grp)
1605 NC_TYPE_INFO_T *type;
1606 char obj_name[NC_MAX_HDF5_NAME + 1];
1610 num_obj = H5Aget_num_attrs(grp->hdf_grpid);
1611 for (i = 0; i < num_obj; i++)
1615 if ((attid = H5Aopen_idx(grp->hdf_grpid, (
unsigned int)i)) < 0)
1617 if (H5Aget_name(attid,
NC_MAX_NAME + 1, obj_name) < 0)
1619 LOG((3,
"reading attribute of _netCDF group, named %s", obj_name));
1625 if (!strcmp(obj_name, NC3_STRICT_ATT_NAME))
1630 if ((retval = nc4_att_list_add(&grp->att)))
1632 for (att = grp->att; att->next; att = att->next)
1637 if (!(att->name = malloc((max_len + 1) *
sizeof(
char))))
1639 strncpy(att->name, obj_name, max_len);
1640 att->name[max_len] = 0;
1641 att->attnum = grp->natts++;
1642 if ((retval = read_hdf5_att(grp, attid, att)))
1646 if ((retval = nc4_att_list_del(&grp->att, att)))
1654 if ((retval = nc4_find_type(grp->nc4_info, att->xtype, &type)))
1657 att->class = type->class;
1662 if (attid > 0 && H5Aclose(attid) < 0)
1670 read_dataset(NC_GRP_INFO_T *grp,
const char *obj_name)
1672 hid_t datasetid = 0;
1673 hid_t spaceid = 0, access_pid = 0;
1675 hsize_t *dims = NULL;
1676 hsize_t *max_dims = NULL;
1678 int dim_without_var = 0;
1683 if ((datasetid = H5Dopen2(grp->hdf_grpid, obj_name, H5P_DEFAULT)) < 0)
1687 if ((access_pid = H5Dget_access_plist(datasetid)) < 0)
1694 if ((spaceid = H5Dget_space(datasetid)) < 0)
1699 if ((ndims = H5Sget_simple_extent_ndims(spaceid)) < 0)
1707 if( (dims = (hsize_t*)malloc(
sizeof(hsize_t)*ndims)) == NULL)
1709 if( (max_dims = (hsize_t*)malloc(
sizeof(hsize_t)*ndims)) == NULL)
1712 if (H5Sget_simple_extent_dims(spaceid, dims, max_dims) < 0)
1716 if ((is_scale = H5DSis_scale(datasetid)) < 0)
1721 if ((retval = read_scale(grp, datasetid, obj_name, dims[0],
1722 max_dims[0], &dim_without_var)))
1730 num_scales = H5DSget_num_scales(datasetid, 0);
1738 if (!dim_without_var)
1739 if ((retval = read_var(grp, datasetid, obj_name, ndims,
1740 is_scale, num_scales, access_pid)))
1743 if (access_pid && H5Pclose(access_pid) < 0)
1748 if (spaceid && H5Sclose(spaceid) < 0)
1754 if(dims) free(dims);
1755 if(max_dims) free(max_dims);
1760 if (access_pid && H5Pclose(access_pid) < 0)
1765 if (datasetid && H5Dclose(datasetid) < 0)
1767 if (spaceid && H5Sclose(spaceid) <0)
1773 if(dims) free(dims);
1774 if(max_dims) free(max_dims);
1828 #define USE_ITERATE_CODE
1829 #ifdef USE_ITERATE_CODE
1832 nc4_rec_read_types_cb(hid_t grpid,
const char *name,
const H5L_info_t *info,
1836 H5I_type_t otype=-1;
1838 NC_GRP_INFO_T *child_grp;
1839 NC_GRP_INFO_T *grp = (NC_GRP_INFO_T *) (_op_data);
1840 NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;
1843 if ((oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0)
1844 return H5_ITER_ERROR;
1846 if ((otype = H5Iget_type( oid ))<0) {
1848 return H5_ITER_ERROR;
1855 if (otype == H5I_GROUP)
1857 LOG((3,
"found group %s", oname));
1858 if (nc4_grp_list_add(&(grp->children), h5->next_nc_grpid++,
1859 grp, grp->nc4_info->controller, oname, &child_grp))
1860 return H5_ITER_ERROR;
1862 if (nc4_rec_read_types(child_grp))
1863 return H5_ITER_ERROR;
1865 else if (otype == H5I_DATATYPE)
1867 LOG((3,
"found datatype %s", oname));
1868 if (read_type(grp, oname))
1869 return H5_ITER_ERROR;
1872 return (H5_ITER_CONT);
1876 nc4_rec_read_types(NC_GRP_INFO_T *grp)
1881 unsigned crt_order_flags = 0;
1882 NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;
1884 assert(grp && grp->name);
1885 LOG((3,
"nc4_rec_read_types: grp->name %s", grp->name));
1889 if (!grp->hdf_grpid)
1893 if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid,
1894 grp->name, H5P_DEFAULT)) < 0)
1899 if ((grp->hdf_grpid = H5Gopen2(grp->nc4_info->hdfid,
1900 "/", H5P_DEFAULT)) < 0)
1904 assert(grp->hdf_grpid > 0);
1906 pid = H5Gget_create_plist(grp->hdf_grpid);
1907 H5Pget_link_creation_order(pid, &crt_order_flags);
1908 if(H5Pclose(pid) < 0)
1911 crt_order_flags = crt_order_flags & H5_INDEX_CRT_ORDER;
1913 if (crt_order_flags == H5_INDEX_CRT_ORDER)
1915 res = H5Literate(grp->hdf_grpid, H5_INDEX_CRT_ORDER, H5_ITER_INC,
1916 &idx, nc4_rec_read_types_cb, (
void *)grp);
1920 if (!idx && !h5->no_write)
1923 res = H5Literate(grp->hdf_grpid, H5_INDEX_NAME, H5_ITER_INC,
1924 &idx, nc4_rec_read_types_cb, (
void *)grp);
1932 nc4_rec_read_vars_cb(hid_t grpid,
const char *name,
const H5L_info_t *info,
1936 H5I_type_t otype=-1;
1938 NC_GRP_INFO_T *child_grp;
1939 NC_GRP_INFO_T *grp = (NC_GRP_INFO_T *) (_op_data);
1941 NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;
1946 if ((oid = H5Oopen(grpid, name, H5P_DEFAULT)) < 0)
1947 return H5_ITER_ERROR;
1949 if ((otype = H5Iget_type( oid ))<0) {
1951 return H5_ITER_ERROR;
1961 LOG((3,
"re-encountering group %s", oname));
1964 for (child_grp = grp->children; child_grp; child_grp = child_grp->next)
1965 if (!strcmp(child_grp->name, oname))
1968 return H5_ITER_ERROR;
1971 if (nc4_rec_read_vars(child_grp))
1972 return H5_ITER_ERROR;
1978 LOG((3,
"found dataset %s", oname));
1982 if ((retval = read_dataset(grp, oname))) {
1984 return H5_ITER_CONT;
1986 return H5_ITER_ERROR;
1991 LOG((3,
"already handled type %s", oname));
1994 LOG((0,
"Unknown object class %d in nc4_rec_read_vars!", otype));
1996 return (H5_ITER_CONT);
2000 nc4_rec_read_vars(NC_GRP_INFO_T *grp)
2006 unsigned crt_order_flags = 0;
2007 NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;
2009 assert(grp && grp->name && grp->hdf_grpid > 0);
2010 LOG((3,
"nc4_rec_read_vars: grp->name %s", grp->name));
2012 pid = H5Gget_create_plist(grp->hdf_grpid);
2013 H5Pget_link_creation_order(pid, &crt_order_flags);
2014 if(H5Pclose(pid) < 0)
2017 crt_order_flags = crt_order_flags & H5_INDEX_CRT_ORDER;
2019 if (crt_order_flags == H5_INDEX_CRT_ORDER)
2021 res = H5Literate(grp->hdf_grpid, H5_INDEX_CRT_ORDER, H5_ITER_INC,
2022 &idx, nc4_rec_read_vars_cb, (
void *)grp);
2026 if (!idx && !h5->no_write)
2029 res = H5Literate(grp->hdf_grpid, H5_INDEX_NAME, H5_ITER_INC,
2030 &idx, nc4_rec_read_vars_cb, (
void *)grp);
2036 if ((retval = read_grp_atts(grp)))
2048 struct nc_hdf5_link_info
2051 H5I_type_t obj_type;
2073 visit_link(hid_t g_id,
const char *name,
const H5L_info_t *info,
2084 strncpy(((
struct nc_hdf5_link_info *)op_data)->name, name,
2088 if ((
id = H5Oopen_by_addr(g_id, info->u.address)) < 0)
2092 if ((((
struct nc_hdf5_link_info *)op_data)->obj_type = H5Iget_type(
id)) < 0)
2096 if (H5Oclose(
id) < 0)
2107 nc4_iterate_link(
int *ordering_checked,
int *creation_ordering,
2108 hid_t grpid, hsize_t *idx,
struct nc_hdf5_link_info *link_info)
2112 if (*creation_ordering)
2117 res = H5Literate(grpid, H5_INDEX_CRT_ORDER, H5_ITER_INC,
2118 idx, visit_link, (
void *)link_info);
2119 if (res < 0 && *ordering_checked)
2124 if (!*creation_ordering || res < 0)
2126 if (H5Literate(grpid, H5_INDEX_NAME, H5_ITER_INC, idx,
2127 visit_link, link_info) != 1)
2131 *creation_ordering = 0;
2134 *ordering_checked = 1;
2140 nc4_rec_read_types(NC_GRP_INFO_T *grp)
2143 NC_HDF5_FILE_INFO_T *h5 = grp->nc4_info;
2144 NC_GRP_INFO_T *child_grp;
2146 struct nc_hdf5_link_info link_info;
2147 int ordering_checked = 0;
2148 int creation_ordering = 1;
2151 assert(grp && grp->name);
2152 LOG((3,
"nc4_rec_read_types: grp->name %s", grp->name));
2156 if (!grp->hdf_grpid)
2160 if ((grp->hdf_grpid = H5Gopen2(grp->parent->hdf_grpid,
2161 grp->name, H5P_DEFAULT)) < 0)
2166 if ((grp->hdf_grpid = H5Gopen2(grp->nc4_info->hdfid,
2167 "/", H5P_DEFAULT)) < 0)
2171 assert(grp->hdf_grpid > 0);
2174 if (H5Gget_num_objs(grp->hdf_grpid, &num_obj) < 0)
2178 for (i = 0; i < num_obj; i++)
2180 if ((retval = nc4_iterate_link(&ordering_checked, &creation_ordering,
2181 grp->hdf_grpid, &idx, &link_info)))
2185 if (!i && !creation_ordering && !h5->no_write)
2189 if (link_info.obj_type == H5I_GROUP)
2191 LOG((3,
"found group %s", link_info.name));
2192 if ((retval = nc4_grp_list_add(&(grp->children), h5->next_nc_grpid++,
2193 grp, grp->file, link_info.name, &child_grp)))
2195 if ((retval = nc4_rec_read_types(child_grp)))
2198 else if (link_info.obj_type == H5I_DATATYPE)
2200 LOG((3,
"found datatype %s", link_info.name));
2201 if ((retval = read_type(grp, link_info.name)))
2213 nc4_rec_read_vars(NC_GRP_INFO_T *grp)
2216 NC_GRP_INFO_T *child_grp;
2217 struct nc_hdf5_link_info link_info;
2219 int ordering_checked = 0;
2220 int creation_ordering = 1;
2223 assert(grp && grp->name && grp->hdf_grpid > 0);
2224 LOG((3,
"nc4_rec_read_vars: grp->name %s", grp->name));
2227 if (H5Gget_num_objs(grp->hdf_grpid, &num_obj) < 0)
2231 for (i = 0; i < num_obj; i++)
2233 if ((retval = nc4_iterate_link(&ordering_checked, &creation_ordering,
2234 grp->hdf_grpid, &idx, &link_info)))
2238 switch(link_info.obj_type)
2241 LOG((3,
"re-encountering group %s", link_info.name));
2244 for (child_grp = grp->children; child_grp; child_grp = child_grp->next)
2245 if (!strcmp(child_grp->name, link_info.name))
2251 if ((retval = nc4_rec_read_vars(child_grp)))
2255 LOG((3,
"found dataset %s", link_info.name));
2259 if ((retval = read_dataset(grp, link_info.name)))
2263 LOG((3,
"already handled type %s", link_info.name));
2266 LOG((0,
"Unknown object class %d in nc4_rec_read_vars!",
2267 link_info.obj_type));
2272 if ((retval = read_grp_atts(grp)))
2283 nc4_open_file(
const char *path,
int mode, MPI_Comm comm,
2284 MPI_Info info, NC *nc)
2286 hid_t fapl_id = H5P_DEFAULT;
2287 unsigned flags = (mode &
NC_WRITE) ?
2288 H5F_ACC_RDWR : H5F_ACC_RDONLY;
2290 NC_HDF5_FILE_INFO_T* nc4_info = NULL;
2292 LOG((3,
"nc4_open_file: path %s mode %d", path, mode));
2295 if(mode & NC_DISKLESS)
2299 if ((retval = nc4_nc4f_list_add(nc, path, mode)))
2301 nc4_info = NC4_DATA(nc);
2302 assert(nc4_info && nc4_info->root_grp);
2307 if ((fapl_id = H5Pcreate(H5P_FILE_ACCESS)) < 0)
2313 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_SEMI))
2316 if (H5Pset_fclose_degree(fapl_id, H5F_CLOSE_STRONG))
2323 if (mode & NC_MPIIO || mode & NC_MPIPOSIX)
2325 nc4_info->parallel++;
2326 if (mode & NC_MPIIO)
2328 LOG((4,
"opening parallel file with MPI/IO"));
2329 if (H5Pset_fapl_mpio(fapl_id, comm, info) < 0)
2334 LOG((4,
"opening parallel file with MPI/posix"));
2335 if (H5Pset_fapl_mpiposix(fapl_id, comm, 0) < 0)
2340 if (H5Pset_cache(fapl_id, 0, nc4_chunk_cache_nelems, nc4_chunk_cache_size,
2341 nc4_chunk_cache_preemption) < 0)
2343 LOG((4,
"nc4_open_file: set HDF raw chunk cache to size %d nelems %d preemption %f",
2344 nc4_chunk_cache_size, nc4_chunk_cache_nelems, nc4_chunk_cache_preemption));
2350 if ((nc4_info->hdfid = H5Fopen(path, flags, fapl_id)) < 0)
2355 nc4_info->no_write++;
2361 if ((retval = nc4_rec_read_types(nc4_info->root_grp)))
2363 if ((retval = nc4_rec_read_vars(nc4_info->root_grp)))
2368 if ((retval = nc4_rec_match_dimscales(nc4_info->root_grp)))
2374 log_metadata_nc(nc);
2378 if (H5Pclose(fapl_id) < 0)
2387 if (fapl_id != H5P_DEFAULT) H5Pclose(fapl_id);
2391 if (!nc4_info)
return retval;
2392 close_netcdf4_file(nc4_info,1);
2394 if (nc4_info->hdfid > 0) H5Fclose(nc4_info->hdfid);
2395 if (nc4_info->root_grp) {
2396 free(nc4_info->root_grp->name);
2397 free(nc4_info->root_grp);
2407 get_netcdf_type_from_hdf4(NC_HDF5_FILE_INFO_T *h5, int32 hdf4_typeid,
2408 nc_type *xtype, NC_TYPE_INFO_T *type_info)
2411 assert(h5 && xtype);
2459 if (hdf4_typeid == DFNT_FLOAT32 || hdf4_typeid == DFNT_FLOAT64)
2460 type_info->class = H5T_FLOAT;
2461 else if (hdf4_typeid == DFNT_CHAR)
2462 type_info->class = H5T_STRING;
2464 type_info->class = H5T_INTEGER;
2466 type_info->nc_typeid = *xtype;
2467 if (type_info->name)
2468 free(type_info->name);
2469 if (!(type_info->name = malloc((strlen(nc_type_name[t]) + 1) *
sizeof(
char))))
2471 strcpy(type_info->name, nc_type_name[t]);
2481 nc4_open_hdf4_file(
const char *path,
int mode, NC *nc)
2484 NC_HDF5_FILE_INFO_T *h5;
2488 int32 num_datasets, num_gatts;
2492 NC_HDF5_FILE_INFO_T* nc4_info = NULL;
2494 LOG((3,
"nc4_open_hdf4_file: path %s mode %d", path, mode));
2498 if (mode & NC_WRITE)
2502 if ((retval = nc4_nc4f_list_add(nc, path, mode)))
2504 nc4_info = NC4_DATA(nc);
2505 assert(nc4_info && nc4_info->root_grp);
2512 if ((h5->sdid = SDstart(path, DFACC_READ)) == FAIL)
2516 if (SDfileinfo(h5->sdid, &num_datasets, &num_gatts))
2520 for (a = 0; a < num_gatts; a++)
2522 int32 att_data_type, att_count;
2523 size_t att_type_size;
2526 if ((retval = nc4_att_list_add(&h5->root_grp->att)))
2528 for (att = h5->root_grp->att; att->next; att = att->next)
2530 att->attnum = grp->natts++;
2536 if (SDattrinfo(h5->sdid, a, att->name, &att_data_type, &att_count))
2538 if ((retval = get_netcdf_type_from_hdf4(h5, att_data_type,
2539 &att->xtype, NULL)))
2541 att->len = att_count;
2544 if ((retval = nc4_get_typelen_mem(h5, att->xtype, 0, &att_type_size)))
2546 if (!(att->data = malloc(att_type_size * att->len)))
2550 if (SDreadattr(h5->sdid, a, att->data))
2555 for (v = 0; v < num_datasets; v++)
2557 int32 data_type, num_atts;
2566 int32 *dimsize = NULL;
2567 size_t var_type_size;
2571 if ((retval = nc4_var_list_add(&grp->var, &var))) {
2575 var->varid = grp->nvars++;
2577 var->written_to = 1;
2580 if ((var->sdsid = SDselect(h5->sdid, v)) == FAIL) {
2590 if (SDgetinfo(var->sdsid, var->name, &rank, NULL, &data_type, &num_atts))
2593 if(!(dimsize = (int32*)malloc(
sizeof(int32)*rank))) {
2598 if (SDgetinfo(var->sdsid, var->name, &rank, dimsize, &data_type, &num_atts)) {
2599 if(dimsize) free(dimsize);
2604 var->hdf4_data_type = data_type;
2607 if (!(var->type_info = calloc(1,
sizeof(NC_TYPE_INFO_T)))) {
2608 if(dimsize) free(dimsize);
2612 if ((retval = get_netcdf_type_from_hdf4(h5, data_type, &var->xtype, var->type_info))) {
2613 if(dimsize) free(dimsize);
2617 if ((retval = nc4_get_typelen_mem(h5, var->xtype, 0, &var_type_size))) {
2618 if(dimsize) free(dimsize);
2622 var->type_info->size = var_type_size;
2623 LOG((3,
"reading HDF4 dataset %s, rank %d netCDF type %d", var->name,
2627 if (!(var->fill_value = malloc(var_type_size))) {
2628 if(dimsize) free(dimsize);
2632 if (SDgetfillvalue(var->sdsid, var->fill_value))
2635 free(var->fill_value);
2636 var->fill_value = NULL;
2642 if (!(var->dim = malloc(
sizeof(NC_DIM_INFO_T *) * var->ndims))) {
2643 if(dimsize) free(dimsize);
2647 if (!(var->dimids = malloc(
sizeof(
int) * var->ndims))) {
2648 if(dimsize) free(dimsize);
2655 for (d = 0; d < var->ndims; d++)
2657 int32 dimid, dim_len, dim_data_type, dim_num_attrs;
2661 if ((dimid = SDgetdimid(var->sdsid, d)) == FAIL) {
2662 if(dimsize) free(dimsize);
2665 if (SDdiminfo(dimid, dim_name, &dim_len, &dim_data_type,
2668 if(dimsize) free(dimsize);
2674 for (dim = grp->dim; dim; dim = dim->next)
2675 if (!strcmp(dim->name, dim_name))
2681 LOG((4,
"adding dimension %s for HDF4 dataset %s",
2682 dim_name, var->name));
2683 if ((retval = nc4_dim_list_add(&grp->dim)))
2687 dim->dimid = grp->nc4_info->next_dimid++;
2692 strcpy(dim->name, dim_name);
2696 dim->len = *dimsize;
2700 var->dimids[d] = dim->dimid;
2704 for (a = 0; a < num_atts; a++)
2706 int32 att_data_type, att_count;
2707 size_t att_type_size;
2710 if ((retval = nc4_att_list_add(&var->att))) {
2711 if(dimsize) free(dimsize);
2714 for (att = var->att; att->next; att = att->next)
2716 att->attnum = var->natts++;
2721 if(dimsize) free(dimsize);
2724 if (SDattrinfo(var->sdsid, a, att->name, &att_data_type, &att_count)) {
2725 if(dimsize) free(dimsize);
2728 if ((retval = get_netcdf_type_from_hdf4(h5, att_data_type,
2729 &att->xtype, NULL))) {
2730 if(dimsize) free(dimsize);
2734 att->len = att_count;
2737 if ((retval = nc4_get_typelen_mem(h5, att->xtype, 0, &att_type_size))) {
2738 if(dimsize) free(dimsize);
2741 if (!(att->data = malloc(att_type_size * att->len))) {
2742 if(dimsize) free(dimsize);
2747 if (SDreadattr(var->sdsid, a, att->data)) {
2748 if(dimsize) free(dimsize);
2752 if(dimsize) free(dimsize);
2758 log_metadata_nc(h5->root_grp->nc4_info->controller);
2766 NC4_open(
const char *path,
int mode,
int basepe,
size_t *chunksizehintp,
2767 int use_parallel,
void *mpidata, NC_Dispatch *dispatch, NC *nc_file)
2774 int comm = 0, info = 0;
2778 assert(nc_file && path);
2780 LOG((1,
"nc_open_file: path %s mode %d comm %d info %d",
2781 path, mode, comm, info));
2786 NC_MPI_INFO *nmi = (NC_MPI_INFO *)mpidata;
2787 comm = nmi->comm; info = nmi->info;
2794 if (H5Eset_auto(NULL, NULL) < 0)
2795 LOG((0,
"Couldn't turn off HDF5 error messages!"));
2796 LOG((1,
"HDF5 error messages turned off!"));
2803 if (mode & ~(NC_WRITE |
NC_SHARE | NC_MPIIO | NC_MPIPOSIX |
2805 (mode & NC_MPIIO && mode & NC_MPIPOSIX))
2812 if(mode & NC_PNETCDF) {
2814 int pnetcdf_nvars, i;
2815 NC_HDF5_FILE_INFO_T* nc4_info;
2818 res = nc4_nc4f_list_add(nc_file, path, mode);
2820 nc4_info = NC4_DATA(nc_file);
2823 res = ncmpi_open(comm, path, mode, info, &(nc_file->int_ncid));
2824 nc4_info->pnetcdf_file++;
2828 res = ncmpi_begin_indep_data(nc_file->int_ncid);
2834 res = ncmpi_inq_nvars(nc_file->int_ncid, &pnetcdf_nvars);
2835 for (i = 0; i < pnetcdf_nvars; i++)
2836 res = ncmpi_inq_varndims(nc_file->int_ncid, i,
2837 &(nc4_info->pnetcdf_ndims[i]));
2844 if ((res = nc_check_for_hdf(path, use_parallel, comm, info, &hdf_file)))
2847 if (hdf_file == NC_HDF5_FILE)
2849 nc_file->int_ncid = nc_file->ext_ncid;
2850 res = nc4_open_file(path, mode, comm, info, nc_file);
2852 else if (hdf_file == NC_HDF4_FILE)
2854 nc_file->int_ncid = nc_file->ext_ncid;
2855 res = nc4_open_hdf4_file(path, mode, nc_file);
2871 NC4_set_fill(
int ncid,
int fillmode,
int *old_modep)
2874 NC_HDF5_FILE_INFO_T* nc4_info;
2876 LOG((2,
"nc_set_fill: ncid 0x%x fillmode %d", ncid, fillmode));
2878 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
2883 if (nc4_info->no_write)
2892 *old_modep = nc4_info->fill_mode;
2894 nc4_info->fill_mode = fillmode;
2898 if (nc4_info->pnetcdf_file)
2899 return ncmpi_set_fill(nc->int_ncid, fillmode, old_modep);
2912 NC_HDF5_FILE_INFO_T* nc4_info;
2914 LOG((1,
"nc_redef: ncid 0x%x", ncid));
2917 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
2923 if (nc4_info->pnetcdf_file)
2924 return ncmpi_redef(nc->int_ncid);
2928 if (nc4_info->flags & NC_INDEF)
2932 if (nc4_info->no_write)
2936 nc4_info->flags |= NC_INDEF;
2948 NC4__enddef(
int ncid,
size_t h_minfree,
size_t v_align,
2949 size_t v_minfree,
size_t r_align)
2951 if (nc4_find_nc_file(ncid,NULL) == NULL)
2954 return NC4_enddef(ncid);
2959 static int NC4_enddef(
int ncid)
2962 NC_HDF5_FILE_INFO_T* nc4_info;
2964 LOG((1,
"nc_enddef: ncid 0x%x", ncid));
2966 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
2971 if (nc4_info->pnetcdf_file)
2974 res = ncmpi_enddef(nc->int_ncid);
2977 if (nc4_info->pnetcdf_access_mode == NC_INDEPENDENT)
2978 res = ncmpi_begin_indep_data(nc->int_ncid);
2984 return nc4_enddef_netcdf4_file(nc4_info);
2990 sync_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
2995 LOG((3,
"sync_netcdf4_file"));
2999 if (h5->flags & NC_INDEF)
3005 h5->flags ^= NC_INDEF;
3014 log_metadata_nc(h5->root_grp->nc4_info->controller);
3020 int bad_coord_order = 0;
3021 if ((retval = nc4_rec_write_types(h5->root_grp)))
3023 if ((retval = nc4_rec_detect_need_to_preserve_dimids(h5->root_grp, &bad_coord_order)))
3025 if ((retval = nc4_rec_write_metadata(h5->root_grp, bad_coord_order)))
3029 H5Fflush(h5->hdfid, H5F_SCOPE_GLOBAL);
3045 NC_HDF5_FILE_INFO_T* nc4_info;
3047 LOG((2,
"nc_sync: ncid 0x%x", ncid));
3049 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
3055 if (nc4_info->pnetcdf_file)
3056 return ncmpi_sync(nc->int_ncid);
3060 if (nc4_info && nc4_info->flags & NC_INDEF)
3064 if ((retval = NC4_enddef(ncid)))
3068 return sync_netcdf4_file(nc4_info);
3075 close_netcdf4_file(NC_HDF5_FILE_INFO_T *h5,
int abort)
3079 assert(h5 && h5->root_grp);
3080 LOG((3,
"close_netcdf4_file: h5->path %s abort %d",
3081 h5->controller->path, abort));
3084 if (h5->flags & NC_INDEF)
3085 h5->flags ^= NC_INDEF;
3089 if (!h5->no_write && !abort)
3090 if ((retval = sync_netcdf4_file(h5)))
3095 if ((retval = nc4_rec_grp_del(&h5->root_grp, h5->root_grp)))
3102 if (SDend(h5->sdid))
3108 if (H5Fclose(h5->hdfid) < 0)
3111 nobjs = H5Fget_obj_count(h5->hdfid, H5F_OBJ_ALL);
3115 }
else if(nobjs > 0) {
3121 LOG((0,
"There are %d HDF5 objects open!", nobjs));
3127 if (H5garbage_collect() < 0)
3151 int delete_file = 0;
3154 NC_HDF5_FILE_INFO_T* nc4_info;
3156 LOG((2,
"nc_abort: ncid 0x%x", ncid));
3159 if (!(nc = nc4_find_nc_file(ncid,&nc4_info)))
3166 if (nc4_info->pnetcdf_file)
3167 return ncmpi_abort(nc->int_ncid);
3171 if (nc4_info->flags & NC_INDEF && !nc4_info->redef)
3179 if ((retval = close_netcdf4_file(nc4_info, 1)))
3195 NC_HDF5_FILE_INFO_T *h5;
3198 LOG((1,
"nc_close: ncid 0x%x", ncid));
3201 if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
3204 assert(nc && h5 && grp);
3212 if (h5->pnetcdf_file)
3213 return ncmpi_close(nc->int_ncid);
3217 if ((retval = close_netcdf4_file(grp->nc4_info, 0)))
3226 NC4_inq(
int ncid,
int *ndimsp,
int *nvarsp,
int *nattsp,
int *unlimdimidp)
3229 NC_HDF5_FILE_INFO_T *h5;
3236 LOG((2,
"nc_inq: ncid 0x%x", ncid));
3239 if ((retval = nc4_find_nc_grp_h5(ncid, &nc, &grp, &h5)))
3242 assert(h5 && grp && nc);
3246 if (h5->pnetcdf_file)
3247 return ncmpi_inq(nc->int_ncid, ndimsp, nvarsp, nattsp, unlimdimidp);
3254 for (dim = grp->dim; dim; dim = dim->next)
3260 for (var = grp->var; var; var= var->next)
3266 for (att = grp->att; att; att = att->next)
3279 for (dim = grp->dim; dim; dim = dim->next)
3282 *unlimdimidp = dim->dimid;
3293 nc4_enddef_netcdf4_file(NC_HDF5_FILE_INFO_T *h5)
3296 LOG((3,
"nc4_enddef_netcdf4_file"));
3299 if (!(h5->flags & NC_INDEF))
3303 h5->flags ^= NC_INDEF;
3308 return sync_netcdf4_file(h5);
3315 if (num_plists || num_spaces)
3324 nc_use_parallel_enabled()