/******************************************************************************\
*                                                                              *
*        Copyright (c) 2003, The Regents of the University of California       *
*      See the file COPYRIGHT for a complete copyright notice and license.     *
*                                                                              *
********************************************************************************
*
* CVS info:
*   $RCSfile: aiori-HDF5.c,v $
*   $Revision: 1.82 $ 
*   $Date: 2007/08/29 00:01:07 $
*   $Author: loewe $
*
* Purpose:
*       Implementation of abstract I/O interface for HDF5.
*
\******************************************************************************/

#include "aiori.h"                        /* abstract IOR interface */
#include <errno.h>                        /* sys_errlist */
#include <stdio.h>                        /* only for fprintf() */
#include <stdlib.h>
#include <sys/stat.h>
#include <unistd.h>
#include <hdf5.h>

#define NUM_DIMS 1                        /* number of dimensions to data set */

/******************************************************************************/
/*
 * HDF5_CHECK will display a custom error message and then exit the program
 */

/*
 * should use MPI_Abort(), not exit(), in this macro; some versions of
 * MPI, however, hang with HDF5 property lists et al. left unclosed
 */

/*
 * for versions later than hdf5-1.6, the H5Eget_[major|minor]() functions
 * have been deprecated and replaced with H5Eget_msg()
 */
#if H5_VERS_MAJOR > 1 && H5_VERS_MINOR > 6
#define HDF5_CHECK(HDF5_RETURN, MSG) do {                                \
    char   resultString[1024];                                           \
                                                                         \
    if (HDF5_RETURN < 0) {                                               \
        fprintf(stdout, "** error **\n");                                \
        fprintf(stdout, "ERROR in %s (line %d): %s.\n",                  \
                __FILE__, __LINE__, MSG);                                \
        strcpy(resultString, H5Eget_major((H5E_major_t)HDF5_RETURN));    \
        if (strcmp(resultString, "Invalid major error number") != 0)     \
            fprintf(stdout, "HDF5 %s\n", resultString);                  \
        strcpy(resultString, H5Eget_minor((H5E_minor_t)HDF5_RETURN));    \
        if (strcmp(resultString, "Invalid minor error number") != 0)     \
            fprintf(stdout, "%s\n", resultString);                       \
        fprintf(stdout, "** exiting **\n");                              \
        exit(-1);                                                        \
    }                                                                    \
} while(0)
#else /* ! (H5_VERS_MAJOR > 1 && H5_VERS_MINOR > 6) */
#define HDF5_CHECK(HDF5_RETURN, MSG) do {                                \
    char   resultString[1024];                                           \
                                                                         \
    if (HDF5_RETURN < 0) {                                               \
        fprintf(stdout, "** error **\n");                                \
        fprintf(stdout, "ERROR in %s (line %d): %s.\n",                  \
                __FILE__, __LINE__, MSG);                                \
        /*                                                               \
	 * H5Eget_msg(hid_t mesg_id, H5E_type_t* mesg_type,              \
	 *            char* mesg, size_t size)                           \
	 */                                                              \
        fprintf(stdout, "** exiting **\n");                              \
        exit(-1);                                                        \
    }                                                                    \
} while(0)
#endif /* H5_VERS_MAJOR > 1 && H5_VERS_MINOR > 6 */
/**************************** P R O T O T Y P E S *****************************/

IOR_offset_t SeekOffset_HDF5  (void *, IOR_offset_t, IOR_param_t *);
void         SetHints         (MPI_Info *, char *);
void         SetupDataSet_HDF5(void *, IOR_param_t *);
void         ShowHints        (MPI_Info *);


/************************** D E C L A R A T I O N S ***************************/

extern int      errno,                                /* error number */
                rank,
                rankOffset,
                verbose;                              /* verbose output */
extern MPI_Comm testComm;


static hid_t xferPropList;       /* xfer property list */
hid_t        dataSet;            /* data set id */ 
hid_t        dataSpace;          /* data space id */ 
hid_t        fileDataSpace;      /* file data space id */ 
hid_t        memDataSpace;       /* memory data space id */ 
int          newlyOpenedFile;    /* newly opened file */

/***************************** F U N C T I O N S ******************************/


/******************************************************************************/
/*
 * Create and open a file through the HDF5 interface.
 */

void *
IOR_Create_HDF5(char        * testFileName,
		IOR_param_t * param)
{
    return IOR_Open_HDF5(testFileName, param);
} /* IOR_Create_HDF5() */


/******************************************************************************/
/*
 * Open a file through the HDF5 interface.
 */

void *
IOR_Open_HDF5(char        * testFileName,
              IOR_param_t * param)
{
    hid_t            accessPropList,
                     createPropList;
    hsize_t          memStart[NUM_DIMS],
                     dataSetDims[NUM_DIMS],
                     memStride[NUM_DIMS],
                     memCount[NUM_DIMS],
                     memBlock[NUM_DIMS],
                     memDataSpaceDims[NUM_DIMS];
    int              tasksPerDataSet;
    unsigned         fd_mode                     = (unsigned)0;
    hid_t           *fd;
    MPI_Comm         comm;
    MPI_Info         mpiHints                    = MPI_INFO_NULL;

    fd = (hid_t *)malloc(sizeof(hid_t));
    if (fd == NULL) ERR("Unable to malloc file descriptor");
    /*
     * HDF5 uses different flags than those for POSIX/MPIIO
     */
    if (param->open == WRITE) { /* WRITE flags */
        param->openFlags = IOR_TRUNC;
    } else {                    /* READ or check WRITE/READ flags */
        param->openFlags = IOR_RDONLY;
    }

    /* set IOR file flags to HDF5 flags */
    /* -- file open flags -- */
    if (param->openFlags & IOR_RDONLY) {fd_mode |= H5F_ACC_RDONLY;}
    if (param->openFlags & IOR_WRONLY) {
        fprintf(stdout, "File write only not implemented in HDF5\n");
    }
    if (param->openFlags & IOR_RDWR)   {fd_mode |= H5F_ACC_RDWR;}
    if (param->openFlags & IOR_APPEND) {
        fprintf(stdout, "File append not implemented in HDF5\n");
    }
    if (param->openFlags & IOR_CREAT)  {fd_mode |= H5F_ACC_CREAT;}
    if (param->openFlags & IOR_EXCL)   {fd_mode |= H5F_ACC_EXCL;}
    if (param->openFlags & IOR_TRUNC)  {fd_mode |= H5F_ACC_TRUNC;}
    if (param->openFlags & IOR_DIRECT) {
        fprintf(stdout, "O_DIRECT not implemented in HDF5\n");
    }

    /* set up file creation property list */
    createPropList = H5Pcreate(H5P_FILE_CREATE);
    HDF5_CHECK(createPropList, "cannot create file creation property list");
    /* set size of offset and length used to address HDF5 objects */
    HDF5_CHECK(H5Pset_sizes(createPropList, sizeof(hsize_t), sizeof(hsize_t)),
               "cannot set property list properly");

    /* set up file access property list */
    accessPropList = H5Pcreate(H5P_FILE_ACCESS);
    HDF5_CHECK(accessPropList, "cannot create file access property list");

    /*
     * someday HDF5 implementation will allow subsets of MPI_COMM_WORLD
     */
    /* store MPI communicator info for the file access property list */
    if (param->filePerProc) {
        comm = MPI_COMM_SELF;
    } else {
        comm = testComm;
    }

    SetHints(&mpiHints, param->hintsFileName);
    /*
     * note that with MP_HINTS_FILTERED=no, all key/value pairs will
     * be in the info object.  The info object that is attached to
     * the file during MPI_File_open() will only contain those pairs
     * deemed valid by the implementation.
     */
    /* show hints passed to file */
    if (rank == 0 && param->showHints) {
         fprintf(stdout, "\nhints passed to access property list {\n");
         ShowHints(&mpiHints);
         fprintf(stdout, "}\n");
    }
    HDF5_CHECK(H5Pset_fapl_mpio(accessPropList, comm, mpiHints),
               "cannot set file access property list");

    /* set alignment */
    HDF5_CHECK(H5Pset_alignment(accessPropList, param->setAlignment,
                                param->setAlignment), "cannot set alignment");

    /* open file */
    if (param->open == WRITE) { /* WRITE */
        *fd = H5Fcreate(testFileName, fd_mode,
                            createPropList, accessPropList);
        HDF5_CHECK(*fd, "cannot create file");
    } else {                    /* READ or CHECK */
        *fd = H5Fopen(testFileName, fd_mode, accessPropList);
        HDF5_CHECK(*fd, "cannot open file");
    }

    /* show hints actually attached to file handle */
    if (param->showHints || (1) /* WEL - this needs fixing */) {
        if (rank == 0 && (param->showHints) /* WEL - this needs fixing */) {
            WARN("showHints not working for HDF5");
        }
    } else {
        MPI_Info mpiHintsCheck = MPI_INFO_NULL;
	hid_t apl;
        apl = H5Fget_access_plist(*fd);
        HDF5_CHECK(H5Pget_fapl_mpio(apl, &comm, &mpiHintsCheck),
	           "cannot get info object through HDF5");
        if (rank == 0) {
            fprintf(stdout,
                    "\nhints returned from opened file (HDF5) {\n");
            ShowHints(&mpiHintsCheck);
            fprintf(stdout, "}\n");
	    if (1 == 1) {    /* request the MPIIO file handle and its hints */
	        MPI_File * fd_mpiio;
	        HDF5_CHECK(H5Fget_vfd_handle(*fd, apl, (void **)&fd_mpiio),
                           "cannot get MPIIO file handle");
                MPI_CHECK(MPI_File_get_info(*fd_mpiio, &mpiHintsCheck),
                          "cannot get info object through MPIIO");
                fprintf(stdout,
                        "\nhints returned from opened file (MPIIO) {\n");
                ShowHints(&mpiHintsCheck);
                fprintf(stdout, "}\n");
	    }
        }
	MPI_CHECK(MPI_Barrier(testComm), "barrier error");
    }

    /* this is necessary for resetting various parameters
       needed for reopening and checking the file */
    newlyOpenedFile = TRUE;

    HDF5_CHECK(H5Pclose(createPropList), "cannot close creation property list");
    HDF5_CHECK(H5Pclose(accessPropList), "cannot close access property list");

    /* create property list for serial/parallel access */
    xferPropList = H5Pcreate(H5P_DATASET_XFER);
    HDF5_CHECK(xferPropList, "cannot create transfer property list");

    /* set data transfer mode */
    if (param->collective) {
        HDF5_CHECK(H5Pset_dxpl_mpio(xferPropList, H5FD_MPIO_COLLECTIVE),
                   "cannot set collective data transfer mode");
    } else {
        HDF5_CHECK(H5Pset_dxpl_mpio(xferPropList, H5FD_MPIO_INDEPENDENT),
                   "cannot set independent data transfer mode");
    }

    /* set up memory data space for transfer */
    memStart[0]         = (hsize_t)0;
    memCount[0]         = (hsize_t)1;
    memStride[0]        = (hsize_t)(param->transferSize / sizeof(IOR_size_t));
    memBlock[0]         = (hsize_t)(param->transferSize / sizeof(IOR_size_t));
    memDataSpaceDims[0] = (hsize_t)param->transferSize;
    memDataSpace = H5Screate_simple(NUM_DIMS, memDataSpaceDims, NULL);
    HDF5_CHECK(memDataSpace, "cannot create simple memory data space");

    /* define hyperslab for memory data space */
    HDF5_CHECK(H5Sselect_hyperslab(memDataSpace, H5S_SELECT_SET,
                                   memStart, memStride, memCount, memBlock),
               "cannot create hyperslab");

    /* set up parameters for fpp or different dataset count */
    if (param->filePerProc) {
        tasksPerDataSet = 1;
    } else {
        if (param->individualDataSets) {
            /* each task in segment has single data set */
            tasksPerDataSet = 1;
        } else {
            /* share single data set across all tasks in segment */
            tasksPerDataSet = param->numTasks;
        }
    }
    dataSetDims[0] = (hsize_t)((param->blockSize / sizeof(IOR_size_t))
                               * tasksPerDataSet);

    /* create a simple data space containing information on size
       and shape of data set, and open it for access */
    dataSpace = H5Screate_simple(NUM_DIMS, dataSetDims, NULL);
    HDF5_CHECK(dataSpace, "cannot create simple data space");
    
    return(fd);
} /* IOR_Open_HDF5() */


/******************************************************************************/
/*
 * Write or read access to file using the HDF5 interface.
 */

IOR_offset_t
IOR_Xfer_HDF5(int            access,
              void         * fd,
              IOR_size_t   * buffer,
              IOR_offset_t   length,
              IOR_param_t  * param)
{
    static int   firstReadCheck   = FALSE,
                 startNewDataSet;
    IOR_offset_t segmentPosition,
                 segmentSize;

    /*
     * this toggle is for the read check operation, which passes through
     * this function twice; note that this function will open a data set
     * only on the first read check and close only on the second
     */
    if (access == READCHECK) {
        if (firstReadCheck == TRUE) {
            firstReadCheck = FALSE;
        } else {
            firstReadCheck = TRUE;
        }
    }

    /* determine by offset if need to start new data set */
    if (param->filePerProc == TRUE) {
        segmentPosition = (IOR_offset_t)0;
        segmentSize = param->blockSize;
    } else {
        segmentPosition = (IOR_offset_t)((rank + rankOffset) % param->numTasks)
                                         * param->blockSize;
        segmentSize = (IOR_offset_t)(param->numTasks) * param->blockSize;
    }
    if ((IOR_offset_t)((param->offset - segmentPosition) % segmentSize) == 0) {
        /*
         * ordinarily start a new data set, unless this is the
         * second pass through during a read check
         */
        startNewDataSet = TRUE;
        if (access == READCHECK && firstReadCheck != TRUE) {
            startNewDataSet = FALSE;
        }
    }

    /* create new data set */
    if (startNewDataSet == TRUE) {
        /* if just opened this file, no data set to close yet */
        if (newlyOpenedFile != TRUE) {
            HDF5_CHECK(H5Dclose(dataSet), "cannot close data set");
            HDF5_CHECK(H5Sclose(fileDataSpace),
                       "cannot close file data space");
        }
        SetupDataSet_HDF5(fd, param);
    }

    SeekOffset_HDF5(fd, param->offset, param);

    /* this is necessary to reset variables for reaccessing file */
    startNewDataSet = FALSE;
    newlyOpenedFile = FALSE;

    /* access the file */
    if (access == WRITE) { /* WRITE */
        HDF5_CHECK(H5Dwrite(dataSet, H5T_NATIVE_LLONG,
                            memDataSpace, fileDataSpace,
                            xferPropList, buffer),
                   "cannot write to data set");
    } else {               /* READ or CHECK */
        HDF5_CHECK(H5Dread(dataSet, H5T_NATIVE_LLONG,
                           memDataSpace, fileDataSpace,
                            xferPropList, buffer),
                   "cannot read from data set");
    }
    return(length);
} /* IOR_Xfer_HDF5() */


/******************************************************************************/
/*
 * Perform fsync().
 */

void
IOR_Fsync_HDF5(void * fd, IOR_param_t * param)
{
    ;
} /* IOR_Fsync_HDF5() */


/******************************************************************************/
/*
 * Close a file through the HDF5 interface.
 */

void
IOR_Close_HDF5(void        * fd,
               IOR_param_t * param)
{
    if (param->fd_fppReadCheck == NULL) {
        HDF5_CHECK(H5Dclose(dataSet), "cannot close data set");
        HDF5_CHECK(H5Sclose(dataSpace), "cannot close data space");
        HDF5_CHECK(H5Sclose(fileDataSpace), "cannot close file data space");
        HDF5_CHECK(H5Sclose(memDataSpace), "cannot close memory data space");
        HDF5_CHECK(H5Pclose(xferPropList),
                   " cannot close transfer property list");
    }
    HDF5_CHECK(H5Fclose(*(hid_t *)fd), "cannot close file");
    free(fd);
} /* IOR_Close_HDF5() */


/******************************************************************************/
/*
 * Delete a file through the HDF5 interface.
 */

void
IOR_Delete_HDF5(char * testFileName, IOR_param_t * param)
{
    if (unlink(testFileName) != 0) WARN("cannot delete file");
} /* IOR_Delete_HDF5() */


/******************************************************************************/
/*
 * Determine api version.
 */

void
IOR_SetVersion_HDF5(IOR_param_t *test)
{
    unsigned major, minor, release;
    if (H5get_libversion(&major, &minor, &release) < 0) {
	WARN("cannot get HDF5 library version");
    } else {
	sprintf(test->apiVersion, "%s-%u.%u.%u",
		test->api, major, minor, release);
    }
#ifndef H5_HAVE_PARALLEL
    strcat(test->apiVersion, " (Serial)");
#else /* H5_HAVE_PARALLEL */
    strcat(test->apiVersion, " (Parallel)");
#endif /* not H5_HAVE_PARALLEL */
} /* IOR_SetVersion_HDF5() */


/************************ L O C A L   F U N C T I O N S ***********************/

/******************************************************************************/
/*
 * Seek to offset in file using the HDF5 interface and set up hyperslab.
 */

IOR_offset_t
SeekOffset_HDF5(void         *fd,
                IOR_offset_t  offset,
                IOR_param_t * param)
{
    IOR_offset_t segmentSize;
    hsize_t      hsStride[NUM_DIMS],
                 hsCount[NUM_DIMS],
                 hsBlock[NUM_DIMS];
    hsize_t      hsStart[NUM_DIMS];

    if (param->filePerProc == TRUE) {
        segmentSize = (IOR_offset_t)param->blockSize;
    } else {
        segmentSize = (IOR_offset_t)(param->numTasks) * param->blockSize;
    }

    /* create a hyperslab representing the file data space */
    if (param->individualDataSets) {
        /* start at zero offset if not */
        hsStart[0] = (hsize_t)((offset % param->blockSize)
                               / sizeof(IOR_size_t));
    } else {
        /* start at a unique offset if shared */
        hsStart[0] = (hsize_t)((offset % segmentSize) / sizeof(IOR_size_t));
    }
    hsCount[0]  = (hsize_t)1;
    hsStride[0] = (hsize_t)(param->transferSize / sizeof(IOR_size_t));
    hsBlock[0]  = (hsize_t)(param->transferSize / sizeof(IOR_size_t));

    /* retrieve data space from data set for hyperslab */
    fileDataSpace = H5Dget_space(dataSet);
    HDF5_CHECK(fileDataSpace, "cannot get data space from data set");
    HDF5_CHECK(H5Sselect_hyperslab(fileDataSpace, H5S_SELECT_SET,
                                   hsStart, hsStride, hsCount, hsBlock),
               "cannot select hyperslab");
    return(offset);
} /* SeekOffset_HDF5() */


/******************************************************************************/
/*
 * Create HDF5 data set.
 */

void
SetupDataSet_HDF5(void        * fd,
                  IOR_param_t * param)
{
    char       dataSetName[MAX_STR];
    hid_t      dataSetPropList;
    int        dataSetID;
    static int dataSetSuffix          = 0;

    /* may want to use an extendable dataset (H5S_UNLIMITED) someday */
    /* may want to use a chunked dataset (H5S_CHUNKED) someday */

    /* need to reset suffix counter if newly-opened file */
    if (newlyOpenedFile) dataSetSuffix = 0;

    /* may want to use individual access to each data set someday */
    if (param->individualDataSets) {
        dataSetID = (rank + rankOffset) % param->numTasks;
    } else {
        dataSetID = 0;
    }

    sprintf(dataSetName, "%s-%04d.%04d", "Dataset", dataSetID, dataSetSuffix++);

    if (param->open == WRITE) { /* WRITE */
        /* create data set */
        dataSetPropList = H5Pcreate(H5P_DATASET_CREATE);
        /* check if hdf5 available */
        #if defined (H5_VERS_MAJOR) && defined (H5_VERS_MINOR)
            /* no-fill option not available until hdf5-1.6.x */
            #if (H5_VERS_MAJOR > 0 && H5_VERS_MINOR > 5)
                if (param->noFill == TRUE) {
                    if (rank == 0 && verbose >= VERBOSE_1) {
                        fprintf(stdout, "\nusing 'no fill' option\n");
                    }
                    HDF5_CHECK(H5Pset_fill_time(dataSetPropList,
                                                H5D_FILL_TIME_NEVER),
                               "cannot set fill time for property list");
                }
            #else
                char errorString[MAX_STR];
                sprintf(errorString, "'no fill' option not available in %s",                            test->apiVersion);
                ERR(errorString);
            #endif
        #else
            WARN("unable to determine HDF5 version for 'no fill' usage");
        #endif
        dataSet = H5Dcreate(*(hid_t *)fd, dataSetName, H5T_NATIVE_LLONG,
                            dataSpace, dataSetPropList);
        HDF5_CHECK(dataSet, "cannot create data set");
    } else { /* READ or CHECK */
        dataSet = H5Dopen(*(hid_t *)fd, dataSetName);
        HDF5_CHECK(dataSet, "cannot create data set");
    }

} /* SetupDataSet_HDF5() */


/******************************************************************************/
/*
 * Use MPIIO call to get file size.
 */

IOR_offset_t
IOR_GetFileSize_HDF5(IOR_param_t * test,
                     MPI_Comm      testComm,
                     char        * testFileName)
{
    return(IOR_GetFileSize_MPIIO(test, testComm, testFileName));
} /* IOR_GetFileSize_HDF5() */