EXAMPLES

The following examples illustrate typical scenarios of manipulating HDF5 files and demonstrate how much simpler it is to solve these in HDFql (across different programming languages). Additionally, a quick start guide covering the most commonly used HDFql operations can be found here, and the complete reference manual is available here.

Example #1

  • Find all datasets existing in an HDF5 file named “data.h5” that start with “temperature” and are of data type float
  • For each dataset found, print its name and read its data
  • Write the data into a file named “output.txt” in an ascending order
  • Each value (belonging to the data) is written in a new line using a UNIX-based end of line (EOL) terminator

#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[1024];
      hdfql_execute(“USE FILE data.h5”);
      hdfql_execute(“SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT”);
      while(hdfql_cursor_next(NULL) == HDFQL_SUCCESS)
      {
            printf(“Dataset found: %s\n”, hdfql_cursor_get_char(NULL));
            sprintf(script, “SELECT FROM %s ORDER ASC INTO UNIX FILE output.txt SPLIT 1”, hdfql_cursor_get_char(NULL));
            hdfql_execute(script);
      }
      hdfql_execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      char script[1024];
      HDFql::execute(“USE FILE data.h5”);
      HDFql::execute(“SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT”);
      while(HDFql::cursorNext() == HDFql::Success)
      {
            printf(“Dataset found: %s\n”, HDFql::cursorGetChar());
            sprintf(script, “SELECT FROM %s ORDER ASC INTO UNIX FILE output.txt SPLIT 1”, HDFql::cursorGetChar());
            HDFql::execute(script);
      }
      HDFql::execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            HDFql.execute(“USE FILE data.h5”);
            HDFql.execute(“SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT”);
            while(HDFql.cursorNext() == HDFql.SUCCESS)
            {
                  System.out.println(“Dataset found: “ + HDFql.cursorGetChar());
                  HDFql.execute(“SELECT FROM “ + HDFql.cursorGetChar() + ” ORDER ASC INTO UNIX FILE output.txt SPLIT 1″);
            }
            HDFql.execute(“CLOSE FILE”);
      }
}

import HDFql

HDFql.execute(“USE FILE data.h5”)
HDFql.execute(“SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT”)
while HDFql.cursor_next() == HDFql.SUCCESS:
      print(“Dataset found: %s” % HDFql.cursor_get_char())
      HDFql.execute(“SELECT FROM %s ORDER ASC INTO UNIX FILE output.txt SPLIT 1” % HDFql.cursor_get_char())
HDFql.execute(“CLOSE FILE”)
 
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            HDFql.Execute(“USE FILE data.h5”);
            HDFql.Execute(“SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT”);
            while(HDFql.CursorNext() == HDFql.Success)
            {
                  System.Console.WriteLine(“Dataset found: {0}”, HDFql.CursorGetChar());
                  HDFql.Execute(“SELECT FROM “ + HDFql.CursorGetChar() + ” ORDER ASC INTO UNIX FILE output.txt SPLIT 1″);
            }
            HDFql.Execute(“CLOSE FILE”);
      }
 

PROGRAM Example
      USE HDFql
      INTEGER :: state
      state = hdfql_execute(“USE FILE data.h5”)
      state = hdfql_execute(“SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT”)
      DO WHILE(hdfql_cursor_next() .EQ. HDFQL_SUCCESS)
            WRITE(*, *) “Dataset found: “, hdfql_cursor_get_char()
            state = hdfql_execute(“SELECT FROM “ // hdfql_cursor_get_char() // ” ORDER ASC INTO UNIX FILE output.txt SPLIT 1″)
      END DO
      state = hdfql_execute(“CLOSE FILE”)
END PROGRAM
 
 
 
 
 

source(“HDFql.R”)

hdfql_execute(“USE FILE data.h5”)
hdfql_execute(“SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT”)
while(hdfql_cursor_next() == HDFQL_SUCCESS)
{
      print(paste(“Dataset found:”, hdfql_cursor_get_char()))
      hdfql_execute(paste(“SELECT FROM”, hdfql_cursor_get_char(), “ORDER ASC INTO UNIX FILE output.txt SPLIT 1”))
}
hdfql_execute(“CLOSE FILE”)
 
 
 
 
 
 

Example #2

  • Create an HDF5 file named “painters.h5”
  • Inside the file, create a group named “picasso” that tracks the creation order of objects stored within it
  • Inside the group, create a dataset named “guernica” of data type int of two dimensions with size 200x150
  • The dataset is organized through chunks of 40x30 and uses Fletcher32 checksum for error detection
  • Write an incremented value (starting from 0) in each position of the dataset
  • Inside the dataset, create an attribute named “subject” of data type UTF8 variable char with an initial value of “guerra civil española”

#include “HDFql.h”

int main(int argc, char *argv[])
{
      int values[200][150];
      char script[1024];
      int x;
      int y;
      hdfql_execute(“CREATE FILE painters.h5”);
      hdfql_execute(“USE FILE painters.h5”);
      hdfql_execute(“CREATE GROUP picasso ORDER TRACKED”);
      hdfql_execute(“CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32”);
      for(x = 0; x < 200; x++)
      {
            for(y = 0; y < 150; y++)
            {
                  values[x][y] = x * 150 + y;
            }
      }
      sprintf(script, “INSERT INTO picasso/guernica VALUES FROM MEMORY %d”, hdfql_variable_transient_register(values));
      hdfql_execute(script);
      hdfql_execute(“CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\”guerra civil española\”)”);
      hdfql_execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      int values[200][150];
      char script[1024];
      int x;
      int y;
      HDFql::execute(“CREATE FILE painters.h5”);
      HDFql::execute(“USE FILE painters.h5”);
      HDFql::execute(“CREATE GROUP picasso ORDER TRACKED”);
      HDFql::execute(“CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32”);
      for(x = 0; x < 200; x++)
      {
            for(y = 0; y < 150; y++)
            {
                  values[x][y] = x * 150 + y;
            }
      }
      sprintf(script, “INSERT INTO picasso/guernica VALUES FROM MEMORY %d”, HDFql::variableTransientRegister(values));
      HDFql::execute(script);
      HDFql::execute(“CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\”guerra civil española\”)”);
      HDFql::execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            Integer values[][] = new Integer[200][150];
            int x;
            int y;
            HDFql.execute(“CREATE FILE painters.h5”);
            HDFql.execute(“USE FILE painters.h5”);
            HDFql.execute(“CREATE GROUP picasso ORDER TRACKED”);
            HDFql.execute(“CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32”);
            for(x = 0; x < 200; x++)
            {
                  for(y = 0; y < 150; y++)
                  {
                        values[x][y] = x * 150 + y;
                  }
            }
            HDFql.execute(“INSERT INTO picasso/guernica VALUES FROM MEMORY “ + HDFql.variableTransientRegister(values));
            HDFql.execute(“CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\”guerra civil española\”)”);
            HDFql.execute(“CLOSE FILE”);
      }
}

import HDFql
import numpy

HDFql.execute(“CREATE FILE painters.h5”)
HDFql.execute(“USE FILE painters.h5”)
HDFql.execute(“CREATE GROUP picasso ORDER TRACKED”)
HDFql.execute(“CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32”)
values = numpy.ndarray([200, 150], numpy.int32)
for x in range(200):
      for y in range(150):
            values[x][y] = x * 150 + y
HDFql.execute(“INSERT INTO picasso/guernica VALUES FROM MEMORY %d” % HDFql.variable_transient_register(values))
HDFql.execute(“CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\”guerra civil española\”)”)
HDFql.execute(“CLOSE FILE”)
 
 
 
 
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            int[,] values = new int[200, 150];
            int x;
            int y;
            HDFql.Execute(“CREATE FILE painters.h5”);
            HDFql.Execute(“USE FILE painters.h5”);
            HDFql.Execute(“CREATE GROUP picasso ORDER TRACKED”);
            HDFql.Execute(“CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32”));
            for(x = 0; x < 200; x++)
            {
                  for(y = 0; y < 150; y++)
                  {
                        values[x, y] = x * 150 + y;
                  }
            }
            HDFql.Execute(“INSERT INTO picasso/guernica VALUES FROM MEMORY “ + HDFql.VariableTransientRegister(values));
            HDFql.Execute(“CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\”guerra civil española\”)”);
            HDFql.Execute(“CLOSE FILE”);
      }
}

PROGRAM Example
      USE HDFql
      CHARACTER :: variable_number
      INTEGER, DIMENSION(200, 150) :: values
      INTEGER :: state
      INTEGER :: x
      INTEGER :: y
      state = hdfql_execute(“CREATE FILE painters.h5”)
      state = hdfql_execute(“USE FILE painters.h5”)
      state = hdfql_execute(“CREATE GROUP picasso ORDER TRACKED”)
      state = hdfql_execute(“CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32”)
      DO x = 1, 150
            DO y = 1, 200
                  values(y, x) = x * 200 + y 201
            END DO
      END DO
      WRITE(variable_number, “(I0)”) hdfql_variable_transient_register(values)
      state = hdfql_execute(“INSERT INTO picasso/guernica VALUES FROM MEMORY “ // variable_number)
      state = hdfql_execute(“CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(“”guerra civil española””)”)
      state = hdfql_execute(“CLOSE FILE”)
END PROGRAM
 
 
 
 

source(“HDFql.R”)

hdfql_execute(“CREATE FILE painters.h5”)
hdfql_execute(“USE FILE painters.h5”)
hdfql_execute(“CREATE GROUP picasso ORDER TRACKED”)
hdfql_execute(“CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32”)
values <- array(dim = c(200, 150))
for(x in 1:150)
{
      for(y in 1:200)
      {
            values[y, x] <- as.integer(x * 200 + y 201)
      }
}
hdfql_execute(paste(“INSERT INTO picasso/guernica VALUES FROM MEMORY”, hdfql_variable_transient_register(values)))
hdfql_execute(“CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\”guerra civil española\”)”)
hdfql_execute(“CLOSE FILE”)
 
 
 
 
 
 
 
 

Example #3

  • Create an HDF5 file named “my_file.h5” and overwrite (i.e. truncate) the file if it already exists
  • Inside the file, create a dataset named “my_dataset” of data type short of two dimensions with size UNLIMITEDx1024
  • The dataset is extendible on its first dimension (to store an unknown volume of data) and compressed with ZLIB
  • Acquire data from a process which returns an array of 1024 shorts on each reading
  • Write the acquired data into the dataset using a hyperslab selection (so that already stored data is not overwritten)

#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[100];
      short values[1024];
      hdfql_execute(“CREATE TRUNCATE FILE my_file.h5”);
      hdfql_execute(“USE FILE my_file.h5”);
      hdfql_execute(“CREATE DATASET my_dataset AS SMALLINT(UNLIMITED, 1024) ENABLE ZLIB”);
      hdfql_variable_register(values);
      while(acquire(values))     // call hypothetical function that acquires data
      {
            sprintf(script, “INSERT INTO my_dataset(-1:::) VALUES FROM MEMORY %d”, hdfql_variable_get_number(values));
            hdfql_execute(script);
            hdfql_execute(“ALTER DIMENSION my_dataset TO +1”);
      }
      hdfql_variable_unregister(values);
      hdfql_execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      char script[100];
      short values[1024];
      HDFql::execute(“CREATE TRUNCATE FILE my_file.h5”);
      HDFql::execute(“USE FILE my_file.h5”);
      HDFql::execute(“CREATE DATASET my_dataset AS SMALLINT(UNLIMITED, 1024) ENABLE ZLIB”);
      HDFql::variableRegister(values);
      while(acquire(values))     // call hypothetical function that acquires data
      {
            sprintf(script, “INSERT INTO my_dataset(-1:::) VALUES FROM MEMORY %d”, HDFql::variableGetNumber(values));
            HDFql::execute(script);
            HDFql::execute(“ALTER DIMENSION my_dataset TO +1”);
      }
      HDFql::variableUnregister(values);
      HDFql::execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            Short values[] = new Short[1024];
            HDFql.execute(“CREATE TRUNCATE FILE my_file.h5”);
            HDFql.execute(“USE FILE my_file.h5”);
            HDFql.execute(“CREATE DATASET my_dataset AS SMALLINT(UNLIMITED, 1024) ENABLE ZLIB”);
            HDFql.variableRegister(values);
            while(acquire(values))     // call hypothetical function that acquires data
            {
                  HDFql.execute(“INSERT INTO my_dataset(-1:::) VALUES FROM MEMORY “ + HDFql.variableGetNumber(values));
                  HDFql.execute(“ALTER DIMENSION my_dataset TO +1”);
            }
            HDFql.variableUnregister(values);
            HDFql.execute(“CLOSE FILE”);
      }
}

import HDFql
import numpy

HDFql.execute(“CREATE TRUNCATE FILE my_file.h5”)
HDFql.execute(“USE FILE my_file.h5”)
HDFql.execute(“CREATE DATASET my_dataset AS SMALLINT(UNLIMITED, 1024) ENABLE ZLIB”)
values = numpy.ndarray([1024], numpy.int16)
HDFql.variable_register(values)
while acquire(values):     # call hypothetical function that acquires data
      HDFql.execute(“INSERT INTO my_dataset(-1:::) VALUES FROM MEMORY %d” % HDFql.variable_get_number(values))
      HDFql.execute(“ALTER DIMENSION my_dataset TO +1”)
HDFql.variable_unregister(values)
HDFql.execute(“CLOSE FILE”)
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            short[] values = new short[1024];
            HDFql.Execute(“CREATE TRUNCATE FILE my_file.h5”);
            HDFql.Execute(“USE FILE my_file.h5”);
            HDFql.Execute(“CREATE DATASET my_dataset AS SMALLINT(UNLIMITED, 1024) ENABLE ZLIB”);
            HDFql.VariableRegister(values);
            while(Acquire(values))     // call hypothetical function that acquires data
            {
                  HDFql.Execute(“INSERT INTO my_dataset(-1:::) VALUES FROM MEMORY “ + HDFql.VariableGetNumber(values));
                  HDFql.Execute(“ALTER DIMENSION my_dataset TO +1”);
            }
            HDFql.VariableUnregister(values);
            HDFql.Execute(“CLOSE FILE”);
      }
}

PROGRAM Example
      USE HDFql
      CHARACTER :: variable_number
      INTEGER(2), DIMENSION(1024) :: values
      INTEGER :: state
      state = hdfql_execute(“CREATE TRUNCATE FILE my_file.h5”)
      state = hdfql_execute(“USE FILE my_file.h5”)
      state = hdfql_execute(“CREATE DATASET my_dataset AS SMALLINT(UNLIMITED, 1024) ENABLE ZLIB”)
      WRITE(variable_number, “(I0)”) hdfql_variable_register(values)
      DO WHILE(acquire(values) .EQ. .TRUE.)     ! call hypothetical function that acquires data
            state = hdfql_execute(“INSERT INTO my_dataset(-1:::) VALUES FROM MEMORY “ // variable_number)
            state = hdfql_execute(“ALTER DIMENSION my_dataset TO +1”)
      END DO
      state = hdfql_variable_unregister(values)
      state = hdfql_execute(“CLOSE FILE”)
END PROGRAM
 
 
 
 

source(“HDFql.R”)

hdfql_execute(“CREATE TRUNCATE FILE my_file.h5”)
hdfql_execute(“USE FILE my_file.h5”)
hdfql_execute(“CREATE DATASET my_dataset AS SMALLINT(UNLIMITED, 1024) ENABLE ZLIB”)
values <- array(dim = c(1024))
while(acquire(values))     # call hypothetical function that acquires data
{
      hdfql_variable_register(values)
      hdfql_execute(paste(“INSERT INTO my_dataset(-1:::) VALUES FROM MEMORY”, hdfql_variable_get_number(values)))
      hdfql_variable_unregister(values)
      hdfql_execute(“ALTER DIMENSION my_dataset TO +1”)
}
hdfql_execute(“CLOSE FILE”)
 
 
 
 
 
 

Example #4

  • Create an HDF5 file named “experiment.h5” in parallel (using MPI)
  • Inside the file, create a dataset named “measurements” of data type int of one dimension (size 4)
  • Write values 0, 10, 20 and 30 in parallel into positions #0 (by MPI process rank 0), #1 (by MPI process rank 1), #2 (by MPI process rank 2) and #3 (by MPI process rank 3) of dataset “measurements” using a point selection
  • The writing is done using four MPI processes (i.e. by launching the program in parallel like, e.g., mpiexec –n 4 my_program)

#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[100];
      int rank;
      hdfql_execute(“CREATE PARALLEL FILE experiment.h5”);
      hdfql_execute(“USE PARALLEL FILE experiment.h5”);
      hdfql_execute(“CREATE DATASET measurements AS INT(4)”);
      rank = hdfql_mpi_get_rank();
      sprintf(script, “INSERT INTO PARALLEL measurements(%d) VALUES(%d)”, rank, rank * 10);
      hdfql_execute(script);
      hdfql_execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      char script[100];
      int rank;
      HDFql::execute(“CREATE PARALLEL FILE experiment.h5”);
      HDFql::execute(“USE PARALLEL FILE experiment.h5”);
      HDFql::execute(“CREATE DATASET measurements AS INT(4)”);
      rank = HDFql::mpiGetRank();
      sprintf(script, “INSERT INTO PARALLEL measurements(%d) VALUES(%d)”, rank, rank * 10);
      HDFql::execute(script);
      HDFql::execute(“CLOSE FILE”);
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            int rank;
            HDFql.execute(“CREATE PARALLEL FILE experiment.h5”);
            HDFql.execute(“USE PARALLEL FILE experiment.h5”);
            HDFql.execute(“CREATE DATASET measurements AS INT(4)”);
            rank = HDFql.mpiGetRank();
            HDFql.execute(“INSERT INTO PARALLEL measurements(“ + rank + “) VALUES(“ + (rank * 10) + “)”);
            HDFql.execute(“CLOSE FILE”);
      }
}

import HDFql

HDFql.execute(“CREATE PARALLEL FILE experiment.h5”)
HDFql.execute(“USE PARALLEL FILE experiment.h5”)
HDFql.execute(“CREATE DATASET measurements AS INT(4)”)
rank = HDFql.mpi_get_rank()
HDFql.execute(“INSERT INTO PARALLEL measurements(%d) VALUES(%d)” % (rank, rank * 10))
HDFql.execute(“CLOSE FILE”)
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string []args)
      {
            int rank;
            HDFql.Execute(“CREATE PARALLEL FILE experiment.h5”);
            HDFql.Execute(“USE PARALLEL FILE experiment.h5”);
            HDFql.Execute(“CREATE DATASET measurements AS INT(4)”);
            rank = HDFql.MpiGetRank();
            HDFql.Execute(“INSERT INTO PARALLEL measurements(“ + rank + “) VALUES(“ + (rank * 10) + “)”);
            HDFql.Execute(“CLOSE FILE”);
      }
}

PROGRAM Example
      USE HDFql
      CHARACTER :: rank
      CHARACTER(2) :: value
      INTEGER :: state
      state = hdfql_execute(“CREATE PARALLEL FILE experiment.h5”)
      state = hdfql_execute(“USE PARALLEL FILE experiment.h5”)
      state = hdfql_execute(“CREATE DATASET measurements AS INT(4)”)
      WRITE(rank, “(I0)”) hdfql_mpi_get_rank()
      WRITE(value, “(I0)”) hdfql_mpi_get_rank() * 10
      state = hdfql_execute(“INSERT INTO PARALLEL measurements(“ // rank // “) VALUES(“ // value // “)”)
      state = hdfql_execute(“CLOSE FILE”)
END PROGRAM
 
 

source(“HDFql.R”)

hdfql_execute(“CREATE PARALLEL FILE experiment.h5”)
hdfql_execute(“USE PARALLEL FILE experiment.h5”)
hdfql_execute(“CREATE DATASET measurements AS INT(4)”)
rank <- hdfql_mpi_get_rank()
hdfql_execute(paste(“INSERT INTO PARALLEL measurements(“, rank, “) VALUES(“, rank * 10, “)”))
hdfql_execute(“CLOSE FILE”)
 
 
 
 
 
 
 

© 2019

(C) 2016-2017