EXAMPLES

The following examples illustrate typical scenarios of manipulating HDF5 files and demonstrate how much simpler it is to solve these in HDFql (across different programming languages). Additionally, a quick start guide covering the most commonly used HDFql operations can be found here, and the complete reference manual is available here.

Example #1

  • Find all datasets existing in an HDF5 file named “data.h5” that start with “temperature” and are of data type float
  • For each dataset found, print its name and read its data
  • Write the data into a file named “output.txt” in an ascending order
  • Each value (belonging to the data) is written in a new line using a UNIX-based end of line (EOL) terminator
#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[1024];
      hdfql_execute("USE FILE data.h5");
      hdfql_execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
      while(hdfql_cursor_next(NULL) == HDFQL_SUCCESS)
      {
            printf("Dataset found: %s\n", hdfql_cursor_get_char(NULL));
            sprintf(script, "SELECT FROM %s ORDER ASC INTO UNIX FILE output.txt SPLIT 1", hdfql_cursor_get_char(NULL));
            hdfql_execute(script);
      }
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}
 

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      HDFql::execute("USE FILE data.h5");
      HDFql::execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
      while(HDFql::cursorNext() == HDFql::Success)
      {
            std::cout << "Dataset found: " << HDFql::cursorGetChar() << std::endl;
            script.str("");
            script << "SELECT FROM " << HDFql::cursorGetChar() << " ORDER ASC INTO UNIX FILE output.txt SPLIT 1";
            HDFql::execute(script);
      }
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            HDFql.execute("USE FILE data.h5");
            HDFql.execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
            while(HDFql.cursorNext() == HDFql.SUCCESS)
            {
                  System.out.println("Dataset found: " + HDFql.cursorGetChar());
                  HDFql.execute("SELECT FROM " + HDFql.cursorGetChar() + " ORDER ASC INTO UNIX FILE output.txt SPLIT 1");
            }
            HDFql.execute("CLOSE FILE");
      }
}
 
 
 

import HDFql

HDFql.execute("USE FILE data.h5")
HDFql.execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT")
while HDFql.cursor_next() == HDFql.SUCCESS:
      print("Dataset found: %s" % HDFql.cursor_get_char())
      HDFql.execute("SELECT FROM %s ORDER ASC INTO UNIX FILE output.txt SPLIT 1" % HDFql.cursor_get_char())
HDFql.execute("CLOSE FILE")
 
 
 
 
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            HDFql.Execute("USE FILE data.h5");
            HDFql.Execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
            while(HDFql.CursorNext() == HDFql.Success)
            {
                  System.Console.WriteLine("Dataset found: {0}", HDFql.CursorGetChar());
                  HDFql.Execute("SELECT FROM " + HDFql.CursorGetChar() + " ORDER ASC INTO UNIX FILE output.txt SPLIT 1");
            }
            HDFql.Execute("CLOSE FILE");
      }
}
 
 
 

PROGRAM Example
      USE HDFql
      INTEGER :: state
      state = hdfql_execute("USE FILE data.h5")
      state = hdfql_execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT")
      DO WHILE(hdfql_cursor_next() == HDFQL_SUCCESS)
            WRITE(*, *) "Dataset found: ", hdfql_cursor_get_char()
            state = hdfql_execute("SELECT FROM " // hdfql_cursor_get_char() // " ORDER ASC INTO UNIX FILE output.txt SPLIT 1")
      END DO
      state = hdfql_execute("CLOSE FILE")
END PROGRAM
 
 
 
 
 
 
 
 
source("HDFql.R")

hdfql_execute("USE FILE data.h5")
hdfql_execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT")
while(hdfql_cursor_next() == HDFQL_SUCCESS)
{
      print(paste("Dataset found:", hdfql_cursor_get_char()))
      hdfql_execute(paste("SELECT FROM", hdfql_cursor_get_char(), "ORDER ASC INTO UNIX FILE output.txt SPLIT 1"))
}
hdfql_execute("CLOSE FILE")
 
 
 
 
 
 
 
 
 

Example #2

  • Create an HDF5 file named “painters.h5”
  • Inside the file, create a group named “picasso” that tracks the creation order of objects stored within it
  • Inside the group, create a dataset named “guernica” of data type int of two dimensions with size 200×150
  • The dataset is organized through chunks of 40×30 and uses Fletcher32 checksum for error detection
  • Write an incremented value (starting from 0) in each position of the dataset
  • Inside the dataset, create an attribute named “subject” of data type UTF8 variable char with an initial value of “guerra civil española”
#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      int values[200][150];
      char script[1024];
      int x;
      int y;
      hdfql_execute("CREATE FILE painters.h5");
      hdfql_execute("USE FILE painters.h5");
      hdfql_execute("CREATE GROUP picasso ORDER TRACKED");
      hdfql_execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32");
      for(x = 0; x < 200; x++)
      {
            for(y = 0; y < 150; y++)
            {
                  values[x][y] = x * 150 + y;
            }
      }
      sprintf(script, "INSERT INTO picasso/guernica VALUES FROM MEMORY %d", hdfql_variable_transient_register(values));
      hdfql_execute(script);
      hdfql_execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      int values[200][150];
      int x;
      int y;
      HDFql::execute("CREATE FILE painters.h5");
      HDFql::execute("USE FILE painters.h5");
      HDFql::execute("CREATE GROUP picasso ORDER TRACKED");
      HDFql::execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32");
      for(x = 0; x < 200; x++)
      {
            for(y = 0; y < 150; y++)
            {
                  values[x][y] = x * 150 + y;
            }
      }
      script << "INSERT INTO picasso/guernica VALUES FROM MEMORY " << HDFql::variableTransientRegister(values);
      HDFql::execute(script);
      HDFql::execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            Integer values[][] = new Integer[200][150];
            int x;
            int y;
            HDFql.execute("CREATE FILE painters.h5");
            HDFql.execute("USE FILE painters.h5");
            HDFql.execute("CREATE GROUP picasso ORDER TRACKED");
            HDFql.execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32");
            for(x = 0; x < 200; x++)
            {
                  for(y = 0; y < 150; y++)
                  {
                        values[x][y] = x * 150 + y;
                  }
            }
            HDFql.execute("INSERT INTO picasso/guernica VALUES FROM MEMORY " + HDFql.variableTransientRegister(values));
            HDFql.execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
            HDFql.execute("CLOSE FILE");
      }
}
 
 

import HDFql
import numpy

HDFql.execute("CREATE FILE painters.h5")
HDFql.execute("USE FILE painters.h5")
HDFql.execute("CREATE GROUP picasso ORDER TRACKED")
HDFql.execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32")
values = numpy.ndarray([200, 150], numpy.int32)
for x in range(200):
      for y in range(150):
            values[x][y] = x * 150 + y
HDFql.execute("INSERT INTO picasso/guernica VALUES FROM MEMORY %d" % HDFql.variable_transient_register(values))
HDFql.execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")")
HDFql.execute("CLOSE FILE")
 
 
 
 
 
 
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            int[,] values = new int[200, 150];
            int x;
            int y;
            HDFql.Execute("CREATE FILE painters.h5");
            HDFql.Execute("USE FILE painters.h5");
            HDFql.Execute("CREATE GROUP picasso ORDER TRACKED");
            HDFql.Execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32"));
            for(x = 0; x < 200; x++)
            {
                  for(y = 0; y < 150; y++)
                  {
                        values[x, y] = x * 150 + y;
                  }
            }
            HDFql.Execute("INSERT INTO picasso/guernica VALUES FROM MEMORY " + HDFql.VariableTransientRegister(values));
            HDFql.Execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
            HDFql.Execute("CLOSE FILE");
      }
}
 
 

PROGRAM Example
      USE HDFql
      CHARACTER :: variable_number
      INTEGER, DIMENSION(200, 150) :: values
      INTEGER :: state
      INTEGER :: x
      INTEGER :: y
      state = hdfql_execute("CREATE FILE painters.h5")
      state = hdfql_execute("USE FILE painters.h5")
      state = hdfql_execute("CREATE GROUP picasso ORDER TRACKED")
      state = hdfql_execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32")
      DO x = 1, 150
            DO y = 1, 200
                  values(y, x) = x * 200 + y 201
            END DO
      END DO
      WRITE(variable_number, "(I0)") hdfql_variable_transient_register(values)
      state = hdfql_execute("INSERT INTO picasso/guernica VALUES FROM MEMORY " // variable_number)
      state = hdfql_execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(""guerra civil española"")")
      state = hdfql_execute("CLOSE FILE")
END PROGRAM
 
 
 
 
 
 
source("HDFql.R")

hdfql_execute("CREATE FILE painters.h5")
hdfql_execute("USE FILE painters.h5")
hdfql_execute("CREATE GROUP picasso ORDER TRACKED")
hdfql_execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32")
values <- array(dim = c(200, 150))
for(x in 1:150)
{
      for(y in 1:200)
      {
            values[y, x] <- as.integer(x * 200 + y 201)
      }
}
hdfql_execute(paste("INSERT INTO picasso/guernica VALUES FROM MEMORY", hdfql_variable_transient_register(values)))
hdfql_execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")")
hdfql_execute("CLOSE FILE")
 
 
 
 
 
 
 
 
 
 

Example #3

  • Create an HDF5 file named “my_file.h5” and overwrite (i.e. truncate) the file if it already exists
  • Inside the file, create a dataset named “my_dataset” of data type short of two dimensions with size UNLIMITEDx1024
  • The dataset is extendible on its first dimension (to store an unknown volume of data) and compressed with ZLIB
  • Acquire data from a process which returns an array of 1024 shorts on each reading
  • Write the acquired data into the dataset using a hyperslab selection (so that already stored data is not overwritten)
#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[100];
      short values[1024];
      hdfql_execute("CREATE TRUNCATE FILE my_file.h5");
      hdfql_execute("USE FILE my_file.h5");
      hdfql_execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
      hdfql_variable_register(values);
      while(acquire(values))     // call hypothetical function that acquires data
      {
            hdfql_execute("ALTER DIMENSION my_dataset TO +1");
            sprintf(script, "INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY %d", hdfql_variable_get_number(values));
            hdfql_execute(script);
      }
      hdfql_variable_unregister(values);
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}
 

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      short values[1024];
      HDFql::execute("CREATE TRUNCATE FILE my_file.h5");
      HDFql::execute("USE FILE my_file.h5");
      HDFql::execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
      HDFql::variableRegister(values);
      while(acquire(values))     // call hypothetical function that acquires data
      {
            HDFql::execute("ALTER DIMENSION my_dataset TO +1");
            script.str("");
            script << "INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " << HDFql::variableGetNumber(values);
            HDFql::execute(script);
      }
      HDFql::variableUnregister(values);
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            Short values[] = new Short[1024];
            HDFql.execute("CREATE TRUNCATE FILE my_file.h5");
            HDFql.execute("USE FILE my_file.h5");
            HDFql.execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
            HDFql.variableRegister(values);
            while(acquire(values))     // call hypothetical function that acquires data
            {
                  HDFql.execute("ALTER DIMENSION my_dataset TO +1");
                  HDFql.execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " + HDFql.variableGetNumber(values));
            }
            HDFql.variableUnregister(values);
            HDFql.execute("CLOSE FILE");
      }
}
 
 
 

import HDFql
import numpy

HDFql.execute("CREATE TRUNCATE FILE my_file.h5")
HDFql.execute("USE FILE my_file.h5")
HDFql.execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB")
values = numpy.ndarray([1024], numpy.int16)
HDFql.variable_register(values)
while acquire(values):     # call hypothetical function that acquires data
      HDFql.execute("ALTER DIMENSION my_dataset TO +1")
      HDFql.execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY %d" % HDFql.variable_get_number(values))
HDFql.variable_unregister(values)
HDFql.execute("CLOSE FILE")
 
 
 
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            short[] values = new short[1024];
            HDFql.Execute("CREATE TRUNCATE FILE my_file.h5");
            HDFql.Execute("USE FILE my_file.h5");
            HDFql.Execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
            HDFql.VariableRegister(values);
            while(Acquire(values))     // call hypothetical function that acquires data
            {
                  HDFql.Execute("ALTER DIMENSION my_dataset TO +1");
                  HDFql.Execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " + HDFql.VariableGetNumber(values));
            }
            HDFql.VariableUnregister(values);
            HDFql.Execute("CLOSE FILE");
      }
}
 
 
 

PROGRAM Example
      USE HDFql
      CHARACTER :: variable_number
      INTEGER(2), DIMENSION(1024) :: values
      INTEGER :: state
      state = hdfql_execute("CREATE TRUNCATE FILE my_file.h5")
      state = hdfql_execute("USE FILE my_file.h5")
      state = hdfql_execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB")
      WRITE(variable_number, "(I0)") hdfql_variable_register(values)
      DO WHILE(acquire(values))     ! call hypothetical function that acquires data
            state = hdfql_execute("ALTER DIMENSION my_dataset TO +1")
            state = hdfql_execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " // variable_number)
      END DO
      state = hdfql_variable_unregister(values)
      state = hdfql_execute("CLOSE FILE")
END PROGRAM
 
 
 
 
 
 
 
source("HDFql.R")

hdfql_execute("CREATE TRUNCATE FILE my_file.h5")
hdfql_execute("USE FILE my_file.h5")
hdfql_execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB")
values <- array(dim = c(1024))
while(acquire(values))     # call hypothetical function that acquires data
{
      hdfql_execute("ALTER DIMENSION my_dataset TO +1")
      hdfql_variable_register(values)
      hdfql_execute(paste("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY", hdfql_variable_get_number(values)))
      hdfql_variable_unregister(values)
}
hdfql_execute("CLOSE FILE")
 
 
 
 
 
 
 
 
 

Example #4

  • Create an HDF5 file named “experiment.h5” in parallel (using MPI)
  • Inside the file, create a dataset named “measurements” of data type int of one dimension with size 4
  • Write values 5, 15, 25 and 35 in parallel into positions #0 (by MPI process rank 0), #1 (by MPI process rank 1), #2 (by MPI process rank 2) and #3 (by MPI process rank 3) of dataset “measurements” using a point selection
  • The writing is done using four MPI processes (i.e. by launching the program in parallel like, e.g., mpiexec –n 4 ./my_program)
#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[100];
      int rank;
      hdfql_execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
      hdfql_execute("CREATE DATASET measurements AS INT(4)");
      rank = hdfql_mpi_get_rank();
      sprintf(script, "INSERT INTO measurements[%d] IN PARALLEL VALUES(%d)", rank, rank * 10 + 5);
      hdfql_execute(script);
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      int rank;
      HDFql::execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
      HDFql::execute("CREATE DATASET measurements AS INT(4)");
      rank = HDFql::mpiGetRank();
      script << "INSERT INTO measurements[" << rank << "] IN PARALLEL VALUES(" << rank * + 5 << ")";
      HDFql::execute(script);
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            int rank;
            HDFql.execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
            HDFql.execute("CREATE DATASET measurements AS INT(4)");
            rank = HDFql.mpiGetRank();
            HDFql.execute("INSERT INTO measurements[" + rank + "] IN PARALLEL VALUES(" + (rank * 10 + 5) + ")");
            HDFql.execute("CLOSE FILE");
      }
}
 
 

import HDFql

HDFql.execute("CREATE AND USE FILE experiment.h5 IN PARALLEL")
HDFql.execute("CREATE DATASET measurements AS INT(4)")
rank = HDFql.mpi_get_rank()
HDFql.execute("INSERT INTO measurements[%d] IN PARALLEL VALUES(%d)" % (rank, rank * 10 + 5))
HDFql.execute("CLOSE FILE")
 
 
 
 
 
 
 
 
 

using AS.HDFql;

public class Example
{
      public static void Main(string []args)
      {
            int rank;
            HDFql.Execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
            HDFql.Execute("CREATE DATASET measurements AS INT(4)");
            rank = HDFql.MpiGetRank();
            HDFql.Execute("INSERT INTO measurements[" + rank + "] IN PARALLEL VALUES(" + (rank * 10 + 5) + ")");
            HDFql.Execute("CLOSE FILE");
      }
}
 
 

PROGRAM Example
      USE HDFql
      CHARACTER :: rank
      CHARACTER(2) :: value
      INTEGER :: state
      state = hdfql_execute("CREATE AND USE FILE experiment.h5 IN PARALLEL")
      state = hdfql_execute("CREATE DATASET measurements AS INT(4)")
      WRITE(rank, "(I0)") hdfql_mpi_get_rank()
      WRITE(value, "(I0)") hdfql_mpi_get_rank() * 10 + 5
      state = hdfql_execute("INSERT INTO measurements[" // rank // "] IN PARALLEL VALUES(" // value // ")")
      state = hdfql_execute("CLOSE FILE")
END PROGRAM
 
 
 
 
source("HDFql.R")

hdfql_execute("CREATE AND USE FILE experiment.h5 IN PARALLEL")
hdfql_execute("CREATE DATASET measurements AS INT(4)")
rank <- hdfql_mpi_get_rank()
hdfql_execute(paste("INSERT INTO measurements[", rank, "] IN PARALLEL VALUES(", rank * 10 + 5, ")"))
hdfql_execute("CLOSE FILE")