EXAMPLES

The following examples illustrate typical scenarios of manipulating HDF5 files and demonstrate how much simpler it is to solve these in HDFql (across different programming languages). Additionally, a quick start guide covering the most commonly used HDFql operations can be found here, and the complete reference manual is available here.

Example #1

Find all datasets existing in an HDF5 file named “data.h5” that start with “temperature” and are of data type float
For each dataset found, print its name and read its data
Write the data into a file named “output.txt” in an ascending order
Each value (belonging to the data) is written in a new line using a UNIX-based end of line (EOL) terminator

#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[1024];
      hdfql_execute("USE FILE data.h5");
      hdfql_execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
      while(hdfql_cursor_next(NULL) == HDFQL_SUCCESS)
      {
            printf("Dataset found: %s\n", hdfql_cursor_get_char(NULL));
            sprintf(script, "SELECT FROM %s ORDER ASC INTO UNIX FILE output.txt SPLIT 1", hdfql_cursor_get_char(NULL));
            hdfql_execute(script);
      }
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      HDFql::execute("USE FILE data.h5");
      HDFql::execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
      while(HDFql::cursorNext() == HDFql::Success)
      {
            std::cout << "Dataset found: " << HDFql::cursorGetChar() << std::endl;
            script.str("");
            script << "SELECT FROM " << HDFql::cursorGetChar() << " ORDER ASC INTO UNIX FILE output.txt SPLIT 1";
            HDFql::execute(script);
      }
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            HDFql.execute("USE FILE data.h5");
            HDFql.execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
            while(HDFql.cursorNext() == HDFql.SUCCESS)
            {
                  System.out.println("Dataset found: " + HDFql.cursorGetChar());
                  HDFql.execute("SELECT FROM " + HDFql.cursorGetChar() + " ORDER ASC INTO UNIX FILE output.txt SPLIT 1");
            }
            HDFql.execute("CLOSE FILE");
      }
}

import HDFql

HDFql.execute("USE FILE data.h5")
HDFql.execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT")
while HDFql.cursor_next() == HDFql.SUCCESS:
print("Dataset found: %s" % HDFql.cursor_get_char())
HDFql.execute("SELECT FROM %s ORDER ASC INTO UNIX FILE output.txt SPLIT 1" % HDFql.cursor_get_char())
HDFql.execute("CLOSE FILE")

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            HDFql.Execute("USE FILE data.h5");
            HDFql.Execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT");
            while(HDFql.CursorNext() == HDFql.Success)
            {
                  System.Console.WriteLine("Dataset found: {0}", HDFql.CursorGetChar());
                  HDFql.Execute("SELECT FROM " + HDFql.CursorGetChar() + " ORDER ASC INTO UNIX FILE output.txt SPLIT 1");
            }
            HDFql.Execute("CLOSE FILE");
      }
}

PROGRAM Example
      USE HDFql
      INTEGER :: state
      state = hdfql_execute("USE FILE data.h5")
      state = hdfql_execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT")
      DO WHILE(hdfql_cursor_next() == HDFQL_SUCCESS)
            WRITE(*, *) "Dataset found: ", hdfql_cursor_get_char()
            state = hdfql_execute("SELECT FROM " // hdfql_cursor_get_char() // " ORDER ASC INTO UNIX FILE output.txt SPLIT 1")
      END DO
      state = hdfql_execute("CLOSE FILE")
END PROGRAM

source("HDFql.R")

hdfql_execute("USE FILE data.h5")
hdfql_execute("SHOW DATASET LIKE **/^temperature WHERE DATA TYPE == FLOAT")
while(hdfql_cursor_next() == HDFQL_SUCCESS)
{
print(paste("Dataset found:", hdfql_cursor_get_char()))
hdfql_execute(paste("SELECT FROM", hdfql_cursor_get_char(), "ORDER ASC INTO UNIX FILE output.txt SPLIT 1"))
}
hdfql_execute("CLOSE FILE")

Example #2

Create an HDF5 file named “painters.h5”
Inside the file, create a group named “picasso” that tracks the creation order of objects stored within it
Inside the group, create a dataset named “guernica” of data type int of two dimensions with size 200×150
The dataset is organized through chunks of 40×30 and uses Fletcher32 checksum for error detection
Write an incremented value (starting from 0) in each position of the dataset
Inside the dataset, create an attribute named “subject” of data type UTF8 variable char with an initial value of “guerra civil española”

#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      int values[200][150];
      char script[1024];
      int x;
      int y;
      hdfql_execute("CREATE FILE painters.h5");
      hdfql_execute("USE FILE painters.h5");
      hdfql_execute("CREATE GROUP picasso ORDER TRACKED");
      hdfql_execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32");
      for(x = 0; x < 200; x++)
      {
            for(y = 0; y < 150; y++)
            {
                  values[x][y] = x * 150 + y;
            }
      }
      sprintf(script, "INSERT INTO picasso/guernica VALUES FROM MEMORY %d", hdfql_variable_transient_register(values));
      hdfql_execute(script);
      hdfql_execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      int values[200][150];
      int x;
      int y;
      HDFql::execute("CREATE FILE painters.h5");
      HDFql::execute("USE FILE painters.h5");
      HDFql::execute("CREATE GROUP picasso ORDER TRACKED");
      HDFql::execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32");
      for(x = 0; x < 200; x++)
      {
            for(y = 0; y < 150; y++)
            {
                  values[x][y] = x * 150 + y;
            }
      }
      script << "INSERT INTO picasso/guernica VALUES FROM MEMORY " << HDFql::variableTransientRegister(values);
      HDFql::execute(script);
      HDFql::execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            Integer values[][] = new Integer[200][150];
            int x;
            int y;
            HDFql.execute("CREATE FILE painters.h5");
            HDFql.execute("USE FILE painters.h5");
            HDFql.execute("CREATE GROUP picasso ORDER TRACKED");
            HDFql.execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32");
            for(x = 0; x < 200; x++)
            {
                  for(y = 0; y < 150; y++)
                  {
                        values[x][y] = x * 150 + y;
                  }
            }
            HDFql.execute("INSERT INTO picasso/guernica VALUES FROM MEMORY " + HDFql.variableTransientRegister(values));
            HDFql.execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
            HDFql.execute("CLOSE FILE");
      }
}

import HDFql
import numpy

HDFql.execute("CREATE FILE painters.h5")
HDFql.execute("USE FILE painters.h5")
HDFql.execute("CREATE GROUP picasso ORDER TRACKED")
HDFql.execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32")
values = numpy.ndarray([200, 150], numpy.int32)
for x in range(200):
for y in range(150):
values[x][y] = x * 150 + y
HDFql.execute("INSERT INTO picasso/guernica VALUES FROM MEMORY %d" % HDFql.variable_transient_register(values))
HDFql.execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")")
HDFql.execute("CLOSE FILE")

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            int[,] values = new int[200, 150];
            int x;
            int y;
            HDFql.Execute("CREATE FILE painters.h5");
            HDFql.Execute("USE FILE painters.h5");
            HDFql.Execute("CREATE GROUP picasso ORDER TRACKED");
            HDFql.Execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32"));
            for(x = 0; x < 200; x++)
            {
                  for(y = 0; y < 150; y++)
                  {
                        values[x, y] = x * 150 + y;
                  }
            }
            HDFql.Execute("INSERT INTO picasso/guernica VALUES FROM MEMORY " + HDFql.VariableTransientRegister(values));
            HDFql.Execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")");
            HDFql.Execute("CLOSE FILE");
      }
}

PROGRAM Example
      USE HDFql
      CHARACTER :: variable_number
      INTEGER, DIMENSION(200, 150) :: values
      INTEGER :: state
      INTEGER :: x
      INTEGER :: y
      state = hdfql_execute("CREATE FILE painters.h5")
      state = hdfql_execute("USE FILE painters.h5")
      state = hdfql_execute("CREATE GROUP picasso ORDER TRACKED")
      state = hdfql_execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32")
      DO x = 1, 150
            DO y = 1, 200
                  values(y, x) = x * 200 + y – 201
            END DO
      END DO
      WRITE(variable_number, "(I0)") hdfql_variable_transient_register(values)
      state = hdfql_execute("INSERT INTO picasso/guernica VALUES FROM MEMORY " // variable_number)
      state = hdfql_execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(""guerra civil española"")")
      state = hdfql_execute("CLOSE FILE")
END PROGRAM

source("HDFql.R")

hdfql_execute("CREATE FILE painters.h5")
hdfql_execute("USE FILE painters.h5")
hdfql_execute("CREATE GROUP picasso ORDER TRACKED")
hdfql_execute("CREATE CHUNKED(40, 30) DATASET picasso/guernica AS INT(200, 150) ENABLE FLETCHER32")
values <- array(dim = c(200, 150))
for(x in 1:150)
{
      for(y in 1:200)
      {
            values[y, x] <- as.integer(x * 200 + y – 201)
      }
}
hdfql_execute(paste("INSERT INTO picasso/guernica VALUES FROM MEMORY", hdfql_variable_transient_register(values)))
hdfql_execute("CREATE ATTRIBUTE picasso/guernica/subject AS UTF8 VARCHAR VALUES(\"guerra civil española\")")
hdfql_execute("CLOSE FILE")

Example #3

Create an HDF5 file named “my_file.h5” and overwrite (i.e. truncate) the file if it already exists
Inside the file, create a dataset named “my_dataset” of data type short of two dimensions with size UNLIMITEDx1024
The dataset is extendible on its first dimension (to store an unknown volume of data) and compressed with ZLIB
Acquire data from a process which returns an array of 1024 shorts on each reading
Write the acquired data into the dataset using a hyperslab selection (so that already stored data is not overwritten)

#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[100];
      short values[1024];
      hdfql_execute("CREATE TRUNCATE FILE my_file.h5");
      hdfql_execute("USE FILE my_file.h5");
      hdfql_execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
      hdfql_variable_register(values);
      while(acquire(values))     // call hypothetical function that acquires data
      {
            hdfql_execute("ALTER DIMENSION my_dataset TO +1");
            sprintf(script, "INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY %d", hdfql_variable_get_number(values));
            hdfql_execute(script);
      }
      hdfql_variable_unregister(values);
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      short values[1024];
      HDFql::execute("CREATE TRUNCATE FILE my_file.h5");
      HDFql::execute("USE FILE my_file.h5");
      HDFql::execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
      HDFql::variableRegister(values);
      while(acquire(values))     // call hypothetical function that acquires data
      {
            HDFql::execute("ALTER DIMENSION my_dataset TO +1");
            script.str("");
            script << "INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " << HDFql::variableGetNumber(values);
            HDFql::execute(script);
      }
      HDFql::variableUnregister(values);
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            Short values[] = new Short[1024];
            HDFql.execute("CREATE TRUNCATE FILE my_file.h5");
            HDFql.execute("USE FILE my_file.h5");
            HDFql.execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
            HDFql.variableRegister(values);
            while(acquire(values))     // call hypothetical function that acquires data
            {
                  HDFql.execute("ALTER DIMENSION my_dataset TO +1");
                  HDFql.execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " + HDFql.variableGetNumber(values));
            }
            HDFql.variableUnregister(values);
            HDFql.execute("CLOSE FILE");
      }
}

import HDFql
import numpy

HDFql.execute("CREATE TRUNCATE FILE my_file.h5")
HDFql.execute("USE FILE my_file.h5")
HDFql.execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB")
values = numpy.ndarray([1024], numpy.int16)
HDFql.variable_register(values)
while acquire(values):     # call hypothetical function that acquires data
      HDFql.execute("ALTER DIMENSION my_dataset TO +1")
      HDFql.execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY %d" % HDFql.variable_get_number(values))
HDFql.variable_unregister(values)
HDFql.execute("CLOSE FILE")

using AS.HDFql;

public class Example
{
      public static void Main(string[] args)
      {
            short[] values = new short[1024];
            HDFql.Execute("CREATE TRUNCATE FILE my_file.h5");
            HDFql.Execute("USE FILE my_file.h5");
            HDFql.Execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB");
            HDFql.VariableRegister(values);
            while(Acquire(values))     // call hypothetical function that acquires data
            {
                  HDFql.Execute("ALTER DIMENSION my_dataset TO +1");
                  HDFql.Execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " + HDFql.VariableGetNumber(values));
            }
            HDFql.VariableUnregister(values);
            HDFql.Execute("CLOSE FILE");
      }
}

PROGRAM Example
      USE HDFql
      CHARACTER :: variable_number
      INTEGER(2), DIMENSION(1024) :: values
      INTEGER :: state
      state = hdfql_execute("CREATE TRUNCATE FILE my_file.h5")
      state = hdfql_execute("USE FILE my_file.h5")
      state = hdfql_execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB")
      WRITE(variable_number, "(I0)") hdfql_variable_register(values)
      DO WHILE(acquire(values))     ! call hypothetical function that acquires data
            state = hdfql_execute("ALTER DIMENSION my_dataset TO +1")
            state = hdfql_execute("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY " // variable_number)
      END DO
      state = hdfql_variable_unregister(values)
      state = hdfql_execute("CLOSE FILE")
END PROGRAM

source("HDFql.R")

hdfql_execute("CREATE TRUNCATE FILE my_file.h5")
hdfql_execute("USE FILE my_file.h5")
hdfql_execute("CREATE DATASET my_dataset AS SMALLINT(0 TO UNLIMITED, 1024) ENABLE ZLIB")
values <- array(dim = c(1024))
while(acquire(values))     # call hypothetical function that acquires data
{
      hdfql_execute("ALTER DIMENSION my_dataset TO +1")
      hdfql_variable_register(values)
      hdfql_execute(paste("INSERT INTO my_dataset[-1:::] VALUES FROM MEMORY", hdfql_variable_get_number(values)))
      hdfql_variable_unregister(values)
}
hdfql_execute("CLOSE FILE")

Example #4

Create an HDF5 file named “experiment.h5” in parallel (using MPI)
Inside the file, create a dataset named “measurements” of data type int of one dimension with size 4
Write values 5, 15, 25 and 35 in parallel into positions #0 (by MPI process rank 0), #1 (by MPI process rank 1), #2 (by MPI process rank 2) and #3 (by MPI process rank 3) of dataset “measurements” using a point selection
The writing is done using four MPI processes (i.e. by launching the program in parallel like, e.g., mpiexec –n 4 ./my_program)

#include <stdlib.h>
#include <stdio.h>
#include “HDFql.h”

int main(int argc, char *argv[])
{
      char script[100];
      int rank;
      hdfql_execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
      hdfql_execute("CREATE DATASET measurements AS INT(4)");
      rank = hdfql_mpi_get_rank();
      sprintf(script, "INSERT INTO measurements[%d] IN PARALLEL VALUES(%d)", rank, rank * 10 + 5);
      hdfql_execute(script);
      hdfql_execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

#include <cstdlib>
#include <iostream>
#include “HDFql.hpp”

int main(int argc, char *argv[])
{
      std::stringstream script;
      int rank;
      HDFql::execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
      HDFql::execute("CREATE DATASET measurements AS INT(4)");
      rank = HDFql::mpiGetRank();
      script << "INSERT INTO measurements[" << rank << "] IN PARALLEL VALUES(" << rank * + 5 << ")";
      HDFql::execute(script);
      HDFql::execute("CLOSE FILE");
      return EXIT_SUCCESS;
}

import as.hdfql.*;

public class Example
{
      public static void main(String args[])
      {
            int rank;
            HDFql.execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
            HDFql.execute("CREATE DATASET measurements AS INT(4)");
            rank = HDFql.mpiGetRank();
            HDFql.execute("INSERT INTO measurements[" + rank + "] IN PARALLEL VALUES(" + (rank * 10 + 5) + ")");
            HDFql.execute("CLOSE FILE");
      }
}

import HDFql

HDFql.execute("CREATE AND USE FILE experiment.h5 IN PARALLEL")
HDFql.execute("CREATE DATASET measurements AS INT(4)")
rank = HDFql.mpi_get_rank()
HDFql.execute("INSERT INTO measurements[%d] IN PARALLEL VALUES(%d)" % (rank, rank * 10 + 5))
HDFql.execute("CLOSE FILE")

using AS.HDFql;

public class Example
{
      public static void Main(string []args)
      {
            int rank;
            HDFql.Execute("CREATE AND USE FILE experiment.h5 IN PARALLEL");
            HDFql.Execute("CREATE DATASET measurements AS INT(4)");
            rank = HDFql.MpiGetRank();
            HDFql.Execute("INSERT INTO measurements[" + rank + "] IN PARALLEL VALUES(" + (rank * 10 + 5) + ")");
            HDFql.Execute("CLOSE FILE");
      }
}

PROGRAM Example
      USE HDFql
      CHARACTER :: rank
      CHARACTER(2) :: value
      INTEGER :: state
      state = hdfql_execute("CREATE AND USE FILE experiment.h5 IN PARALLEL")
      state = hdfql_execute("CREATE DATASET measurements AS INT(4)")
      WRITE(rank, "(I0)") hdfql_mpi_get_rank()
      WRITE(value, "(I0)") hdfql_mpi_get_rank() * 10 + 5
      state = hdfql_execute("INSERT INTO measurements[" // rank // "] IN PARALLEL VALUES(" // value // ")")
      state = hdfql_execute("CLOSE FILE")
END PROGRAM

source("HDFql.R")

hdfql_execute("CREATE AND USE FILE experiment.h5 IN PARALLEL")
hdfql_execute("CREATE DATASET measurements AS INT(4)")
rank <- hdfql_mpi_get_rank()
hdfql_execute(paste("INSERT INTO measurements[", rank, "] IN PARALLEL VALUES(", rank * 10 + 5, ")"))
hdfql_execute("CLOSE FILE")