Example Program and Makefile for XC40

Here is a simple example program and makefile for building on XC40 systems. When you compile your code for our XC40 systems, you are normally cross-compiling for the compute nodes (Intel Xeon Phi 2nd Generation [KNL]), rather than directly compiling for the login nodes (non-KNL). The default, you are using the Intel compiler suite. We also provide the Cray, GNU, and Clang compiler suites via modules.

C Program

/* File pi.c */
#include <mpi.h>
#include <math.h>
#include <stdio.h>

int main(int argc, char** argv)
{
  int n, myid, numprocs, i;
  double PI25DT = 3.141592653589793238462643;
  double mypi, pi, h, sum, x;

  MPI_Init(&argc, &argv);
  MPI_Comm_size(MPI_COMM_WORLD, &numprocs);
  MPI_Comm_rank(MPI_COMM_WORLD, &myid);

  /* Number of intervals */
  n = 25;

  MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD);
  h = 1.0 / (double) n;
  sum = 0.0;
  for ( i = myid + 1; i <= n; i += numprocs )
  {
       x = h * ((double) i - 0.5);
       sum += ( 4.0 / (1.0 + x*x) );
  }
  mypi = h * sum;
  MPI_Reduce(&mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD);

  if (myid == 0)
  {
      printf("pi is approximately %.16f, Error is %.16f\n",
               pi, fabs(pi - PI25DT));
  }

  MPI_Finalize();
  return 0;
}

Makefile

# File makefile
CC = cc

OPTFLAGS = -O3
CFLAGS = $(OPTFLAGS) -g

all: pi

pi: pi.o
	$(CC) $(CFLAGS) -o pi pi.o

pi.o: pi.o
	$(CC) $(CFLAGS) -c pi.c

clean:
	rm pi pi.o

Usage

Session screenshot shows building and running, using an example job script:

thetalogin6(pi)> ls
makefile  pi.c  runpi.bash*

thetalogin6(pi)> make
cc -O3 -g -c pi.c
cc -O3 -g -o pi pi.o

thetalogin6(pi)> ls
makefile  pi*  pi.c  pi.o  runpi.bash*

thetalogin6(pi)> cat runpi.bash
#!/bin/bash
echo "Starting Cobalt job script"
export n_nodes=$COBALT_JOBSIZE
export n_mpi_ranks_per_node=64
export n_mpi_ranks=$(($n_nodes * $n_mpi_ranks_per_node))
aprun -n $n_mpi_ranks -N $n_mpi_ranks_per_node -cc depth -d 4 -j 4 ./pi

thetalogin6(pi)> qsub -A AProject -n 1 -t 20 runpi.bash
106040

thetalogin6(pi)> qstat -u user1
JobID   JobName  User   WallTime  QueuedTime  RunTime  Nodes  State   attrs                                Queue    Score    Project
=======================================================================================================================================
106040  N/A      user1  00:20:00  00:00:05    N/A      1      queued  {'numa': 'quad', 'mcdram': 'cache'}  default   51.0    AProject

thetalogin6(pi)> qstat -u user1
JobID   JobName  User   WallTime  QueuedTime  RunTime   Nodes  State     attrs                                Queue    Score    Project
==========================================================================================================================================
106040  N/A      user1  00:20:00  00:00:25    00:00:19  1      starting  {'numa': 'quad', 'mcdram': 'cache'}  default   51.1    AProject

thetalogin6(pi)> qstat -u user1
JobID   JobName  User   WallTime  QueuedTime  RunTime  Nodes  State   attrs                                Queue    Score    Project
=======================================================================================================================================
106040  N/A      user1  00:20:00  00:00:32    00:00:59  1      running  {'numa': 'quad', 'mcdram': 'cache'}  default   51.2    AProject

thetalogin6(pi)> ls -lt
total 18240
-rw-r--r-- 1 user1 cobalt    1671 Jun 27 23:08 106040.cobaltlog
-rw-r--r-- 1 user1 users      185 Jun 27 23:07 106040.output
-rw-r--r-- 1 user1 users        0 Jun 27 23:07 106040.error
-rwxr-xr-x 1 user1 users  9300576 Jun 27 23:06 pi*
-rw-r--r-- 1 user1 users    10256 Jun 27 23:06 pi.o
-rw-r--r-- 1 user1 users      152 Jun 27 23:05 makefile
-rwxr-xr-x 1 user1 users      237 Jun 27 23:05 runpi.bash*
-rw-r--r-- 1 user1 users      816 Jun 27 22:20 pi.c

thetalogin6(pi)> cat 106040.output
Starting Cobalt job script
pi is approximately 3.1417259869152536, Error is 0.0001333333254605
Application 3373484 resources: utime ~6s, stime ~6s, Rss ~5036, inblocks ~0, outblocks ~8
thetalogin6(pi)>