Here is a simple example program and makefile for building on XC40 systems. When you compile your code for our XC40 systems, you are normally cross-compiling for the compute nodes (Intel Xeon Phi 2nd Generation [KNL]), rather than directly compiling for the login nodes (non-KNL). The default, you are using the Intel compiler suite. We also provide the Cray, GNU, and Clang compiler suites via modules.
C Program
/* File pi.c */ #include <mpi.h> #include <math.h> #include <stdio.h> int main(int argc, char** argv) { int n, myid, numprocs, i; double PI25DT = 3.141592653589793238462643; double mypi, pi, h, sum, x; MPI_Init(&argc, &argv); MPI_Comm_size(MPI_COMM_WORLD, &numprocs); MPI_Comm_rank(MPI_COMM_WORLD, &myid); /* Number of intervals */ n = 25; MPI_Bcast(&n, 1, MPI_INT, 0, MPI_COMM_WORLD); h = 1.0 / (double) n; sum = 0.0; for ( i = myid + 1; i <= n; i += numprocs ) { x = h * ((double) i - 0.5); sum += ( 4.0 / (1.0 + x*x) ); } mypi = h * sum; MPI_Reduce(&mypi, &pi, 1, MPI_DOUBLE, MPI_SUM, 0, MPI_COMM_WORLD); if (myid == 0) { printf("pi is approximately %.16f, Error is %.16f\n", pi, fabs(pi - PI25DT)); } MPI_Finalize(); return 0; }
Makefile
# File makefile CC = cc OPTFLAGS = -O3 CFLAGS = $(OPTFLAGS) -g all: pi pi: pi.o $(CC) $(CFLAGS) -o pi pi.o pi.o: pi.o $(CC) $(CFLAGS) -c pi.c clean: rm pi pi.o
Usage
Session screenshot shows building and running, using an example job script:
thetalogin6(pi)> ls makefile pi.c runpi.bash* thetalogin6(pi)> make cc -O3 -g -c pi.c cc -O3 -g -o pi pi.o thetalogin6(pi)> ls makefile pi* pi.c pi.o runpi.bash* thetalogin6(pi)> cat runpi.bash #!/bin/bash echo "Starting Cobalt job script" export n_nodes=$COBALT_JOBSIZE export n_mpi_ranks_per_node=64 export n_mpi_ranks=$(($n_nodes * $n_mpi_ranks_per_node)) aprun -n $n_mpi_ranks -N $n_mpi_ranks_per_node -cc depth -d 4 -j 4 ./pi thetalogin6(pi)> qsub -A AProject -n 1 -t 20 runpi.bash 106040 thetalogin6(pi)> qstat -u user1 JobID JobName User WallTime QueuedTime RunTime Nodes State attrs Queue Score Project ==================================================================================== 106040 N/A user1 00:20:00 00:00:05 N/A 1 queued {'numa': 'quad', 'mcdram': 'cache'} default 51.0 AProject thetalogin6(pi)> qstat -u user1 JobID JobName User WallTime QueuedTime RunTime Nodes State attrs Queue Score Project ==================================================================================== 106040 N/A user1 00:20:00 00:00:25 00:00:19 1 starting {'numa': 'quad', 'mcdram': 'cache'} default 51.1 AProject thetalogin6(pi)> qstat -u user1 JobID JobName User WallTime QueuedTime RunTime Nodes State attrs Queue Score Project ==================================================================================== 106040 N/A user1 00:20:00 00:00:32 00:00:59 1 running {'numa': 'quad', 'mcdram': 'cache'} default 51.2 AProject thetalogin6(pi)> ls -lt total 18240 -rw-r--r-- 1 user1 cobalt 1671 Jun 27 23:08 106040.cobaltlog -rw-r--r-- 1 user1 users 185 Jun 27 23:07 106040.output -rw-r--r-- 1 user1 users 0 Jun 27 23:07 106040.error -rwxr-xr-x 1 user1 users 9300576 Jun 27 23:06 pi* -rw-r--r-- 1 user1 users 10256 Jun 27 23:06 pi.o -rw-r--r-- 1 user1 users 152 Jun 27 23:05 makefile -rwxr-xr-x 1 user1 users 237 Jun 27 23:05 runpi.bash* -rw-r--r-- 1 user1 users 816 Jun 27 22:20 pi.c thetalogin6(pi)> cat 106040.output Starting Cobalt job script pi is approximately 3.1417259869152536, Error is 0.0001333333254605 Application 3373484 resources: utime ~6s, stime ~6s, Rss ~5036, inblocks ~0, outblocks ~8 thetalogin6(pi)>