LAMMPS

Overview

LAMMPS is a general-purpose molecular dynamics software package for massively parallel computers. As described on the LAMMPS website https://lammps.sandia.gov, LAMMPS is highly scalable and used for molecular simulations of solid-state, soft matter, coarse-grained, and mesoscopic systems with dozens of user-developed extensions.

Using LAMMPS at ALCF

ALCF provides assistance with build instructions, compiling executables, submitting jobs, and providing prebuilt binaries. Some older versions of LAMMPS are available in /soft/applications/lammps. The Makefiles provided below can be used for local installations of newer versions of LAMMPS with different user packages and/or with local modifications. For questions, contact us at support@alcf.anl.gov.

How to Obtain the Code

LAMMPS is an open-source code which can be download at https://lammps.sandia.gov/download.html

Building on Blue Gene/Q

After  [LAMMPS] has been downloaded and unpacked, you should see a directory whose name is of the form lammps-<version>. Makefiles for BG/Q are provided in lammps-<version>/src/MAKE/MACHINES and those Makefiles can be updated as needed. The first file is called Makefile.bgq.

# bgq = IBM Blue Gene/Q, multiple compiler options, native MPI, ALCF FFTW3

SHELL = /bin/bash
.SUFFIXES: .cpp .u

# ---------------------------------------------------------------------
# build rules and dependencies
# do not edit this section
# select which compiler by editing Makefile.bgq.details

include ../MAKE/MACHINES/Makefile.bgq.details

include Makefile.package.settings
include Makefile.package

EXTRA_INC = $(LMP_INC) $(PKG_INC) $(MPI_INC) $(FFT_INC) $(JPG_INC) $(PKG_SYSINC)
EXTRA_PATH = $(PKG_PATH) $(MPI_PATH) $(FFT_PATH) $(JPG_PATH) $(PKG_SYSPATH)
EXTRA_LIB = $(PKG_LIB) $(MPI_LIB) $(FFT_LIB) $(JPG_LIB) $(PKG_SYSLIB)
EXTRA_CPP_DEPENDS = $(PKG_CPP_DEPENDS)
EXTRA_LINK_DEPENDS = $(PKG_LINK_DEPENDS)

# Path to src files

vpath %.cpp ..
vpath %.h ..

# Link target

$(EXE): $(OBJ) $(EXTRA_LINK_DEPENDS)
        $(LINK) $(LINKFLAGS) $(EXTRA_PATH) $(OBJ) $(EXTRA_LIB) $(LIB) -o $(EXE)
        $(SIZE) $(EXE)

# Library targets

lib:    $(OBJ) $(EXTRA_LINK_DEPENDS)
        $(ARCHIVE) $(ARFLAGS) $(EXE) $(OBJ)

shlib:  $(OBJ) $(EXTRA_LINK_DEPENDS)
        $(CC) $(CCFLAGS) $(SHFLAGS) $(SHLIBFLAGS) $(EXTRA_PATH) -o $(EXE) \
        $(OBJ) $(EXTRA_LIB) $(LIB)

# Compilation rules

%.o:%.cpp $(EXTRA_CPP_DEPENDS)
        $(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<

%.d:%.cpp $(EXTRA_CPP_DEPENDS)
        $(CC) $(CCFLAGS) $(EXTRA_INC) $(DEPFLAGS) $< > $@

%.o:%.cu $(EXTRA_CPP_DEPENDS)
        $(CC) $(CCFLAGS) $(SHFLAGS) $(EXTRA_INC) -c $<

# Individual dependencies

depend : fastdep.exe $(SRC)
        @./fastdep.exe $(EXTRA_INC) -- $^ > .depend || exit 1

fastdep.exe: ../DEPEND/fastdep.c
        cc -0 -o $@ $<

sinclude .depend

The second file is called Makefile.bgq.details and contains specifics for each compiler (xl, gcc, and llvm).

# multiple compiler options for BGQ

# ---------------------------------------------------------------------
# compiler/linker settings
# specify flags and libraries needed for your compiler

# uncomment one and only one of the following three lines
# to choose a compiler toolchain

#COMPILER = GCC
#COMPILER = LLVM
COMPILER = XLC

ifeq ($(COMPILER),XLC)
CC       = mpixlcxx_r
CCFLAGS  = -O3 -qarch=qp -qtune=qp -qsmp=omp -qsimd=auto -qhot=level=2 -qprefetch -qunroll=yes
FC       = mpixlf90_r
FFLAGS   = -O3 -qarch=qp -qtune=qp -qsimd=auto -qhot=level=2 -qprefetch -qunroll=yes -qsmp=omp -qextname -qnosave
DEPFLAGS = -M -qmakedep=gcc
FFTW     = /soft/libraries/alcf/current/xl/FFTW3
endif

ifeq ($(COMPILER),GCC)
CC       = mpicxx
CCFLAGS  = -O3 -fopenmp
FC       = mpif90
FFLAGS   = -O3 -fopenmp
DEPFLAGS = -M
FFTW     = /soft/libraries/alcf/current/gcc/FFTW3
endif

ifeq ($(COMPILER),LLVM)
CC       = mpiclang++
CCFLAGS  = -O3 -fopenmp
DEPFLAGS = -M
FC       = /bin/false
FFLAGS   = LLVM does not have a Fortran front-end!
FFTW     = /soft/libraries/alcf/current/xl/FFTW3
endif

LINK      = $(CC)
LINKFLAGS = $(CCFLAGS)

ifeq ($(COMPILER),XLC)
  MASS_LIB    = ${IBM_MAIN_DIR}/xlmass/bg/7.3/bglib64
  XLF_LIB     = ${IBM_MAIN_DIR}/xlf/bg/14.1/bglib64
  XLSMP_LIB   = ${IBM_MAIN_DIR}/xlsmp/bg/3.1/bglib64
  LIB        += -L${MASS_LIB} -L${XLF_LIB} -L${XLSMP_LIB}
  LIB        += -lmassv -lmass
  LIB        += -lxlf90_r -lxlsmp -lxlopt -lxlfmath -lxl
endif

ifeq ($(COMPILER),GCC)
# libm is definitely slower than libmass...
  LIB += -lm -lgfortran
endif

ifeq ($(COMPILER),LLVM)
    SLEEF_DIR = /home/projects/llvm/sleef
    LIB += -L${SLEEF_DIR}/lib -lsleef
endif

SIZE       = size

ARCHIVE    = ar
ARFLAGS    = -rc

# BGQ should not use shared libraries

SHFLAGS    =
SHLIBFLAGS =

# ---------------------------------------------------------------------
# LAMMPS-specific settings, all OPTIONAL
# specify settings for LAMMPS features you will use
# if you change any -D setting, do full re-compile after "make clean"

# LAMMPS ifdef settings
# see possible settings in Section 2.2 (step 4) of manual

LMP_INC = -DLAMMPS_GZIP

# MPI library
# see discussion in Section 2.2 (step 5) of manual
# MPI wrapper compiler/linker can provide this info
# can point to dummy MPI library in src/STUBS as in Makefile.serial
# use -D MPICH and OMPI settings in INC to avoid C++ lib conflicts
# INC = path for mpi.h, MPI compiler settings
# PATH = path for MPI library
# LIB = name of MPI library

MPI_INC    =
MPI_PATH   =
MPI_LIB    =

MPI_INC += -DMPICH_SKIP_MPICXX
MPI_LIB +=

# FFT library
# see discussion in Section 2.2 (step 6) of manaul
# can be left blank to use provided KISS FFT library
# INC = -DFFT setting, e.g. -DFFT_FFTW, FFT compiler settings
# PATH = path for FFT library
# LIB = name of FFT library

FFT_INC  = -I$(FFTW)/include -DFFT_FFTW3 -DFFT_SINGLE
FFT_PATH = -L$(FFTW)/lib
FFT_LIB  = -lfftw3f

# JPEG and/or PNG library
# see discussion in Section 2.2 (step 7) of manual
# only needed if -DLAMMPS_JPEG or -DLAMMPS_PNG listed with LMP_INC
# INC = path(s) for jpeglib.h and/or png.h
# PATH = path(s) for JPEG library and/or PNG library
# LIB = name(s) of JPEG library and/or PNG library

JPG_INC  =       
JPG_PATH =
JPG_LIB  =

depend : fastdep.exe $(SRC)
        @./fastdep.exe $(EXTRA_INC) -- $^ > .depend || exit 1

fastdep.exe: ../DEPEND/fastdep.c
        cc -0 -o $@ $<

sinclude .depend

The compiler of choice can be selected by uncommenting the corresponding line at the top of Makefile.bgq.details and updating your .soft file to include the appropriate mpiwrapper (e.g. +mpiwrapper-xl.legacy.ndebug). LAMMPS can then be compiled from the lammps-<version>/src directory using "make bgq -j 8".

Running Jobs on Blue Gene/Q

The following is a valid invocation of LAMMPS on the ALCF Blue Gene/Q systems.

   qsub -t 1:00:00 -n 1024 -A project_name --mode c32 ./lmp_bgq -in in_274.MS -log log.lammps

This is a valid invocation of a LAMMPS build using the USER-OMP package.

   qsub -t 1:00:00 -n 1024 -A project_name --mode c16 --env OMP_NUM_THREADS=4 ./lmp_bgq -sf omp -in in_274.MS -log log.lammps