ASCI RED F90 example

Compile,run a f90 application on cougar compute nodes

You must already have accounts and be logged onto sasn100.sandia.gov or janus.sandia.gov. We are already logged into sasn100. The PGI F90 compiler is invoked with:

cif90  


Acknowledgement and Disclaimer

which is a alias for .

pgf90 -cougar

Below I have a sample mpi application: 5 source files, a Makefile and output from compiling and running the program.

First, the main routine:

sasn100 ~/f90 103 > cat test90.f
! ------------------------------------------------------------------------------
! ------------------------------------------------------------------------------
      program test90
        use define_kind
        implicit none

! ----- purpose

! ----- parameters

        integer, parameter :: MODVER = 19970221
        integer, parameter :: MVERSN = 19970220

! ----- code

        call mysubroutine('test90',MODVER,MVERSN)

        call exit
      end

And next the subroutine 'mysubroutine':

 
sasn100 ~/f90 104 > cat mysubroutine.f
! ------------------------------------------------------------------------------
      subroutine mysubroutine(prblm,modver,mversn)
        use define_kind
        implicit none
        external cerror 
        external dclock
        interface 
         
         subroutine f90mpi(my_pe,num_pe)
          integer, intent(out) :: my_pe,num_pe
         end subroutine f90mpi
        end interface

! ----- calling arguments

        character, intent(in) :: prblm*(*)
        integer, intent(in) :: modver, mversn

! ----- functions
        real(REAL8) :: dclock

! ----- local variables

        integer :: my_node_num, total_nodes
        real(REAL8) :: tsave, secold

! ----- code
        call f90mpi(my_node_num,total_nodes)
        if(my_node_num == 0)then
         tsave = dclock()
         print *,'mysubroutine of program ',prblm,modver,mversn
         print *,'PI = ', PI
! ----- call c subroutine
         
         call cerror('this should go to stderr')
         secold = dclock()
         print *,'we took ',secold-tsave,' seconds'
        endif
        print *,'f90mpi tells me I am node ',  &
          my_node_num,' of ',total_nodes


      end subroutine mysubroutine

Here is the c subroutine we call.

sasn100 ~/f90 105 > cat ciolib.c

#include 
#include 
#include 

#ifdef ASCIRED
#define cerror       cerror_
#endif

void cerror(char *msg)
{
        fprintf(stderr,"%s\n",msg);
}

The mpi subroutine has to be compiled in 'fixed' mode as it uses the f77 format include statement.

sasn100 ~/f90 106 > cat f90mpi.f
! ------------------------------------------------------------------------------
      subroutine f90mpi(my_pe,num_pe)
        use define_kind
        implicit none
        include 'mpif.h'

! ----- calling arguments

        integer, intent(out) :: my_pe,num_pe

! ----- local variables
        integer :: ierr

! ----- code
       call mpi_init(ierr)
       call mpi_comm_size(MPI_COMM_WORLD,num_pe,ierr)
       call mpi_comm_rank(MPI_COMM_WORLD,my_pe,ierr)
       call mpi_barrier(MPI_COMM_WORLD,ierr)
       print *,'hello from node ', my_pe,' of ',num_pe
       call mpi_finalize(ierr)


      end subroutine f90mpi

And the last part of the application is the module where define_kind is defined.

sasn100 ~/f90 107 > cat data_modules.f
! ------------------------------------------------------------------------------


      module define_kind

        implicit none
        save
        public

        integer, parameter :: INT4  = SELECTED_INT_KIND(8)
        integer, parameter :: INT6  = SELECTED_INT_KIND(12)
        integer, parameter :: INT8  = SELECTED_INT_KIND(16)
        integer, parameter :: REAL4 = SELECTED_REAL_KIND(5)
        integer, parameter :: REAL8 = SELECTED_REAL_KIND(12)

        real(REAL8), parameter :: ZERO    = 0.0_REAL8
        real(REAL8), parameter :: QUARTER = 0.25_REAL8
        real(REAL8), parameter :: HALF    = 0.5_REAL8
        real(REAL8), parameter :: ONE     = 1.0_REAL8
        real(REAL8), parameter :: TWO     = 2.0_REAL8
        real(REAL8), parameter :: THREE   = 3.0_REAL8
        real(REAL8), parameter :: FOUR    = 4.0_REAL8
        real(REAL8), parameter :: FIVE    = 5.0_REAL8
        real(REAL8), parameter :: SIX     = 6.0_REAL8
        real(REAL8), parameter :: SEVEN   = 7.0_REAL8
        real(REAL8), parameter :: EIGHT   = 8.0_REAL8
        real(REAL8), parameter :: NINE    = 9.0_REAL8
        real(REAL8), parameter :: TEN     = 10.0_REAL8

        real(REAL8), parameter :: PI      = 3.14159265358979323846_REAL8
        real(REAL8), parameter :: TWOPI   = TWO*PI
        real(REAL8), parameter :: THIRD   = ONE/THREE

        integer :: io_stat
        integer :: mype, numpe

      end module define_kind
! ------------------------------------------------------------------------------
sasn100 ~/f90 108 >

And what application is complete without a good Makefile. Note that any line that begins with _tab_ is actually supposed to be indented with a tab. You MUST replace the string _tab_ with a tab or you will get make errors when you run make.

SHELL=/bin/sh

# Following is for ASCI Red TFLOPS
ARCHITECTURE=ASCIRED


FC=cif90
FOPTIMIZE=  -c -O2
FFLAGS=$(FOPTIMIZE)
FREE=-Mfree
FIXED=-Mfixed

# This is for the c compiler
CC=cicc
COPTIMIZE=-O2
CFLAGS=$(COPTIMIZE) -D$(ARCHITECTURE)

# This is the linker that will be used by f90 at link time.
LD=$(FC)
LDFLAGS=
LDLIBS= -lmpi
LDMODULES=$(MODULES)

RM=rm -f
CP=cp
CPP=/usr/ccs/lib/cpp
CPPFLAGS=-P -D$(ARCHITECTURE)

.SUFFIXES:
.SUFFIXES: .F .f .c .o

# this rule assumes files with names like test.F need to be
# preprocessed by cpp
.F.o:
        $(CP) $*.F $*.c && $(CPP) $(CPPFLAGS) $*.c > $*.f && $(RM) $*.c
        $(FC) $(FFLAGS) $(FREE) $*.f
# This is the standard f90 (free) compile rule
.f.o:
        $(FC) $(FFLAGS) $(FREE) $<
# This invokes the c compiler above
.c.o:
        $(CC) -c $(CFLAGS) $<

PROG=test90.x

MODULES=data_modules.o

OBJS= test90.o \
      mysubroutine.o \
      f90mpi.o \
      ciolib.o

EXTRAS=
LDEXTRAS=


$(PROG): $(MODULES) $(LDEXTRAS) $(OBJS)
        @echo '*** Loading: ***'
        $(LD) $(LDFLAGS) -o $(PROG) $(LDMODULES)  $(LDEXTRAS) $(OBJS) $(LDLIBS)

# Dependencies
$(OBJS): $(MODULES)
#$(EXTRAS): $(MODULES)

clean:
        $(RM) $(MODULES) $(OBJS)
        $(RM) *.mod
        $(RM) *.T *.lst *.ii
        $(RM) core

# add any special compile rules below

# List the mpi file separately since it must be compiled -fixed
f90mpi.o: f90mpi.f
        $(FC) $(FFLAGS) $(FIXED) $<

Now to compile the application:

sasn100 ~/f90 109 > make
pgf90 -cougar -g -c -O2 -Mfree data_modules.f
pgf90 -cougar -g -c -O2 -Mfree test90.f
pgf90 -cougar -g -c -O2 -Mfree mysubroutine.f
pgf90 -cougar -g -c -O2 -Mfixed f90mpi.f
pgcc  -cougar -g -O2 -DASCIRED ciolib.c
*** Loading: ***
pgf90 -cougar  -o test90.x data_modules.o   test90.o  mysubroutine.o  f90mpi.o  ciolib.o -lmpi
Linking:
sasn100 ~/f90 110 >

To run the program, we have to login to janus.sandia.gov.

Then yod the program onto 4 compute nodes.

janus ~/f90 102 > yod -sz 4 test90.x 
 hello from node  0  of  4
 hello from node  1  of  4
 hello from node  2  of  4
 hello from node  3  of  4
 mysubroutine of program test90 19970221 19970220
 f90mpi tells me I am node  1  of  4
 f90mpi tells me I am node  2  of  4
 f90mpi tells me I am node  3  of  4
 PI =    3.1415926535897931
this should go to stderr
 we took    0.1799166549899383  seconds
 f90mpi tells me I am node  0  of  4
janus ~/f90 103 >

That's it!

janus 104 > logout