main.f

Go to the documentation of this file.

! -*- f90 -*-
!vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv!
!                                                                      !
!  MODULE: MAIN                                                        !
!                                                                      !
!  Purpose: Main module for top level mfix subroutines.                !
!                                                                      !
!^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^!

      MODULE main

      use exit, only: mfix_exit

!-----------------------------------------------
! Module variables
!-----------------------------------------------
! Final value of CPU time.
      DOUBLE PRECISION :: cpu1
! time used for computations.
      DOUBLE PRECISION :: cputime_used, walltime_used
! DISTIO variable for specifying the mfix version
      CHARACTER(LEN=512) :: version
! environment variable
!$ CHARACTER(LEN=512) :: omp_num_threads
!$ INTEGER :: length
!$ INTEGER :: status

! Number of iterations
      INTEGER :: nit_total
! used for activating check_data_30
      INTEGER :: ncheck, dncheck

! Flag to save results and cleanly exit.
      LOGICAL :: exit_signal = .false.

      CHARACTER(LEN=80), DIMENSION(100) :: cmd_line_args
      INTEGER :: cmd_line_args_count = 0

      CONTAINS

      SUBROUTINE initialize

#ifdef MPI
      USE mpi, only: mpi_comm_world, mpi_barrier  ! ignore-depcomp
#endif
      USE cdist, only: bdoing_postmfix
      USE cdist, only: bglobalnetcdf, bstart_with_one_res, bdist_io, bwrite_netcdf
      USE check, only: check_mass_balance
      USE check_data_cg, only: check_bc_flags, report_best_processor_size
      USE coeff, only: init_coeff
      USE compar, only: mpierr, mype, pe_io
      USE cont, only: do_cont
      USE cutcell, only: cartesian_grid, re_indexing, set_corner_cells
      USE discretelement, only: discrete_element
      USE drag, only: f_gs
      USE error_manager, only: err_msg, flush_err_msg
      USE error_manager, only: init_err_msg, finl_err_msg
      USE fldvar, only: rop_g, rop_s
      USE funits, only: dmp_log, unit_log, unit_res
      USE machine, only: start_log, end_log
      USE machine, only: wall_time, pc_quickwin, machine_cons, get_run_id, start_log, end_log
      USE mfix_netcdf, only: mfix_usingnetcdf
      USE output, only: dbgprn_layout
      USE output_man, only: init_output_vars, output_manager
      USE parallel_mpi, only: parallel_init, parallel_fin
      USE param1, only: n_spx, undefined, zero
      USE pgcor, only: d_e, d_n, d_t, phase_4_p_g, switch_4_p_g
      USE physprop, only: mmax
      USE pscor, only: e_e, e_n, e_t, do_p_s, phase_4_p_s, mcp, switch_4_p_s
      USE qmom_kinetic_equation, only: qmomk
      USE read_input, only: get_data
      USE run, only: id_version, ier
      USE run, only: automatic_restart, call_usr, dem_solids, dt_max, dt_min
      USE run, only: iter_restart, nstep, pic_solids, run_type, dt, shear, time, v_sh
      USE time_cpu, only: cpu00, wall0
      USE time_cpu, only: cpu_io, cpu_nlog, cpu0, cpuos, time_nlog
      USE vtk, only: write_vtk_files

      IMPLICIT NONE

!$    INTEGER num_threads, threads_specified, omp_id
!$    INTEGER omp_get_num_threads
!$    INTEGER omp_get_thread_num

      ! Temporary storage for DT
      DOUBLE PRECISION :: DT_tmp
      ! Save TIME in input file for RESTART_2
      DOUBLE PRECISION :: TIME_SAVE

      INTEGER :: LL, MM

! DISTIO
! If you change the value below in this subroutine, you must also
! change it in write_res0.f and the value should also be consistent
! with the check in read_res0
      version = 'RES = 01.6'

      bdoing_postmfix = .false.

! Invoke MPI initialization routines and get rank info.
      CALL parallel_init
      CALL gen_log_basename

! we want only PE_IO to write out common error messages
      dmp_log = (mype == pe_io)

! set the version.release of the software
      id_version = '2016-1'

! set automatic restart flag to false
!      AUTOMATIC_RESTART = .FALSE.
!      ITER_RESTART      = 1

! specify the number of processors to be used
!$        call get_environment_variable("OMP_NUM_THREADS",omp_num_threads,length,status, .true.)
!$      if (status.eq.0 .and. length.ne.0) then
!$        read(omp_num_threads,*) threads_specified
!$      else
!$        WRITE(*,'(A,$)') 'Enter the number of threads to be used for SMP: '
!$        READ(*,*) threads_specified
!$      endif

!$      call omp_set_num_threads(threads_specified)

! Find the number of processors used
!$omp  parallel
!$      num_threads = omp_get_num_threads()
!$      omp_id = omp_get_thread_num()
!$      if(omp_id.eq.0) Write(*,*)' Number of threads used for SMP = ',  num_threads
!$omp  end parallel

! Set machine dependent constants
      CALL machine_cons

! Get the date and time. They give the unique run_id in binary output
! files
      CALL get_run_id

! AEOLUS: stop trigger mechanism to terminate MFIX normally before batch
! queue terminates. timestep at the beginning of execution
      CALL cpu_time (cpu00)
      wall0 = wall_time()

! Read input data, check data, do computations for IC and BC locations
! and flows, and set geometry parameters such as X, X_E, DToDX, etc.
      CALL get_data

! Write the initial part of the standard output file
      CALL write_out0
      IF(.NOT.cartesian_grid)  CALL write_flags

! Write the initial part of the special output file(s)
      CALL write_usr0

!$    CALL START_LOG
!$    IF(DMP_LOG)WRITE (UNIT_LOG, *) ' '
!$    IF(DMP_LOG)WRITE (UNIT_LOG, *) ' Number of processors used = ', threads_specified
!$    IF(DMP_LOG)WRITE (UNIT_LOG, *) ' '
!$    CALL END_LOG

!  setup for PC quickwin application
      CALL pc_quickwin

      CALL init_err_msg('MFIX')


! if not netcdf writes asked for ... globally turn off netcdf
      if(mfix_usingnetcdf()) then
         bglobalnetcdf = .false.
         do ll = 1,20
            if (bwrite_netcdf(ll)) bglobalnetcdf = .true.
         enddo
      endif

      dt_tmp = dt
      SELECT CASE (trim(run_type))

      CASE ('NEW')
! Write the initial part of the restart files
         CALL write_res0
         DO ll = 1, n_spx
            CALL write_spx0 (ll, 0)
         ENDDO

      CASE ('RESTART_1')
! Read the time-dependent part of the restart file
         CALL read_res1
         WRITE(err_msg, 1010) time, nstep
         CALL flush_err_msg()

      CASE ('RESTART_2')
         time_save = time
! DISTIO
         if (mype .ne. pe_io .and. bdist_io .and. bstart_with_one_res) then
            write (unit_res,rec=1) version
            write (unit_res,rec=2) 4
            write (unit_res,rec=3) 4
         endif

         CALL read_res1
         time = time_save

1010     FORMAT('Message 1010: Read in data from .RES file for TIME = ',&
              g12.5,/'Time step number (NSTEP) =',i7)

         WRITE(err_msg, 1010) time, nstep
         CALL flush_err_msg()

         CALL write_res0

! Writing the RES1 and SPX1 can only be done here when re-indexing is turned off
! This will be done after the cell re-indexing is done later in this file.
! This allows restarting independently of the re-indexing setting between
! the previous and current run.
         IF(.NOT.re_indexing) THEN
            CALL write_res1
            DO ll = 1, n_spx
               CALL write_spx0 (ll, 0)
               CALL write_spx1 (ll, 0)
            END DO
            call write_netcdf(0,0,time)
         ENDIF

      CASE DEFAULT
         CALL start_log
         IF(dmp_log)WRITE (unit_log, *) &
              ' MFIX: Do not know how to process'
         IF(dmp_log)WRITE (unit_log, *) ' RUN_TYPE in data file'
         CALL end_log
         call mfix_exit(mype)

      END SELECT

#ifdef MPI
      call mpi_barrier(mpi_comm_world,mpierr)
#endif

      IF (dt_tmp /= undefined) THEN
         dt = max(dt_min,min(dt_max,dt))
      ELSE
         dt = dt_tmp
      ENDIF

! Set arrays for computing indices. A secondary call is made
! after cut cell-preprocessing to update array indices.
      IF(cartesian_grid) THEN
         CALL set_increments
         CALL set_increments3
      ENDIF

!      IF(.NOT.RE_INDEXING) CALL WRITE_IJK_VALUES

! Set the flags for wall surfaces impermeable and identify flow
! boundaries using FLAG_E, FLAG_N, and FLAG_T
      CALL set_flags1

!  Update flags for Cartesian_GRID.
      IF(cartesian_grid) CALL check_bc_flags

! Calculate cell volumes and face areas
      IF(.NOT.cartesian_grid) CALL set_geometry1

! Find corner cells and set their face areas to zero
      IF(.NOT.cartesian_grid)  THEN
         CALL get_corner_cells()
      ELSE
         IF (set_corner_cells)  CALL get_corner_cells ()
      ENDIF

! Set constant physical properties
      CALL set_constprop

! Set initial conditions
      CALL set_ic

! Set point sources.
      CALL set_ps

! Set boundary conditions
      CALL zero_norm_vel
      CALL set_bc0

! Cartesian grid implementation
      IF(cartesian_grid) CALL cg_set_bc0

! Set gas mixture molecular weight
      CALL set_mw_mix_g

! Set the pressure field for a fluidized bed
      IF (run_type == 'NEW') CALL set_fluidbed_p

! Initialize densities.
      IF (run_type == 'NEW') CALL set_ro_g
      IF (run_type == 'NEW') CALL set_ro_s

! Initialize time dependent boundary conditions
      CALL set_bc1

! Check the field variable data and report errors.
      IF(.NOT.cartesian_grid)  CALL check_data_20

!=======================================================================
! JFD: START MODIFICATION FOR RE-INDEXING CELLS
!=======================================================================
      IF(cartesian_grid.AND.re_indexing) THEN

         IF(mype == pe_io) THEN
            WRITE(*,"(72('='))")
            WRITE(*,*)' RE-INDEXING CELLS FOR CARTESIAN GRID...'
         ENDIF
         CALL re_index_arrays


         !IF(myPE == PE_IO)print*,'Calling REPORT_BEST_IJK_SIZE:'
         !CALL REPORT_BEST_IJK_SIZE
         CALL report_best_processor_size
         !IF(myPE == PE_IO)print*,'Exiting MFIX after REPORT_BEST_IJK_SIZE.'

         IF(mype == pe_io) WRITE(*,"(72('='))")

! In case of a RESTART_2, write the RES1 and SPX1 files here
! This was commented out earlier in this file.
         IF(run_type == 'RESTART_2') THEN
            CALL write_res1
            DO ll = 1, n_spx
               CALL write_spx0 (ll, 0)
               CALL write_spx1 (ll, 0)
            END DO
            call write_netcdf(0,0,time)
         ENDIF
      ENDIF
!=======================================================================
! JFD: END MODIFICATION FOR RE-INDEXING CELLS
!=======================================================================

! Setup VTK data for regular (no cut cells) grid
      IF(.NOT.cartesian_grid.AND.write_vtk_files) CALL setup_vtk_no_cutcell

      IF(discrete_element) CALL make_arrays_des
      IF(qmomk) CALL qmomk_make_arrays

! Set the inflow/outflow BCs for DEM solids
      IF(dem_solids) CALL set_bc_dem
! Set the inflow/outflow BC for PIC solids
      IF(pic_solids) CALL set_bc_pic

! Set the inital properties of each particle.
      IF(dem_solids) CALL set_ic_dem

! AEOLUS: debug prints
      if (dbgprn_layout .or. bdist_io) then
         !write (*,*) myPE , ' E.4 ... version = ' , version(1:33)
         call debug_write_layout()
         call write_parallel_info()
      endif

! Initializations for CPU time calculations in iterate
      cpuos = 0.
      CALL cpu_time (cpu1)
      cpu_nlog = cpu1
      time_nlog = time - dt

! Get the initial value of CPU time
      CALL cpu_time (cpu0)

! Find the solution of the equations from TIME to TSTOP at
! intervals of DT

!-----------------------------------------------

      ncheck  = nstep
      dncheck = 1
      cpu_io  = zero
      nit_total = 0

      CALL init_output_vars

! Parse residual strings
      CALL parse_resid_string ()

! Call user-defined subroutine to set constants, check data, etc.
      IF (call_usr) CALL usr0

      CALL rrates_init()

! Calculate all the coefficients once before entering the time loop
      CALL init_coeff(ier)

      DO mm=1, mmax
         f_gs(1,mm) = zero
      ENDDO

! Remove undefined values at wall cells for scalars
      CALL undef_2_0 (rop_g)
      DO mm = 1, mmax
         CALL undef_2_0 (rop_s(1,mm))
      ENDDO

! Initialize d's and e's to zero
      DO mm = 0, mmax
         d_e(1,mm) = zero
         d_n(1,mm) = zero
         d_t(1,mm) = zero
      ENDDO
      e_e(:) = zero
      e_n(:) = zero
      e_t(:) = zero

! calculate shear velocities if periodic shear BCs are used
      IF(shear) CALL cal_d(v_sh)

! Initialize check_mass_balance.  This routine is not active by default.
! Specify a reporting interval (hard-wired in the routine) to activate
! the routine.
      Call check_mass_balance (0)

! sof modification: now it's only needed to do this once before time-loop
! Mark the phase whose continuity will be solved and used to correct
! void/volume fraction in calc_vol_fr (see subroutine for details)
      CALL mark_phase_4_cor (phase_4_p_g, phase_4_p_s, do_cont, mcp,&
           do_p_s, switch_4_p_g, switch_4_p_s)

      END SUBROUTINE initialize

      SUBROUTINE finalize

      USE cutcell, only: cartesian_grid
      USE dashboard
      USE cut_cell_preproc, only: close_cut_cell_files
      USE error_manager, only: finl_err_msg
      USE machine, only: wall_time
      USE parallel_mpi, only: parallel_fin
      USE run, only: dt, call_usr, dt_min, get_tunit, tunit
      USE time_cpu
      IMPLICIT NONE

! Call user-defined subroutine after time-loop.
      IF (call_usr) CALL usr3

! Get the final value of CPU time.  The difference gives the
! CPU time used for the computations.
      CALL cpu_time (cpu1)

! Compute the CPU time and write it out in the .OUT file.
      cputime_used = cpu1 - cpu0 - cpu_io
      walltime_used = wall_time() - wall0
      CALL write_out3 (cputime_used, walltime_used, cpu_io)

! JFD: cartesian grid implementation
      IF(write_dashboard) THEN
         IF(dt>=dt_min) THEN
            run_status = 'Complete.'
         ELSE
            run_status = 'DT < DT_MIN.  Recovery not possible!'
         ENDIF
         CALL get_tunit(cputime_used,tunit)
         CALL update_dashboard(0,cputime_used,tunit)
      ENDIF
      IF(cartesian_grid)  CALL close_cut_cell_files

! Finalize and terminate MPI
      call parallel_fin

      CALL finl_err_msg

      END SUBROUTINE finalize

!vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv!
!                                                                      !
!  Subroutine: GEN_LOG_BASENAME                                        !
!  Author: Aytekin Gel                                Date: 19-SEP-03  !
!                                                                      !
!  Purpose: Generate the file base for DMP logs.                       !
!                                                                      !
!^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^!
      SUBROUTINE gen_log_basename

      use compar, only: mype
      use compar, only: fbname

      implicit none

! Variables for generating file basename with processor id
      INTEGER :: i1, i10, i100, i1000, i10000

! PAR_I/O Generate file basename for LOG files
      i10000 = int(mype/10000)
      i1000  = int((mype-i10000*10000)/1000)
      i100   = int((mype-i10000*10000-i1000*1000)/100)
      i10    = int((mype-i10000*10000-i1000*1000-i100*100)/10)
      i1     = int((mype-i10000*10000-i1000*1000-i100*100-i10*10)/1)

      i10000 = i10000 + 48
      i1000  = i1000  + 48
      i100   = i100   + 48
      i10    = i10    + 48
      i1     = i1     + 48

      fbname=char(i10000)//char(i1000)//char(i100)//char(i10)//char(i1)

      RETURN
      END SUBROUTINE gen_log_basename

!vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvC
!                                                                      C
!  Module name: debug_write()                                          C
!  Purpose: Write out full geometry index setup information for the
!  case
!                                                                      C
!  Author: Aytekin Gel                                Date: 19-SEP-03  C
!  Reviewer:                                          Date:            C
!                                                                      C
!                                                                      C
!  Literature/Document References:                                     C
!                                                                      C
!  Variables referenced:                                               C
!  Variables modified:                                                 C
!                                                                      C
!  Local variables:                                                    C
!                                                                      C
!^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^C

   SUBROUTINE debug_write_layout()

!-----------------------------------------------
! Modules
!-----------------------------------------------
      USE cdist
      USE compar
      USE functions
      USE funits
      USE geometry
      USE indices
      USE leqsol
      USE mpi_utility
      USE parallel
      USE param
      USE param1
      USE run
      USE sendrecv
      USE sendrecv3
      USE time_cpu
      IMPLICIT NONE
!-----------------------------------------------
! Local Variables
!-----------------------------------------------
! phase index
      INTEGER :: M
! indices
      INTEGER :: i, j, k, ijk, ijk_GL, ijk_PROC, ijk_IO
!
      integer :: indxA, indxA_gl, indxB, indxB_gl, indxC, indxC_gl
      integer :: indxD, indxD_gl, indxE, indxE_gl, indxF, indxF_gl
      integer :: indxG, indxG_gl, indxH, indxH_gl
!
      logical :: amgdbg = .true.

      character(LEN=80) :: fname

!DISTIO
!      fname = "layout_xxxx.txt"
!      write (fname(8:11),'(i4.4)') myPE
      fname = "layout_xxxxx.txt"
      write (fname(8:12),'(i5.5)') mype
      open (unit=11,file=fname,status='unknown')

      write (11,*) ' ********************************************'
      write (11,*) ' ********************************************'
      write (11,*) ' ********************************************'
      write (11,*) ' ********************************************'
      write (11,*) ' '
      write (11,*) ' '
      write (11,*) ' myPE =           ' , mype
      write (11,*) ' '
      write (11,*) ' '


      IF (amgdbg .OR. bdist_io) THEN
         write(11,.AND..AND."('BLK1: Running from istart3,iend3  jstart3, jend3  kstart3, kend3')")
         write(11,"(' (   i ,    j,     k) =>    ijk      ijk_GL     ijk_PROC    ijk_IO')")
         write(11,"(' ====================      =====     =======    ========    ======')")
         DO k = kstart3, kend3
            DO i = istart3,iend3
               DO j = jstart3, jend3
                  ijk = funijk(i,j,k)
                  ijk_gl = funijk_gl(i,j,k)
                  ijk_proc = funijk_proc(i,j,k,mype)
                  ijk_io = funijk_io(i,j,k)
                  write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',4(I8,' , '))") &
                       i,j,k,ijk,ijk_gl,ijk_proc,ijk_io
               ENDDO
            ENDDO
         ENDDO

         write(11,"(/,/,'BLK2: Print out Bottom, South, West, East, North, Top neighbors')")
         write(11,"(' (   i ,    j,     k) =>    ijk    ijk_GL    B_of    S_of    W_of    E_of    N_of    T_of')")
         write(11,"(' ====================      =====   =======  ======  ======  ======  ======  ======  ======')")
         DO k = kstart3, kend3
            DO i = istart3,iend3
               DO j = jstart3, jend3
                  ijk = funijk(i,j,k)
                  ijk_gl = funijk_gl(i,j,k)
                  write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',2(I7,' , '),6(I7,2X))") &
                       i,j,k,ijk,ijk_gl,bottom_of(ijk),south_of(ijk),west_of(ijk),&
                       east_of(ijk),north_of(ijk),top_of(ijk)
               ENDDO
            ENDDO
         ENDDO

         write(11,"(/,/,'BLK3: Print out km, jm, im, ip, jp, kp neighbors')")
         write(11,"(' (   i ,    j,     k) =>    ijk    ijk_GL    km_of   jm_of   im_of   ip_of   jp_of   kp_of')")
         write(11,"(' ====================      =====   =======  ======  ======  ======  ======  ======  ======')")
         DO k = kstart3, kend3
            DO i = istart3,iend3
               DO j = jstart3, jend3
                  ijk = funijk(i,j,k)
                  ijk_gl = funijk_gl(i,j,k)
                  write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',2(I7,' , '),6(I7,2X))") &
                       i,j,k,ijk,ijk_gl,km_of(ijk),jm_of(ijk),im_of(ijk),&
                       ip_of(ijk),jp_of(ijk),kp_of(ijk)
               ENDDO
            ENDDO
         ENDDO

         write(11,"(/,'BLK4a: Active Fluid Cells:FLUID_AT(ijk)=.T.',/,&
              &           ' (   i ,    j,     k) =>    ijk  [   x ,     ,     z]')")
         write(11,"(' ====================      =====  ====================')")
         DO ijk = ijkstart3, ijkend3
            i = i_of(ijk)
            j = j_of(ijk)
            k = k_of(ijk)

            !         IF (FLOW_AT_E(IJK)) THEN
            IF (fluid_at(ijk)) THEN
               !          write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8)") I,J,K,ijk
               write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8,' [',E12.5,',',E12.5,' ]')") i,j,k,ijk,x(i),z(k)
            ENDIF
         ENDDO

         write(11,.NOT."(/,'BLK4b: Cells that are (WALL_AT(IJK)) = .T.',/,&
              &           ' (   i ,    j,     k) =>    ijk  [   x ,     ,     z]')")
         write(11,"(' ====================      =====  ====================')")
         DO ijk = ijkstart3, ijkend3
            i = i_of(ijk)
            j = j_of(ijk)
            k = k_of(ijk)

            IF (.NOT.wall_at(ijk)) THEN
               !          write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8)") I,J,K,ijk
               write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8,' [',E12.5,',',E12.5,' ]')") i,j,k,ijk,x(i),z(k)
            ENDIF
         ENDDO

         DO k = kstart3, kend3
            DO i = istart3,iend3
               DO j = jstart3, jend3
                  ijk = funijk(i,j,k)
                  ijk_gl = funijk_gl(i,j,k)

                  if (i == istart2 .AND. j == jstart2) then
                     indxa = ijk
                     indxa_gl = ijk_gl
                  endif
                  if (i == istart1 .AND. j == jstart1) then
                     indxe = ijk
                     indxe_gl = ijk_gl
                  endif
                  if (i == istart2 .AND. j == jend2) then
                     indxb = ijk
                     indxb_gl = ijk_gl
                  endif
                  if (i == istart1 .AND. j == jend1) then
                     indxf = ijk
                     indxf_gl = ijk_gl
                  endif
                  if (i == iend1 .AND. j == jstart1) then
                     indxh = ijk
                     indxh_gl = ijk_gl
                  endif
                  if (i == iend2 .AND. j == jstart2) then
                     indxd = ijk
                     indxd_gl = ijk_gl
                  endif
                  if (i == iend1 .AND. j == jend1) then
                     indxg = ijk
                     indxg_gl = ijk_gl
                  endif
                  if (i == iend2 .AND. j == jend2) then
                     indxc = ijk
                     indxc_gl = ijk_gl
                  endif
               ENDDO
            ENDDO
            write(11,"('BLK5:')")
            write(11,"(57('='))")
            write(11,"('k= ',I5,/,57('='))") k
            write(11,"('B= ',I5,' (',I7,')',20X,'C= ',I5,' (',I7,')',/)") indxb, indxb_gl, &
                 indxc, indxc_gl
            !        write(UNIT_LOG,"(' \',34X,'/')")
            !        write(UNIT_LOG,"(2X,'\',32X,'/')")
            write(11,"(3X,'F= ',I5,' (',I7,')',12X,'G= ',I5,' (',I7,')')") indxf, indxf_gl, &
                 indxg, indxg_gl
            write(11,"(4(9X,'|',29X,'|',/))")
            write(11,"(3X,'E= ',I5,' (',I7,')',12X,'H= ',I5,' (',I7,')',/)") indxe, indxe_gl, &
                 indxh, indxh_gl
            !        write(UNIT_LOG,"(2X,'/',32X,'\')")
            !        write(UNIT_LOG,"('/',34X,'\')")
            write(11,"('A= ',I5,' (',I7,')',20X,'D= ',I5,' (',I7,')',/,/)") indxa, indxa_gl, &
                 indxd, indxd_gl

            !        write(UNIT_LOG,"(' (',I4,' , ',I4,' , ',I4,') => ',2(I7,' , '),6(I7,2X))") &
            !                                         i,j,k,ijk,ijk_GL,bottom_of(ijk),south_of(ijk),west_of(ijk),&
            !                                        east_of(ijk),north_of(ijk),top_of(ijk)

         ENDDO

         !      write(UNIT_LOG,"(/,' (   i ,    j,     k) =>    ijk (Active Fluid)')")
         !      write(UNIT_LOG,"(' ====================      =====')")
         !       DO ijk = ijkstart3, ijkend3
         !         I = I_OF(IJK)
         !         J = J_OF(IJK)
         !         K = K_OF(IJK)

         !         IF (FLOW_AT_E(IJK)) THEN
         !         IF (FLUID_AT(IJK)) THEN
         !           write(UNIT_LOG,"(' (',I4,' , ',I4,' , ',I4,') => ',I8)") I,J,K,ijk
         !         ENDIF
         !      END DO


      endif   ! end if(amgdbg .or. bdist_io)

      m = 0
      !      CALL WRITE_AB_M (A_M, B_M, IJKMAX2, M, IER)

      IF (amgdbg .OR. bdist_io) THEN
         write(11,"(/,/,'BLK6: ========= ORIGINAL MFIX VARIABLES ===========')")
         write(11,"('PE ',I5,': imin1  = ',I6,3X,'imax1= ',I6,/,'PE ',I5,': jmin1  = ',I6,3X,'jmax1= ',I6)") &
              mype,imin1,imax1,mype,jmin1,jmax1
         write(11,"('PE ',I5,': kmin1  = ',I6,3X,'kmax1= ',I6)") mype,kmin1,kmax1
         write(11,"('-----')")
         write(11,"('PE ',I5,': imin2  = ',I6,3X,'imax2= ',I6,/,'PE ',I5,': jmin2  = ',I6,3X,'jmax2= ',I6)") &
              mype,imin2,imax2,mype,jmin2,jmax2
         write(11,"('PE ',I5,': kmin2  = ',I6,3X,'kmax2= ',I6)") mype,kmin2,kmax2
         write(11,"('----- Below xxx3 set is DMP extension ------------')")
         write(11,"('PE ',I5,': imin3  = ',I6,3X,'imax3= ',I6,/,'PE ',I5,': jmin3  = ',I6,3X,'jmax3= ',I6)") &
              mype,imin3,imax3,mype,jmin3,jmax3
         write(11,"('PE ',I5,': kmin3  = ',I6,3X,'kmax3= ',I6)") mype,kmin3,kmax3
         write(11,"('----- End of Below xxx3 set is DMP extension -----')")
         !      write(11,"('PE ',I5,': ijkmax2= ',I6)") myPE,ijkmax2
         write(11,"('PE ',I5,': ijmax2 = ',I6)") mype,ijmax2
         write(11,"('PE ',I5,': ijkmin1= ',I6,' ijkmax1= ',I12)") mype,ijkmin1, ijkmax1
         write(11,"('PE ',I5,':          ',6X,' ijkmax2= ',I12)") mype,ijkmax2
         write(11,"('PE ',I5,':          ',6X,' ijkmax3= ',I12)") mype,ijkmax3
         write(11,"('PE ',I5,': ijkmin4= ',I6,' ijkmax4= ',I12)") mype,ijkmin4, ijkmax4


         write(11,"(/,/,' ========= DMP EXTENSION VARIABLES ===========')")
         !      write(UNIT_LOG,"('PE ',I5,': ijksize  = ',I6)") myPE,ijksize
         write(11,"('PE ',I5,': ijksize3 = ',I6,3X,'ijksize3_all = ',I6)") mype,ijksize3,ijksize3_all(mype)
         write(11,"('PE ',I5,': ijksize4 = ',I6,3X,'ijksize4_all = ',I6)") mype,ijksize4,ijksize4_all(mype)
         write(11,"('PE ',I5,': ijkstart3  = ',I6,3X,'ijkend3  = ',I6)") mype,ijkstart3, ijkend3
         write(11,"('PE ',I5,': ijkstart3_all = ',I6,3X,'ijkstart4_all = ',I6)") mype,ijkstart3_all(mype),ijkstart4_all(mype)
         write(11,"('PE ',I5,': istart_all = ',I6,3X,'iend_all = ',I6,/,'PE ',I5,': jstart_all = ',I6,3X,'jend_all = ',I6)") &
              mype,istart_all(mype),iend_all(mype),mype,jstart_all(mype),jend_all(mype)
         write(11,"('PE ',I5,': kstart_all = ',I6,3X,'kend_all = ',I6,/,'----------------------')") &
              mype,kstart_all(mype),kend_all(mype)

         write(11,"('PE ',I5,': istart1_all= ',I6,3X,'iend1_all= ',I6,/,'PE ',I5,': jstart1_all= ',I6,3X,'jend3_all= ',I6)") &
              mype,istart1_all(mype),iend1_all(mype),mype,jstart1_all(mype),jend1_all(mype)
         write(11,"('PE ',I5,': kstart1_all= ',I6,3X,'kend1_all= ',I6,/,'----------------------')") &
              mype,kstart1_all(mype),kend1_all(mype)

         write(11,"('PE ',I5,': istart2_all= ',I6,3X,'iend2_all= ',I6,/,'PE ',I5,': jstart2_all= ',I6,3X,'jend3_all= ',I6)") &
              mype,istart2_all(mype),iend2_all(mype),mype,jstart2_all(mype),jend2_all(mype)
         write(11,"('PE ',I5,': kstart2_all= ',I6,3X,'kend2_all= ',I6,/,'----------------------')") &
              mype,kstart2_all(mype),kend2_all(mype)

         write(11,"('PE ',I5,': istart3_all= ',I6,3X,'iend3_all= ',I6,/,'PE ',I5,': jstart3_all= ',I6,3X,'jend3_all= ',I6)") &
              mype,istart3_all(mype),iend3_all(mype),mype,jstart3_all(mype),jend3_all(mype)
         write(11,"('PE ',I5,': kstart3_all= ',I6,3X,'kend3_all= ',I6,/,'----------------------')") &
              mype,kstart3_all(mype),kend3_all(mype)

         write(11,"('PE ',I5,': istart1= ',I6,3X,'iend1= ',I6,/,'PE ',I5,': jstart1= ',I6,3X,'jend1= ',I6)") &
              mype,istart1,iend1,mype,jstart1,jend1
         write(11,"('PE ',I5,': kstart1= ',I6,3X,'kend1= ',I6,/,'----------------------')") &
              mype,kstart1,kend1
         write(11,"('PE ',I5,': istart2= ',I6,3X,'iend2= ',I6,/,'PE ',I5,': jstart2= ',I6,3X,'jend2= ',I6)") &
              mype,istart2,iend2,mype,jstart2,jend2
         write(11,"('PE ',I5,': kstart2= ',I6,3X,'kend2= ',I6,/,'----------------------')") &
              mype,kstart2,kend2
         write(11,"('PE ',I5,': istart3= ',I6,3X,'iend3= ',I6,/,'PE ',I5,': jstart3= ',I6,3X,'jend3= ',I6)") &
              mype,istart3,iend3,mype,jstart3,jend3
         write(11,"('PE ',I5,': kstart3= ',I6,3X,'kend3= ',I6,/,'----------------------')") &
              mype,kstart3,kend3

      ENDIF   ! end if(amgdbg .or. bdist_io)

      close(unit=11)


      RETURN
   END SUBROUTINE debug_write_layout



   !vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvC

   !^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^C

   SUBROUTINE write_parallel_info()

      !-----------------------------------------------
      !   M o d u l e s
      !-----------------------------------------------
      USE compar
      USE functions
      USE funits
      USE geometry
      USE indices
      USE leqsol
      USE mpi_utility
      USE parallel
      USE param
      USE param1
      USE run
      USE sendrecv
      USE sendrecv3
      USE time_cpu
      IMPLICIT NONE
      !-----------------------------------------------
      ! Dummy arguments
      !-----------------------------------------------
      ! Local Variables
      !-----------------------------------------------
      ! phase index
      INTEGER :: M
      ! indices
      INTEGER :: i, j, k, ijk, ijk_GL, ijk_PROC, ijk_IO
      !
      character(LEN=80) :: fname
      !-----------------------------------------------

      !DISTIO
      !      fname = "p_info_xxxx.txt"
      !      write (fname(8:11),'(i4.4)') myPE
      fname = "p_info_xxxxx.txt"
      write (fname(8:12),'(i5.5)') mype
      open (unit=11,file=fname,status='unknown')

      write (11,*) mype , ' = myPE'

      write (11,*) mype , istart3,iend3
      write (11,*) mype , jstart3,jend3
      write (11,*) mype , kstart3,kend3

      write(11,.AND..AND."('BLK1: Running from istart3,iend3  jstart3, jend3  kstart3, kend3')")
      write(11,"(' (   i ,    j,     k)       ijk      ijk_GL     ijk_PROC    ijk_IO')")
      write(11,"(' ====================      =====     =======    ========    ======')")
      DO k = kstart3, kend3
         DO i = istart3,iend3
            DO j = jstart3, jend3
               ijk = funijk(i,j,k)
               ijk_gl = funijk_gl(i,j,k)
               ijk_proc = funijk_proc(i,j,k,mype)
               ijk_io = funijk_io(i,j,k)
               write(11,"('  ',I4,'   ',I4,'   ',I4,'     ',4(I8,'   '))" ) &
                    i,j,k,ijk,ijk_gl,ijk_proc,ijk_io
            ENDDO
         ENDDO
      ENDDO

      m = 0
      !      CALL WRITE_AB_M (A_M, B_M, IJKMAX2, M, IER)

      close(unit=11)

      RETURN
   END SUBROUTINE write_parallel_info

   !vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv!
   !                                                                      !
   !  SUBROUTINE: DO_MPI_BCAST                                            !
   !                                                                      !
   !  Purpose: Used by pymfix for broadcasting commands.                  !
   !                                                                      !
   !^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^!
   function do_mpi_bcast(str)
#ifdef MPI
      use mpi ! ignore-depcomp
#endif
      implicit none

      ! TODO: return a dynamically allocated string, instead of fixed size of 100,000 bytes
      character(len=100000),intent(in) :: str
      character :: aa(100000)
      character :: do_mpi_bcast(100000)
      integer :: ii
      integer :: ierr

      do ii = 1,len(str)
         aa(ii) = str(ii:ii)
      end do

#ifdef MPI
      call mpi_bcast(aa,100000,mpi_character,0,mpi_comm_world,ierr)
#endif

      do ii = 1,100000
         do_mpi_bcast(ii:ii) = aa(ii)
      end do

   end function do_mpi_bcast

   subroutine do_write_dbg_vtu_and_vtp_files
      implicit none
      call write_dbg_vtu_and_vtp_files
   end subroutine do_write_dbg_vtu_and_vtp_files

   subroutine do_backupres
      use output_man, only: backup_res
      implicit none
      call backup_res
   end subroutine do_backupres

   subroutine do_reinit(filename)
      use reinit, only: reinitialize
      implicit none
      ! filename of uploaded mfix.dat file
      character(len=*), intent(in) :: filename
      call reinitialize(filename)
   end subroutine do_reinit

   subroutine do_abort
      use compar, only: mype
      implicit none
      call mfix_exit(mype)
   end subroutine do_abort

   !vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv!
   !  Subroutine: ADD_COMMAND_LINE_ARGUMENT                               !
   !  Author: M.Meredith                                 Date: 03-FEB-16  !
   !                                                                      !
   !  Purpose: Save command line arguments in CMD_LINE_ARGS array.        !
   !           Used by both mfix.f and pymfix.                            !
   !                                                                      !
   !^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^!
   SUBROUTINE add_command_line_argument(ARG)
      implicit none
      CHARACTER(LEN=80), INTENT(IN) :: ARG

      cmd_line_args_count = cmd_line_args_count + 1

      if (cmd_line_args_count > 100) THEN
         print *,"TOO MANY COMMAND LINE ARGUMENTS"
         stop
      ENDIF

      cmd_line_args(cmd_line_args_count) = arg

   END SUBROUTINE add_command_line_argument

END MODULE main