File: RELATIVE:/../../../mfix.git/model/mfix.f

1     !vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv!
2     !                                                                      !
3     !  Subroutine: MFIX                                                    !
4     !  Author: M. Syamlal                                 Date: 29-JAN-92  !
5     !                                                                      !
6     !  Purpose: The main module in the MFIX program                        !
7     !                                                                      !
8     !^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^!
9     !
10     !> \mainpage Multiphase Flow with Interphase eXchanges
11     !!
12     !! MFIX is a general-purpose computer code developed at the National
13     !! Energy Technology Laboratory, NETL, for describing the hydrodynamics,
14     !! heat transfer, and chemical reactions in fluid-solid systems.
15     !!
16     !! It has been used for describing bubbling and circulating fluidized
17     !! beds and spouted beds. MFiX calculations give transient data on the
18     !! three-dimensional distribution of pressure, velocity, temperature,
19     !! and species mass fractions. MFiX code is based on a generally
20     !! accepted set of multiphase flow equations. The code is used as a
21     !! "test-stand" for testing and developing multiphase flow constitutive
22     !!  equations.
23     !!
24     !! \section Notice
25     !! Neither the United States Government nor any agency thereof, nor any
26     !! of their employees, makes any warranty, expressed or implied, or
27     !! assumes any legal liability or responsibility for the accuracy,
28     !! completeness, or usefulness of any information, apparatus, product,
29     !! or process disclosed or represents that its use would not infringe
30     !! privately owned rights.
31     !!
32     !! * MFIX is provided without any user support for applications in the
33     !!   user's immediate organization. It should not be redistributed in
34     !!   whole or in part.
35     !!
36     !! * The use of MFIX is to be acknowledged in any published paper based
37     !!   on computations using this software by citing the MFIX theory
38     !!   manual. Some of the submodels are being developed by researchers
39     !!   outside of NETL. The use of such submodels is to be acknowledged
40     !!   by citing the appropriate papers of the developers of the submodels.
41     !!
42     !! * The authors would appreciate receiving any reports of bugs or other
43     !!   difficulties with the software, enhancements to the software, and
44     !!   accounts of practical applications of this software.
45     !!
46     !! \section Disclaimer
47     !! This report was prepared as an account of work sponsored by an agency
48     !! of the United States Government. Neither the United States Government
49     !! nor any agency thereof, nor any of their employees, makes any
50     !! warranty, express or implied, or assumes any legal liability or
51     !! responsibility for the accuracy, completeness, or usefulness of any
52     !! information, apparatus, product, or process disclosed, or represents
53     !! that its use would not infringe privately owned rights. Reference
54     !! herein to any specific commercial product, process, or service by
55     !! trade name, trademark, manufacturer, or otherwise does not
56     !! necessarily constitute or imply its endorsement, recommendation, or
57     !! favoring by the United States Government or any agency thereof. The
58     !! views and opinions of authors expressed herein do not necessarily
59     !! state or reflect those of the United States Government or any
60     !! agency thereof.
61     
62           PROGRAM MFIX
63     
64     !-----------------------------------------------
65     ! Modules
66     !-----------------------------------------------
67           USE MFIX_netcdf
68           USE cdist
69           USE compar
70           USE cutcell
71           USE dashboard
72           USE discretelement
73           USE fldvar
74           USE functions
75           USE funits
76           USE machine
77           USE mfix_pic
78           USE mpi_utility
79           USE output
80           USE parallel_mpi
81           USE param
82           USE param1
83           USE qmom_kinetic_equation
84           USE quadric
85           USE run
86           USE time_cpu
87     
88           USE vtk, only : WRITE_VTK_FILES
89     
90           use error_manager
91     
92           IMPLICIT NONE
93     !-----------------------------------------------
94     ! Local variables
95     !-----------------------------------------------
96     ! Final value of CPU time.
97           DOUBLE PRECISION :: CPU1
98     ! time used for computations.
99           DOUBLE PRECISION :: CPUTIME_USED, WALLTIME_USED
100     ! CPU time unit.
101           CHARACTER(LEN=4) :: TUNIT
102     ! Save TIME in input file for RESTART_2
103           DOUBLE PRECISION :: TIME_SAVE
104     ! Temporary storage for DT
105           DOUBLE PRECISION :: DT_tmp
106     ! loop counter
107           INTEGER :: L
108     ! DISTIO variable for specifying the mfix version
109           CHARACTER(LEN=512) :: version
110     ! environment variable
111     !$      CHARACTER(LEN=512) :: omp_num_threads
112     !$      INTEGER :: length
113     !$      INTEGER :: status
114     
115     !$      INTEGER num_threads, threads_specified, omp_id
116     !$      INTEGER omp_get_num_threads
117     !$      INTEGER omp_get_thread_num
118     
119     ! C Function
120           INTERFACE
121              SUBROUTINE INIT_CMD_SOCKET(port) BIND ( C )
122                use, INTRINSIC :: iso_c_binding
123                CHARACTER(KIND=C_CHAR), INTENT(IN) :: port(*)
124              END SUBROUTINE INIT_CMD_SOCKET
125              SUBROUTINE INIT_LOG_SOCKET(port) BIND ( C )
126                use, INTRINSIC :: iso_c_binding
127                CHARACTER(KIND=C_CHAR), INTENT(IN) :: port(*)
128              END SUBROUTINE INIT_LOG_SOCKET
129           END INTERFACE
130     
131     !-----------------------------------------------
132     
133     ! DISTIO
134     ! If you change the value below in this subroutine, you must also
135     ! change it in write_res0.f and the value should also be consistent
136     ! with the check in read_res0
137           version = 'RES = 01.6'
138     
139           bDoing_postmfix = .false.
140     
141     ! Invoke MPI initialization routines and get rank info.
142           CALL PARALLEL_INIT
143           CALL GEN_LOG_BASENAME
144     
145     ! we want only PE_IO to write out common error messages
146           DMP_LOG = (myPE == PE_IO)
147     
148     ! set the version.release of the software
149           ID_VERSION = '2015-2'
150     
151     ! set automatic restart flag to false
152     !      AUTOMATIC_RESTART = .FALSE.
153     !      ITER_RESTART      = 1
154     
155           ! create libmsockets server
156     #ifdef socket
157           CALL INIT_CMD_SOCKET("7777"//CHAR(0))
158           CALL INIT_LOG_SOCKET("8888"//CHAR(0))
159     #endif
160     
161     ! specify the number of processors to be used
162     !$        call get_environment_variable("OMP_NUM_THREADS",omp_num_threads,length,status, .true.)
163     !$      if (status.eq.0 .and. length.ne.0) then
164     !$        read(omp_num_threads,*) threads_specified
165     !$      else
166     !$        WRITE(*,'(A,$)') 'Enter the number of threads to be used for SMP: '
167     !$        READ(*,*) threads_specified
168     !$      endif
169     
170     !$      call omp_set_num_threads(threads_specified)
171     
172     ! Find the number of processors used
173     !$omp  parallel
174     !$      num_threads = omp_get_num_threads()
175     !$      omp_id = omp_get_thread_num()
176     !$      if(omp_id.eq.0) Write(*,*)' Number of threads used for SMP = ',  num_threads
177     !$omp  end parallel
178     
179     
180     ! Set machine dependent constants
181           CALL MACHINE_CONS
182     
183     ! Get the date and time. They give the unique run_id in binary output
184     ! files
185           CALL GET_RUN_ID
186     
187     ! AEOLUS: stop trigger mechanism to terminate MFIX normally before batch
188     ! queue terminates. timestep at the beginning of execution
189           CALL CPU_TIME (CPU00)
190           WALL0 = WALL_TIME()
191     
192     ! Read input data, check data, do computations for IC and BC locations
193     ! and flows, and set geometry parameters such as X, X_E, DToDX, etc.
194           CALL GET_DATA
195     
196     ! Write the initial part of the standard output file
197           CALL WRITE_OUT0
198           IF(.NOT.CARTESIAN_GRID)  CALL WRITE_FLAGS
199     
200     ! Write the initial part of the special output file(s)
201           CALL WRITE_USR0
202     
203     !$    CALL START_LOG
204     !$    IF(DMP_LOG)WRITE (UNIT_LOG, *) ' '
205     !$    IF(DMP_LOG)WRITE (UNIT_LOG, *) ' Number of processors used = ', threads_specified
206     !$    IF(DMP_LOG)WRITE (UNIT_LOG, *) ' '
207     !$    CALL END_LOG
208     
209     !  setup for PC quickwin application
210           CALL PC_QUICKWIN
211     
212     
213           CALL INIT_ERR_MSG('MFIX')
214     
215     
216       101 CONTINUE
217     
218     
219     ! if not netcdf writes asked for ... globally turn off netcdf
220           if (MFIX_usingNETCDF()) then
221              bGlobalNetcdf = .false.
222              do L = 1,20
223                 if (bWrite_netcdf(L)) bGlobalNetcdf = .true.
224              enddo
225           endif
226     
227     
228           IF(AUTOMATIC_RESTART) THEN
229              RUN_TYPE = 'RESTART_1'
230              AUTOMATIC_RESTART = .FALSE.
231              ITER_RESTART = ITER_RESTART + 1
232              CALL CHECK_INITIAL_CONDITIONS
233              CALL CHECK_BOUNDARY_CONDITIONS
234              CALL CHECK_INTERNAL_SURFACES
235              CALL CHECK_POINT_SOURCES
236              CALL CHECK_CHEMICAL_RXNS
237              CALL SET_FLAGS
238              CALL SET_CONSTPROP
239           ENDIF
240     
241           DT_TMP = DT
242           SELECT CASE (TRIM(RUN_TYPE))
243     
244           CASE ('NEW')
245     ! Write the initial part of the restart files
246              CALL WRITE_RES0
247              DO L = 1, N_SPX
248                 CALL WRITE_SPX0 (L, 0)
249              ENDDO
250     
251            CASE ('RESTART_1')
252     ! Read the time-dependent part of the restart file
253              CALL READ_RES1
254              WRITE(ERR_MSG, 1010) TIME, NSTEP
255              CALL FLUSH_ERR_MSG()
256     
257           CASE ('RESTART_2')
258              TIME_SAVE = TIME
259     ! DISTIO
260              if (myPE .ne. PE_IO .and. bDist_IO .and. bStart_with_one_res) then
261                  write (unit_res,rec=1) version
262                  write (unit_res,rec=2) 4
263                  write (unit_res,rec=3) 4
264              endif
265     
266              CALL READ_RES1
267              TIME = TIME_SAVE
268     
269              WRITE(ERR_MSG, 1010) TIME, NSTEP
270              CALL FLUSH_ERR_MSG()
271     
272              CALL WRITE_RES0
273     
274     ! Writing the RES1 and SPX1 can only be done here when re-indexing is turned off
275     ! This will be done after the cell re-indexing is done later in this file.
276     ! This allows restarting independently of the re-indexing setting between
277     ! the previous and current run.
278              IF(.NOT.RE_INDEXING) THEN
279                 CALL WRITE_RES1
280                 DO L = 1, N_SPX
281                    CALL WRITE_SPX0 (L, 0)
282                    CALL WRITE_SPX1 (L, 0)
283                 END DO
284                 call write_netcdf(0,0,time)
285              ENDIF
286     
287           CASE DEFAULT
288              CALL START_LOG
289              IF(DMP_LOG)WRITE (UNIT_LOG, *) &
290                 ' MFIX: Do not know how to process'
291              IF(DMP_LOG)WRITE (UNIT_LOG, *) ' RUN_TYPE in data file'
292              CALL END_LOG
293              call mfix_exit(myPE)
294     
295           END SELECT
296     
297           call MPI_Barrier(MPI_COMM_WORLD,mpierr)
298     
299           IF (DT_TMP /= UNDEFINED) THEN
300              DT = MAX(DT_MIN,MIN(DT_MAX,DT))
301           ELSE
302              DT = DT_TMP
303           ENDIF
304     
305     ! Set arrays for computing indices. A secondary call is made
306     ! after cut cell-preprocessing to update array indices.
307           IF(CARTESIAN_GRID) THEN
308              CALL SET_INCREMENTS
309              CALL SET_INCREMENTS3
310           ENDIF
311     
312     
313     !      IF(.NOT.RE_INDEXING) CALL WRITE_IJK_VALUES
314     
315     
316     ! Set the flags for wall surfaces impermeable and identify flow
317     ! boundaries using FLAG_E, FLAG_N, and FLAG_T
318           CALL SET_FLAGS1
319     
320     !  Update flags for Cartesian_GRID.
321           IF(CARTESIAN_GRID) CALL CHECK_BC_FLAGS
322     
323     ! Calculate cell volumes and face areas
324           IF(.NOT.CARTESIAN_GRID)  THEN
325              CALL SET_GEOMETRY1
326     !       ELSE
327     !         CALL SET_GEOMETRY
328            ENDIF
329     
330     ! Find corner cells and set their face areas to zero
331           IF(.NOT.CARTESIAN_GRID)  THEN
332              CALL GET_CORNER_CELLS()
333           ELSE
334              IF (SET_CORNER_CELLS)  CALL GET_CORNER_CELLS ()
335           ENDIF
336     
337     ! Set constant physical properties
338           CALL SET_CONSTPROP
339     
340     ! Set initial conditions
341           CALL SET_IC
342     
343     ! Set point sources.
344           CALL SET_PS
345     
346     ! Set boundary conditions
347           CALL ZERO_NORM_VEL
348           CALL SET_BC0
349     !      IF(DISCRETE_ELEMENT) CALL MAKE_ARRAYS_DES
350     
351     ! JFD: cartesian grid implementation
352           IF(CARTESIAN_GRID) CALL CG_SET_BC0
353     !      IF(DEM_SOLIDS) CALL SET_BC_DEM
354     
355     ! Set gas mixture molecular weight
356           CALL SET_MW_MIX_G
357     
358     ! Set the pressure field for a fluidized bed
359           IF (RUN_TYPE == 'NEW') CALL SET_FLUIDBED_P
360     
361     ! Initialize densities.
362           IF (RUN_TYPE == 'NEW') CALL SET_RO_G
363           IF (RUN_TYPE == 'NEW') CALL SET_RO_S
364     
365     ! Initialize time dependent boundary conditions
366           CALL SET_BC1
367     
368     ! Check the field variable data and report errors.
369           IF(.NOT.CARTESIAN_GRID)  CALL CHECK_DATA_20
370     
371     
372     !=======================================================================
373     ! JFD: START MODIFICATION FOR RE-INDEXING CELLS
374     !=======================================================================
375           IF(CARTESIAN_GRID.AND.RE_INDEXING) THEN
376     
377              IF(myPE == PE_IO) THEN
378                 WRITE(*,"(72('='))")
379                 WRITE(*,*)' RE-INDEXING CELLS FOR CARTESIAN GRID...'
380              ENDIF
381              CALL RE_INDEX_ARRAYS
382     
383     
384     !      IF(myPE == PE_IO)print*,'Calling REPORT_BEST_IJK_SIZE:'
385     !       CALL REPORT_BEST_IJK_SIZE
386            CALL REPORT_BEST_PROCESSOR_SIZE
387     !      IF(myPE == PE_IO)print*,'Exiting MFIX after REPORT_BEST_IJK_SIZE.'
388     
389     
390              IF(myPE == PE_IO) WRITE(*,"(72('='))")
391     
392     ! In case of a RESTART_2, write the RES1 and SPX1 files here
393     ! This was commented out earlier in this file.
394              IF(RUN_TYPE == 'RESTART_2') THEN
395                 CALL WRITE_RES1
396                 DO L = 1, N_SPX
397                    CALL WRITE_SPX0 (L, 0)
398                    CALL WRITE_SPX1 (L, 0)
399                 END DO
400                 call write_netcdf(0,0,time)
401              ENDIF
402           ENDIF
403     
404     !=======================================================================
405     ! JFD: END MODIFICATION FOR RE-INDEXING CELLS
406     !=======================================================================
407     
408     ! Setup VTK data for regular (no cut cells) grid
409           IF(.NOT.CARTESIAN_GRID.AND.WRITE_VTK_FILES) CALL SETUP_VTK_NO_CUTCELL
410     
411           IF(DISCRETE_ELEMENT) CALL MAKE_ARRAYS_DES
412           IF(QMOMK) CALL QMOMK_MAKE_ARRAYS
413     
414     ! Set the inflow/outflow BCs for DEM solids
415           IF(DEM_SOLIDS) CALL SET_BC_DEM
416     ! Set the inflow/outflow BC for PIC solids
417           IF(PIC_SOLIDS) CALL SET_BC_PIC
418     
419     ! Set the inital properties of each particle.
420           IF(DEM_SOLIDS) CALL SET_IC_DEM
421     
422     ! AEOLUS: debug prints
423           if (DBGPRN_LAYOUT .or. bdist_io) then
424     !     write (*,*) myPE , ' E.4 ... version = ' , version(1:33)
425              call debug_write_layout()
426              call write_parallel_info()
427           endif
428     
429     ! Initializations for CPU time calculations in iterate
430           CPUOS = 0.
431           CALL CPU_TIME (CPU1)
432           CPU_NLOG = CPU1
433           TIME_NLOG = TIME - DT
434     
435     ! Get the initial value of CPU time
436           CALL CPU_TIME (CPU0)
437     
438     ! Find the solution of the equations from TIME to TSTOP at
439     ! intervals of DT
440           CALL TIME_MARCH
441           IF(AUTO_RESTART.AND.AUTOMATIC_RESTART&
442              .AND.ITER_RESTART.LE.10) GOTO 101
443     
444     ! Call user-defined subroutine after time-loop.
445           IF (CALL_USR) CALL USR3
446     
447     ! Get the final value of CPU time.  The difference gives the
448     ! CPU time used for the computations.
449           CALL CPU_TIME (CPU1)
450     
451     ! Compute the CPU time and write it out in the .OUT file.
452           CPUTIME_USED = CPU1 - CPU0 - CPU_IO
453           WALLTIME_USED = WALL_TIME() - WALL0
454           CALL WRITE_OUT3 (CPUTIME_USED, WALLTIME_USED, CPU_IO)
455     
456     ! JFD: cartesian grid implementation
457           IF(WRITE_DASHBOARD) THEN
458              IF(DT>=DT_MIN) THEN
459                 RUN_STATUS = 'Complete.'
460              ELSE
461                 RUN_STATUS = 'DT < DT_MIN.  Recovery not possible!'
462              ENDIF
463              CALL GET_TUNIT(CPUTIME_USED,TUNIT)
464              CALL UPDATE_DASHBOARD(0,CPUTIME_USED,TUNIT)
465           ENDIF
466           IF(CARTESIAN_GRID)  CALL CLOSE_CUT_CELL_FILES
467     
468     ! Finalize and terminate MPI
469           call parallel_fin
470     
471           CALL FINL_ERR_MSG
472     
473           STOP
474     
475      1000 FORMAT(/1X,'MFIX ',A,' Simulation:'/)
476     
477      1010 FORMAT('Message 1010: Read in data from .RES file for TIME = ',&
478              G12.5,/'Time step number (NSTEP) =',I7)
479     
480           END PROGRAM MFIX
481     
482     !vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv!
483     !                                                                      !
484     !  Subroutine: GEN_LOG_BASENAME                                        !
485     !  Author: Aytekin Gel                                Date: 19-SEP-03  !
486     !                                                                      !
487     !  Purpose: Generate the file base for DMP logs.                       !
488     !                                                                      !
489     !^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^!
490           SUBROUTINE GEN_LOG_BASENAME
491     
492           use compar, only: myPE
493           use compar, only: fbname
494     
495           implicit none
496     
497     ! Variables for generating file basename with processor id
498           INTEGER :: i1, i10, i100, i1000, i10000
499     
500     ! PAR_I/O Generate file basename for LOG files
501           i10000 = int(myPE/10000)
502           i1000  = int((myPE-i10000*10000)/1000)
503           i100   = int((myPE-i10000*10000-i1000*1000)/100)
504           i10    = int((myPE-i10000*10000-i1000*1000-i100*100)/10)
505           i1     = int((myPE-i10000*10000-i1000*1000-i100*100-i10*10)/1)
506     
507           i10000 = i10000 + 48
508           i1000  = i1000  + 48
509           i100   = i100   + 48
510           i10    = i10    + 48
511           i1     = i1     + 48
512     
513           fbname=char(i10000)//char(i1000)//char(i100)//char(i10)//char(i1)
514     
515           RETURN
516           END SUBROUTINE GEN_LOG_BASENAME
517     
518     
519     
520     
521     !vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvC
522     !                                                                      C
523     !  Module name: debug_write()                                          C
524     !  Purpose: Write out full geometry index setup information for the
525     !  case
526     !                                                                      C
527     !  Author: Aytekin Gel                                Date: 19-SEP-03  C
528     !  Reviewer:                                          Date:            C
529     !                                                                      C
530     !                                                                      C
531     !  Literature/Document References:                                     C
532     !                                                                      C
533     !  Variables referenced:                                               C
534     !  Variables modified:                                                 C
535     !                                                                      C
536     !  Local variables:                                                    C
537     !                                                                      C
538     !^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^C
539     
540           SUBROUTINE debug_write_layout()
541     
542     !-----------------------------------------------
543     ! Modules
544     !-----------------------------------------------
545           USE param
546           USE param1
547           USE parallel
548           USE matrix
549           USE geometry
550           USE compar
551           USE mpi_utility
552           USE sendrecv
553           USE sendrecv3
554           USE indices
555           USE leqsol
556           USE cdist
557           USE funits
558           USE run
559           USE time_cpu
560           USE functions
561           IMPLICIT NONE
562     !-----------------------------------------------
563     ! Local Variables
564     !-----------------------------------------------
565     ! phase index
566           INTEGER :: M
567     ! indices
568           INTEGER :: i, j, k, ijk, ijk_GL, ijk_PROC, ijk_IO
569     !
570           integer :: indxA, indxA_gl, indxB, indxB_gl, indxC, indxC_gl
571           integer :: indxD, indxD_gl, indxE, indxE_gl, indxF, indxF_gl
572           integer :: indxG, indxG_gl, indxH, indxH_gl
573     !
574           logical :: amgdbg = .TRUE.
575     
576           character(LEN=80) :: fname
577     
578     !DISTIO
579     !      fname = "layout_xxxx.txt"
580     !      write (fname(8:11),'(i4.4)') myPE
581           fname = "layout_xxxxx.txt"
582           write (fname(8:12),'(i5.5)') myPE
583           open (unit=11,file=fname,status='unknown')
584     
585           write (11,*) ' ********************************************'
586           write (11,*) ' ********************************************'
587           write (11,*) ' ********************************************'
588           write (11,*) ' ********************************************'
589           write (11,*) ' '
590           write (11,*) ' '
591           write (11,*) ' myPE =           ' , myPE
592           write (11,*) ' '
593           write (11,*) ' '
594     
595     
596         IF (AMGDBG .OR. bDist_IO) THEN
597           write(11,"('BLK1: Running from istart3,iend3 .AND. jstart3, jend3 .AND. kstart3, kend3')")
598           write(11,"(' (   i ,    j,     k) =>    ijk      ijk_GL     ijk_PROC    ijk_IO')")
599           write(11,"(' ====================      =====     =======    ========    ======')")
600           DO k = kstart3, kend3
601             DO i = istart3,iend3
602               DO j = jstart3, jend3
603                  ijk = FUNIJK(i,j,k)
604                  ijk_GL = FUNIJK_GL(i,j,k)
605                  ijk_PROC = FUNIJK_PROC(i,j,k,myPE)
606                  ijk_IO = FUNIJK_IO(i,j,k)
607                  write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',4(I8,' , '))") &
608                                              i,j,k,ijk,ijk_GL,ijk_PROC,ijk_IO
609               ENDDO
610             ENDDO
611           ENDDO
612     
613           write(11,"(/,/,'BLK2: Print out Bottom, South, West, East, North, Top neighbors')")
614           write(11,"(' (   i ,    j,     k) =>    ijk    ijk_GL    B_of    S_of    W_of    E_of    N_of    T_of')")
615           write(11,"(' ====================      =====   =======  ======  ======  ======  ======  ======  ======')")
616           DO k = kstart3, kend3
617             DO i = istart3,iend3
618               DO j = jstart3, jend3
619                  ijk = FUNIJK(i,j,k)
620                  ijk_GL = FUNIJK_GL(i,j,k)
621                  write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',2(I7,' , '),6(I7,2X))") &
622                                              i,j,k,ijk,ijk_GL,bottom_of(ijk),south_of(ijk),west_of(ijk),&
623                                              east_of(ijk),north_of(ijk),top_of(ijk)
624               ENDDO
625             ENDDO
626           ENDDO
627     
628           write(11,"(/,/,'BLK3: Print out km, jm, im, ip, jp, kp neighbors')")
629           write(11,"(' (   i ,    j,     k) =>    ijk    ijk_GL    km_of   jm_of   im_of   ip_of   jp_of   kp_of')")
630           write(11,"(' ====================      =====   =======  ======  ======  ======  ======  ======  ======')")
631           DO k = kstart3, kend3
632             DO i = istart3,iend3
633               DO j = jstart3, jend3
634                  ijk = FUNIJK(i,j,k)
635                  ijk_GL = FUNIJK_GL(i,j,k)
636                  write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',2(I7,' , '),6(I7,2X))") &
637                                              i,j,k,ijk,ijk_GL,km_of(ijk),jm_of(ijk),im_of(ijk),&
638                                              ip_of(ijk),jp_of(ijk),kp_of(ijk)
639               ENDDO
640             ENDDO
641           ENDDO
642     
643           write(11,"(/,'BLK4a: Active Fluid Cells:FLUID_AT(ijk)=.T.',/,&
644          &           ' (   i ,    j,     k) =>    ijk  [   x ,     ,     z]')")
645           write(11,"(' ====================      =====  ====================')")
646            DO ijk = ijkstart3, ijkend3
647              I = I_OF(IJK)
648              J = J_OF(IJK)
649              K = K_OF(IJK)
650     
651     !         IF (FLOW_AT_E(IJK)) THEN
652              IF (FLUID_AT(IJK)) THEN
653     !          write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8)") I,J,K,ijk
654                write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8,' [',E12.5,',',E12.5,' ]')") I,J,K,ijk,X(i),Z(k)
655              ENDIF
656           ENDDO
657     
658           write(11,"(/,'BLK4b: Cells that are (.NOT.WALL_AT(IJK)) = .T.',/,&
659          &           ' (   i ,    j,     k) =>    ijk  [   x ,     ,     z]')")
660           write(11,"(' ====================      =====  ====================')")
661            DO ijk = ijkstart3, ijkend3
662              I = I_OF(IJK)
663              J = J_OF(IJK)
664              K = K_OF(IJK)
665     
666              IF (.NOT.WALL_AT(IJK)) THEN
667     !          write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8)") I,J,K,ijk
668                write(11,"(' (',I4,' , ',I4,' , ',I4,') => ',I8,' [',E12.5,',',E12.5,' ]')") I,J,K,ijk,X(i),Z(k)
669              ENDIF
670           ENDDO
671     
672           DO k = kstart3, kend3
673             DO i = istart3,iend3
674               DO j = jstart3, jend3
675                  ijk = FUNIJK(i,j,k)
676                  ijk_GL = FUNIJK_GL(i,j,k)
677     
678                  if (i == istart2 .AND. j == jstart2) then
679                      indxA = ijk
680                      indxA_gl = ijk_GL
681                  endif
682                  if (i == istart1 .AND. j == jstart1) then
683                      indxE = ijk
684                      indxE_gl = ijk_GL
685                  endif
686                  if (i == istart2 .AND. j == jend2) then
687                      indxB = ijk
688                      indxB_gl = ijk_GL
689                  endif
690                  if (i == istart1 .AND. j == jend1) then
691                      indxF = ijk
692                      indxF_gl = ijk_GL
693                  endif
694                  if (i == iend1 .AND. j == jstart1) then
695                      indxH = ijk
696                      indxH_gl = ijk_GL
697                  endif
698                  if (i == iend2 .AND. j == jstart2) then
699                      indxD = ijk
700                      indxD_gl = ijk_GL
701                  endif
702                  if (i == iend1 .AND. j == jend1) then
703                      indxG = ijk
704                      indxG_gl = ijk_GL
705                  endif
706                  if (i == iend2 .AND. j == jend2) then
707                      indxC = ijk
708                      indxC_gl = ijk_GL
709                  endif
710               ENDDO
711             ENDDO
712             write(11,"('BLK5:')")
713             write(11,"(57('='))")
714             write(11,"('k= ',I5,/,57('='))") k
715             write(11,"('B= ',I5,' (',I7,')',20X,'C= ',I5,' (',I7,')',/)") indxB, indxB_gl, &
716                             indxC, indxC_gl
717     !        write(UNIT_LOG,"(' \',34X,'/')")
718     !        write(UNIT_LOG,"(2X,'\',32X,'/')")
719             write(11,"(3X,'F= ',I5,' (',I7,')',12X,'G= ',I5,' (',I7,')')") indxF, indxF_gl, &
720                             indxG, indxG_gl
721             write(11,"(4(9X,'|',29X,'|',/))")
722             write(11,"(3X,'E= ',I5,' (',I7,')',12X,'H= ',I5,' (',I7,')',/)") indxE, indxE_gl, &
723                             indxH, indxH_gl
724     !        write(UNIT_LOG,"(2X,'/',32X,'\')")
725     !        write(UNIT_LOG,"('/',34X,'\')")
726             write(11,"('A= ',I5,' (',I7,')',20X,'D= ',I5,' (',I7,')',/,/)") indxA, indxA_gl, &
727                             indxD, indxD_gl
728     
729     !        write(UNIT_LOG,"(' (',I4,' , ',I4,' , ',I4,') => ',2(I7,' , '),6(I7,2X))") &
730     !                                         i,j,k,ijk,ijk_GL,bottom_of(ijk),south_of(ijk),west_of(ijk),&
731     !                                        east_of(ijk),north_of(ijk),top_of(ijk)
732     
733           ENDDO
734     
735     !      write(UNIT_LOG,"(/,' (   i ,    j,     k) =>    ijk (Active Fluid)')")
736     !      write(UNIT_LOG,"(' ====================      =====')")
737     !       DO ijk = ijkstart3, ijkend3
738     !         I = I_OF(IJK)
739     !         J = J_OF(IJK)
740     !         K = K_OF(IJK)
741     
742     !         IF (FLOW_AT_E(IJK)) THEN
743     !         IF (FLUID_AT(IJK)) THEN
744     !           write(UNIT_LOG,"(' (',I4,' , ',I4,' , ',I4,') => ',I8)") I,J,K,ijk
745     !         ENDIF
746     !      END DO
747     
748     
749         endif   ! end if(amgdbg .or. bdist_io)
750     
751           M = 0
752     !      CALL WRITE_AB_M (A_M, B_M, IJKMAX2, M, IER)
753     
754         IF (AMGDBG .OR. bDist_IO) THEN
755           write(11,"(/,/,'BLK6: ========= ORIGINAL MFIX VARIABLES ===========')")
756           write(11,"('PE ',I5,': imin1  = ',I6,3X,'imax1= ',I6,/,'PE ',I5,': jmin1  = ',I6,3X,'jmax1= ',I6)") &
757                  myPE,imin1,imax1,myPE,jmin1,jmax1
758           write(11,"('PE ',I5,': kmin1  = ',I6,3X,'kmax1= ',I6)") myPE,kmin1,kmax1
759           write(11,"('-----')")
760           write(11,"('PE ',I5,': imin2  = ',I6,3X,'imax2= ',I6,/,'PE ',I5,': jmin2  = ',I6,3X,'jmax2= ',I6)") &
761                  myPE,imin2,imax2,myPE,jmin2,jmax2
762           write(11,"('PE ',I5,': kmin2  = ',I6,3X,'kmax2= ',I6)") myPE,kmin2,kmax2
763           write(11,"('----- Below xxx3 set is DMP extension ------------')")
764           write(11,"('PE ',I5,': imin3  = ',I6,3X,'imax3= ',I6,/,'PE ',I5,': jmin3  = ',I6,3X,'jmax3= ',I6)") &
765                  myPE,imin3,imax3,myPE,jmin3,jmax3
766           write(11,"('PE ',I5,': kmin3  = ',I6,3X,'kmax3= ',I6)") myPE,kmin3,kmax3
767           write(11,"('----- End of Below xxx3 set is DMP extension -----')")
768     !      write(11,"('PE ',I5,': ijkmax2= ',I6)") myPE,ijkmax2
769           write(11,"('PE ',I5,': ijmax2 = ',I6)") myPE,ijmax2
770           write(11,"('PE ',I5,': ijkmin1= ',I6,' ijkmax1= ',I12)") myPE,ijkmin1, ijkmax1
771           write(11,"('PE ',I5,':          ',6X,' ijkmax2= ',I12)") myPE,ijkmax2
772           write(11,"('PE ',I5,':          ',6X,' ijkmax3= ',I12)") myPE,ijkmax3
773           write(11,"('PE ',I5,': ijkmin4= ',I6,' ijkmax4= ',I12)") myPE,ijkmin4, ijkmax4
774     
775     
776           write(11,"(/,/,' ========= DMP EXTENSION VARIABLES ===========')")
777     !      write(UNIT_LOG,"('PE ',I5,': ijksize  = ',I6)") myPE,ijksize
778           write(11,"('PE ',I5,': ijksize3 = ',I6,3X,'ijksize3_all = ',I6)") myPE,ijksize3,ijksize3_all(myPE)
779           write(11,"('PE ',I5,': ijksize4 = ',I6,3X,'ijksize4_all = ',I6)") myPE,ijksize4,ijksize4_all(myPE)
780           write(11,"('PE ',I5,': ijkstart3  = ',I6,3X,'ijkend3  = ',I6)") myPE,ijkstart3, ijkend3
781           write(11,"('PE ',I5,': ijkstart3_all = ',I6,3X,'ijkstart4_all = ',I6)") myPE,ijkstart3_all(myPE),ijkstart4_all(myPE)
782           write(11,"('PE ',I5,': istart_all = ',I6,3X,'iend_all = ',I6,/,'PE ',I5,': jstart_all = ',I6,3X,'jend_all = ',I6)") &
783                  myPE,istart_all(myPE),iend_all(myPE),myPE,jstart_all(myPE),jend_all(myPE)
784           write(11,"('PE ',I5,': kstart_all = ',I6,3X,'kend_all = ',I6,/,'----------------------')") &
785                  myPE,kstart_all(myPE),kend_all(myPE)
786     
787           write(11,"('PE ',I5,': istart1_all= ',I6,3X,'iend1_all= ',I6,/,'PE ',I5,': jstart1_all= ',I6,3X,'jend3_all= ',I6)") &
788                  myPE,istart1_all(myPE),iend1_all(myPE),myPE,jstart1_all(myPE),jend1_all(myPE)
789           write(11,"('PE ',I5,': kstart1_all= ',I6,3X,'kend1_all= ',I6,/,'----------------------')") &
790                  myPE,kstart1_all(myPE),kend1_all(myPE)
791     
792           write(11,"('PE ',I5,': istart2_all= ',I6,3X,'iend2_all= ',I6,/,'PE ',I5,': jstart2_all= ',I6,3X,'jend3_all= ',I6)") &
793                  myPE,istart2_all(myPE),iend2_all(myPE),myPE,jstart2_all(myPE),jend2_all(myPE)
794           write(11,"('PE ',I5,': kstart2_all= ',I6,3X,'kend2_all= ',I6,/,'----------------------')") &
795                  myPE,kstart2_all(myPE),kend2_all(myPE)
796     
797           write(11,"('PE ',I5,': istart3_all= ',I6,3X,'iend3_all= ',I6,/,'PE ',I5,': jstart3_all= ',I6,3X,'jend3_all= ',I6)") &
798                  myPE,istart3_all(myPE),iend3_all(myPE),myPE,jstart3_all(myPE),jend3_all(myPE)
799           write(11,"('PE ',I5,': kstart3_all= ',I6,3X,'kend3_all= ',I6,/,'----------------------')") &
800                  myPE,kstart3_all(myPE),kend3_all(myPE)
801     
802           write(11,"('PE ',I5,': istart1= ',I6,3X,'iend1= ',I6,/,'PE ',I5,': jstart1= ',I6,3X,'jend1= ',I6)") &
803                  myPE,istart1,iend1,myPE,jstart1,jend1
804           write(11,"('PE ',I5,': kstart1= ',I6,3X,'kend1= ',I6,/,'----------------------')") &
805                  myPE,kstart1,kend1
806           write(11,"('PE ',I5,': istart2= ',I6,3X,'iend2= ',I6,/,'PE ',I5,': jstart2= ',I6,3X,'jend2= ',I6)") &
807                  myPE,istart2,iend2,myPE,jstart2,jend2
808           write(11,"('PE ',I5,': kstart2= ',I6,3X,'kend2= ',I6,/,'----------------------')") &
809                  myPE,kstart2,kend2
810           write(11,"('PE ',I5,': istart3= ',I6,3X,'iend3= ',I6,/,'PE ',I5,': jstart3= ',I6,3X,'jend3= ',I6)") &
811                  myPE,istart3,iend3,myPE,jstart3,jend3
812           write(11,"('PE ',I5,': kstart3= ',I6,3X,'kend3= ',I6,/,'----------------------')") &
813                  myPE,kstart3,kend3
814     
815         ENDIF   ! end if(amgdbg .or. bdist_io)
816     
817           close(unit=11)
818     
819     
820           RETURN
821           END SUBROUTINE DEBUG_WRITE_LAYOUT
822     
823     
824     
825     !vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvC
826     
827     !^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^C
828     
829           SUBROUTINE write_parallel_info()
830     
831     !-----------------------------------------------
832     !   M o d u l e s
833     !-----------------------------------------------
834           USE param
835           USE param1
836           USE parallel
837           USE matrix
838           USE geometry
839           USE compar
840           USE mpi_utility
841           USE sendrecv
842           USE sendrecv3
843           USE indices
844           USE leqsol
845           USE funits
846           USE run
847           USE time_cpu
848           USE functions
849           IMPLICIT NONE
850     !-----------------------------------------------
851     ! Dummy arguments
852     !-----------------------------------------------
853     ! Local Variables
854     !-----------------------------------------------
855     ! phase index
856           INTEGER :: M
857     ! indices
858           INTEGER :: i, j, k, ijk, ijk_GL, ijk_PROC, ijk_IO
859     !
860           character(LEN=80) :: fname
861     !-----------------------------------------------
862     
863     !DISTIO
864     !      fname = "p_info_xxxx.txt"
865     !      write (fname(8:11),'(i4.4)') myPE
866           fname = "p_info_xxxxx.txt"
867           write (fname(8:12),'(i5.5)') myPE
868           open (unit=11,file=fname,status='unknown')
869     
870           write (11,*) myPe , ' = myPE'
871     
872           write (11,*) myPE , istart3,iend3
873           write (11,*) myPE , jstart3,jend3
874           write (11,*) myPE , kstart3,kend3
875     
876           write(11,"('BLK1: Running from istart3,iend3 .AND. jstart3, jend3 .AND. kstart3, kend3')")
877           write(11,"(' (   i ,    j,     k)       ijk      ijk_GL     ijk_PROC    ijk_IO')")
878           write(11,"(' ====================      =====     =======    ========    ======')")
879           DO k = kstart3, kend3
880             DO i = istart3,iend3
881               DO j = jstart3, jend3
882                  ijk = FUNIJK(i,j,k)
883                  ijk_GL = FUNIJK_GL(i,j,k)
884                  ijk_PROC = FUNIJK_PROC(i,j,k,myPE)
885                  ijk_IO = FUNIJK_IO(i,j,k)
886                  write(11,"('  ',I4,'   ',I4,'   ',I4,'     ',4(I8,'   '))" ) &
887                                              i,j,k,ijk,ijk_GL,ijk_PROC,ijk_IO
888               ENDDO
889             ENDDO
890           ENDDO
891     
892           M = 0
893     !      CALL WRITE_AB_M (A_M, B_M, IJKMAX2, M, IER)
894     
895           close(unit=11)
896     
897           RETURN
898           END SUBROUTINE write_parallel_info
899     
900