#!/usr/bin/env python

#====================================================================
#
# File:         respect.in/respect.py 
#
# Licensing:    Copyright (c) 2010-2019 by the authors of ReSpect.
#
#               All Rights Reserved.
#
#               This source code is part of the ReSpect program package.
#               It is provided under a written license and may be used,
#               copied, transmitted, or stored only in accordance to the
#               conditions of that written license.
#
#               In particular, no part of the source code or compiled modules may
#               be distributed outside the research group of the license holder.
#               This means also that persons (e.g. post-docs) leaving the research
#               group of the license holder may not take any part of ReSpect,
#               including modified files, with him/her, unless that person has
#               obtained his/her own license.
#
#               For information on how to get a license, as well as the
#               author list and the complete list of contributors to the
#               ReSpect program, see: www.respectprogram.org
#
#
# Description:  launch ReSpect script 
#
# Author:       Stanislav Komorovsky
#
# Revision:     4.0
#
#====================================================================

import argparse
import getpass
import glob
import os
import shutil
import subprocess
import sys
import tarfile
import tempfile
import time

# ------------------------------------------------------------------------------#
#                       --------  INITIALIZE   --------                         #
# ------------------------------------------------------------------------------#
PIN      = os.getpid()          # Process identification number
USER     = getpass.getuser()    # User identification
machine  = os.uname()[1]        # Machine identification
mpi      = False         # MPI
omp      = True      # OMP
profile  = False   # Profiling
sections = ( \
            'scf'
           ,'pscf'
           ,'pscf-bands'
           ,'tdscf'
           ,'nsr'
           ,'tddft'
           ,'sscc'
           ,'cs'
           ,'visual'
           ,'lmo'
           ,'gt'
           ,'hfcc'
           ,'cpp'
           ,'mossbauer'
           ,'el-dipole'
           ,'ang-moments'
           ,'efg'
           ,'pefg'
           )

# ------------------------------------------------------------------------------#
#                      --------  DEFINE CLASSES  --------                       #
# ------------------------------------------------------------------------------#
class Logger:
    def __init__(self, stdout, filename):
        self.stdout = stdout
        self.logfile = open(filename, 'w')
    def write_all(self, text):
        self.stdout.write(text)
        self.stdout.flush()
        self.logfile.write(text)
        self.logfile.flush()
    def write_std(self, text):
        self.stdout.write(text)
        self.stdout.flush()
    def write_log(self, text):
        self.logfile.write(text)
        self.logfile.flush()
    def close(self):
        self.stdout.close()
        self.logfile.close()
    def flush(self):
        pass

# ------------------------------------------------------------------------------#
#                  --------  PARSE INPUT ARGUMENTS  --------                    #
# ------------------------------------------------------------------------------#
usage = 'respect --inp=input_file(.inp)'

parser = argparse.ArgumentParser( usage, formatter_class=argparse.ArgumentDefaultsHelpFormatter )

parser.add_argument('--inp',
                   action='store',
                   type=str,
                   dest='input',
                   default=None,
                   help='Name of the input file (without ".inp").',
                   metavar='[file]')

parser.add_argument('--guess',
                   action='store',
                   type=str,
                   dest='guess',
                   default=None,
                   help='Define the file used as guess.',
                   metavar='[file]')

parser.add_argument('--restart',
                   action='store_true',
                   dest='restart',
                   default=False,
                   help='Restart the calculation from its previous run.')

parser.add_argument('--start-data',
                   action='store',
                   type=str,
                   dest='start_data',
                   default=None,
                   help='Specify the file of the previous calculation step.',
                   metavar='[file]')

parser.add_argument('--scratch',
                   action='store',
                   type=str,
                   dest='scratch',
                   default=None,
                   help='Path to the scratch directory.',
                   metavar='[dir]')

parser.add_argument('--put',
                   action='store',
                   type=str,
                   dest='put',
                   default=None,
                   help='Copy files from the working directory to scratch directory.',
                   metavar='[file]')

parser.add_argument('--valgrind',
                   action='store_true',
                   dest='valgrind',
                   default=False,
                   help='Run with valgrind.')

parser.add_argument('--recover-job',
                   action='store',
                   type=str,
                   dest='recover',
                   default=None,
                   help='Path to the scratch directory to be recovered.',
                   metavar='[dir]')

parser.add_argument('--np',
                   action='store',
                   type=str,
                   dest='np',
                   default=None,
                   help='Distribution of MPI parallel processes. This argument is mandatory '
                       +'for parallel ReSpect builds with the MPI parallelization enabled. '
                       +'Format of the argument is either a) "#processes" or b) "#nodes x #processes-per-node". '
                       +'For example, --np=10 invokes 10 unbound MPI processes, whereas --np=5x2 '  
                       +'invokes 10 MPI processes bound to 5 nodes, each node running 2 processes. '
                       +'For hybrid MPI+OpenMP parallel builds, format b) is mandatory!',
                   metavar='[string]')

parser.add_argument('--nt',
                   action='store',
                   type=str,
                   dest='nt',
                   default=None,
                   help='Number of OpenMP parallel threads per single process. The argument is '
                       +'mandatory for parallel ReSpect builds with the OpenMP parallelization enabled.',
                   metavar='[integer]')

parser.add_argument('--mem',
                   action='store',
                   type=str,
                   dest='mem',
                   default=None,
                   help='Size of the private stack memory per single OpenMP thread (in megabytes).',
                   metavar='[integer]')

for parser_section in sections:

    parser.add_argument('--' + parser_section,
                       action='store_true',
                       dest='section_' + parser_section,
                       default=False,
                       help='Run the ' + parser_section + ' part of the calculation.')

# read .respectrc file if exists
# add its content to options from command line when parsing
file_commands = []
respectrc = os.path.expanduser('~') + '/.respectrc'
if os.path.isfile( respectrc ):
    with open( respectrc, 'r' ) as f:
        while True:
            line = f.readline()
            if line == '': break
            file_commands = file_commands + line.split()

try:
    args = parser.parse_args( file_commands + sys.argv[1:] )
except SystemExit:
    print( "\n\n ERROR in respect script arguments. Check arguments of respect,\n" + \
               "       or testrs.py script, and arguments in the ~/.respectrc file.\n\n")
    sys.exit(165)

# ------------------------------------------------------------------------------#
#           --------  Create checkpoint file from scratch  --------             #
# ------------------------------------------------------------------------------#
if args.recover is not None:

    # initialize
    output_file = os.path.join( os.path.realpath('.'), 'respect_checkpoint_file.50' )
    scratch_dir = os.path.abspath( os.path.expanduser(args.recover) )
    if not os.path.isdir(scratch_dir):
        sys.exit( "\n ERROR: Scratch directory does not exist: %s \n" % args.recover )

    # print
    print('\n    Scratch directory   = %s' % scratch_dir)
    print(  '    Recovered data file = %s' % output_file)

    # check if output exists
    if os.path.isfile(output_file):
        remove_out = False
        user_input = raw_input('\nFile respect_checkpoint_file.50 exists, overwrite? [Y/n]:')
        if user_input.lower() == 'n': sys.exit()
        if user_input.strip() == '':  remove_out = True
        if user_input.lower() == 'y': remove_out = True
        if not remove_out:
            sys.exit( "\nUnknown option: %s \n" % user_input )

    # add file to tar archive
    def add_file(dir_name, file_name, tar_file):
        fullpath = os.path.join(dir_name, file_name)
        try:
            tar_file.add( name=fullpath, arcname=file_name )
        except:
            pass

    # create new data file (respect_checkpoint_file.50)
    job_info = os.path.join(scratch_dir,'job.info')
    job_stat = open(job_info).read()
    if os.path.isfile( job_info ):
        if ('finish' in job_stat) or ('checkpoint' in job_stat):
            tar = tarfile.open( output_file, "w" )
            for sl in sections:
                add_file( scratch_dir, 'grid.'+sl, tar )
                for i in range(10):
                    fname = 'charlie.'+sl+'.'+str(i)
                    add_file( scratch_dir, fname, tar )
            add_file( scratch_dir, "grid.11", tar )
            tar.close()
        else:
            sys.exit( "\n ERROR: Unable to create checkpoint file (data was not yet created)\n" )
    else:
        sys.exit( "\n ERROR: Unable to create checkpoint file (job.info not present)\n" )

    # success
    print('\n    Checkpoint file was created successfully!\n')

    # exit script
    sys.exit(1)

# ------------------------------------------------------------------------------#
#                         --------  MAIN  --------                              #
# ------------------------------------------------------------------------------#

#
# Get section name
#
calc_only   = 0
section     = None
section_set = False

for element in sections:
    if getattr( args, 'section_' + element ):
        if section_set:
            sys.exit( "\n\n ERROR: at least two sections were specified:\n" + \
                          "        --%s and --%s\n" % (section.lower(), element) + \
                          "        --> specify exactly one section\n\n" )
        section     = element.lower()
        section_set = True
        calc_only   = 1

if not section:
    sys.exit( "\n\n ERROR: Property argument is not defined. Available property arguments:\n" + \
                  "        %s \n\n" % (sections,) )

#
# Basic sanity checks
#
def check_if_file_exists(args_file,option):
    if not os.path.isfile(args_file):
        sys.exit( "\n\n ERROR: %s file does not exist: %s \n\n" % (option, args_file) )

if args.input is None:
    sys.exit( "\n\n ERROR: no input file was specified\n" + \
                  "        --> use the \"--inp\" option to specify the input file \n\n" )
else:
    check_if_file_exists(args.input+'.inp','--inp')

if args.put is not None:
    check_if_file_exists(args.put,'--put')

if args.start_data is not None:
    for ff in args.start_data.split():
        check_if_file_exists(ff+'.50','--start-data')
    ic = len(args.start_data.split())
    if (ic!=3) and (ic!=1):
        sys.exit( "\n\n ERROR: Exactly one or three files are expected in start-data option \n\n" )

if args.scratch is None:
    print( "\n\n ERROR: Scratch directory was not specified\n" + \
               "        --> use the \"--scratch\" option to specify the scratch directory\n" + \
               "        --> you can specify --scratch option in file ~/.respectrc to avoid\n" + \
               "            setting it at each run of the respect script\n\n" )
    sys.exit(165)

if (args.start_data is not None) and section == 'scf':
    sys.exit( "\n\n ERROR: start-data option was specified for the --scf calculation \n" + \
                  "        --> remove the start-data flag or choose a different type of the calculation\n\n" )

if args.restart:
    if args.start_data is not None:
        sys.exit( "\n\n ERROR: it is not allowed to set both start-data and restart options\n\n" )
    if args.guess is not None:
        sys.exit( "\n\n ERROR: it is not allowed to set both guess and restart options\n\n" )


# nt/np -- omp/mpi compatibility check
if mpi:
   if (args.np is None):
      print( "\n\n ERROR: ReSpect is compiled with MPI ==> --np argument is mandatory!\n\n"+ \
                 "        Binary type:  MPI        --> %r\n" % ( mpi )+ \
                 "                      OpenMP     --> %r\n" % ( omp )+ \
                 "                      Sequential --> %r\n\n" % ( not omp and not mpi ) )
      sys.exit(165)

else:
   if (args.np is not None):
      print( "\n\n ERROR: ReSpect is not compiled with MPI ==> --np argument is redundant!\n\n"+ \
                 "        Binary type:  MPI        --> %r\n" % ( mpi )+ \
                 "                      OpenMP     --> %r\n" % ( omp )+ \
                 "                      Sequential --> %r\n\n" % ( not omp and not mpi ) )
      sys.exit(165)


if omp:
   if (args.nt is None):
      print( "\n\n ERROR: ReSpect is compiled with OpenMP ==> --nt argument is mandatory!\n\n"+ \
                 "        Binary type:  MPI        --> %r\n" % ( mpi )+ \
                 "                      OpenMP     --> %r\n" % ( omp )+ \
                 "                      Sequential --> %r\n\n" % ( not omp and not mpi ) )
      sys.exit(165)

else:
   if (args.nt is not None):
      print( "\n\n ERROR: ReSpect is not compiled with OpenMP ==> --nt argument is redundant!\n\n"+ \
                 "        Binary type:  MPI        --> %r\n" % ( mpi )+ \
                 "                      OpenMP     --> %r\n" % ( omp )+ \
                 "                      Sequential --> %r\n\n" % ( not omp and not mpi ) )
      sys.exit(165)

   if (args.mem is not None):
      print( "\n\n ERROR: ReSpect is not compiled with OpenMP ==> --mem argument is redundant!\n\n"+ \
                 "        Binary type:  MPI        --> %r\n" % ( mpi )+ \
                 "                      OpenMP     --> %r\n" % ( omp )+ \
                 "                      Sequential --> %r\n\n" % ( not omp and not mpi ) )
      sys.exit(165)


if mpi and omp:
   if len(args.np.split('x'))!=2:
      print( "\n\n ERROR: incorrect format of the argument --np for hybrid parallel build (see --help)!\n" + \
               "        --np=%s \n\n" % args.np )
      sys.exit(165)
   


#
# Filenames and directory names
#
file_inp = args.input + '.inp'

input_tmp = args.input

if args.start_data is not None:
    if len(args.start_data.split()) == 1:
        start_data_tmp = args.start_data
    else:
        start_data_tmp = args.start_data.split()[0].replace('_X','').replace('_Y','').replace('_Z','')
    file_out  = start_data_tmp + '-' + input_tmp + '.out_' + section
    file_50   = start_data_tmp + '-' + input_tmp + '.50'
    file_name = start_data_tmp + '-' + input_tmp
else:
    file_out  = input_tmp + '.out_' + section
    file_50   = input_tmp + '.50'
    file_name = input_tmp

work_dir    = os.path.realpath('.') + '/'
build_dir   = os.path.dirname( os.path.abspath(__file__) )
executable  = os.path.join( build_dir, 'ReSpect-mDKS.x' )
path_to_inp = os.path.join( work_dir, file_inp )

file_50 = os.path.join( work_dir, file_50 )

# it is not allowed to have both start-data and inp.50 file
if (args.start_data is not None) and os.path.isfile(os.path.join(work_dir,input_tmp+'.50')):
    sys.exit( "\n\n ERROR: Both start-data option and "+(input_tmp+'.50')+" file are present.\n" + \
                  "        To avoid confusion about the origin of the data either\n"             + \
                  "        remove %s.50 file or do not use the --start-data option.\n\n" % (input_tmp) )

# check input file
if not os.path.isfile(path_to_inp):
    sys.exit( "\n\n ERROR: input file does not exist: %s \n\n" % path_to_inp )

# respect must run from build directory
if not os.path.exists( executable ):
    print("\n\n ERROR: ReSpect executable not found. You are probably trying to run\n" + \
              "        respect script out of the build or install directory.\n" + \
              "        Execute respect script directly from these directories.\n" + \
              "        The respect script you were trying to run is in:\n" + \
              "        %s\n\n" % build_dir )
    sys.exit(165)

#
# create and move to scratch directory
#
if not os.path.exists(args.scratch):
    print( "\n\n ERROR: Invalid scratch directory: %s \n\n" % args.scratch )
    sys.exit(165)
if not os.access(args.scratch, os.W_OK):
    print( "\n\n ERROR: You don't have permission to write to the scratch directory:\n" + \
               "        %s \n\n" % args.scratch )
    sys.exit(165)

dir_prefix  = os.path.join( args.scratch, 'tmp' + str(PIN) + '.' )
dir_suffix  = '.' + USER

scratch_dir = tempfile.mkdtemp( prefix=dir_prefix, suffix=dir_suffix )
os.chdir(scratch_dir)

#
# set mpi/omp and memory
#
if args.np is None:
   nr_ppn  = '-'
   nr_proc = '-'
else:
   value = args.np.split('x')
   if len(value)==1:
     nr_ppn  = '-'
     nr_proc = str(int(value[0]))
   elif len(value)==2:
     nr_nds  = int(value[0])
     nr_ppn  = int(value[1])
     nr_proc = str(nr_nds*nr_ppn)
     nr_ppn  = str(nr_ppn)
   else:
      print( "\n\n ERROR: incorrect format of the argument --np (see --help)!\n" + \
               "        --np=%s \n\n" % args.np )
      sys.exit(165)

if args.nt is     None: nr_ompt = os.environ.get('OMP_NUM_THREADS','1')
if args.nt is not None: nr_ompt = args.nt

if args.mem is     None: nr_ompm = os.environ.get('OMP_STACKSIZE','2048m')
if args.mem is not None: nr_ompm = str(args.mem)+'m'

# set the number of threads and size of stack memory (avail for OpenMP spec-3.0)
# bind threads to physical processors (available for Intel)
if omp: os.environ['KMP_AFFINITY']    = 'compact'
if omp: os.environ['OMP_STACKSIZE']   = str(nr_ompm) 
if omp: os.environ['OMP_NUM_THREADS'] = str(nr_ompt)


# ------------------------------------------------------------------------------ #
#                   Script is working in scratch directory                       #
# ------------------------------------------------------------------------------ #
def copy_file( old_file, new_file ):
    if os.path.exists( os.path.join( scratch_dir, old_file ) ):
        old_file_path =  os.path.join( scratch_dir, old_file )
        new_file_path =  os.path.join( work_dir,    new_file )
        shutil.copy( old_file_path, new_file_path )

def exit_ReSpect_from_scratch( exitid = 1 ):
    copy_file( 'OUTPUT', file_out )
    if exitid == 1:
        sys.exit()
    else:
        sys.exit(exitid)
    return

#
# initialize OUTPUT file
#
output_file = os.path.join( scratch_dir, 'OUTPUT' )
logger = Logger( sys.stdout, output_file )
sys.stdout = logger

# add all command line arguments
arg = ' '
for i in range( 1, len(sys.argv) ):
    arg = arg + sys.argv[i] + ' '
# from file .respectrc add only arguments not present in command line
for i in range( len(file_commands) ):
    argument = file_commands[i].split('=')[0]
    if '--' in argument:
        argument_defined_by_user = False
        for j in range( 1, len(sys.argv) ):
            if argument == sys.argv[j].split('=')[0]: argument_defined_by_user = True
    if not argument_defined_by_user: arg = arg + file_commands[i] + ' '

#
# initialize timer
#
starttime = time.time()

#
# valgrind
#
valgrind = args.valgrind

#
# first print
#
logger.write_log('##############################################################################\n')
logger.write_log('                              Environment setting\n')
logger.write_log('##############################################################################\n')
logger.write_log('\n Job setting:\n    ')

logger.write_std('\nStarting time: %s\n' % time.strftime("%Y-%m-%d %H:%M:%S") )
logger.write_std('\n    ReSpect program, version 5.3.0 (2024)\n')

logger.write_all('\n    Call arguments   are   %s' % arg        )
logger.write_all('\n    Input file name   is   %s' % file_inp   )
logger.write_all('\n    Binary  directory is   %s' % build_dir  )
logger.write_all('\n    Work    directory is   %s' % work_dir   )
logger.write_all('\n    Scratch directory is   %s' % scratch_dir)
logger.write_all('\n    Machine name      is   %s' % machine    )
if omp: logger.write_all('\n    OMP stack memory size  %s'                   % nr_ompm )
if omp: logger.write_all('\n    OMP parallel job with  %s threads'           % nr_ompt )
if mpi:
   if nr_ppn=='-':
        logger.write_all('\n    MPI parallel job with  %s unbound processes' % nr_proc )
   else:
        logger.write_all('\n    MPI parallel job with  %s processes bound'   % nr_proc\
                        +     ' on %i nodes'                                 % (int(nr_proc)/int(nr_ppn)) )
        logger.write_all('\n    MPI processes per node %s'                   % nr_ppn  )
logger.write_all('\n    Output  file will be   %s' % work_dir+file_out )

# ===============================================================
#                Prepare files for running ReSpect
# ===============================================================

#
# Read input file to find proper section
# Copy input file to scratch dir
#
respect_inp   = open( 'INPUT', 'w' )
copy_line     = False
section_found = False
indentation   = -1
with open( path_to_inp, 'r' ) as user_inp_file:
    il = 0
    for line in user_inp_file:
        il = il + 1
        # check if only basic ASCII character are used
        for c in line:
            if (ord(c)>127) or (ord(c)<0):
                logger.write_all( "\n\n ERROR: Input contains non-ASCII characters on line %i\n" \
                                      "        line --> %s\n\n" % (il, repr(line)) )
                exit_ReSpect_from_scratch(165)
        # stop if hard tabs are used
        if '\t' in line:
            logger.write_all( "\n\n ERROR: Hard tabs are not allowed in the respect input,\n" \
                                  "        use soft tabs (spaces) instead.\n" \
                                  "        --> tab detected on line %i\n" \
                                  "        --> %s\n\n" % ( il, line.rstrip() ) )
            exit_ReSpect_from_scratch(165)
        # skip empty and commented lines
        if not ( not line or line.isspace() or (line.lstrip()[0] == '#') ):
            if indentation >= line.index(line.lstrip()[0]): copy_line = False
        # find section
        if section == line.split(':')[0].lstrip().rstrip().lower():
            if indentation == -1: indentation = line.index(line.lstrip()[0])
            copy_line     = True
            section_found = True
        # copy section to INPUT file
        if copy_line:
            respect_inp.write( line )
    user_inp_file.seek(0)
respect_inp.close()

if not section_found:
    logger.write_all( "\n\n ERROR: couldn't find the \"%s\" section in the input file\n" % section + \
                          "        %s\n\n" % path_to_inp )
    exit_ReSpect_from_scratch()

#
# preparing of the basis set file
#
def listdir_fullpath(d):
    return [os.path.join(d, f) for f in sorted(os.listdir(d))]

mdks_dir        = os.path.join(build_dir,'basis/mdks')
basis_file_list = listdir_fullpath(mdks_dir)
file_inp_dir    = os.path.join(work_dir,os.path.dirname(file_inp))
basis_here      = os.path.join(file_inp_dir,'BASIS')
if os.path.exists(basis_here): basis_file_list.append( basis_here )

with open('BASIS', 'w') as out_file:
    for fname in basis_file_list:
        with open(fname) as in_file:
            for line in in_file:
                out_file.write(line)

#
# preparing data for atomic restart and EHT
#
shutil.copy(     os.path.join(build_dir,'basis/EHT_CORE/BASCORE'), 'ATCORE'   )
shutil.copy(     os.path.join(build_dir,'ehmacc.x'),               'ehmacc.x' )
shutil.copytree( os.path.join(build_dir,'basis/libmos'),           'libmos'   )

#
# prepare .50 data on scratch
#
def untar( key, data_50_file ):
    try:
        tar = tarfile.open(data_50_file,'r:')
    except:
        logger.write_all( "\n\n ERROR: unavailable restart or start-data file %s\n\n" % data_50_file )
        exit_ReSpect_from_scratch()
    if key == 'all':
        tar.extractall(path=scratch_dir)
    else:
        try:
            tar.extract(member=key,path=scratch_dir)
        except:
            pass
    tar.close()

def shutil_move( old_dir, old_file, new_dir, new_file ):
    old_path = os.path.join(old_dir,old_file)
    new_path = os.path.join(new_dir,new_file)
    if os.path.exists(old_path):
        shutil.move( old_path, new_path )

def shutil_copy( old_dir, old_file, new_dir, new_file ):
    old_path = os.path.join(old_dir,old_file)
    new_path = os.path.join(new_dir,new_file)
    if os.path.exists(old_path):
        shutil.copy( old_path, new_path )

# restart files
if args.restart:
    untar( 'all', file_50 )
    if not os.path.isfile( os.path.join(scratch_dir,'charlie.'+section+'.0') ):
        logger.write_all( "\n\n ERROR: restart option was specified but no previous\n" + \
                              "        run of the "+section+" section was detected.\n\n" )
        exit_ReSpect_from_scratch()
    for i in range(10):
        fname = 'charlie.'+section+'.'+str(i)
        shutil_move( scratch_dir, fname, scratch_dir, fname+'.rest' )

# start-data files
if args.start_data is not None:
    if len(args.start_data.split()) == 3:
        file_X = os.path.join( work_dir, args.start_data.split()[0] + '.50' )
        file_Y = os.path.join( work_dir, args.start_data.split()[1] + '.50' )
        file_Z = os.path.join( work_dir, args.start_data.split()[2] + '.50' )
        untar( 'all',           file_X )
        untar( 'charlie.scf.2', file_Y )
        untar( 'charlie.lmo.2', file_Y )
        untar( 'charlie.scf.3', file_Z )
        untar( 'charlie.lmo.3', file_Z )
    else:
        start_data_file = os.path.join( work_dir, args.start_data + '.50' )
        untar( 'all', start_data_file )

# without restart or start-data just untar .50 file
if (args.start_data is None) and (not args.restart):
    common_file = os.path.join( work_dir, args.input + '.50' )
    if os.path.exists( common_file ):
        untar( 'all', common_file )
        tar = tarfile.open(common_file,'r:')
        fnames = tar.getnames()
        for f in fnames:
            if section in f: shutil_move( scratch_dir, f, scratch_dir, f+'.old' )
        tar.close()

#
# put files to scratch directory
#
if args.put is not None:
    for file in args.put.split():
        if os.path.exists( os.path.join(scratch_dir,file) ):
            logger.write_all( "\n\n ERROR: File %s already exists in the scratch directory\n" % (file) + \
                                  "        Remove --put option or rename the file.\n\n" )
            exit_ReSpect_from_scratch()
        shutil.copy( os.path.join(work_dir,file), os.path.join(scratch_dir,file) )

#
# parse pcm input if it is part of the put string
#
if args.put is not None:
    path_to_pcm = os.path.join(scratch_dir,'pcmsolver.inp')
    if os.path.isfile(path_to_pcm):
        sys.path.append( os.path.join(build_dir, '') )
        import pcmsolver
        pcmsolver.parse_pcm_input('pcmsolver.inp', write_out=True)

# ===============================================================
#                        Run the program
# ===============================================================

logger.write_std('\n\n    Job is being run ...\n\n')
logger.write_log('\n\n\n System setting:\n\n    ')

# check if stack size is set to unlimited
stack = subprocess.Popen( 'ulimit -s', shell=True, executable='/bin/bash', stdin  = subprocess.PIPE,
                                                                           stdout = subprocess.PIPE,
                                                                           stderr = subprocess.PIPE )
stdout_coded, stderr_coded = stack.communicate()
out = stdout_coded.decode('UTF-8')
if out:
    if out.strip('\n') != 'unlimited':
        logger.write_all( "\n\n ERROR: the current stack size limit is = %s\n" \
                              "        To safely run the ReSpect program set the stack size\n" \
                              "        to unlimited in your run/submit script using command:\n" \
                              "        ulimit -s unlimited\n\n" % out.strip('\n') )
        exit_ReSpect_from_scratch(165)

# report system setting
command = 'ulimit -a; echo "omp_thread_limit:"$OMP_THREAD_LIMIT; echo "omp_stacksize:"$OMP_STACKSIZE'
child = subprocess.Popen( command, shell=True, executable='/bin/bash', stdin  = subprocess.PIPE,
                                                                       stdout = subprocess.PIPE,
                                                                       stderr = subprocess.PIPE )
stdout_coded, stderr_coded = child.communicate()
out = stdout_coded.decode('UTF-8')
err = stderr_coded.decode('UTF-8')
if out: logger.write_log(out.replace('\n','\n    '))
if err: logger.write_all('\n\nERROR: '+err+'\n\n')

# gathering mpi_arg for MPI parallel jobs
#newer formalism suitable for OpenMPI (1.8+)
if mpi and not omp:
   if( nr_ppn=='-' ): mpi_arg = "mpirun -np " + nr_proc
   if( nr_ppn!='-' ): mpi_arg = "mpirun -np " + nr_proc + " --map-by ppr:" + nr_ppn + ":node:pe=1"  # + " --report-bindings"

# gathering mpi_arg for hybrid parallel jobs
#newer formalism suitable for OpenMPI (1.8+)
if mpi and omp:
   mpi_arg = "mpirun -np " + nr_proc + " --map-by ppr:" + nr_ppn + ":node:pe=" + nr_ompt  # + " --report-bindings"

# valgrind
valgrind_arg = "valgrind --error-limit=no --tool=memcheck --track-origins=yes \
                --show-reachable=yes --leak-check=full --log-file=valgrind.log"
input_file   = "< INPUT"
time_arg     = "time"

command = ''
if not valgrind : command = command +     time_arg + ' '
if     mpi      : command = command +      mpi_arg + ' '
if     valgrind : command = command + valgrind_arg + ' '

command = command + executable + ' ' + input_file

#  THE ACTUAL CALL
child = subprocess.Popen( command, shell=True, executable='/bin/bash', stdin  = subprocess.PIPE,
                                                                       stdout = logger.logfile,
                                                                       stderr = subprocess.PIPE )
_, stderr_coded = child.communicate()
err = stderr_coded.decode('UTF-8')
logger.write_log(err)

# PROFILING
if profile:
    command = 'gprof ' + executable
    child = subprocess.Popen( command, shell=True, executable='/bin/bash', stdin  = subprocess.PIPE,
                                                                           stdout = logger.logfile,
                                                                           stderr = subprocess.PIPE )
    _, stderr_coded = child.communicate()
    err = stderr_coded.decode('UTF-8')
    logger.write_log(err)


# ==============================================================
#                 Now copy outputs and clean up
# ==============================================================

# copy output data to work dir
copy_file('valgrind.log', file_name + '.vlog'         ) # valgrind log file
copy_file(  'molden.log', file_name + '.molden'       ) # molden output
copy_file(      'initmo', file_name + '.mos'          )
copy_file(       'DEBUG', file_name + '.dbg_'+section ) # long output for debugging
copy_file(         'TST', file_name + '.TST'          ) # test file

# Files produced by PCMSolver
copy_file(  'cavity.npz', file_name + '.npz'      ) # NumPy compressed cavity file
# GeomView cavity file
off   = [filename for filename in os.listdir(scratch_dir) if filename.startswith('cavity.off')]
if off:
    copy_file(  off[0], file_name + '.off'      )
# Cavity generator report
pedra = [filename for filename in os.listdir(scratch_dir) if filename.startswith('PEDRA.OUT')]
if pedra:
    copy_file( pedra[0], file_name + '.PEDRA.OUT')
# Export ReSpect Data file (just for collaboration with Ben and Daniel)
txt = [filename for filename in os.listdir(scratch_dir) if filename.startswith('rsd_')]
for f in txt:
    copy_file( f, file_name + '.' + f )

# raw and cube data files
suffix = ['cube','raw','vti']
for suff in suffix:
  list_cubes = glob.glob('*'+suff)
  for f in list_cubes:
      if f == suff:
          copy_file( f, file_name + '.'+suff )
      else:
          copy_file( f, file_name + '_' + f[0:-len(suff)] + '.'+suff )

# backup old and create new data file (file_name.50)
def add_file(dir_name, file_name, tar_file):
    fullpath = os.path.join(dir_name, file_name)
    try:
        tar_file.add( name=fullpath, arcname=file_name )
    except:
        pass

job_info = os.path.join(scratch_dir,'job.info')
if os.path.isfile( job_info ):
    job_stat = open(job_info).read()
    if ('finish' in job_stat) or ('checkpoint' in job_stat):
        # backup
        if os.path.exists( file_50 ):
            shutil.move( file_50, file_50 + '.backup' )
        # create new data file
        tar = tarfile.open( file_50, "w" )
        for sl in sections:
            add_file( scratch_dir, 'grid.'+sl, tar )
            for i in range(10):
                fname = 'charlie.'+sl+'.'+str(i)
                add_file( scratch_dir, fname, tar )
        add_file( scratch_dir, "grid.11", tar )
        tar.close()


# copy unit test files
list_testF90  = glob.glob('test.F90*')
for f in list_testF90: copy_file( f, file_name + '_' + f )

# last prints
elp_time = (time.time() - starttime)
m, s = divmod(elp_time, 60)
h, m = divmod(m, 60)
logger.write_std('    ReSpect job is done ( elapsed time %d:%02d:%02d )\n' % (h, m, s))
logger.write_std('\nEnding time:   %s\n\n' % time.strftime("%Y-%m-%d %H:%M:%S"))

# finalize logger before copying main output file
logger.close()
copy_file( 'OUTPUT', file_out )

# finalize
os.chdir(work_dir)
shutil.rmtree(scratch_dir)
