#!/bin/bash
# This is the script to run the specjapp benchmark
#
# reqparam value TYPE : WAS|DB2|CLIENT|DRIVER|EMULATOR : should this instance of the benchmark be the was, db2, client, driver, or emulator tier (driver and emulator are the two client functions seperated to facilitate multiple drivers)
# reqparam value MASTER : WAS|DB2|CLIENT|DRIVER|EMULATOR : which tier is the master for the benchmark
# reqparam value DB2_IP : the ip address of the db2 tier
# reqparam value WAS_IP : the ip address of the was tier
# reqparam value CLIENT_IP : the ip address of the client tier
# reqparam value IR : the injection rate for the run
# reqparam value RUN_LENGTH : SHORT|LONG : the length of the run to make
# reqparam value DB2_BARRIER_IP : the ip address of the db2 tier to use for barrier synchronization
# reqparam value WAS_BARRIER_IP : the ip address of the was tier to use for barrier synchronization
# reqparam value CLIENT_BARRER_IP : the ip address of the client tier to use for barrier synchronization
# reqparam value TPM_ENABLED : 0|1 : whether or not to enable Tivoli Performance Monitor in the WAS tier
# optparam value SPRAYER_IP : the ip address of the sprayer tier
# optparam value SPRAYER_BARRIER_IP : the ip addres of the sprayer tier to use for barrier sychrinization

. /etc/autobench.conf || . functions

LOG_LABEL="SPECjAppServer2004"

# track if the driver.sh script is ever actually exectured (keeps from uploading outputs from a different run if this one never starts)
RUN_INITIATED=0

# get my hostname
MY_NAME=`hostname`

# function to print out the proper usage of teh specjapp autobench script
function usage()
{
  echo
  echo "specjapp autobench script"
  echo
  echo "required parameters: <TYPE> <MASTER> <DB2_IP> <WAS_IP> <CLIENT_IP> <IR> <RUN_LENGTH> <DB2_BARRIER_IP> <WAS_BARRIER_IP> <CLIENT_BARRIER_IP> <TPM_ENABLED> [<SPRAYER_IP> <SPRAYER_BARRIER_IP>]"
  echo
  echo "<TYPE> specifies the node type the script is running on, possible values are WAS, DB2, CLIENT, DRIVER, or EMULATOR"
  echo "<MASTER> specifies which tier is the master for the benchmark run, possible values are WAS, DB2, CLIENT, DRIVER, or EMULATOR"
  echo "<DB2_IP> specifies the ip address for the db2 tier"
  echo "<WAS_IP> specifies the ip address for the was tier"
  echo "<CLIENT_IP> specifies the ip address for the client tier"
  echo "<IR> specifies the injection rate for the benchmark run"
  echo "<RUN_LENGTH> specifies the length of the run, possible values are SHORT and LOG"
  echo "<DB2_BARRIER_IP> specifies the ip address the db2 tier will use for barrier communication"
  echo "<WAS_BARRIER_IP> specifies the ip address the was tier will use for barrier communication"
  echo "<CLIENT_BARRIER_IP> specifies the ip address the client tier will use for barrier communication"
  echo "<TPM_ENABLED> specifies that TPM should be enabled on was tier with a 1 and disabled with a 0"
  echo "<SPRAYER_IP> specifies the ip address of the sprayer tier"
  echo "<SPRAYER_BARRIER_IP> specifies the ip address the sprayer tier will use for barrier communication"  
  echo 
}

# check for the presence of nptl, returns 0 on true and non zero otherwise
function has_nptl()
{
  getconf GNU_LIBPTHREAD_VERSION 2>&1 | grep NPTL > /dev/null 2>&1
}

has_nptl
if [ $? -eq 0 ]
then
  log "$LOG_LABEL: node has nptl"
  NPTL_HANDLER="LD_ASSUME_KERNEL=2.2.5 LANG=$LC_CTYPE "
else
  log "$LOG_LABEL: node does not have nptl"
  NPTL_HANDLER=""
fi


# Process the arguments #########################################
if [ $# -lt 11 ]
then
  usage
  exit 1
fi

TYPE=$1
MASTER=$2
DB2_IP=$3
WAS_IP=$4
CLIENT_IP=$5
IR=$6
RUN_LENGTH=$7
DB2_BARRIER_IP=$8
WAS_BARRIER_IP=$9
CLIENT_BARRIER_IP=${10}
TPM_ENABLED=${11}
SPRAYER_IP=${12}
SPRAYER_BARRIER_IP=${13}

if [ "$SPRAYER_IP" != "" -a "$SPRAYER_BARRIER_IP" != "" ]
then
  log "$LOG_LABEL: using sprayer for this run"
  SPRAYER_PRESENT=1
else
  log "$LOG_LABEL: no sprayer for this run"
  SPRAYER_PRESENT=0
fi

case "$TYPE" in
  "db2"|"DB2")
    TYPE="DB2"
    ;;
  "was"|"WAS")
    TYPE="WAS"
    ;;
  "client"|"CLIENT")
    TYPE="CLIENT"
    ;;
  "sprayer"|"SPRAYER")
    TYPE="SPRAYER"
    ;;
  "driver"|"DRIVER")
    TYPE="DRIVER"
    ;;
  "emulator"|"EMULATOR")
    TYPE="EMULATOR"
    ;;
  *)
    log "$LOG_LABEL: type $TYPE not recognized"
    ;;
esac

case "$MASTER" in
  "db2"|"DB2")
    MASTER="DB2"
    ;;
  "was"|"WAS")
    MASTER="WAS"
    ;;
  "client"|"CLIENT")
    MASTER="CLIENT"
    ;;
  "sprayer"|"SPRAYER")
    MASTER="SPRAYER"
    ;;
  "driver"|"DRIVER")
    MASTER="DRIVER"
    ;;
  "emulator"|"EMULATOR")
    MASTER="EMULATOR"
    ;;
  *)
    log "$LOG_LABEL: master $MASTER not recognized"
    ;;
esac
# end of process the arguments ##################################


# These values are lifted from the run.properties file, they are required on all tiers but the file is only on the
# DB2 and Client tiers
case "$RUN_LENGTH" in
  "LONG")
    RAMP_UP=600
    STEADY_STATE=3600
    RAMP_DOWN=300
    ;;
  "SHORT"|*)
    RAMP_UP=360
    STEADY_STATE=840
    RAMP_DOWN=60
    ;;
esac

# trigger time must be greater than the sum of the thread startup, ir rate * 10, ir rate * 3
# trigger fluff adds an arbitrary amount of leeway
THREAD_START=100
TRIGGER_FLUFF=1
TRIGGER_TIME=$(( ( 10 * $IR + 3 * $IR + $THREAD_START + $TRIGGER_FLUFF ) / 3 ))

# Time to wait for backgrounded processes to complete
WAIT_TIME=300


# Local Functions ###############################################

# looks for a line in FILE containing FIND and replaces it with REPLACE, if it doesn't locate FIND then just appends
# REPLACE to the end of FILE
function addreplace()
{
  FIND=$1
  REPLACE=$2
  FILE=$3

  if [ `grep ${FIND} ${FILE} | wc -l` -lt 1 ]
  then
    echo -e $REPLACE >> $FILE
  else
    cat $FILE | sed "s/.*${FIND}.*$/${REPLACE}/" > ${FILE}.tmp
    mv ${FILE}.tmp $FILE
  fi
}

# waits for $2 seconds before killing the process specified by $1, it checks every WAIT_STEP seconds to see if the
# process is still alive and returns if not, the $3 argument is used to specifiy that all java instances should be killed
# if the timeout is exceeded
function mywaitkill()
{
  PID=$1
  TOTAL_TIME=$2
  WAIT_STEP=10
  WAIT_STEPS=$(( $TOTAL_TIME / $WAIT_STEP ))

  for element in $(seq 1 $WAIT_STEPS)
  do
    sleep $WAIT_STEP
    if [ `ps -p ${PID} | wc -l` -lt 2 ]
    then
      return
    fi
  done

  log "$LOG_LABEL: killing PID=${PID} after waiting for $TOTAL_TIME seconds"
  kill -9 $PID
  if [ $# == 3 ]
  then
    log "$LOG_LABEL: killing all java processes"
    killall -9 java    
  fi
}

function mycleanup()
{
  STATE=$1

  log "$LOG_LABEL: processing cleanup at $STATE"

  case "$TYPE" in
    "DB2")
      su - db2inst1 -c "db2stop && db2start"
      ;;
    "WAS")
      /bin/sh -c "${NPTL_HANDLER} ${WAS_INSTALL}/bin/stopServer.sh server1&"
      mywaitkill $! $WAIT_TIME "java"
      killall -9 java
      rm -Rf ${WAS_INSTALL}/tranlog/*
      ${MQM_INSTALL}/bin/runmqsc WAS_${MY_NAME}_server1 < ${SPECJAPPKIT}/mq_clean.in
      ;;
    "CLIENT"|"EMULATOR")
      /bin/sh -c "${NPTL_HANDLER} ${WAS_INSTALL}/bin/stopServer.sh server1&"
      mywaitkill $! $WAIT_TIME "java"
      killall -9 java
      killall -9 rmiregistry
      ;;
    "SPRAYER")
      ${IHS_INSTALL}/bin/apachectl stop
      killall -9 httpd
      ;;
    "DRIVER")
      killall -9 java
      ;;
  esac

  if [ $STATE == "start" -o $STATE == "purge" ]
  then
    case "$TYPE" in
      "CLIENT"|"DRIVER")
        rm ${SPECJAPPKIT}/orbtrc.*
        ;;
    esac

    case "$TYPE" in
      "CLIENT"|"WAS"|"EMULATOR")
          rm ${WAS_INSTALL}/logs/server1/*
          rm ${WAS_INSTALL}/javacore*
        ;;
      "SPRAYER")
        rm ${IHS_INSTALL}/logs/*
        ;;
    esac
  else
    # this handles an error case where the profiler would still be running when a barrier times out causing the
    # the profiler processes to be migrated back up the automation process tree causing hangs
    stopprofilers
  fi

  log "$LOG_LABEL: finishing cleanup at $STATE"
}

function report()
{
  log "$LOG_LABEL: reporting"

  case "$TYPE" in
    "CLIENT"|"DRIVER")
      if [ $RUN_INITIATED -eq 1 ]
      then
        SPECJAPP_RUN_NUMBER=`ls -1tr ${SPECJAPPKIT}/output|tail -n 1`
        mkdir ${LOGDIR}/benchmark/specjapp_output.${RUN_NUMBER}
        cp ${SPECJAPPKIT}/output/${SPECJAPP_RUN_NUMBER}/* ${LOGDIR}/benchmark/specjapp_output.${RUN_NUMBER}
      fi
      mkdir ${LOGDIR}/benchmark/orbtrc.${RUN_NUMBER}
      mv ${SPECJAPPKIT}/orbtrc.* ${LOGDIR}/benchmark/orbtrc.${RUN_NUMBER}
      ;;
    "SPRAYER")
      mkdir ${LOGDIR}/benchmark/ihs_logs.${RUN_NUMBER}
      mv ${IHS_INSTALL}/logs/* ${LOGDIR}/benchmark/ihs_logs.${RUN_NUMBER}
      cp ${IHS_INSTALL}/conf/httpd.conf ${LOG_DIR}/benchmark/httpd.conf.${RUN_NUMBER}
      ;;
    "WAS")
      db2level > ${LOGDIR}/benchmark/db2level.${RUN_NUMBER}
      ;;
  esac

  case "$TYPE" in
    "CLIENT"|"WAS"|"EMULATOR")
      mkdir ${LOGDIR}/benchmark/was_logs.${RUN_NUMBER}
      mv ${WAS_INSTALL}/logs/server1/* ${LOGDIR}/benchmark/was_logs.${RUN_NUMBER}
      mkdir ${LOGDIR}/benchmark/was_config.${RUN_NUMBER}
      cp -r ${WAS_INSTALL}/config/* ${LOGDIR}/benchmark/was_config.${RUN_NUMBER}
      mkdir ${LOGDIR}/benchmark/javacores.${RUN_NUMBER}
      mv ${WAS_INSTALL}/javacore* ${LOGDIR}/benchmark/javacores.${RUN_NUMBER}
      ;;
  esac

  case "$TYPE" in
    "CLIENT"|"DB2"|"DRIVER")
      mkdir ${LOGDIR}/benchmark/specjapp_config.${RUN_NUMBER}
      cp -r ${SPECJAPPKIT}/config/* ${LOGDIR}/benchmark/specjapp_config.${RUN_NUMBER}
      ;;
  esac

  # record the run ir rate for easy access during post processing
  echo $IR > ${LOGDIR}/benchmark/run_ir.${RUN_NUMBER}

  # record node type for post processing
  echo $TYPE > ${LOGDIR}/benchmark/node_type

  # record the sysctl values for the run
  sysctl -A > ${LOGDIR}/benchmark/sysctl.${RUN_NUMBER}

  # record the ulimit values for the run
  ulimit -a > ${LOGDIR}/benchmark/ulimit.${RUN_NUMBER}

  log "$LOG_LABEL: reporting done"
}

# function the handle barrier timeout cases
function handle_barrier()
{
  BARRIER_NUMBER=$1
  shift 1

  ${@:1:3} ${4}_${BARRIER_NUMBER} ${@:5}
  RS=$?

  if [ $RS -ne 0 ]
  then
    mycleanup "barrier"
    report
    exit 1
  fi
}

# function to mirror the output of the driver to the console and to a log file prepending a timestamp to each line
# this function assumes that output is being piped to it -- I think ;-)
function shadow()
{
  OUT_FILE=$1

  while read line
  do
    DATE=`date +%x-%T`
    echo "$DATE $LOG_LABEL $line"
  done | tee $OUT_FILE
}

# funtion to handle an alternate way of running WAS
# the start_server1.sh script is created by running "./startServer.sh server1 -script"
function run_was()
{
  if [ -e ${WAS_INSTALL}/bin/start_server1.sh ]
  then
    log "$LOG_LABEL: starting WAS with start_server1.sh"
    /bin/sh -c "${NPTL_HANDLER} ${WAS_INSTALL}/bin/start_server1.sh 2>&1" | shadow $LOGDIR/benchmark/run_was_out.${RUN_NUMBER} &
    sleep 180
  else
    log "$LOG_LABEL: starting WAS with startServer.sh"
    /bin/sh -c "${NPTL_HANDLER} ${WAS_INSTALL}/bin/startServer.sh server1"
  fi
}

# setup stuff ###################################################

# log to describe run parameters
log "$LOG_LABEL: System uptime: `uptime`"
log "$LOG_LABEL: Starting run at `date`"
log "$LOG_LABEL: Node Type:       $TYPE"
log "$LOG_LABEL: Master Node:     $MASTER"
log "$LOG_LABEL: DB2 IP:          $DB2_IP"
log "$LOG_LABEL: WAS IP:          $WAS_IP"
log "$LOG_LABEL: Client IP:       $CLIENT_IP"
if [ $SPRAYER_PRESENT -eq 1 ]
then
  log "$LOG_LABEL: Sprayer IP:      $SPRAYER_IP"
fi
log "$LOG_LABEL: Injection Rate:  $IR"
log "$LOG_LABEL: Run Length:      $RUN_LENGTH"
log "$LOG_LABEL: Barrier Info:"
log "$LOG_LABEL: DB2 $DB2_BARRIER_IP"
log "$LOG_LABEL: WAS $WAS_BARRIER_IP"
log "$LOG_LABEL: Client $CLIENT_BARRIER_IP"  
if [ $SPRAYER_PRESENT -eq 1 ]
then
  log "$LOG_LABEL: Sprayer $SPRAYER_BARRIER_IP"
fi

BARRIER_TIMEOUT=2400
case "$MASTER" in
  "DB2")
    BARRIER_ALL="barrier -t ${BARRIER_TIMEOUT} BARRIER_ALL ${DB2_BARRIER_IP} ${WAS_BARRIER_IP} ${CLIENT_BARRIER_IP} ${SPRAYER_BARRIER_IP}"
    ;;
  "WAS")
    BARRIER_ALL="barrier -t ${BARRIER_TIMEOUT} BARRIER_ALL ${WAS_BARRIER_IP} ${DB2_BARRIER_IP} ${CLIENT_BARRIER_IP} ${SPRAYER_BARRIER_IP}"
    ;;
  "CLIENT")
    BARRIER_ALL="barrier -t ${BARRIER_TIMEOUT} BARRIER_ALL ${CLIENT_BARRIER_IP} ${WAS_BARRIER_IP} ${DB2_BARRIER_IP} ${SPRAYER_BARRIER_IP}"
    ;;
  "SPRAYER")
    BARRIER_ALL="barrier -t ${BARRIER_TIMEOUT} BARRIER_ALL ${SPRAYER_BARRIER_IP} ${CLIENT_BARRIER_IP} ${WAS_BARRIER_IP} ${DB2_BARRIER_IP}"
    ;;
  "DRIVER")
    ;;
  "EMULATOR")
    ;;
esac

BARRIER_WAS_DB2="barrier -t ${BARRIER_TIMEOUT} BARRIER_WAS_DB2 ${WAS_BARRIER_IP} ${DB2_BARRIER_IP}"

log "$LOG_LABEL: BARRIER_ALL=${BARRIER_ALL}"
log "$LOG_LABEL: BARRIER_WAS_DB2=${BARRIER_WAS_DB2}"


# Install support files #########################################
getcommand doprofilers
doprofilers install


# Prepare the benchmark #########################################
WAS_INSTALL="/opt/WebSphere/AppServer"
IHS_INSTALL="/opt/IBMIHS"
MQM_INSTALL="/opt/mqm"
SPECJAPPKIT="/SPECjAppServer2004"
BARRIER_NUM=0

# save the benchmark script
cp ${AUTODIR}/scripts/benchmarks/specjapp ${LOGDIR}/benchmark/specjapp

case "$TYPE" in
  "WAS")
    # source the db2 profile script to ensure the db2 environment is setup for WAS
    . /home/db2inst1/sqllib/db2profile
    ;;
esac

mycleanup "start"


# Purge the MQ queues ##########################################

(( BARRIER_NUM += 1 ))
handle_barrier $BARRIER_NUM $BARRIER_ALL

case "$TYPE" in
  "WAS")
    log "$LOG_LABEL: Purging MQ Series queues"
    run_was
    mycleanup "purge"
    ;;
esac

(( BARRIER_NUM += 1 ))
handle_barrier $BARRIER_NUM $BARRIER_ALL


# Apply Tuning parameters  #####################################

if [ -e ${SPECJAPPKIT}/sjas4_tune.sh ]
then
  log "$LOG_LABEL: applying tuning parameters from ${SPECJAPPKIT}/sjas4_tune.sh"
  # source the tuning script since some of the tuning parameters are specific to the shell environment
  # running the script will cause the parameters to only be valid for that subprocess's shell environment
  # once the subprocess exits the parameters will be lost
  . ${SPECJAPPKIT}/sjas4_tune.sh
  cp ${SPECJAPPKIT}/sjas4_tune.sh ${LOGDIR}/benchmark/sjas4_tune.${RUN_NUMBER}
fi

################################################################


# fix hosts file here
case "$TYPE" in
  "SPRAYER")
    addreplace "specdelivery" "${WAS_IP}\tspecdelivery\t# WAS server"        /etc/hosts
    log "$LOG_LABEL: /etc/hosts fixed"
    cp /etc/hosts $LOGDIR/benchmark/etc_hosts.${RUN_NUMBER}
    ;;
  "WAS"|"CLIENT"|"DRIVER"|"EMULATOR")
    if [ $SPRAYER_PRESENT -eq 1 ]
    then
      if [ "$TYPE" == "WAS" ]
      then
        addreplace "specdelivery" "${WAS_IP}\tspecdelivery\t# WAS server"        /etc/hosts
      else
        addreplace "specdelivery" "${SPRAYER_IP}\tspecdelivery\t# sprayer machine" /etc/hosts
      fi
      addreplace "specemulator" "${CLIENT_IP}\tspecemulator\t# client machine" /etc/hosts
      addreplace "specdriver"   "${CLIENT_IP}\tspecdriver\t# client machine"   /etc/hosts
      addreplace "specdb"       "${DB2_IP}\tspecdb\t# db2 machine"             /etc/hosts
    else
      addreplace "specdelivery" "${WAS_IP}\tspecdelivery\t# WAS server"        /etc/hosts
      addreplace "specemulator" "${CLIENT_IP}\tspecemulator\t# client machine" /etc/hosts
      addreplace "specdriver"   "${CLIENT_IP}\tspecdriver\t# client machine"   /etc/hosts
      addreplace "specdb"       "${DB2_IP}\tspecdb\t# db2 machine"             /etc/hosts
    fi
    log "$LOG_LABEL: /etc/hosts fixed"
    cp /etc/hosts $LOGDIR/benchmark/etc_hosts.${RUN_NUMBER}
    ;;
esac

(( BARRIER_NUM += 1 ))
handle_barrier $BARRIER_NUM $BARRIER_ALL

case "$TYPE" in
  "DB2")
    mkdir $LOGDIR/benchmark/db2_config.${RUN_NUMBER}
    chmod 777 $LOGDIR/benchmark/db2_config.${RUN_NUMBER}

    # by this logic, the script is currently capped at an artificial maximum injection rate of 1000 (pretty BIG)
    if [ $IR -lt 101 ]
    then
      if [ $IR -lt 11 ]
      then
        DB_IR=$IR
      else
        for step in 20 30 40 50 60 70 80 90 100
        do
          if [ $IR -lt $step -o $IR -eq $step ]
          then
            DB_IR=$step
            break
          fi
        done
      fi
    else
      for step in 200 300 400 500 600 700 800 900 1000
      do
        if [ $IR -lt $step -o $IR -eq $step ]
        then
          DB_IR=$step
          break
        fi
      done
    fi

    su - db2inst1 -c "pushd ${SPECJAPPKIT}/schema/db2 > /dev/null; \
                      if [ -d /archive/archive_${DB_IR} ]; \
                      then \
                        ./restore.sh ${DB_IR}; \
                      else \
                        ./rebuild.sh ${DB_IR}; \
                      fi; \
                      ./db2tune.sh; \
                      date; \
                      ./getdbparms.sh ${LOGDIR}/benchmark/db2_config.${RUN_NUMBER}; \
                      cp ./db2tune.sh ${LOGDIR}/benchmark/db2_config.${RUN_NUMBER}; \
                      popd > /dev/null"
    log "$LOG_LABEL: database loaded with DB_IR=$DB_IR for IR=$IR"
    (( BARRIER_NUM += 1 ))
    handle_barrier $BARRIER_NUM $BARRIER_WAS_DB2
    ;;
  "CLIENT"|"DRIVER"|"EMULATOR")
    (( BARRIER_NUM += 1 ))
    if [ "$TYPE" != "EMULATOR" ]
    then
      # setup values in run.properties
      FILE="${SPECJAPPKIT}/config/run.properties"
      log "$LOG_LABEL: setting txRate = $IR"
      log "$LOG_LABEL: setting doAudit = 1"
      log "$LOG_LABEL: setting triggerTime = $TRIGGER_TIME"
      log "$LOG_LABEL: setting msBetweenThreadStart = $THREAD_START"
      log "$LOG_LABEL: setting rampUp = $RAMP_UP"
      log "$LOG_LABEL: setting stdyState = $STEADY_STATE"
      log "$LOG_LABEL: setting rampDown = $RAMP_DOWN"
      cat $FILE | sed "s/txRate = .*$/txRate = ${IR}/" | \
                  sed "s/doAudit = .*$/doAudit = 1/" | \
                  sed "s/triggerTime = .*$/triggerTime = ${TRIGGER_TIME}/" | \
                  sed "s/msBetweenThreadStart = .*$/msBetweenThreadStart = ${THREAD_START}/" | \
                  sed "s/rampUp = .*$/rampUp = ${RAMP_UP}/" | \
                  sed "s/stdyState = .*$/stdyState = ${STEADY_STATE}/" | \
                  sed "s/rampDown = .*$/rampDown = ${RAMP_DOWN}/" > ${FILE}.tmp
      mv ${FILE}.tmp $FILE
    fi
    if [ "$TYPE" != "DRIVER" ]
    then
      run_was
      log "$LOG_LABEL: WAS started"
    fi
    ;;
  "WAS")
    (( BARRIER_NUM += 1 ))
    handle_barrier $BARRIER_NUM $BARRIER_WAS_DB2
    CFG_FILE="${WAS_INSTALL}/config/cells/${MY_NAME}/nodes/${MY_NAME}/servers/server1/server.xml"
    if [ $TPM_ENABLED -eq 1 ]
    then
      log "$LOG_LABEL: turning on Tivoli Performance Monitor"
      perl -p -i -e "s/PMIService_1\" enable=\"false\"/PMIService_1\" enable=\"true\"/g" ${CFG_FILE}
      perl -p -i -e "s/PMIService_1\" enable=\"true\" initialSpecLevel=\"beanModule=[N|M|H]:cacheModule=[N|M|H]:connectionPoolModule=[N|M|H]:j2cModule=[N|M|H]:jvmRuntimeModule=[N|M|H]:orbPerfModule=[N|M|H]:servletSessionsModule=[N|M|H]:systemModule=[N|M|H]:threadPoolModule=[N|M|H]:transactionModule=[N|M|H]:webAppModule=[N|M|H]:webServicesModule=[N|M|H]:wlmModule=[N|M|H]:wsgwModule=[N|M|H]\"/PMIService_1\" enable=\"true\" initialSpecLevel=\"beanModule=H:cacheModule=H:connectionPoolModule=H:j2cModule=H:jvmRuntimeModule=M:orbPerfModule=H:servletSessionsModule=H:systemModule=H:threadPoolModule=H:transactionModule=H:webAppModule=H:webServicesModule=H:wlmModule=H:wsgwModule=H\"/g" ${CFG_FILE}
    else
      log "$LOG_LABEL: turning off Tivoli Performance Monitor"
      perl -p -i -e "s/PMIService_1\" enable=\"true\"/PMIService_1\" enable=\"false\"/g" ${CFG_FILE}
    fi
    run_was
    log "$LOG_LABEL: WAS started"
    sleep 180 # delay the script to make sure that WAS is fully initialized, experimental
    ;;
  "SPRAYER")
    (( BARRIER_NUM += 1 ))
    ${IHS_INSTALL}/bin/apachectl start
    ;;
esac

(( BARRIER_NUM += 1 ))
handle_barrier $BARRIER_NUM $BARRIER_ALL


# Run the benchmark #############################################
case "$TYPE" in
  "DB2"|"WAS"|"SPRAYER"|"EMULATOR")
    sleep $(( $TRIGGER_TIME + $RAMP_UP ))
    startprofilers
    log "$LOG_LABEL: profilers started"
    sleep $STEADY_STATE
    stopprofilers
    log "$LOG_LABEL: profilers stopped"
    sleep $RAMP_DOWN
    ;;
  "CLIENT"|"DRIVER")
    pushd $SPECJAPPKIT > /dev/null
    log "$LOG_LABEL: run started"
    /bin/sh -c "${NPTL_HANDLER} ./bin/driver.sh 2>&1" | shadow $LOGDIR/benchmark/specjappout.${RUN_NUMBER} &
    DRIVER_PID=$!
    RUN_INITIATED=1
    sleep $(( $TRIGGER_TIME + $RAMP_UP ))
    startprofilers
    log "$LOG_LABEL: profilers started"
    sleep $STEADY_STATE
    stopprofilers
    log "$LOG_LABEL: profilers stopped"
    sleep $RAMP_DOWN
    wait $DRIVER_PID 
    log "$LOG_LABEL: run finished"
    popd > /dev/null
    ;;
esac

doprofilers report
doprofilers postprocess

(( BARRIER_NUM += 1 ))
handle_barrier $BARRIER_NUM $BARRIER_ALL


# Finish the benchmark ##########################################
mycleanup "finish"


# Handle Reporting ##############################################
report
