make_tests.sh

#!/bin/bash

#for user doc, check scripts/00-template
base_args=""

GNU_TIME=/usr/bin/time
GNU_DATE=date
GNU_TIMEOUT=timeout
#GNU_SED=sed
DIFF=diff
GCOV=gcov
READLINK=readlink
UNBUFFER="${UNBUFFER:-}"

EXTERNAL_MEDIA_AVAILABLE=1

platform=`uname -s`

if [ $platform = "Darwin" ] ; then
GNU_TIME=gtime
GNU_DATE=gdate
GNU_TIMEOUT=gtimeout
READLINK=greadlink
fi

GPAC_CPU="x86"
cpu_arch=`uname -m`

if [ $cpu_arch = "armv4l" ] ; then
GPAC_CPU="arm"
elif [ $cpu_arch = "arm" ] ; then
GPAC_CPU="arm"
elif [ $cpu_arch = "arm64" ] ; then
GPAC_CPU="arm"
elif [ $cpu_arch = "aarch64" ] ; then
GPAC_CPU="arm"
fi

#if the script in launched from elsewhere, main_dir still needs to be the script directory
main_dir="$(dirname $($READLINK -f $0))"
cd $main_dir

#if launched from an absolute path, set all paths as absolute (will break on cygwin)
rel_main_dir="."
if [[ "$0" = /* ]]; then
  rel_main_dir=$main_dir
fi

generate_hash=0
generate_hash_suffix=""
global_test_ui=0
log_after_fail=0
verbose=0
enable_timeout=0
skip_next_hash_test=0
do_playback=0
do_clean=0
quiet=0
do_clean_hash=0
check_only=0
disable_hash=0
strict_mode=0
track_stack=0
speed=1
single_test_name=""
keep_temp_dir=0
store_video=0
quick_mode=0
gpac_profile=""
skip_all_tests=0
dump_js=0
current_script=""

DEF_DUMP_DUR=10
DEF_DUMP_SIZE="200x200"
DEF_TIMEOUT=20

echo "" > all_tests.js

#remote location of resource files: all media files, hash files and generated videos
REFERENCE_DIR="https://download.tsi.telecom-paristech.fr/gpac/gpac_test_suite/resources"
#dir where all external media are stored
EXTERNAL_MEDIA_DIR="$rel_main_dir/external_media"
#dir where all hashes are stored
HASH_DIR="$rel_main_dir/hash_refs"
#dir where all specific test rules (override of defaults, positive tests, ...) are stored
RULES_DIR="$rel_main_dir/rules"
#dir where all referenced videos are stored
SCRIPTS_DIR="$rel_main_dir/scripts"
#dir where all referenced videos are stored
VIDEO_DIR_REF="$rel_main_dir/external_videos_refs"

#dir where all local media data (ie from git repo) is stored
MEDIA_DIR="$rel_main_dir/media"
#local dir where all data will be generated (except hashes and referenced videos)
LOCAL_OUT_DIR="$rel_main_dir/results"

#dir where all test videos are generated
VIDEO_DIR="$LOCAL_OUT_DIR/videos"
#dir where all logs are generated
LOGS_DIR="$LOCAL_OUT_DIR/logs"
#temp dir for any test
INTERN_TEMP_DIR="$LOCAL_OUT_DIR/temp"
TEMP_DIR=$INTERN_TEMP_DIR

HAS_GPAC_NODE="no"

ALL_REPORTS="$LOCAL_OUT_DIR/all_results.xml"
ALL_LOGS="$LOCAL_OUT_DIR/all_logs.txt"

TEST_ERR_FILE="$TEMP_DIR/err_exit"

rm -f "$TEST_ERR_FILE" 2> /dev/null
rm -f "$LOGS_DIR/*.sh" 2> /dev/null

if [ ! -e $LOCAL_OUT_DIR ] ; then
mkdir $LOCAL_OUT_DIR
fi

if [ ! -e $HASH_DIR ] ; then
mkdir $HASH_DIR
fi

if [ ! -e $VIDEO_DIR ] ; then
mkdir $VIDEO_DIR
fi

if [ ! -e $VIDEO_DIR_REF ] ; then
mkdir $VIDEO_DIR_REF
fi


if [ ! -e $LOGS_DIR ] ; then
mkdir $LOGS_DIR
fi

if [ ! -e $RULES_DIR ] ; then
mkdir $RULES_DIR
fi

if [ ! -e $INTERN_TEMP_DIR ] ; then
mkdir $INTERN_TEMP_DIR
fi

L_ERR=1
L_WAR=2
L_INF=3
L_DEB=4

log()
{
 if [ $quiet = 1 ]; then
  return;
 fi

 if [ $TERM = "cygwin" ]; then

  if [ $1 = $L_ERR ]; then
    echo -ne "\033[31m"
  elif [ $1 = $L_WAR ]; then
    echo -ne "\033[32m"
  elif [ $1 = $L_INF ]; then
    echo -ne "\033[34m"
  elif [ $verbose = 0 ]; then
    echo -ne "\033[0m"
    return
  fi

  echo $2
  echo -ne "\033[0m"

 else

  if [ $1 = $L_ERR ]; then
    tput setaf 1
  elif [ $1 = $L_WAR ]; then
    tput setaf 2
  elif [ $1 = $L_INF ]; then
    tput setaf 3
  elif [ $verbose = 0 ]; then
    tput sgr0
    return
  fi

  echo $2
  tput sgr0

 fi
}

print_usage ()
{
echo "GPAC Test Suite Usage: $0 [<options>] [<SCRIPTS>]"
echo ""
echo "*** Test suite validation options"
echo "  -clean [ARG]:          removes all results (logs, stat cache and video). If ARG is specified, only clean tests generated by scripts ARG."
echo "  -re [ARG]:             removes all results (logs, stat cache and video) and run. If ARG is specified, only clean and rerun tests generated by scripts ARG."
echo "  -no-hash:              runs test suite without hash checking."
echo ""
echo "*** Test suite generation options"
echo "  -clean-hash [ARG]:     removes all generated hash, logs, stat cache and videos. If ARG is specified, only clean tests generated by scripts ARG."
echo "  -hash:                 regenerate tests with missing hash files."
echo "  -hash=SUFFIX:          regenerate tests with missing hash files. Appends '-alt-SUFFIX' to hash name for alternate hash (e.g.: for non standard platforms)"
echo "  -uirec:                generates UI event traces."
echo "  -uiplay:               replays all recorded UI event traces."
echo "  -speed=N:              sets playback speed for -uiplay. Default is 1."
echo ""
echo "*** General options"
echo "  -strict:               stops at the first failed test"
echo "  -warn:                 dump logs after each failed test (used for travisCI)"
echo "  -videos:               stores all videos from playback tests in $VIDEO_DIR"
echo "  -keep-tmp or -tmp:     keeps tmp folder used in tests (erased by default)"
echo "  -sync-hash:            syncs all remote reference hashes with local base"
echo "  -git-hash:             syncs all remote reference hashes from git with local base"
echo "  -sync-media:           syncs all remote media with local base (warning this can be long)"
echo "  -sync-refs:            syncs all remote reference videos with local base (warning this can be long)"
echo "  -sync-before:          syncs all remote resources with local base (warning this can be long) before running the tests"
echo "  -check:                check test suites (names of each test is unique)"
echo "  -track-stack:          track stack in malloc and turns on -warn option"
echo "  -play:                 executes playback tests (not all tests use playback tests)"
echo "  -p=NAME:               sets execution profile of all gpac apps to NAME. Use '-p=0' to disable profiles from local storage (eg for parallel executions of test suite)"
echo "  -test=NAME             only executes given test"
echo "  -precommit             alias for -sync-before -git-hash -warn. Before commit/push, you should run ./make_tests -precommit"
echo "  -quick                 only executes the first test of each script"
echo "  -timeout=SECONDS       sets the timeout for each test command (default: 20)"

echo "  SCRIPTS                only runs the scripts provided as arguments, by default runs everything in $SCRIPTS_DIR"
echo "  -v:                    set verbose output"
echo "  -quiet:                only print each test result"
echo "  -h:                    print this help"
}


#performs mirroring of media and references hash & videos
sync_media ()
{
 log $L_INF "- Mirroring $REFERENCE_DIR/media/ to $EXTERNAL_MEDIA_DIR"
 if [ ! -e $EXTERNAL_MEDIA_DIR ] ; then
  mkdir $EXTERNAL_MEDIA_DIR
 fi
 cd $EXTERNAL_MEDIA_DIR
 wget -q -m -nH --no-parent --cut-dirs=4 --reject "*.gif" --reject "index.html*" --restrict-file-names=nocontrol "$REFERENCE_DIR/media/"
 cd "$main_dir"
}

sync_hash ()
{
return 0
log $L_INF "- Mirroring reference hashes from from github to $HASH_DIR"
cd $HASH_DIR
if [ ! -d ".git" ]; then
  rm -f *
  git clone https://github.com/gpac/gpac-test-hash.git .
else
  git fetch origin
  git reset --hard origin/master
fi
cd "$main_dir"
}


#performs mirroring of media and references hash & videos
sync_refs ()
{
log $L_INF "- Mirroring reference videos from $REFERENCE_DIR to $VIDEO_DIR_REF"
cd $VIDEO_DIR_REF
wget -q -m -nH --no-parent --cut-dirs=4 --reject "*.gif" "$REFERENCE_DIR/video_refs/"
cd "$main_dir"
}

url_arg=()

#Parse arguments
for i in $* ; do
 case $i in
 "-hash")
  generate_hash=1;;
 -hash=*)
  generate_hash=1
  generate_hash_suffix="${i#-hash=}"
  ;;
 "-play-all")
   ;;
 "-clean")
   do_clean=1;;
 "-re")
   do_clean=2;;
 "-clean-hash")
   do_clean_hash=1;;
 "-uirec")
  global_test_ui=1;;
 "-uiplay")
  global_test_ui=2;;
 -speed*)
  speed="${i#-speed=}"
  ;;
 "-keep-avi")
  ;;
 "-videos")
  store_video=1;;
 "-keep-tmp")
  keep_temp_dir=1;;
 "-tmp")
  keep_temp_dir=1;;
 "-no-hash")
  disable_hash=1;;
 "-strict")
  strict_mode=1;;
 "-sync-hash")
  sync_hash
  exit;;
 "-git-hash")
  sync_hash;;
 "-sync-media")
  sync_media;;
 "-sync-refs")
  sync_refs
  exit;;
 "-sync-before")
  sync_media;;
 "-check")
  check_only=1;;
 "-warn")
  log_after_fail=1;;
 "-track-stack")
  track_stack=1;;
 "-noplay")
  ;;
 "-play")
  do_playback=1;;
"-quick")
 quick_mode=1;;

"-quiet")
 quiet=1;;

 -test*)
  single_test_name="${i#-test=}"
  ;;
 "-v")
  verbose=1;;
 -p=*)
  gpac_profile="${i#-p=}"
  ;;
 "-precommit")
  sync_media
  sync_hash
  log_after_fail=1
  ;;
 -timeout=*)
  DEF_TIMEOUT="${i#-timeout=}"
  ;;
 "-h")
  print_usage
  exit;;
 -*)
   log $L_ERR "Unknown Option \"$i\" - check usage (-h)"
   exit;;
 *)
  url_arg+=("$i")
 ;;
esac
done

if [ $check_only != 0 ] ; then
 do_clean_hash=0
 do_clean=0
 global_test_ui=0
fi

#Clean all hashes and reference videos
if [ $do_clean_hash != 0 ] ; then

 #force cleaning as well
 do_clean=1

 #when using specific scripts, cleanup will be done in test_begin()
 if [ "${#url_arg[@]}" -eq 0 ] ; then
  read -p "This will remove all referenced videos and hashes. Are you sure (y/n)?" choice
  if [ $choice != "y" ] ; then
   log $L_ERR "Canceled"
   exit
  fi
  log $L_INF "Deleting SHA-1 Hashes"
  rm -rf $HASH_DIR/* 2> /dev/null
  rm -rf $VIDEO_DIR_REF/* 2> /dev/null
 fi
fi

#Clean all cached results and generated videos
if [ $do_clean != 0 ] ; then
 rm -f $ALL_REPORTS > /dev/null
 rm -f $ALL_LOGS > /dev/null
 rm -rf $INTERN_TEMP_DIR/* 2> /dev/null
 if [ "${#url_arg[@]}" -eq 0 ] ; then
  echo "Deleting cache (logs, stats and videos)"
  find "$main_dir/$LOGS_DIR/" -mindepth 1 -delete > /dev/null
  rm -rf $VIDEO_DIR/* 2> /dev/null
  exit
 fi
fi

log $L_INF "Checking test suite config"

if [ $generate_hash = 0 ] ; then
  log $L_INF "- Enabling hash tests"
fi

if [ ! -e $EXTERNAL_MEDIA_DIR ] ; then
EXTERNAL_MEDIA_AVAILABLE=0
elif [ ! -e $EXTERNAL_MEDIA_DIR/counter ] ; then
EXTERNAL_MEDIA_AVAILABLE=0
fi

if [ $EXTERNAL_MEDIA_AVAILABLE = 0 ] ; then
 log $L_WAR "- External media dir unavailable - you may sync it using -sync-media"
else
 log $L_INF "- External media dir available"
fi

#test for GNU time
$GNU_TIME ls > /dev/null 2>&1
res=$?
if [ $res != 0 ] ; then
log $L_ERR "GNU time not found (ret $res) - exiting"
exit 1
fi

#test for GNU date
$GNU_DATE > /dev/null 2>&1
res=$?
if [ $res != 0 ] ; then
log $L_ERR "GNU date not found (ret $res) - exiting"
exit 1
fi

#test for timeout
$GNU_TIMEOUT 1.0 ls > /dev/null 2>&1
res=$?
if [ $res != 0 ] ; then
 log $L_ERR "GNU timeout not found (ret $res) - some tests may hang forever ..."
 enable_timeout=0
else
enable_timeout=1
fi

node -v > /dev/null 2>&1
res=$?
if [ $res = 0 ] ; then
  TMPFILE=$(mktemp)

cat <<EOF > $TMPFILE
const gpac = require('gpac');
console.log("Welcome to GPAC NodeJS !\nVersion: " + gpac.version);
EOF

  node $TMPFILE > /dev/null 2>&1
  node_res=$?
  rm $TMPFILE
  if [ $node_res = 0 ] ; then
    HAS_GPAC_NODE="yes"
  fi

fi


if [ $check_only = 0 ] ; then

#check MP4Box and gpac (use default args, not custom ones because of -mem-track)
MP4Box -h > /dev/null 2>&1
res=$?
if [ $res != 0 ] ; then
log $L_ERR "MP4Box not found (ret $res) - exiting"
exit 1
fi

GPAC_OSTYPE="unknown"

gpac -p=0 -h > /dev/null 2>&1
res=$?
if [ $res != 0 ] ; then
log $L_ERR "gpac not found (ret $res) - exiting"
#exit 1
else
 bininfo=`gpac -h bin 2>&1`
 config=`echo $bininfo | grep GPAC_64`
 if [ -z "$config" ] ; then
  bits=32
 else
  bits=64
 fi

 config=`echo $bininfo | grep GPAC_CONFIG_LINUX`
 if [ -n "$config" ] ; then
  GPAC_OSTYPE="lin$bits"
 else
  config=`echo $bininfo | grep GPAC_CONFIG_DARWIN`
  if [ -n "$config" ] ; then
  GPAC_OSTYPE="osx$bits"
  else
   config=`echo $bininfo | grep GPAC_CONFIG_WIN`
   if [ -n "$config" ] ; then
    GPAC_OSTYPE="win$bits"
   fi
  fi
 fi
fi

#check mem tracking is supported
res=`MP4Box -mem-track -h 2>&1 | grep "WARNING"`
if [ -n "$res" ]; then
  log $L_WAR "- GPAC not compiled with memory tracking"
else
 log $L_INF "- Enabling memory tracking"
 if [ $track_stack = 1 ]; then
  base_args="$base_args -mem-track-stack"
  log_after_fail=1
 else
  base_args="$base_args -mem-track"
 fi
fi

#set check-props by default if supported
has_props=`gpac -hh core | grep check-props`
if [ "$has_props" != "" ] ; then
  base_args="$base_args -check-props"
  log $L_INF "- Enabling property checking"
fi

fi
log $L_INF "- CPU config: $GPAC_CPU - OS: $GPAC_OSTYPE"
#end check_only

echo ""


#reassign our default programs
MP4BOX="MP4Box -noprog -for-test $base_args"
GPAC="gpac $base_args -noprog -for-test -no-reassign "


if [ "$gpac_profile" != "" ] ; then
log $L_INF "- GPAC profile: $gpac_profile"
MP4BOX="$MP4BOX -p=$gpac_profile"
GPAC="$GPAC -p=$gpac_profile"
fi

$MP4BOX -version 2> $INTERN_TEMP_DIR/version.txt
VERSION="`grep 'GPAC version' $INTERN_TEMP_DIR/version.txt | head -1 | cut -d ' ' -f 5-` "
rm $INTERN_TEMP_DIR/version.txt
log $L_INF "GPAC version: $VERSION"
log $L_INF ""

#reset all the possible return values
reset_stat ()
{
 EXECUTION_STATUS="N/A"
 RETURN_VALUE="N/A"
 MEM_TOTAL_AVG="N/A"
 MEM_RESIDENT_AVG="N/A"
 MEM_RESIDENT_MAX="N/A"
 CPU_PERCENT="N/A"
 CPU_ELAPSED_TIME="N/A"
 CPU_USER_TIME="N/A"
 CPU_KERNEL_TIME="N/A"
 PAGE_FAULTS="N/A"
 FILE_INPUTS="N/A"
 SOCKET_MSG_REC="N/A"
 SOCKET_MSG_SENT="N/A"
}

test_start_ts=0
#begin a test with name $1 and using hashes called $1-$2 ... $1-$N
test_begin ()
{
  if [ $# -gt 1 ] ; then
   log $L_ERR "> in script $current_script line $BASH_LINENO"
   log $L_ERR "	@test_begin takes only two arguments - wrong call (first arg is $1)"
  fi

  if [ $dump_js = 1 ] ; then
   test_skip=0
   echo '{ "name": "'$1'", "tests": [' >> all_tests.js
   return
  fi


 test_skip=0
 result=""
 TEST_NAME=$1
 reference_hash_valid="$HASH_DIR/$TEST_NAME-valid-hash"

 log $L_DEB "Starting test $TEST_NAME"

 TEMP_DIR=$INTERN_TEMP_DIR

 report="$TEMP_DIR/$TEST_NAME-temp.txt"
 LOGS="$LOGS_DIR/$TEST_NAME-logs.txt-new"
 final_report="$LOGS_DIR/$TEST_NAME-passed.xml"
 test_ui=0
 test_start_ts=`$GNU_DATE +%s%N`


 if [ $do_clean != 0 ] ; then
  if [ $do_clean_hash != 0 ] ; then
   rm -rf $HASH_DIR/$TEST_NAME* 2> /dev/null
   rm -rf $VIDEO_DIR_REF/$TEST_NAME* 2> /dev/null
   rm -rf $reference_hash_valid 2> /dev/null
  fi
  rm -rf $LOGS_DIR/$TEST_NAME* > /dev/null
  rm -rf $VIDEO_DIR/$TEST_NAME* 2> /dev/null
  if [ $do_clean != 2 ] ; then
   test_skip=1
   return
  fi
 fi

 if [ $quick_mode != 0 ] ; then
  if [ $skip_all_tests != 0 ] ; then
   test_skip=1
   return
  fi
  skip_all_tests=1
 fi

 if [ $check_only != 0 ] ; then
  test_ui=0
  report="$TEMP_DIR/$TEST_NAME.test"
  if [ -f $report ] ; then
   log $L_ERR "Test $TEST_NAME already exists - please fix ($current_script)"
   rm -rf $TEMP_DIR/* 2> /dev/null
   exit
  fi
  echo "" > $report
  test_skip=1
  return
 fi

 if [ $keep_temp_dir != 0 ] ; then
 	TEMP_DIR="$INTERN_TEMP_DIR/$TEST_NAME"
	if [ ! -e $TEMP_DIR ] ; then
		mkdir $TEMP_DIR
	fi
 fi

 #reset defaults
 dump_dur=$DEF_DUMP_DUR
 dump_size=$DEF_DUMP_SIZE
 test_timeout="-k $DEF_TIMEOUT $DEF_TIMEOUT"

 test_skip=0
 single_test=0

 test_args="$@"
 test_nb_args=$#
 skip_play_hash=0
 subtest_idx=0
 nb_subtests=0
 test_ui=$global_test_ui

 test_stats="$LOGS_DIR/$TEST_NAME-stats.sh"

 #if error in strict mode, mark the test as skippable using value 2
 if [ $strict_mode = 1 ] ; then
  if [ -f $TEST_ERR_FILE ] ; then
   test_skip=2
  fi
 fi

 if [ $generate_hash = 1 ] ; then
  #skip test only if reference hash is marked as valid
  if [ -f "$reference_hash_valid" ] ; then
   log $L_DEB "Reference hash found for test $TEST_NAME - skipping hash generation"
   test_skip=1
  fi
 elif [ $test_ui != 0 ] ; then
   test_skip=0
 elif [ $test_skip = 0 ] ; then
  #skip test only if final report is present (whether we generate hashes or not)
  if [ -f "$final_report" ] ; then
   if [ -f "$test_stats" ] ; then
    log $L_DEB "$TEST_NAME already passed - skipping"
    test_skip=1
   else
    log $L_WAR "$TEST_NAME already passed but missing stats.sh - regenerating"
   fi
  fi
 fi

 if [ "$single_test_name" != "" ] && [ "$single_test_name" != "$TEST_NAME" ] ; then
   test_ui=0
   test_skip=1
 fi

 if [ $test_skip != 0 ] ; then
   #stats.sh may be missing when generating hashes and that's not an error
   if [ -f "$test_stats" ] ; then
    has_skip=`grep -w "TEST_SKIP" $test_stats`

    if [ "$has_skip" = "" ]; then
		echo "TEST_SKIP=$test_skip" >> $test_stats
    fi
   fi
   test_skip=1
 elif [ $test_ui != 0 ] ; then
   #in UI test mode don't check cache status, always run the tests
   test_skip=1
 else
  echo "*** $TEST_NAME logs (GPAC version $VERSION) - test date $(date '+%d/%m/%Y %H:%M:%S') ***" > $LOGS
  echo "" >> $LOGS
 fi


 rules_sh=$RULES_DIR/$TEST_NAME.sh
 if [ -f $rules_sh ] ; then
  source $rules_sh
 fi

 if [ $test_skip != 0 ] ; then
  TEMP_DIR=$INTERN_TEMP_DIR
 fi
}

mark_test_error ()
{
 if [ $strict_mode = 1 ] ; then
  echo "" > $TEST_ERR_FILE
  log $L_ERR "Error test $TEST_NAME subtest $SUBTEST_NAME - aborting"
 fi
}

test_end_ts=0
#ends test - gather all logs/stats produced and generate report
test_end ()
{
 #wait for all sub-tests to complete (some may use subshells)
 wait

 test_end_ts=`$GNU_DATE +%s%N`
 test_runtime=$((test_end_ts-test_start_ts))
 test_runtime=$(($test_runtime / 1000000))

  if [ $dump_js = 1 ] ; then
   test_skip=0
   echo '{}]},' >> all_tests.js
   return
  fi

 TEMP_DIR=$INTERN_TEMP_DIR

  if [ $# -gt 0 ] ; then
   log $L_ERR "> in test $TEST_NAME in script $current_script line $BASH_LINENO"
   log $L_ERR "	@test_end takes no argument - wrong call"
  fi

 if [ $test_skip = 1 ] ; then
  return
 fi

 test_stats="$LOGS_DIR/$TEST_NAME-stats.sh"
 echo "" > $test_stats
 stat_xml_temp="$TEMP_DIR/$TEST_NAME-statstemp.xml"
 echo "" > $stat_xml_temp

 test_fail=0
 test_leak=0
 test_exec_na=0
 nb_subtests=0
 nb_test_hash=0
 nb_hash_fail=0
 nb_hash_missing=0

 if [ "$result" != "" ] ; then
  test_fail=1
 fi

# makes glob on non existing files to expand to null
# enabling loops on nonexisting* to be empty
shopt -s nullglob

 #gather all stats per subtests
 for i in $TEMP_DIR/$TEST_NAME-stats-*.sh ; do
  reset_stat
  RETURN_VALUE=0
  SUBTEST_NAME=""
  COMMAND_LINE=""
  SUBTEST_IDX=0

  nb_subtests=$((nb_subtests + 1))

  source $i

  echo "  <stat subtest=\"$SUBTEST_NAME\" execution_status=\"$EXECUTION_STATUS\" return_status=\"$RETURN_STATUS\" mem_total_avg=\"$MEM_TOTAL_AVG\" mem_resident_avg=\"$MEM_RESIDENT_AVG\" mem_resident_max=\"$MEM_RESIDENT_MAX\" cpu_percent=\"$CPU_PERCENT\" cpu_elapsed_time=\"$CPU_ELAPSED_TIME\" cpu_user_time=\"$CPU_USER_TIME\" cpu_kernel_time=\"$CPU_KERNEL_TIME\" page_faults=\"$PAGE_FAULTS\" file_inputs=\"$FILE_INPUTS\" socket_msg_rec=\"$SOCKET_MSG_REC\" socket_msg_sent=\"$SOCKET_MSG_SENT\" return_value=\"$RETURN_VALUE\">" >> $stat_xml_temp

  XML_CMD_LINE=${COMMAND_LINE//&/&amp;}

  echo "   <command_line>$XML_CMD_LINE</command_line>" >> $stat_xml_temp
  echo "  </stat>" >> $stat_xml_temp

  test_ok=1

  if [ $RETURN_VALUE -eq 1 ] ; then
   result="$SUBTEST_NAME:Fail $result"
   test_ok=0
   test_fail=$((test_fail + 1))
  elif [ $RETURN_VALUE -eq 2 ] ; then
   result="$SUBTEST_NAME:MemLeak $result"
   test_ok=0
   test_leak=$((test_leak + 1))
  elif [ $RETURN_VALUE != 0 ] ; then
   # 124 is gnu timeout sigterm
   # 137 is 128 from gnu time + 9 sigkill from timeout
   # both mean timeout for us
   if [[ $enable_timeout -ne 0 ]] && [[ $RETURN_VALUE -eq 124 || $RETURN_VALUE -eq 137 ]]; then
    result="$SUBTEST_NAME:Timeout $result"
   else
    result="$SUBTEST_NAME:Fail(ret code $RETURN_VALUE) $result"
   fi
   test_ok=0
   test_fail=$((test_fail + 1))
  fi

  if [ $log_after_fail = 1 ] ; then
   if [ $test_ok = 0 ] ; then
    sublog=$LOGS_DIR/$TEST_NAME-logs-$SUBTEST_IDX-$SUBTEST_NAME.txt
    if [ -f $sublog ] ; then
	 cat $sublog 2> stderr
    fi
   fi
  fi
  rm -f $i > /dev/null
 done

 # list all logs files
 for i in $LOGS_DIR/$TEST_NAME-logs-*.txt; do
  wa=`grep "Wrong argument value" $i`
  #exception for this test where we test bad formatting for coverage
  if [ "$wa" != "" ] && [ "$TEST_NAME" != "gpac-filter-dump_props" ] ; then
    result="$result CMDLINE_INVALID"
    if [ $test_ok != 0 ] ; then
      test_ok=0
      test_fail=$((test_fail + 1))
    fi
  fi
  cat $i >> $LOGS
 done
 rm -f $LOGS_DIR/$TEST_NAME-logs-*.txt > /dev/null


 #gather all hashes for this test
 for i in $TEMP_DIR/$TEST_NAME-stathash-*.sh ; do
  if [ -f $i ] ; then
   HASH_TEST=""
   HASH_NOT_FOUND=0
   HASH_FAIL=0
   SRC_NOT_FOUND=0

   source $i
   nb_test_hash=$((nb_test_hash + 1))
   if [ $HASH_NOT_FOUND -eq 1 ] ; then
    result="$HASH_TEST:HashNotFound $result"
    nb_hash_missing=$((nb_hash_missing + 1))
    test_exec_na=$((test_exec_na + 1))
   elif [ $SRC_NOT_FOUND -eq 1 ] ; then
    result="$HASH_TEST:HashSourceNotFound $result"
    test_ok=0
    nb_hash_fail=$((nb_hash_fail + 1))
    test_exec_na=$((test_exec_na + 1))
   elif [ $HASH_FAIL -eq 1 ] ; then
    result="$HASH_TEST:HashFail $result"
    test_ok=0
    nb_hash_fail=$((nb_hash_fail + 1))
    test_exec_na=$((test_exec_na + 1))
   fi
  fi
  rm -f $i > /dev/null
 done

 if [ "$result" = "" ] ; then
  result="OK"
 fi

 if [ ! -f $TEST_ERR_FILE ] ; then
  if [ $generate_hash = 1 ] ; then
    log $L_DEB "Test $TEST_NAME $nb_subtests subtests and $nb_test_hash hashes"
    nb_hashes=$((nb_test_hash + nb_test_hash))
  	#only allow no hash if only one subtest
    if [ $subtest_idx -gt 1 ] && [ $nb_hashes -lt $subtest_idx ] ; then
     log $L_ERR "Test $TEST_NAME has too few hash tests: $nb_hashes for $nb_subtests subtests - please fix"
     result="NOT ENOUGH HASHES"
    else
		echo "ok" > $reference_hash_valid
	fi
#    if [ $nb_test_hash -gt 15 ] ; then
#     log $L_WAR "Test $TEST_NAME has too many subtests with hashes ($nb_test_hash), not efficient for hash generation - consider rewriting $current_script"
#	fi
  fi
 fi


 echo " <test name=\"$TEST_NAME\" result=\"$result\" date=\"$(date '+%d/%m/%Y %H:%M:%S')\" runtime=\"$test_runtime\">" > $report
 cat $stat_xml_temp >> $report
 rm -f $stat_xml_temp > /dev/null
 echo " </test>" >> $report

 echo "TEST_FAIL=$test_fail" >> $test_stats
 echo "TEST_EXEC_NA=$test_exec_na" >> $test_stats
 echo "SUBTESTS_LEAK=$test_leak" >> $test_stats
 echo "NB_HASH_SUBTESTS=$nb_test_hash" >> $test_stats
 echo "NB_HASH_SUBTESTS_MISSING=$nb_hash_missing" >> $test_stats
 echo "NB_HASH_SUBTESTS_FAIL=$nb_hash_fail" >> $test_stats

 echo "NB_SUBTESTS=$nb_subtests" >> $test_stats

 if [ "$result" == "OK" ] ; then
  mv $report "$LOGS_DIR/$TEST_NAME-passed-new.xml"

  echo "$TEST_NAME: $result"
 else
  mv $report "$LOGS_DIR/$TEST_NAME-failed.xml"
  mark_test_error

  log $L_ERR "$TEST_NAME: $result"
 fi

 #cleanup temp dir
 if [ $keep_temp_dir != 1 ] ; then
  if [ $CLEAN_TEMP != 0 ] ; then
   rm -rf $TEMP_DIR/* 2> /dev/null
  fi
 fi

shopt -u nullglob
}


#@do_test execute the command line given $1 using GNU time and store stats with return value, command line ($1) and subtest name ($2)
ret=0
do_test ()
{
  skip_next_hash_test=0

  if [ $dump_js = 1 ] ; then
   test_skip=0
   echo '{ "subtest": "'$2'", "cmd": "'$1'"},' >> all_tests.js
   return
  fi


  if [ $# -gt 2 ] ; then
   log $L_ERR "> in test $TEST_NAME in script $current_script line $BASH_LINENO"
   log $L_ERR "	@do_test takes only two arguments - wrong call (first arg $1)"
  fi

 if [ $strict_mode = 1 ] ; then
  if [ -f $TEST_ERR_FILE ] ; then
   return
  fi
 fi

 if [ $test_skip = 1 ] ; then
  return
 fi

 log L_DEB "executing $1"

 subtest_idx=$((subtest_idx + 1))

 log_subtest="$LOGS_DIR/$TEST_NAME-logs-$subtest_idx-$2.txt"
 stat_subtest="$INTERN_TEMP_DIR/$TEST_NAME-stats-$subtest_idx-$2.sh"
 SUBTEST_NAME=$2


echo "" > $log_subtest
echo "*** Subtest \"$2\": executing \"$1\" ***" >> $log_subtest

timeout_args=""
if [ $enable_timeout != 0 ] ; then
timeout_args="$GNU_TIMEOUT $test_timeout"
fi

$UNBUFFER $GNU_TIME -o $stat_subtest -f 'EXECUTION_STATUS="OK"\n RETURN_STATUS=%x\n MEM_TOTAL_AVG=%K\n MEM_RESIDENT_AVG=%t\n MEM_RESIDENT_MAX=%M\n CPU_PERCENT=%P\n CPU_ELAPSED_TIME=%E\n CPU_USER_TIME=%U\n CPU_KERNEL_TIME=%S\n PAGE_FAULTS=%F\n FILE_INPUTS=%I\n SOCKET_MSG_REC=%r\n SOCKET_MSG_SENT=%s' $timeout_args $1 >> $log_subtest 2>&1
rv=$?

echo "SUBTEST_NAME=$2" >> $stat_subtest
echo "SUBTEST_IDX=$subtest_idx" >> $stat_subtest

#regular error, check if this is a negative test.
if [ $rv -eq 1 ] ; then
 if [ $single_test = 1 ] ; then
  negative_test_stderr=$RULES_DIR/$TEST_NAME-stderr.txt
 else
  negative_test_stderr=$RULES_DIR/$TEST_NAME-$2-stderr.txt
 fi
 if [ -f $negative_test_stderr ] ; then
  #look for all lines in -stderr file, if one found consider this a success
  while read line ; do
   res_err=`grep -o "$line" $log_subtest`
   if [ -n "$res_err" ]; then
    echo "Negative test detected, reverting to success (found \"$res_err\" in stderr)" >> $log_subtest
    rv=0
    skip_next_hash_test=1
    echo "" > $stat_subtest
    break
   fi
  #remove windows style endlines as they may cause some problems
  #also remove empty lines otherwise grep always matches
  done < <(tr -d '\r' <$negative_test_stderr | sed '/^$/d' )
 fi
fi

#override generated stats if error, since gtime may put undesired lines in output file which would break sourcing
if [ $rv != 0 ] ; then
echo "SUBTEST_NAME=$2" > $stat_subtest
echo "SUBTEST_IDX=$subtest_idx" >> $stat_subtest
mark_test_error
fi

echo "RETURN_VALUE=$rv" >> $stat_subtest
echo "COMMAND_LINE=\"$1\"" >> $stat_subtest

echo "" >> $log_subtest
ret=$rv
}
#end do_test

#do_play_test: playback the input video (arg2) and audio (arg3) to vout/aout, and produces the MP4 file for these files
do_play_test ()
{
  if [ $# -gt 3 ] ; then
   log $L_ERR "> in test $TEST_NAME in script $current_script line $BASH_LINENO"
   log $L_ERR "	@do_play_test takes only 3 arguments - wrong call (first arg $1)"
   return
  fi
  name=$1

 audio_args=""
 if [ $# -gt 2 ] ; then
  audio_args=$3
 fi

 if [ $do_playback = 1 ] ; then
  args=""
  if [ "$2" != "" ] ; then
   args="$args -i $2 vout"
  fi
  if [ "$audio_args" != "" ] ; then
   if [ "$audio_args" != "-" ] ; then
    args="$args -i $audio_args aout"
   else
    args="$args aout"
   fi
  fi

  if [ "$args" = "" ] ; then
   return
  fi
  $GPAC $args 2> /dev/null
 fi


 if [ $store_video = 1 ] ; then
  args=""
  if [ "$2" != "" ] ; then
   args="$args -i $2 enc:FID=V:c=avc:b=500k:pfmt=yuv"
  fi
  if [ "$audio_args" != "" ] ; then
   if [ "$audio_args" != "-" ] ; then
    args="$args -i $audio_args"
   fi
   args="$args enc:FID=A:c=aac:b=64k"
  fi

  if [ "$args" = "" ] ; then
   return
  fi
  name=$(basename $1)
  vid_name="$VIDEO_DIR/$name.mp4"
  $GPAC $args -o $vid_name:SID=V,A 2> /dev/null
 fi
}

#end do_play_test

#@do_hash_test: generates a hash for $1 file , compare it to HASH_DIR/$TEST_NAME$2.hash
do_hash_test ()
{
  if [ $dump_js = 1 ] ; then
    echo '{ "hash": "'$TEST_NAME$2.hash'", "file": "'$1'"},' >> all_tests.js
    return
  fi
  if [ $skip_next_hash_test = 1 ] ; then
    skip_next_hash_test=0
    return
  fi

  if [ $skip_next_hash_test = 1 ] ; then
    skip_next_hash_test=0
    return
  fi

  if [ $# -gt 3 ] ; then
   log $L_ERR "> in test $TEST_NAME in script $current_script line $BASH_LINENO"
   log $L_ERR "	@do_hash_test takes only three arguments - wrong call (first arg is $1)"
  fi
 if [ $strict_mode = 1 ] ; then
  if [ -f $TEST_ERR_FILE ] ; then
   return
  fi
 fi

 is_bin=0
 if [ $# -gt 2 ] ; then
  is_bin=$3
 fi

 if [ $test_skip  = 1 ] ; then
  return
 fi
 log L_DEB "Generating hash for $1"

 if [ $disable_hash = 1 ] ; then
  return
 fi

 STATHASH_SH="$INTERN_TEMP_DIR/$TEST_NAME-stathash-$2.sh"

 test_hash="$INTERN_TEMP_DIR/$TEST_NAME-$2-test.hash"
 ref_hash="$HASH_DIR/$TEST_NAME-$2.hash"

 echo "HASH_TEST=$2" > $STATHASH_SH

 #redefine log subtest var using the hash name in case the subtest was a subscript (do_test &)
 log_subt="$LOGS_DIR/$TEST_NAME-logs-$subtest_idx-$2.txt"

 echo "Computing $1  ($2) hash: " >> $log_subt
 file_to_hash="$1"

 if [ $is_bin != 1 ] ; then
  # for text files, we remove potential CR chars
  # to prevent having different hashes on different platforms
  if [ -n "$(file -b $1 | grep text)" ] ||  [ ${1: -4} == ".lsr" ] ||  [ ${1: -4} == ".svg" ] ; then
   file_to_hash="to_hash_$(basename $1)"
   if [ -f $1 ]; then
	tr -d '\r' <  "$1" > "$file_to_hash"
   fi
  fi
 fi

 $MP4BOX -hash -std $file_to_hash > $test_hash 2>> $log_subt

 if [ "$file_to_hash" != "$1" ] && [ -f "$file_to_hash" ]; then
  rm "$file_to_hash"
 fi

 if [ $generate_hash = 0 ] ; then
  if [ ! -f $1 ] ; then
   echo "SRC_NOT_FOUND=1" >> $STATHASH_SH
   echo "HASH_FAIL=0" >> $STATHASH_SH
   echo "not found $1"
   return
  fi

  echo "SRC_NOT_FOUND=0" >> $STATHASH_SH
  if [ ! -f $ref_hash ] ; then
   echo "HASH_NOT_FOUND=1" >> $STATHASH_SH
   return
  fi

  echo "HASH_NOT_FOUND=0" >> $STATHASH_SH

  $DIFF $test_hash $ref_hash > /dev/null
  rv=$?

  if [ $rv != 0 ] ; then
   hashres=0
   fhash=`hexdump -ve '1/1 "%.2X"' $ref_hash`
   echo "Hash fail, ref hash $ref_hash was $fhash"  >> $log_subt
   shopt -s nullglob
   for alt_ref in "$ref_hash-alt"* ; do
    $DIFF $test_hash $alt_ref > /dev/null
    rv_alt=$?
    if [ $rv_alt != 0 ] ; then
      fhash=`hexdump -ve '1/1 "%.2X"' $alt_ref`
      echo "Hash alt fail, alt ref $alt_ref was $fhash"  >> $log_subt
    else
      hashres=1
      echo "Hash alt OK with alt ref $alt_ref"  >> $log_subt
      break
    fi
   done
   shopt -u nullglob
   if [ $hashres != 0 ] ; then
    echo "Hash OK for $1"  >> $log_subt
    echo "HASH_FAIL=0" >> $STATHASH_SH
   else
    echo "HASH_FAIL=1" >> $STATHASH_SH
   fi
  else
   echo "Hash OK for $1"  >> $log_subt
   echo "HASH_FAIL=0" >> $STATHASH_SH
  fi
  rm $test_hash

 else

  ref_hash_suffix="$ref_hash"
  if [ -n "$generate_hash_suffix" ] ; then
    ref_hash_suffix="$ref_hash_suffix-alt-$generate_hash_suffix"
  fi

  if [ -f "$ref_hash" ] ; then

    # don't overwrite base ref if no diff (keeps mtime)
    # also don't write alt hash if it's equal to base hash
    $DIFF $test_hash $ref_hash > /dev/null
    rv=$?
    if [ $rv != 0 ] ; then
      if [ -n "$generate_hash_suffix" -a -f "$ref_hash_suffix" ] ; then
        $DIFF $test_hash $ref_hash_suffix > /dev/null
        rv=$?
        if [ $rv != 0 ] ; then
          # case 1: we're using suffix + hash exist and is different => overwrite
          cp $test_hash $ref_hash_suffix
        fi
      else
        # case 2: either not using suffix or suffix hash different => write hash file
        cp $test_hash $ref_hash_suffix
      fi

    fi

  else
    # if no base ref create it even if we're using alt hashes
    cp $test_hash $ref_hash
    [ -n "$generate_hash_suffix" ] && cp $test_hash $ref_hash_suffix
  fi

  rm -f $test_hash
 fi
}
#end do_hash_test

do_hash_test_bin ()
{
	do_hash_test $1 $2 1
}

#compare hashes of $1 and $2, return 0 if OK, error otherwise
do_compare_file_hashes ()
{
  if [ $dump_js = 1 ] ; then
    echo '{ "hash": "compare", "file1": "'$1'", "file2" : "'$2'"},' >> all_tests.js
    return
  fi

  if [ $# -gt 2 ] ; then
   log $L_ERR "> in test $TEST_NAME in script $current_script line $BASH_LINENO"
   log $L_ERR "	@do_compare_file_hashes takes only two arguments - wrong call (first arg is $1)"
  fi
test_hash_first="$INTERN_TEMP_DIR/$TEST_NAME-$(basename $1).hash"
test_hash_second="$INTERN_TEMP_DIR/$TEST_NAME-$(basename $2).hash"

$MP4BOX -hash -std $1 > $test_hash_first 2> /dev/null
$MP4BOX -hash -std $1 > $test_hash_second 2> /dev/null
$DIFF $test_hash_first $test_hash_first > /dev/null

rv=$?
if [ $rv != 0 ] ; then
echo "Hash fail between $1 and $2"  >> $log_subtest
else
echo "Same Hash for $1 and $2"  >> $log_subtest
fi

rm $test_hash_first
rm $test_hash_second

return $rv

}
#end do_compare_file_hashes


#@single_test: performs a single test without hash with $1 command line and $2 test name
single_test ()
{
  if [ $# -gt 2 ] ; then
   log $L_ERR "> in test $TEST_NAME in script $current_script line $BASH_LINENO"
   log $L_ERR "	@single_test takes only two arguments - wrong call (first arg is $1)"
  fi
test_begin "$2"
if [ $test_skip  = 1 ] ; then
return
fi
single_test=1
do_test "$1" "single"
test_end
}


#@test_ui_test: if $test_ui is 1 records user input on $1 playback (10 sec) and stores in $RULES_DIR/$1-ui.xml. If $test_ui is 2, plays back the recorded stream
do_ui_test()
{
  if [ $# -gt 2 ] ; then
   log $L_ERR "> in test $TEST_NAME in script $current_script line $BASH_LINENO"
   log $L_ERR "	@test_ui_test takes one or two arguments - wrong call (first arg is $1)"
  fi

 if [ $test_ui = 0 ] ; then
  return
 fi

 if [ "$single_test_name" != "" ] && [ "$single_test_name" != "$TEST_NAME" ] ; then
   log $L_DEB "skipping ui test $TEST_NAME"
   return
 fi

 src=$1
 if [ $# -gt 1 ] ; then
  FULL_SUBTEST="$TEST_NAME-$2"
 else
  FULL_SUBTEST="$TEST_NAME"
 fi

 SUBTEST=$2
 ui_stream=$RULES_DIR/$FULL_SUBTEST-ui.xml

 if [ $test_ui = 1 ] ; then
  if [ -f $ui_stream ]; then
   log $L_DEB "User input trace present for $FULL_SUBTEST - skipping"
   return
  fi
  echo "Recording user input for $FULL_SUBTEST"
  echo "Recording user input for $FULL_SUBTEST (file $src) into trace $ui_stream" >> $ALL_LOGS
  $GPAC -runfor=$dump_dur -mp4c $src -size=$dump_size -no-save -cfg=Validator:Mode=Record -cfg=Validator:Trace=$ui_stream 2>> $ALL_LOGS
  rv=$?
 else
  if [ ! -f $ui_stream ]; then
   log $L_WAR "User input trace not found for $FULL_SUBTEST - skipping playback"
   return
  fi
  echo "Playing user input for $FULL_SUBTEST"
  echo "Playing user input for $FULL_SUBTEST (file $src) from trace $ui_stream" >> $ALL_LOGS
  dur=$(($dump_dur / $speed))
  $GPAC -runfor=$dur -mp4c $src -size=$dump_size -speed=$speed -no-save -cfg=Validator:Mode=Play -cfg=Validator:Trace=$ui_stream 2>> $ALL_LOGS
  rv=$?
 fi
 #regular error, check if this is a negative test
 if [ $rv != 0 ] ; then
  log $L_ERR "Error executing UI test for $FULL_SUBTEST (source file $src - test name $TEST_NAME)"
#  if [ $strict_mode = 1 ] ; then
#   exit
#  fi
 fi
}
#end test_ui_test


#start of our tests
start_ts=`$GNU_DATE +%s%N`
start_date="$(date '+%d/%m/%Y %H:%M:%S')"

if [ $generate_hash = 1 ] ; then
 log $L_INF "Generating Test Suite SHA-1 Hashes"
elif [ $do_clean = 1 ] ; then
 log $L_INF "Cleaning Test Suite"
elif [ $check_only = 1 ] ; then
 log $L_INF "Checking Test Suite Names"
elif [ $global_test_ui = 1 ] ; then
 log $L_INF "Generating User Input traces"
elif [ $global_test_ui = 2 ] ; then
 log $L_INF "Playing User Input traces"
else
 log $L_INF "Evaluating Test Suite"
fi

print_end()
{

  if [ $quiet = 1 ]; then
   return;
  fi

end_ts=`$GNU_DATE +%s%N`
runtime=$((end_ts-start_ts))

ms=$(($runtime / 1000000))
secs=$(($ms / 1000))
ms=$(($ms - $secs*1000))
h=$(($secs / 3600))
secs=$(($secs - $h*3600))
m=$(($secs / 60))
secs=$(($secs - $m*60))

printf "$1 in %02d:%02d:%02d:%03d\n" $h $m $secs $ms
}

#gather all tests reports and build our final report
finalize_make_test()
{

#we are cleaning, nothing to do
if [ $do_clean = 1 ] ; then
 print_end "Cleanup done"
 return
fi

#ui tests nothing to do
if [ $global_test_ui != 0 ] ; then
 print_end "UI Tests done"
 return
fi

#create logs and final report
echo "Logs for GPAC test suite - execution date $start_date" > $ALL_LOGS

echo '<?xml version="1.0" encoding="UTF-8"?>' > $ALL_REPORTS
echo '<?xml-stylesheet href="stylesheet.xsl" type="text/xsl"?>' >> $ALL_REPORTS
echo "<GPACTestSuite version=\"$VERSION\" platform=\"$platform\" start_date=\"$start_date\" end_date=\"$(date '+%d/%m/%Y %H:%M:%S')\">" >> $ALL_REPORTS


if [ $keep_temp_dir != 1 ] ; then
   rm -rf $TEMP_DIR/* 2> /dev/null
fi

#count all tests using generated -stats.sh
TESTS_SKIP=0
TESTS_TOTAL=0
TESTS_DONE=0
TESTS_PASSED=0
TESTS_FAILED=0
TESTS_LEAK=0
TESTS_EXEC_NA=0

SUBTESTS_FAIL=0
SUBTESTS_EXEC_NA=0
SUBTESTS_DONE=0
SUBTESTS_LEAK=0
SUBTESTS_HASH=0
SUBTESTS_HASH_FAIL=0
SUBTESTS_HASH_MISSING=0

for i in $LOGS_DIR/*-stats.sh ; do
if [ -f $i ] ; then

#reset stats
TEST_SKIP=0
TEST_EXEC_NA=0
SUBTESTS_LEAK=0
NB_HASH_SUBTESTS=0
NB_HASH_SUBTESTS_MISSING=0
NB_HASH_SUBTESTS_FAIL=0
NB_SUBTESTS=0

#load stats
source $i

#test not run due to error in strict mode
if [ $TEST_SKIP = 2 ] ; then
rm -f $i > /dev/null
continue;
fi

TESTS_TOTAL=$((TESTS_TOTAL + 1))
if [ $TEST_SKIP = 0 ] ; then
 TESTS_DONE=$((TESTS_DONE + 1))
else
 TESTS_SKIP=$((TESTS_SKIP + $TEST_SKIP))
fi

if [ $TEST_FAIL = 0 ] ; then
  if [ $TEST_EXEC_NA = 0 ] && [ $SUBTESTS_LEAK = 0 ]; then
    TESTS_PASSED=$((TESTS_PASSED + 1))
  else
    rm -f $i > /dev/null
    if [ $TEST_EXEC_NA != 0 ] ; then
      TESTS_EXEC_NA=$((TESTS_EXEC_NA + 1))
    else
      TESTS_LEAK=$((TESTS_LEAK + 1))
    fi
  fi
else
  TESTS_FAILED=$((TESTS_FAILED + 1))
  rm -f $i > /dev/null
fi


SUBTESTS_FAIL=$((SUBTESTS_FAIL + $TEST_FAIL))
SUBTESTS_EXEC_NA=$((SUBTESTS_EXEC_NA + $TEST_EXEC_NA))
SUBTESTS_DONE=$((SUBTESTS_DONE + $NB_SUBTESTS))
SUBTESTS_LEAK=$((SUBTESTS_LEAK + $SUBTESTS_LEAK))
SUBTESTS_HASH=$((SUBTESTS_HASH + $NB_HASH_SUBTESTS))
SUBTESTS_HASH_FAIL=$((SUBTESTS_HASH_FAIL + $NB_HASH_SUBTESTS_FAIL))
SUBTESTS_HASH_MISSING=$((SUBTESTS_HASH_MISSING + $NB_HASH_SUBTESTS_MISSING))

fi

done

echo "<TestSuiteResults NumTests=\"$TESTS_TOTAL\" NumSubtests=\"$SUBTESTS_DONE\" TestsPassed=\"$TESTS_PASSED\" TestsFailed=\"$TESTS_FAILED\" TestsLeaked=\"$TESTS_LEAK\" TestsUnknown=\"$TESTS_EXEC_NA\" HashFailed=\"$SUBTESTS_HASH_FAIL\" HashMissing=\"$SUBTESTS_HASH_MISSING\" />" >> $ALL_REPORTS

#gather all failed reports first
for i in $LOGS_DIR/*-failed.xml; do
 if [ -f $i ] ; then
  cat $i >> $ALL_REPORTS
  echo "" >> $ALL_REPORTS
  rm $i
 fi
done

#gather all new reports
for i in $LOGS_DIR/*-passed-new.xml; do
 if [ -f $i ] ; then
  cat $i >> $ALL_REPORTS
  echo "" >> $ALL_REPORTS
  #move new report to final name
  n=${i%"-new.xml"}
  n="$n.xml"
  mv "$i" "$n"
 fi
done

echo '</GPACTestSuite>' >> $ALL_REPORTS

#cat all logs
for i in $LOGS_DIR/*-logs.txt-new; do
 if [ -f $i ] ; then
  cat $i >> $ALL_LOGS
  echo "" >> $ALL_LOGS
  #move new report to final name
  n=${i%".txt-new"}
  n="$n.txt"
  mv "$i" "$n"
 fi
done

if [ $TESTS_TOTAL = 0 ] ; then
log $L_INF "No tests executed"
else


pc1=$((100*TESTS_DONE/TESTS_TOTAL))
pc2=$((100*TESTS_SKIP/TESTS_TOTAL))
log $L_INF "Number of Tests $TESTS_TOTAL - $SUBTESTS_DONE subtests - Executed: $TESTS_DONE ($pc1 %) - Cached: $TESTS_SKIP ($pc2 %)"


if [ $TESTS_DONE = 0 ] ; then
 TESTS_DONE=$TESTS_TOTAL
fi
if [ $SUBTESTS_DONE = 0 ] ; then
 SUBTESTS_DONE=$TESTS_TOTAL
fi

 pc=$((100*TESTS_PASSED/TESTS_TOTAL))
 log $L_INF "Tests passed $TESTS_PASSED ($pc %) - $SUBTESTS_DONE sub-tests"

 # the follwing % are in subtests
 if [ $SUBTESTS_FAIL != 0 ] ; then
  pc=$((100*SUBTESTS_FAIL/SUBTESTS_DONE))
  log $L_ERR "Tests failed $TESTS_FAILED ($pc % of subtests)"
 fi

 if [ $SUBTESTS_LEAK != 0 ] ; then
  pc=$((100*SUBTESTS_LEAK/SUBTESTS_DONE))
  log $L_WAR "Tests Leaked $TESTS_LEAK ($pc % of subtests)"
 fi

 if [ $SUBTESTS_EXEC_NA != 0 ] ; then
  pc=$((100*SUBTESTS_EXEC_NA/SUBTESTS_DONE))
  log $L_WAR "Tests Unknown $TESTS_EXEC_NA ($pc % of subtests)"
 fi

 if [ $SUBTESTS_HASH_FAIL != 0 ] ; then
  pc=$((100*SUBTESTS_HASH_FAIL/SUBTESTS_DONE))
  log $L_WAR "Tests HASH total $SUBTESTS_HASH - fail $SUBTESTS_HASH_FAIL ($pc % of subtests)"
 fi

 if [ $SUBTESTS_HASH_MISSING != 0 ] ; then
  pc=$((100*SUBTESTS_HASH_MISSING/$SUBTESTS_HASH))
  log $L_WAR "Missing hashes $SUBTESTS_HASH_MISSING / $SUBTESTS_HASH ($pc % hashed subtests)"
 fi


fi

print_end "Generation done"

} #end finalize_make_test



# trap ctrl-c and generate reports
trap ctrl_c_trap INT

ctrl_c_trap() {
	echo "CTRL-C trapped - cleanup and building up reports"
	local pids=$(jobs -pr)
	[ -n "$pids" ] && kill $pids
	finalize_make_test
	exit
}

# disable nullglobing in case sub-scripts aren't made for it
shopt -u nullglob


# by default run everything in scripts/ dir
scripts_torun="$SCRIPTS_DIR/*.sh"

# if a subset of tests was provided, run only these
if [ "${#url_arg[@]}" -gt 0 ] ; then
  scripts_torun="${url_arg[@]}"
fi

#run our tests
for i in $scripts_torun ; do
if [ $verbose = 1 ] ; then
 log $L_DEB "Source script: $i"
fi
CLEAN_TEMP=0
current_script=$i
skip_all_tests=0
source $i
cd $main_dir
#break if error
if [ $strict_mode = 1 ] ; then
 #wait for all tests to be done before checking error marker
 wait
 if [ -f $TEST_ERR_FILE ] ; then
  break
 fi
fi

#cleanup temp dir
if [ $keep_temp_dir != 1 ] ; then
   rm -rf $TEMP_DIR/* 2> /dev/null
fi

done


#wait for all tests to be done, since some tests may use subshells
wait

if [ $check_only != 0 ] ; then
 rm -rf $TEMP_DIR/* 2> /dev/null
 exit
fi


finalize_make_test