mercury/tests/Mmake.common

#-----------------------------------------------------------------------------#
# Mmake.common - shared Mmake variables and rules for the test directories.
#
# The including Mmakefile must set the following variables:
# THIS_DIR - the name of the directory containing tests
#		(e.g. benchmarks, hard_coded/typeclasses)
# TESTS_DIR - the directory containing this file.
# SUBDIRS - the names of any subdirectories containing tests.
# PROGS - the names of the main modules of the test programs.
# TESTS- the names of the tests. For each test there must be targets
#	test.depend, test.runtest and test.realclean, unless the test ends
#	in `-nodepend', in which case the `-nodepend' suffix will be stripped
#	off and the test.depend target is not required.
#
# If the variable ERROR_FILE is set, only the tests which failed in
# the test run which produced the specified error log file will be run.
#
#-----------------------------------------------------------------------------#

MAIN_TARGET = runtests

# Set up to test a particular workspace.
ifdef WORKSPACE

# Make sure we don't get the installed versions of the libraries.
LINK_STATIC = yes
include $(WORKSPACE)/Mmake.workspace

endif

#
# Note: Mmake lets you override MCFLAGS for a particular file by setting
# MCFLAGS-foo.  Similarly, you can override GRADEFLAGS for a particular
# file by setting both GRADEFLAGS-foo.
#

DIFF_OPTS=-c

# Override this with `RUN_RECOMPILATION_TESTS=no'
# if your compiler cannot run smart recompilation.
# This is only necessary for compilers built in deep
# profiling grades, because smart recompilation uses
# exception handling, which does not yet work with
# deep profiling.
RUN_RECOMPILATION_TESTS=yes

# The Java interpreter.
JAVA=java

# We avoid picking up parameter settings from $HOME/.mdbrc that could
# cause spurious differences between the outputs of the debugger test
# cases and their expected outputs.

# We suppress the printing of the banner, because different workspaces
# may get different version numbers printed in it. This would otherwise be
# the source of irrelevant difference between the actual and expected outputs.

MDB = HOME=/nonexistent MERCURY_SUPPRESS_MDB_BANNER=yes mdb

# Debugger test cases can standardize the reported event numbers and call
# sequence numbers by using $(MDB_STD) instead of $(MDB) on the command line.

MDB_STD = MERCURY_OPTIONS="$$MERCURY_OPTIONS -de" $(MDB)

ifeq ($(TARGET_ASM),yes)
PARAMS_MSG = in grade $(GRADE) with --target asm
else
PARAMS_MSG = in grade $(GRADE)
endif

-include $(TESTS_DIR)/Mmake.params

# Avoid trying to make this file with `mmc --make' if it doesn't exist.
$(TESTS_DIR)/Mmake.params: ;

#-----------------------------------------------------------------------------#

.PRECIOUS: %.res %.err_res %.out %.err

#
# If there is a `.inp' file, then we pipe that in as the command's input.
# Then we run the command, with stdout and stderr both redirected to the 
# `.out' file.  Finally if the command fails (returns non-zero exit status),
# we print out the contents of the `.out' file.  We use `grep . $@ /dev/null'
# to print out the contents, because that precedes each line of output with
# the filename, which is helpful when running a parallel make.
#
ifneq "$(findstring java,$(GRADE))" ""

%.out: %.class
	{ [ -f $*.inp ] && cat $*.inp; } | $(JAVA) $* > $@ 2>&1 \
		|| { grep . $@ /dev/null; exit 1; }

else

%.out: %
	{ [ -f $*.inp ] && cat $*.inp; } | ./$< > $@ 2>&1 || \
		{ grep . $@ /dev/null; exit 1; }

endif

#
# For some test cases, there is more than one valid output.
# We try matching the output with the `.exp' file, and if that
# doesn't succeed, and there are `.exp2', `.exp3', `.exp4' or `.exp5'
# files, then we try matching against those too.  If none succeed,
# the shortest of the diffs is put into the `.res' file.
#
%.res: %.exp %.out
	@echo "Comparing $*.out with $*.exp*,"
	@echo "	results in $@"
	@-rm -f $@ $*.res[1-5]
	@{ diff $(DIFF_OPTS) $*.exp $*.out > $*.res1 && \
		echo "Matched $*.exp" && \
		cp $*.res1 $@; } || \
	{ test -f $*.exp2 && \
		diff $(DIFF_OPTS) $*.exp2 $*.out > $*.res2 && \
		echo "Matched $*.exp2" && \
		cp $*.res2 $@; } || \
	{ test -f $*.exp3 && \
		diff $(DIFF_OPTS) $*.exp3 $*.out > $*.res3 && \
		echo "Matched $*.exp3" && \
		cp $*.res3 $@; } || \
	{ test -f $*.exp4 && \
		diff $(DIFF_OPTS) $*.exp4 $*.out > $*.res4 && \
		echo "Matched $*.exp4" && \
		cp $*.res4 $@; } || \
	{ test -f $*.exp5 && \
		diff $(DIFF_OPTS) $*.exp5 $*.out > $*.res5 && \
		echo "Matched $*.exp5" && \
		cp $*.res5 $@; } || \
	{ shortest=`wc -l $*.res[1-5] | grep -v total | sort -n | \
		head -1 | awk '{ print $$2; }' `; \
		echo "** $*.out did not match the expected output"; \
		echo "** (closest match was $$shortest)"; \
		cp $$shortest $@; \
		cat $@; \
		exit 1; }

# Some tests have more than one possible valid output, so
# we allow the test to pass if it matches any of the .err_exp* files.
%.err_res: %.err %.err_exp
	@-rm -f $@
	@diff $(DIFF_OPTS) $*.err_exp $*.err > $@ || \
		{ [ -f $*.err_exp2 ] && \
		  diff $(DIFF_OPTS) $*.err_exp2 $*.err > $@; } || \
		{ [ -f $*.err_exp3 ] && \
		  diff $(DIFF_OPTS) $*.err_exp3 $*.err > $@; } || \
		{ [ -f $*.err_exp4 ] && \
		  diff $(DIFF_OPTS) $*.err_exp4 $*.err > $@; } || \
		{ cat $*.err && exit 1; }

#-----------------------------------------------------------------------------#

MERCURY_MAIN_MODULES = $(PROGS)

ERROR_OUTPUT_FILE = runtests.errs

realclean_local: clean_logs clean_errors
clean_local: clean_out clean_res clean_zip

# XXX what is this target for??
clean_mc: clean_c clean_o clean_out clean_res 

clean_out:
	rm -f *.out

clean_res:
	rm -f *.res*

clean_logs:
	rm -f *.log

clean_errors:
	rm -f $(ERROR_OUTPUT_FILE) FAILED_TESTS

# Remove gzipped executables for tests which have failed.
ifeq ($(TESTS),)
clean_zip: ;
else
clean_zip:
	for test in $(TESTS); do \
		prog=`basename $$test -nodepend`; \
		rm -f $$prog.gz $$prog.exe.gz; \
	done
endif

#
# The `foo' targets make recursively make `foo' in all subdirectories.
# before making `foo_local' in the current directory 
# 

.PHONY: runtests runtests_local runtests_subdirs
.PHONY: check check_local check_subdirs
.PHONY: realclean_subdirs clean_subdirs

realclean:      realclean_subdirs
clean:          clean_subdirs
all:		runtests

check:		runtests
check_local:	runtests_local
check_subdirs:	runtests_subdirs

#
# We want to run all the tests, even if some fail, so
# run the tests using `mmake -k'.
# 
runtests:
	+@if mmake -k runtests_local runtests_subdirs; then \
		echo ALL TESTS SUCCEEDED; \
		rm -f $(ERROR_OUTPUT_FILE); \
	else \
		for subdir in $(SUBDIRS) x; do \
		    if [ -f $$subdir/$(ERROR_OUTPUT_FILE) ]; then \
			cat $$subdir/$(ERROR_OUTPUT_FILE) \
				>> $(ERROR_OUTPUT_FILE); \
		    fi; \
		    if [ -f $$subdir/FAILED_TESTS ]; then \
			sed "s@^@$$subdir/@" $$subdir/FAILED_TESTS \
				>> FAILED_TESTS; \
		    fi; \
		done; \
		echo SOME TESTS FAILED: see FAILED_TESTS and $(ERROR_OUTPUT_FILE); \
		exit 1; \
	fi

#
# If the variable ERROR_FILE is set, only the tests which failed in
# the test run which produced the specified error log file will be run.
#
ifdef ERROR_FILE
ifndef TESTS_TO_RUN
TESTS_TO_RUN := $(shell awk '/^FAILED TEST/ { print $$3 }' $(ERROR_FILE))
export TESTS_TO_RUN
endif
endif

ifdef TESTS_TO_RUN
THIS_DIR_TESTS_TO_RUN := $(shell echo $(patsubst $(THIS_DIR)/%,%,\
		$(filter $(THIS_DIR)/%,$(TESTS_TO_RUN))) \
		| tr ' ' '\n' | grep -v /)
else
THIS_DIR_TESTS_TO_RUN := $(strip $(TESTS))
endif

ifeq ($(THIS_DIR_TESTS_TO_RUN),)

runtests_local: ;

else

# Run multiple rm commands at once to speed up
# `mmake realclean' on slow NFS filesystems.
# XXX We used to do this, but if the tests are being
# run with parallel make it results a huge number of
# processes being run because make's jobserver mechanism
# is disabled if `-j' is specified on the command line
# of a sub-make.
# RM_JFACTOR=-j10
RM_JFACTOR=

#
# Clean up after any previous test run.
#
.PHONY: start_runtests_local
start_runtests_local:
	@echo STARTING tests in $(THIS_DIR) $(PARAMS_MSG)
	@echo cleaning up the directory before the tests
	+@if ls -lt | head -2 | egrep CLEAN > /dev/null 2>&1; then \
		rm -f CLEAN > /dev/null 2>&1; \
	else \
		rm -f CLEAN > /dev/null 2>&1; \
		mmake $(RM_JFACTOR) realclean_local > /dev/null 2>&1; \
		rm -f *.d *.dep *.int *.int2 *.int3 > /dev/null 2>&1; \
		rm -f *.date *.date3 *.opt *.optdate > /dev/null 2>&1; \
		rm -f *.trans_opt *.trans_opt_date > /dev/null 2>&1; \
	fi
	@> FAILED_TESTS

#
# Run a single test, cleaning up if it succeeds, or producing
# a log file and gzipping the executable the test fails.
#
.PHONY: %.log
%.log: start_runtests_local
	+@test=$(*:%-nodepend=%); \
	echo RUNNING TEST $(THIS_DIR)/$* $(PARAMS_MSG) | tee $@; \
	case $* in \
		*-nodepend) \
			rm -f $*.failed;; \
		*) \
			rm -f $*.failed; \
			{ mmake $$test.depend 2>&1 || touch $*.failed; } \
				| tee -a $@ ;; \
	esac; \
	if [ -f $*.failed ]; then \
		echo FAILED TEST $(THIS_DIR)/$* $(PARAMS_MSG) \
			| tee -a $@; \
		echo $* >> FAILED_TESTS; \
		exit 1; \
	else \
		{ mmake $$test.runtest 2>&1 || touch $*.failed; } \
			| tee -a $@; \
		if [ -f $*.failed ]; then \
			rm -f $*.failed; \
		    	if [ -f $$test ]; then \
				rm -f $$test.gz $$test.exe.gz; \
				gzip $$test || gzip $$test.exe; \
			fi; \
			echo FAILED TEST $(THIS_DIR)/$* $(PARAMS_MSG) \
				| tee -a $@; \
			echo $* >> FAILED_TESTS; \
			exit 1; \
		else \
			rm -f $@; \
			rm -f $$test.out* $$test.*res*; \
			mmake $$test.realclean > /dev/null 2>&1; \
			echo PASSED TEST $(THIS_DIR)/$* $(PARAMS_MSG); \
		fi; \
	fi

# XXX The tests cannot be run in parallel because where several test
# cases depend on the same target, the different invocations of mmake
# above may all attempt to create the target at the same time.
# Also, `mmake $$test.realclean' would be run before all users
# of the target are finished with it.
TEST_LOGS = $(THIS_DIR_TESTS_TO_RUN:%=%.log)
runtests_local:
	@ \
	mmake -k -j1 $(TEST_LOGS); \
	touch DUMMY.log; \
	cat *.log > $(ERROR_OUTPUT_FILE); \
	rm -f DUMMY.log; \
	if [ ! -s $(ERROR_OUTPUT_FILE) ]; then \
		echo "PASSED tests in $(THIS_DIR) $(PARAMS_MSG)"; \
		echo cleaning up the directory after the tests; \
		mmake $(RM_JFACTOR) realclean_local > /dev/null 2>&1; \
		rm core > /dev/null 2>&1; \
		touch CLEAN; \
	else \
		echo "FAILED tests in $(THIS_DIR) $(PARAMS_MSG)"; \
		exit 1; \
	fi

endif	# THIS_DIR_TESTS_TO_RUN != ""

ifeq ($(SUBDIRS),)

runtests_subdirs realclean_subdirs clean_subdirs : ;

else

RUNTESTS_IN_SUBDIRS=$(SUBDIRS:%=runtests_in_%)
runtests_subdirs: $(RUNTESTS_IN_SUBDIRS)

.PHONY: $(RUNTESTS_IN_SUBDIRS)
$(RUNTESTS_IN_SUBDIRS): runtests_in_%:
	+cd $* && mmake runtests

realclean_subdirs:
	+succeeded=true; \
	for dir in $(SUBDIRS); do \
	    (cd $$dir && mmake realclean) || succeeded=false; \
	done
	case $$succeeded in false) exit 1 ;; esac

clean_subdirs:
	+succeeded=true; \
	for dir in $(SUBDIRS); do \
		(cd $$dir && mmake clean) || succeeded=false; \
	done; \
	case $$succeeded in false) exit 1 ;; esac

endif	# SUBDIRS != ""

#-----------------------------------------------------------------------------#