# *created  "Fri Oct 16 11:51:38 2015" *by "Paul E. Black"
# *modified "Thu Dec 24 10:43:37 2015" *by "Paul E. Black"
#
# SATE V Ockham Criteria
#
# Run extractions and checks for null pointer dereference class
#

all: checks statistics discrimination

checks: checkWsubseteqU checkFcapBisNull

SITES_FILE = ../ExtractSites/pointer_deref_sites.xml
XML2CSV = ../ExtractSites/sitesxml2csv.py -f $(SITES_FILE) -w nullPtrDeref

# (1) Produce a list of read buffer sites in CSV format
nullPtrDeref_universe.csv: $(SITES_FILE)
	@echo "(1) Reformat null pointer sites into CSV format: U"
	-mv -f $@ $@.bak
	# sort -u needed since buggy/good and variable info is discarded. Some 
	# cases, like CWE126_Buffer_Overread__char_alloca_memcpy_01.c, have 
	# both a buggy and a good site (with different vars) on the same line.
	python $(XML2CSV) | sort -u > $@

# (2) Select read outside buffer warnings
frama_c_nullPtrDeref.csv:  ../ExtractWarnings/frama_c_warnings.csv
	@echo "(2) Select null pointer dereference warnings: W"
	-mv -f $@ $@.bak
	grep nullPtrDeref $^ > $@

# (3) Check W \subseteq U
checkWsubseteqU: nullPtrDeref_notsubset.csv

nullPtrDeref_notsubset.csv: nullPtrDeref_universe.csv frama_c_nullPtrDeref.csv
	@echo "(3) Check W subseteq U"
	-mv -f $@ $@.bak
	comm -23 frama_c_nullPtrDeref.csv nullPtrDeref_universe.csv > $@
	wc -l $@
	test -e $@ -a ! -s $@

# (4) Select buggy sites and reformat into CSV format
nullPtrDeref_buggy.csv: $(SITES_FILE)
	@echo "(4) Select buggy sites and reformat into CSV format: B"
	-mv -f $@ $@.bak
	# -u needed since same bugs are reported in both runs
	python $(XML2CSV) -b | sort -u > $@

# (5) Compute list of Findings: F = U - W
nullPtrDeref_findings.csv: nullPtrDeref_universe.csv frama_c_nullPtrDeref.csv
	@echo "(5) Compute list of Findings: F = U - W"
	-mv -f $@ $@.bak
	comm -23 nullPtrDeref_universe.csv frama_c_nullPtrDeref.csv > $@

# (6) Check F \cap B = \null
checkFcapBisNull: nullPtrDeref_false_find.csv

nullPtrDeref_false_find.csv: nullPtrDeref_buggy.csv nullPtrDeref_findings.csv
	@echo "(6) Check F cap B = null"
	-mv -f $@ $@.bak
	comm -12 nullPtrDeref_findings.csv nullPtrDeref_buggy.csv > $@
	wc -l $@
	test -e $@ -a ! -s $@

statistics: nullPtrDeref_universe.csv frama_c_nullPtrDeref.csv \
	nullPtrDeref_findings.csv nullPtrDeref_buggy.csv \
	nullPtrDeref_notsubset.csv nullPtrDeref_false_find.csv
	-mv -f $@ $@.bak
	@echo "Class: Null Pointer Dereference" | tee -a $@
	@echo -n "Number of sites (|U|) " | tee -a $@
	@wc -l < nullPtrDeref_universe.csv | tee -a $@
	@echo -n "Number of warnings (|W|) " | tee -a $@
	@wc -l < frama_c_nullPtrDeref.csv | tee -a $@
	@echo -n "Number of warnings not in universe (|W - U|) " | tee -a $@
	@wc -l < nullPtrDeref_notsubset.csv | tee -a $@
	@echo -n "Number of findings (|F|) " | tee -a $@
	@wc -l < nullPtrDeref_findings.csv | tee -a $@
	@echo -n "Number of buggy sites (|B|) " | tee -a $@
	@wc -l < nullPtrDeref_buggy.csv | tee -a $@
	@echo -n "Number of buggy findings (|F cap B|) " | tee -a $@
	@wc -l < nullPtrDeref_false_find.csv | tee -a $@

Ufile = nullPtrDeref_universe.csv
Wfile = frama_c_nullPtrDeref.csv
Ffile = nullPtrDeref_findings.csv
# prefixes of those test cases that are made for this class
Prefixes = '^CWE476_NULL_Pointer_Dereference_'
# remove multi-file test case suffixes, e.g. 64a.c, 64b.c, 64c.c -> 64, as well
# as information we don't need, like line numbers and weakness class name
REMOVE_SUFFIXES = perl -pwe 's/[a-g]?[.]c, [\d]+, [a-zA-Z]+//'

discrimination: $(Ufile) $(Ffile) $(Wfile)
	@echo "Class: Null Pointer Dereference" | tee -a $@
	@echo -n "Number of test cases made for this class " | tee -a $@
	@grep -e $(Prefixes) $(Ufile) | $(REMOVE_SUFFIXES) | uniq | wc -l | tee -a $@
	@echo -n "Maximum number of points " | tee -a $@
	@# i.e., pertinent test cases with good sites that don't have warnings
	@grep -e $(Prefixes) $(Ffile) | $(REMOVE_SUFFIXES) | uniq | wc -l | tee -a $@
	@echo -n "Minimum number of points " | tee -a $@
	@# i.e., pertinent test cases with only one (relevant) warning
	@grep -e $(Prefixes) $(Wfile) | $(REMOVE_SUFFIXES) | uniq -c | grep '   1 ' | wc -l | tee -a $@
	@# Read the file back in and compute the percentages
	@perl -nwe 'if(/Number \D*(\d+)/){$$t=$$1};if(/Maximum \D*(\d+)/){$$max=$$1};if(/Minimum \D*(\d+)/){$$min=$$1};END{$$maxpts=100*$$max/$$t;$$minpts=100*$$min/$$t;print "Discrimination is between $$minpts% and $$maxpts%\n"}' $@ | tee -a $@

clean:
	rm -f *.bak nullPtrDeref_universe.csv frama_c_nullPtrDeref.csv \
	nullPtrDeref_notsubset.csv nullPtrDeref_buggy.csv \
	nullPtrDeref_findings.csv nullPtrDeref_false_find.csv \
	statistics

# end of Makefile
