#!/usr/bin/perl -w
# $Id$
#             *created  "Tue Mar 11 11:26:20 2014" *by "Paul E. Black"';
$versionMod=' *modified "Mon Nov 30 10:43:31 2015" *by "Paul E. Black"';

#-----------------------------------------------------------------------------
#
$shortDescription = '
    Extract all warnings from Frama-C reports for SATE V Ockham criteria.
    Output a comma-separated, line-oriented format.
';
#
$progName = "extract_warnings";
$useMsg = "use: $progName [--help] [--version] file1.xml file2.xml";
$minOperands = 2;
$maxOperands = 2;
$typUse = "./$progName ../frama_c_report.allocfail.xml ../frama_c_report.succalloc.xml | sort -u > frama_c_warnings.csv";
#
# This software was developed at the National Institute of Standards 
# and Technology by employees of the Federal Government in the course 
# of their official duties.  Pursuant to title 17 Section 105 of the 
# United States Code this software is not subject to copyright 
# protection and is in the public domain. 
# 
# We would appreciate acknowledgment if the software is used.
#
# Paul E. Black  paul.black@nist.gov or p.black@acm.org
#	http://hissa.nist.gov/~black/
#
#-----------------------------------------------------------------------------

# $Log$

#------------------------------------------------------------------------------
#	Command line handling
#------------------------------------------------------------------------------

while ($#ARGV >= 0) {
    if ($ARGV[0] =~ /^--?h(e(lp?)?)?/) {
	print "$useMsg\n";
	print "    where\n";
	print "\t--version Print version and exit\n";
	print "\t--help    Print this message and exit\n";
	print "\t--        End of options (e.g., file begins with -)\n";
	print "$shortDescription\n";
	print "typical use:\n";
	print "    $typUse\n";
	exit 0;
    } elsif ($ARGV[0] =~ /^--?v(e(r(s(i(on?)?)?)?)?)?$/) {
	print "$versionMod\n";
	exit 0;
    } elsif ($ARGV[0] eq "--") {
	shift;
	last;
    } elsif ($ARGV[0] =~ /^-/) {
	print "unknown option: $ARGV[0]\n";
	print "$useMsg\n";
	exit 1;
    } else {
	# end of options
	last;
    }
}

$numberOfOperands = 1 + $#ARGV;
if ($numberOfOperands < $minOperands || $maxOperands < $numberOfOperands) {
    print "Wrong number of operands\n";
    print "$useMsg\n";
    exit 1;
}

$filename1 = $ARGV[0];
checkfile($filename1);

$filename2 = $ARGV[1];
checkfile($filename2);

# check that the "file" is ok: readable, a real file, etc.
sub checkfile {
    my($fname) = @_;

    if (! -e $fname) {
	print "Not found: $fname\n";
	exit 1;
    }
    if (! -f $fname || -l $fname) {
	print "Not a file: $fname\n";
	exit 1;
    }
    if (! -r $fname) {
	print "File not readable: $fname\n";
	exit 1;
    }
}

###############################################################################
#	program proper
###############################################################################

$Trace = 0;

sub printATvals {
    my(%localatvals) = @_;

    while (my ($key, $value) = each(%localatvals)) {
	print " $key=\"$value\"";
    }
}

# the Perl XML Parser package is documented at
# http://search.cpan.org/~msergeant/XML-Parser-2.36/Parser.pm
use XML::Parser;

###### XML handler subroutines ######

sub handle_start {
    my($expatParam, $elementParam, %atvals) = @_;

    #print %atvals;print "\n";

    if ($elementParam eq "weakness") {
	if (! defined $atvals{"id"}) {
	    print "Missing 'id' attribute in\n";
	    print "<weakness"; printATvals(%atvals); print ">\n";
	    die "Stopped";
	}
	$globalWeaknessID = $atvals{"id"};

#	if ($globalWeaknessID == 27843) {
#	    $Trace = 1;
#	}

    } elsif ($elementParam eq "name") {
	if (defined $globalWeaknessName) {
	    print "Multiple 'name' attributes\n";
	    die "Stopped";
	}
	#print "Start 'name'\n";
    } elsif ($elementParam eq "location") {
	if (defined $globalLine) {
	    print "Multiple 'location' attributes\n";
	    die "Stopped";
	}
	if (! defined $atvals{"path"}) {
	    print "Missing 'path' attribute in\n";
	    print "<location"; printATvals(%atvals); print ">\n";
	    die "Stopped";
	}
	if (! defined $atvals{"line"}) {
	    print "Missing 'line' attribute in\n";
	    print "<location"; printATvals(%atvals); print ">\n";
	    die "Stopped";
	}
	my($pathname) = $atvals{"path"};
	$pathname =~ s|.*/||;
	$globalFileName = $pathname;
	$globalLine = $atvals{"line"};
	#print "Location $globalFileName $globalLine\n";
    } elsif ($elementParam eq "textoutput") {
	#print "<textoutput>\n" if $Trace;
    }
}

sub handle_text {
    my($expatParam, $string) = @_;

    if ($concatToGlobalCdata) {
	$globalCdata .= $string;
    }

    if ($string ne "\n") {
	$globalString = $string;
	print "String=>$string<\n" if $Trace;
    }
}

sub handle_cdataStart {
    my($expatParam) = @_;

    $concatToGlobalCdata = 1;
    $globalCdata = "";
}

sub handle_cdataEnd {
    my($expatParam) = @_;

    print "CDATA[$globalCdata" . "]\n" if $Trace;

    $concatToGlobalCdata = 0;
}

sub handle_end {
    my($expatParam, $elementParam, %atvals) = @_;

    #print %atvals;print "\n";

    if ($elementParam eq "weakness") {
	# end of a weakness (warning) found. Produce a line

	my($weaknessClass) = "UNKNOWN";
	undef $assertExpression; # don't use data from previous cycle

	# extract assert expression from the <textoutput>CDATA</textoutput>
	#            assert(\valid(&data))
	if ($textoutputCdata =~ m/assert\(\\[a-z_]+\(([^;]+)\)\)/) {
	    $assertExpression = $1;
	    #print "$textoutputCdata\n";
	    #print "Success $assertExpression\n";
	}
	#            assert(valid arguments for snprintf())
	 elsif ($textoutputCdata =~ m/assert\(([a-z ]+\(\))\)/) {
	    $assertExpression = $1;
	    #print "$textoutputCdata\n";
	    #print "Success $assertExpression\n";
	}
	#            assert \valid(tmp_1);
	 elsif ($textoutputCdata =~ m/assert \\[a-z_]+\(([^;]+)\);/) {
	    $assertExpression = $1;
	    #print "$textoutputCdata\n";
	    #print "Success $assertExpression\n";
	}
	# warning: accessing out of bounds index [0..2147483647]. assert data < 10;
	# code is: printIntLine(buffer[data]);
	 elsif ($textoutputCdata =~ m/warning: accessing out of bounds index (\[-?[.0-9]+\]|\{-?[15]0?\})[.] assert (0 \S+ )?(data|i)(_[01])? < [15]0/) {
	    $assertExpression = "buffer"; # Juliet is very regular!
	}
	# Function memmove: precondition 'valid_dst' got status unknown
	# code is:    memmove(dest, data, strlen(data)*sizeof(char));
	 elsif ($textoutputCdata =~ m/Function mem(move|cpy): precondition 'valid_dst' got status (unknown|invalid)/) {
	    # Juliet is very regular!
	    $assertExpression = "data"; # this is only an approximation
	}
	#            assert -2147483648 ≤ 100/data ≤ 2147483647;
	  elsif ($textoutputCdata =~ m/warning: [^;]+ assert ([^;]+);/) {
	    # don't redefine assertExpression
	    if (defined $assertExpression) {
		print "Trying to reassign assertExpression\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current assertExpression: $assertExpression\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $assertExpression = $1;
	}

	#
	# look for write outside buffer warnings
	#
	if (($globalWeaknessName eq "invalid memory access" &&
	    # these are *reads*, not writes
	    $textoutputCdata !~ m/mem(move|cpy): precondition 'valid_src' got status (invalid|unknown)/ &&
	    # these are reads
	    $textoutputCdata !~ m/warning: out of bounds read/ &&
	    # I think asserts beginning with & are null pointer complaints
	    $textoutputCdata !~ m/out of bounds write. assert \\valid\(&/ &&
	    # "accessing out of bounds index" is a read OR a write
	    ($textoutputCdata !~ m/warning: accessing out of bounds index/ ||
	     # use filename-specific filter for now to remove reads
	     $globalFileName !~ m/CWE12[67]_Buffer_(Ov|Und)erread/) &&

	    # calloc clearing data in case of allocation failure is null ptr, not write outside buffer
	    (!($textoutputCdata =~ m/ calloc / &&
	     # only when allocation fails
	     $inputFileName =~ m/frama_c_report.allocfail.xml/)) &&

	     # NULL ptr deref of a pointer variable
	     $globalFileName !~ m/^CWE476/ &&
	    1) ||

	    # write outside buffer by snprintf
	    ($globalWeaknessName eq "invalid arguments to library function" &&
	     # snprintf is mentioned
	     $textoutputCdata =~ m/assert\(valid arguments for snprintf\(\)\)/ &&
	     $textoutputCdata =~ m/stack: snprintf ::/ &&
	    1)
	    ) {
	    # ok, this is a write-outside-buffer warning

	    if (! defined $assertExpression) {
		print "NO ASSERT EXPRESSION FOUND: $globalWeaknessID $globalLine in $globalFileName\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    # heuristics to change the expression to a variable
	    my($theVariable) = $assertExpression;
	    # (char *)src+(0..(unsigned long)n-1)
	    $theVariable =~ s/^\([^)]+\)//; # remove type cast
	    # &data->intOne
	    $theVariable =~ s/^&//; # remove address-of operators
	    # data+(0..(unsigned int)((int)(100-1))-1)
	    $theVariable =~ s/[+][^>]*$//; # remove index range?

	    $weaknessClass = "writeOutBuf";

	        # don't match on variable name
#	    if ($assertExpression !~ m/tmp(_[24])?<<[36]0/) {
#		print "$globalFileName, $globalLine, writeOutBuf, $theVariable\n";
#		print "$globalFileName, $globalLine, writeOutBuf\n";
# it is not critical to check the variable name since we're not printing it
#		if ($theVariable !~ m/^[_a-zA-Z][_a-zA-Z0-9]*(->[_a-zA-Z0-9]+)?$/) {
		    # oops, we do not recognize this variable
#		    print "UNRECOGNIZED VARIABLE EXPRESSION\n";
#		    print "ASSERT: $assertExpression\n";
#		    print "WEAKNESS: $globalWeaknessName\n";
#		    print "CDATA: $textoutputCdata\n";
#		    die;
#		}
#	    }
	}

	#
	# look for write-what-where warnings
	#
	if (($globalWeaknessName eq "invalid memory access" &&
	    $textoutputCdata =~ m/\[kernel\] warning: out of bounds write\. assert \\valid\(&/ &&
	    1)
	    ) {
	    # ok, this is a write-what-where warning

# Athos' python extract for this warning checked that the file name started
# with CWE123_Write_What_Where_Condition and only then considered it to be
# a write-what-where (cdata is the above CData match):
#      cwe123 = re.search('CWE123_Write_What_Where_Condition', weakness.find('output').find('textoutput').text)
#      if(cdata and cwe123):
#                .
#                .
#        file_line_pairs.append(file_line_pair)


	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to another class\n";
		print "ID: $globalWeaknessID $globalLine in $globalFileName\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "writeWhatWhere";
	}

	#
	# look for read outside buffer warnings
	#
	if (($globalWeaknessName eq "invalid memory access" &&
	    (($textoutputCdata =~ m/out of bounds read/ &&
	      !(# these are NULL ptr deref in alloc fail
		$textoutputCdata =~ m/valid_read\(&twoIntsStructPointer->intOne\)/ ||
	        $textoutputCdata =~ m/valid_read\(&structTwoIntsStruct->int(One|Two)\)/ ||
	        $textoutputCdata =~ m/valid_read\(&data->intOne\)/ ||
	        $textoutputCdata =~ m/valid_read\(&data_(0|1)->intOne\)/ ||
		# these are NULL ptr deref of a pointer variable
		$globalFileName =~ m/^CWE476/
	       )) || # end out of bounds read
	    (($textoutputCdata =~ m/mem(move|cpy): precondition 'valid_src' got status (invalid|unknown)/ &&
	      # use filename-specific filter to identify reads
	      #$globalFileName =~ m/CWE12[67]_Buffer_(Ov|Und)erread/ &&
	     1)) || # end of accessing out of bounds index
	    (($textoutputCdata =~ m/accessing out of bounds index/ &&
	      # use filename-specific filter to identify reads
	      $globalFileName =~ m/CWE12[67]_Buffer_(Ov|Und)erread/ &&
	     1)) # end of accessing out of bounds index
	    )) || # end of invalid memory access
	    ($globalWeaknessName eq "invalid arguments to library function" &&
	     $textoutputCdata =~ m/assert\(valid arguments for printf\(\)\)/ &&
	     1) # end of invalid arguments to library function
#      struct_access_3 = re.search("Function (memcpy|memmove): precondition 'valid_src' got status invalid", weakness.find('output').find('textoutput').text)
	   ) {
	    # ok, this is a read-outside-buffer warning

	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class readOutBuf\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "readOutBuf";

	    # 36 files with unintentional flaw: read after end of const SRC_STR
	    if ($globalFileName =~ m/CWE121_Stack_Based_Buffer_Overflow__char_type_overrun_mem(move|cpy)_/) {
		$weaknessClass = "EXCLUDE-A";
	    }
	}

	#
	# Look for null pointer dereference warnings
	#
	if ($globalWeaknessName eq "invalid memory access" &&
	    # It is very hard to distinguish null pointer dereference (NPD)
	    # from read or write outside buffer based on the output from
	    # Frama-C. So we only consider sites in this subdirectory.
	    $globalFileName =~ m/CWE476_NULL_Pointer_Dereference__/) {

	    # ok, this is a null pointer dereference warning

	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class nullPtrDeref\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "nullPtrDeref";
	}

	#
	# Look for integer overflow warnings
	#
	if ($globalWeaknessName eq "undefined arithmetic overflow") {
	    # we need assertExpression in the actual check (below), so
	    # make sure it is defined in this context first.
	    if (! defined $assertExpression) {
		print "NO ASSERT EXPRESSION FOUND: $globalWeaknessID $globalLine in $globalFileName\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }
	}

	if ($globalWeaknessName eq "undefined arithmetic overflow"
	    #  integer division overflows only when dividing by zero. Since 
	    # that is divide by zero, which is more specific, we exclude
	    # division as a site for integer overflow.  Same goes for %.
	     && $assertExpression !~ m|[/%]|) {

	    # ok, this is an integer overflow warning

	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class intoflow\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "intoflow";
	}

	# exclude test cases that use RAND32, since this causes the 
	# execution to be undefined for some 76 test cases.
	if ($globalFileName =~ m/^CWE12[47]_Buffer_Under(write|read)__CWE839_rand_/) {
	    $weaknessClass = "EXCLUDE-B";
	}

	# exclude RAND32 left shift warnings
	if ($textoutputCdata =~ m/signed overflow. assert (\(long long\))?tmp(_\d+)?<<[36]0/) {
	    $weaknessClass = "EXCLUDE-C";
	}

	#
	# Look for divide by zero warnings
	#
	undef $globalDivOpinExpr; # that is, Division Operator in Expression
	if ($globalWeaknessName eq "floating-point NaN or infinity") {
	    # above weakness counts only if this has a "division" operator
	    if ($textoutputCdata =~ m/assert \\is_finite\(([^;]+);/) {
		my $expression = $1;
		# remove very last close paren - can't do it in the regex above
		# because some entries have parens: (double)(100.0/(double)data)
		$expression =~ s/\)$//; 

		# search for a "division operator"
		if ($expression =~ m|[/%]|) {
		    $globalDivOpinExpr = 1;
		}
	    }
	}
	if ($globalWeaknessName eq "division by zero" ||
	    $globalWeaknessName eq "floating-point NaN or infinity" && $globalDivOpinExpr) {

	    # ok, this is a divide by zero warning

	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class divByZero\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "divByZero";
	}

	#
	# Look for use of uninitialized variable warnings
	#
	if ($globalWeaknessName eq "reading from uninitialized lvalue") {
	    # extract info on what variable was not initialized from the CDATA
	    undef $uninitLval;
	    if ($textoutputCdata =~ m/assert \\initialized\(([^;]+);/) {
		$uninitLval = $1;
		# remove very last close paren - can't do it in regex because
		# some entries have parens: &(data+i)->intOne
		$uninitLval =~ s/\)$//; 

		####### kludge for Juliet 1.2
		# The Frama-C reports have the structure name instead of
		# the variable name.  This ADDS information in those cases.
		if ($uninitLval =~ m/structTwoIntsStruct->intTwo/) {
		    $uninitLval = "data";
		}
	    }

	    if (! defined $uninitLval) {
		print "NO UNINITVAL FOUND: $globalWeaknessID $globalLine in $globalFileName\n";
		die;
	    }

	    # heuristics to change the lval to a variable
	    my($uninitVar) = $uninitLval;
	    $uninitVar =~ s/^&//;
	    $uninitVar =~ s/([+]i(_[01])?|[.]intOne)$//;
	    $uninitVar =~ s/\(data\+i(_[01])?\)->intOne/data/;

	    # not CWE457 if warning concerns failed memory allocation
	    if ($uninitVar !~ m/^tmp_[01]$/ &&
		$uninitVar !~ m/^src$/ &&
		$uninitVar !~ m/^s$/) {

		# ok, this is a use of uninitialized variable warning

		# make sure this assignment doesn't contradict any other
		if ($weaknessClass ne "UNKNOWN") {
		    print "Trying to reassign warning to class uninitVar\n";
		    print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		    print "Current weakness class: $weaknessClass\n";
		    print "WEAKNESS: $globalWeaknessName\n";
		    print "CDATA: $textoutputCdata\n";
		    die;
		}

		$weaknessClass = "uninitVar";
	    }
	}

	#
	# Look for return of stack address warnings
	#
	if ($globalWeaknessName eq "making use of address of object past its lifetime") {
	    # ok, this is a return of stack address warning

	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class retStackAddr\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "retStackAddr";
	}

	#
	# Look for warnings caused by calloc() when allocation fails.  I doubt
	# actual calloc() code tries to zero memory if allocation fails, so I
	# think these are really model artifacts.
	#
	if ($globalWeaknessName eq "invalid memory access" &&
	    $textoutputCdata =~ m/ calloc / &&
	    $inputFileName =~ m/frama_c_report.allocfail.xml/) {

	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class EXCLUDE-D\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "EXCLUDE-D";
	}

	#
	# these warnings are not considered in SATE V Ockham Criteria
	#
	if ($globalWeaknessName eq "overflow in conversion" ||
	    $globalWeaknessName eq "passing INT_MIN to standard function abs()") {
	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class EXCLUDE-E\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "EXCLUDE-E";
	}

	#
	# these warnings are artifacts of Frama-C model of snprintf()
	#
	if ($globalWeaknessName eq "invalid arguments to library function" &&
	    $textoutputCdata =~ m/user error: Too many arguments for format:/ &&
	    $textoutputCdata =~ m/stack: snprintf ::/) {
	    # make sure this assignment doesn't contradict any other
	    if ($weaknessClass ne "UNKNOWN") {
		print "Trying to reassign warning to class EXCLUDE-F\n";
		print "ID: $globalWeaknessID line $globalLine in $globalFileName\n";
		print "Current weakness class: $weaknessClass\n";
		print "WEAKNESS: $globalWeaknessName\n";
		print "CDATA: $textoutputCdata\n";
		die;
	    }

	    $weaknessClass = "EXCLUDE-F";
	}

	print "$globalFileName, $globalLine, $weaknessClass\n";

	$Trace = 0;

	undef $globalWeaknessName;
	undef $globalWeaknessID;
	undef $globalFileName;
	undef $globalLine;
	undef $textoutputCdata;
	undef $assertExpression;
	undef $weaknessClass;
    } elsif ($elementParam eq "name") {
	#print "End 'name'\n";
	if (! defined $globalString) {
	    print "Missing weakness name string in\n";
	    print "<weakness id=\"$globalWeaknessID\">\n";
	    die "Stopped";
	}
	$globalWeaknessName = $globalString;
	#print "$globalWeaknessID $globalWeaknessName\n";
    } elsif ($elementParam eq "textoutput") {
	#print "</textoutput>\n" if $Trace;

	if ($globalFileName eq "io.c") {
	    # The weakness was reported in io.c.  Instead, use the 
	    # first "regular" file (in the stack), e.g.,
#                    stack: printf :: testcasesupport/io.c:13 <-
#                           printLine :: testcases/CWE126_Buffer_Overread/s01/CWE126_Buffer_Overread__CWE170_char_loop_17.c:38 <-
#                           CWE126_Buffer_Overread__CWE170_char_loop_17_bad
	    if ($globalCdata =~ m|testcases/\w+(/s\d\d)?/([\w.]+):(\d+) |) {
		$globalFileName = $2;
		$globalLine = $3;
		die "Cannot get filename: $globalCdata Stopped" if ! defined $globalFileName;
		die "Cannot get line number: $globalCdata Stopped" if ! defined $globalLine;
	    } else {
		print "No replacement filename in ID $globalWeaknessID\n";
		die;
	    }
	}

	# keep around any CDATA in textoutput
	$textoutputCdata = $globalCdata;
	# replace wide "not identical to" with !==
	# NOT IDENTICAL TO documented on http://www.fileformat.info/info/unicode/block/mathematical_operators/images.htm
	# cat -v changed it to M-bM-^IM-" but I can't manage to match
#	$notIdenticalTo = "" . chr(128+98); This should be M-b, but doesn't match
#	$notIdenticalTo = "" . chr(2262); This doesn't match
#	$notIdenticalTo = "" . chr(95); # This works, and replaces _
#	$textoutputCdata =~ s/$notIdenticalTo/!==/g;
	# replace all wide characters with "???"
	$textoutputCdata =~ s/[^[:ascii:]]/???/g;

    }

    undef $globalString;
    undef $globalCdata;
}

# read the XML "site" file
sub readFramaCReportFile {
    my($fname) = @_;
    $inputFileName = $fname;

    $xmlParser = new XML::Parser(
	Handlers => {
	    Start	=> \&handle_start,
	    Char	=> \&handle_text,
	    CdataStart	=> \&handle_cdataStart,
	    CdataEnd	=> \&handle_cdataEnd,
	    End 	=> \&handle_end
	});
    $xmlParser->parsefile($fname);
}

readFramaCReportFile($filename1);
readFramaCReportFile($filename2);

# end of $Source$
