View Javadoc
1   /**
2    * <copyright>
3    *  Copyright 1997-2002 BBNT Solutions, LLC
4    *  under sponsorship of the Defense Advanced Research Projects Agency (DARPA).
5    *
6    *  This program is free software; you can redistribute it and/or modify
7    *  it under the terms of the Cougaar Open Source License as published by
8    *  DARPA on the Cougaar Open Source Website (www.cougaar.org).
9    *
10   *  THE COUGAAR SOFTWARE AND ANY DERIVATIVE SUPPLIED BY LICENSOR IS
11   *  PROVIDED 'AS IS' WITHOUT WARRANTIES OF ANY KIND, WHETHER EXPRESS OR
12   *  IMPLIED, INCLUDING (BUT NOT LIMITED TO) ALL IMPLIED WARRANTIES OF
13   *  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE, AND WITHOUT
14   *  ANY WARRANTIES AS TO NON-INFRINGEMENT.  IN NO EVENT SHALL COPYRIGHT
15   *  HOLDER BE LIABLE FOR ANY DIRECT, SPECIAL, INDIRECT OR CONSEQUENTIAL
16   *  DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE OF DATA OR PROFITS,
17   *  TORTIOUS CONDUCT, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
18   *  PERFORMANCE OF THE COUGAAR SOFTWARE.
19   * </copyright>
20   *
21   * Created on Aug 26, 2002
22   */
23  package net.sourceforge.pmd.stat;
24  
25  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.MINIMUM_DESCRIPTOR;
26  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.SIGMA_DESCRIPTOR;
27  import static net.sourceforge.pmd.lang.rule.stat.StatisticalRule.TOP_SCORE_DESCRIPTOR;
28  import static org.junit.Assert.assertEquals;
29  import static org.junit.Assert.assertTrue;
30  
31  import java.util.ArrayList;
32  import java.util.Collections;
33  import java.util.Iterator;
34  import java.util.List;
35  import java.util.Random;
36  
37  import junit.framework.AssertionFailedError;
38  import net.sourceforge.pmd.Report;
39  import net.sourceforge.pmd.Rule;
40  import net.sourceforge.pmd.RuleContext;
41  import net.sourceforge.pmd.lang.DummyLanguageModule;
42  import net.sourceforge.pmd.lang.LanguageRegistry;
43  import net.sourceforge.pmd.lang.ast.DummyNode;
44  import net.sourceforge.pmd.lang.ast.Node;
45  import net.sourceforge.pmd.lang.rule.stat.StatisticalRule;
46  
47  import org.junit.Before;
48  import org.junit.Ignore;
49  import org.junit.Test;
50  
51  /**
52   * This class tests the Statistical Rules in PMD.
53   * <p/>
54   * The idea is, that we fill up 999 datapoints into
55   * the Stat Rule, and then throw random parameters
56   * at it.
57   * <p/>
58   * The three parameters which are checked are:
59   * sigma - # Sigmas over the mean.
60   * topscore - Only the top 5 or so items.
61   * minimum - Only things of score 10 or better
62   * <p/>
63   * When more than one parameter is lumped together, then
64   * we expect the one which would return the fewest to
65   * determine what gets sent back.
66   * <p/>
67   * So, we throw each collection of parameters, where each
68   * one is a different order into the system.  We check the
69   * results off of what the smallest value should be.
70   * <p/>
71   * If you are going to work with StatisticalRule any, please
72   * bump the "NUM_TESTS" number up to something like 128.  That
73   * way you are more likely to identify problems.  It is set low
74   * now to make building and running tests easier (when we aren't
75   * touching the file.)
76   * <p/>
77   * Note also, that when verifying the Sigma, I wasn't quite able
78   * to determine how many results it would return (it would vary
79   * from -2 to 2 of what I expected.)  That is what the delta
80   * parameter on the verify method takes.  If you can figure it
81   * out exactly, (without stealing code from the StatRule) then
82   * feel free to change it and tighten the deltas.
83   */
84  public class StatisticalRuleTest  {
85  
86      private static final int POINTS = 100;
87  
88      private DataPoint[] points = new DataPoint[POINTS];
89      private MockStatisticalRule IUT = null;
90      private String testName = "";//FIXME - why/when was this added. It was never set.
91      private Random random = new Random();
92  
93      public static final double MAX_MINIMUM = POINTS;
94      public static final double NO_MINIMUM = -1.0;
95      public static final double MAX_SIGMA = 5.0;
96      public static final double NO_SIGMA = -1.0;
97      public static final int MIN_TOPSCORE = 0;
98      public static final int NO_TOPSCORE = -1;
99  
100 
101     public static final double MEAN = 49.5;
102     public static final double SIGMA = 29.0115;
103     public static final int NUM_TESTS = 1;
104 
105     public static final double DELTA = 0.005;
106 
107 
108     @Before
109     public void setUp() {
110         IUT = new MockStatisticalRule();
111         if (testName.endsWith("0")) {
112             for (int i = 0; i < POINTS; i++) {
113                 points[i] = new DataPoint();
114                 points[i].setScore(1.0 * i);
115                 DummyNode s = new DummyNode(1);
116                 s.testingOnly__setBeginLine(i);
117                 s.testingOnly__setBeginColumn(1);
118                 points[i].setNode(s);
119                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
120 
121                 IUT.addDataPoint(points[i]);
122             }
123         } else if (testName.endsWith("1")) {
124             for (int i = POINTS - 1; i >= 0; i--) {
125                 points[i] = new DataPoint();
126                 points[i].setScore(1.0 * i);
127                 DummyNode s = new DummyNode(1);
128                 s.testingOnly__setBeginLine(i);
129                 s.testingOnly__setBeginColumn(1);
130                 points[i].setNode(s);
131                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
132 
133                 IUT.addDataPoint(points[i]);
134             }
135         } else {
136             List<DataPoint> lPoints = new ArrayList<DataPoint>();
137             for (int i = 0; i < POINTS; i++) {
138                 points[i] = new DataPoint();
139                 points[i].setScore(1.0 * i);
140                 DummyNode s = new DummyNode(1);
141                 s.testingOnly__setBeginLine(i);
142                 s.testingOnly__setBeginColumn(1);
143                 s.testingOnly__setBeginColumn(1);
144                 points[i].setNode(s);
145                 points[i].setMessage("DataPoint[" + Integer.toString(i) + "]");
146 
147                 lPoints.add(points[i]);
148             }
149 
150             Collections.shuffle(lPoints);
151             for (int i = 0; i < POINTS; i++) {
152                 IUT.addDataPoint(lPoints.get(i));
153             }
154         }
155 
156     }
157 
158     /**
159      * This test verifies that the Stat rule creates a Metric,
160      * with the proper values.
161      */
162     @Test
163     public void testMetrics() throws Throwable {
164         Report report = makeReport(IUT);
165         Iterator<Metric> metrics = report.metrics();
166 
167         assertTrue(metrics.hasNext());
168         Metric m = metrics.next();
169 
170         assertEquals("net.sourceforge.pmd.stat.MockStatisticalRule", m.getMetricName());
171 
172         assertEquals(0.0, m.getLowValue(), 0.05);
173         assertEquals(POINTS - 1.0, m.getHighValue(), 0.05);
174         assertEquals(MEAN, m.getAverage(), 0.05);
175         assertEquals(SIGMA, m.getStandardDeviation(), 0.05);
176     }
177 
178     /**
179      * This returns a Random value for Sigma which will
180      * return some values.
181      */
182     public double randomSigma() {
183         return random.nextDouble() * 1.0;
184     }
185 
186     /**
187      * This returns a Random value for Sigma which value
188      * is greater than the parameter.
189      */
190     public double randomSigma(int minimum) {
191         double minSigma = ((POINTS - 1 - minimum) - MEAN) / SIGMA;
192 
193         if ((minSigma <= 0) || (minSigma > 2))
194             return randomSigma();
195 
196         return minSigma + (random.nextDouble() * (2 - minSigma));
197     }
198 
199     /**
200      * This returns the expected number of results when
201      * the Sigma rating is the smallest.
202      */
203     public int expectedSigma(double sigma) {
204         long expectedMin = Math.round(MEAN + (sigma * SIGMA));
205 
206         if (((POINTS - 1) - expectedMin) < 0)
207             return 0;
208         return (POINTS - 1) - (int) expectedMin;
209     }
210 
211     /**
212      * This generates a random minimum value for testing.
213      */
214     public double randomMinimum() {
215         return random.nextDouble() * (POINTS - 1);
216     }
217 
218     /**
219      * This generates a random minimum value for which fewer
220      * results would be returned.
221      */
222     public double randomMinimum(int minimum) {
223         double diffTarget = 1.0 * (POINTS - 1 - minimum);
224         return (random.nextDouble() * minimum) + diffTarget;
225     }
226 
227     /**
228      * This returns the expected number of reports.
229      * <p/>
230      * If the Minimum comes in at 521.569 then we expect
231      * 522, 523, ... 999 will pass.
232      */
233     public int expectedMinimum(double minimum) {
234         Double d = Double.valueOf(minimum);
235         return POINTS - 1 - d.intValue();
236     }
237 
238     @Test
239     public void testExpectedMinimum() {
240         for (int i = 0; i < POINTS - 1; i++) {
241             assertEquals("Integer Min", POINTS - 1 - i, expectedMinimum(i * 1.0));
242             assertEquals("Double Min", POINTS - 1 - i, expectedMinimum((i * 1.0) + 0.5));
243         }
244     }
245 
246     /**
247      * This returns a random value for Top Score.
248      */
249     public int randomTopScore() {
250         return random.nextInt(POINTS - 1);
251     }
252 
253     /**
254      * This will return a random value for the Top Score
255      * which will return more than the minimum provided.
256      */
257     public int randomTopScore(double target) {
258         if (target < 0)
259             return 0;
260 
261         return random.nextInt(Double.valueOf(target).intValue());
262     }
263 
264     /**
265      * This will return the expected number of results
266      * with the given Top Score.
267      */
268     public int expectedTopScore(int target) {
269         return target;
270     }
271 
272     // Test Single Datapoint
273     @Test
274     public void testSingleDatapoint() {
275         StatisticalRule IUT = new MockStatisticalRule();
276 
277         DataPoint point = new DataPoint();
278         point.setScore(POINTS + 1.0);
279         DummyNode s = new DummyNode(1);
280         s.testingOnly__setBeginLine(POINTS + 1);
281         s.testingOnly__setBeginColumn(1);
282         point.setNode(s);
283         point.setMessage("SingleDataPoint");
284 
285         IUT.setProperty(MINIMUM_DESCRIPTOR, (double)POINTS);
286 
287         IUT.addDataPoint(point);
288 
289         Report report = makeReport(IUT);
290 
291         assertEquals("Expecting only one result", 1, report.size());
292     }
293 
294     // Okay, we have three properties we need to
295     // test in Combination:
296     //  S = Sigma
297     //  T = Top Score
298     //  M = Minimum
299     //
300     // They are listed in decreasing order of what
301     // to expect.
302     //
303     // Thus testSM() should have the Sigma less than
304     // the minimum, so we expect the Minimum # of results.
305     //
306 
307     @Test
308     public void testS() throws Throwable {
309         verifyResults(MAX_SIGMA, NO_MINIMUM, NO_TOPSCORE, 0, 2);
310 
311         for (int i = 0; i < NUM_TESTS; i++) {
312             double sigma = randomSigma();
313             verifyResults(sigma, -1.0, -1, expectedSigma(sigma), 2);
314         }
315     }
316 
317     @Test
318     public void testS1() throws Throwable {
319         testS();
320     }
321 
322     @Test
323     public void testS2() throws Throwable {
324         testS();
325     }
326 
327     @Test
328     public void testS3() throws Throwable {
329         testS();
330     }
331 
332     @Test
333     public void testS4() throws Throwable {
334         testS();
335     }
336 
337     @Test
338     public void testS5() throws Throwable {
339         testS();
340     }
341 
342 
343     @Test
344     public void testT() throws Throwable {
345         verifyResults(NO_SIGMA, NO_MINIMUM, MIN_TOPSCORE, 0, 0);
346 
347         for (int i = 0; i < NUM_TESTS; i++) {
348             int topScore = randomTopScore();
349             verifyResults(-1.0, -1.0, topScore, expectedTopScore(topScore), 0);
350         }
351     }
352 
353     @Test
354     public void testT1() throws Throwable {
355         testT();
356     }
357 
358     @Test
359     public void testT2() throws Throwable {
360         testT();
361     }
362 
363     @Test
364     public void testT3() throws Throwable {
365         testT();
366     }
367 
368     @Test
369     public void testT4() throws Throwable {
370         testT();
371     }
372 
373     @Test
374     public void testT5() throws Throwable {
375         testT();
376     }
377 
378     @Test
379     public void testM() throws Throwable {
380         verifyResults(NO_SIGMA, MAX_MINIMUM, NO_TOPSCORE, 0, 0);
381 
382         for (int i = 0; i < NUM_TESTS; i++) {
383             double minimum = randomMinimum();
384             verifyResults(-1.0, minimum, -1, expectedMinimum(minimum), 0);
385         }
386     }
387 
388     @Test
389     public void testM1() throws Throwable {
390         testM();
391     }
392 
393     @Test
394     public void testM2() throws Throwable {
395         testM();
396     }
397 
398     @Test
399     public void testM3() throws Throwable {
400         testM();
401     }
402 
403     @Test
404     public void testM4() throws Throwable {
405         testM();
406     }
407 
408     @Test
409     public void testM5() throws Throwable {
410         testM();
411     }
412 
413     @Test
414     public void testST() throws Throwable {
415         verifyResults(randomSigma(), NO_MINIMUM, MIN_TOPSCORE, 0, 0);
416 
417         for (int i = 0; i < NUM_TESTS; i++) {
418             double sigma = randomSigma();
419             int topScore = randomTopScore(expectedSigma(sigma));
420 
421             verifyResults(sigma, NO_MINIMUM, topScore, expectedTopScore(topScore), 0);
422         }
423     }
424 
425     @Test
426     public void testST1() throws Throwable {
427         testST();
428     }
429 
430     @Test
431     public void testST2() throws Throwable {
432         testST();
433     }
434 
435     @Test
436     public void testST3() throws Throwable {
437         testST();
438     }
439 
440     @Test
441     public void testST4() throws Throwable {
442         testST();
443     }
444 
445     @Test
446     public void testST5() throws Throwable {
447         testST();
448     }
449 
450     @Test
451     public void testTS() throws Throwable {
452         verifyResults(MAX_SIGMA, NO_MINIMUM, randomTopScore(), 0, 0);
453 
454         for (int i = 0; i < NUM_TESTS; i++) {
455             int topScore = randomTopScore();
456             double sigma = randomSigma(expectedTopScore(topScore));
457 
458             verifyResults(sigma, -1.0, topScore, expectedSigma(sigma), 2);
459         }
460     }
461 
462     @Test
463     public void testTS1() throws Throwable {
464         testTS();
465     }
466 
467     @Test
468     public void testTS2() throws Throwable {
469         testTS();
470     }
471 
472     @Test
473     public void testTS3() throws Throwable {
474         testTS();
475     }
476 
477     @Test
478     public void testTS4() throws Throwable {
479         testTS();
480     }
481 
482     @Test
483     public void testTS5() throws Throwable {
484         testTS();
485     }
486 
487     @Test
488     public void testSM() throws Throwable {
489         verifyResults(randomSigma(), MAX_MINIMUM, NO_TOPSCORE, 0, 0);
490         for (int i = 0; i < NUM_TESTS; i++) {
491             double sigma = randomSigma();
492             double minimum = randomMinimum(expectedSigma(sigma));
493 
494             verifyResults(sigma, minimum, -1, expectedMinimum(minimum), 0);
495         }
496 
497     }
498 
499     @Test
500     public void testSM1() throws Throwable {
501         testSM();
502     }
503 
504     @Test
505     public void testSM2() throws Throwable {
506         testSM();
507     }
508 
509     @Test
510     public void testSM3() throws Throwable {
511         testSM();
512     }
513 
514     @Test
515     public void testSM4() throws Throwable {
516         testSM();
517     }
518 
519     @Test
520     public void testSM5() throws Throwable {
521         testSM();
522     }
523 
524 
525     @Test
526     public void testMS() throws Throwable {
527         verifyResults(MAX_SIGMA, randomMinimum(), NO_TOPSCORE, 0, 0);
528         for (int i = 0; i < NUM_TESTS; i++) {
529             double minimum = randomMinimum();
530             double sigma = randomSigma(expectedMinimum(minimum));
531 
532             verifyResults(sigma, minimum, -1, expectedSigma(sigma), 2);
533         }
534     }
535 
536     @Test
537     public void testMS1() throws Throwable {
538         testMS();
539     }
540 
541     @Test
542     public void testMS2() throws Throwable {
543         testMS();
544     }
545 
546     @Test
547     public void testMS3() throws Throwable {
548         testMS();
549     }
550 
551     @Test
552     public void testMS4() throws Throwable {
553         testMS();
554     }
555 
556     @Test
557     public void testMS5() throws Throwable {
558         testMS();
559     }
560 
561 
562     @Test
563     public void testTM() throws Throwable {
564         verifyResults(NO_SIGMA, MAX_MINIMUM, randomTopScore(), 0, 0);
565         for (int i = 0; i < NUM_TESTS; i++) {
566             int topScore = randomTopScore();
567             double minimum = randomMinimum(expectedTopScore(topScore));
568 
569             verifyResults(NO_SIGMA, minimum, topScore, expectedMinimum(minimum), 0);
570         }
571     }
572 
573     @Test
574     public void testTM1() throws Throwable {
575         testTM();
576     }
577 
578     @Test
579     public void testTM2() throws Throwable {
580         testTM();
581     }
582 
583     @Test
584     public void testTM3() throws Throwable {
585         testTM();
586     }
587 
588     @Test
589     public void testTM4() throws Throwable {
590         testTM();
591     }
592 
593     @Test
594     public void testTM5() throws Throwable {
595         testTM();
596     }
597 
598 
599     @Test
600     public void testMT() throws Throwable {
601         verifyResults(NO_SIGMA, randomMinimum(), MIN_TOPSCORE, 0, 0);
602         for (int i = 0; i < NUM_TESTS; i++) {
603             double minimum = randomMinimum();
604             int topScore = randomTopScore(expectedMinimum(minimum));
605 
606             verifyResults(NO_SIGMA, minimum, topScore, expectedTopScore(topScore), 0);
607         }
608     }
609 
610     @Test
611     public void testMT1() throws Throwable {
612         testMT();
613     }
614 
615     @Test
616     public void testMT2() throws Throwable {
617         testMT();
618     }
619 
620     @Test
621     public void testMT3() throws Throwable {
622         testMT();
623     }
624 
625     @Test
626     public void testMT4() throws Throwable {
627         testMT();
628     }
629 
630     @Test
631     public void testMT5() throws Throwable {
632         testMT();
633     }
634 
635 
636     @Test
637     public void testSTM() throws Throwable {
638         double sigma = randomSigma();
639         verifyResults(sigma, MAX_MINIMUM, randomTopScore(expectedSigma(sigma)), 0, 0);
640 
641         for (int i = 0; i < NUM_TESTS; i++) {
642             sigma = randomSigma();
643             int topScore = randomTopScore(expectedSigma(sigma));
644             double minimum = randomMinimum(expectedTopScore(topScore));
645 
646             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
647         }
648     }
649 
650     @Test
651     public void testSTM1() throws Throwable {
652         testSTM();
653     }
654 
655     @Test
656     public void testSTM2() throws Throwable {
657         testSTM();
658     }
659 
660     @Test
661     public void testSTM3() throws Throwable {
662         testSTM();
663     }
664 
665     @Test
666     public void testSTM4() throws Throwable {
667         testSTM();
668     }
669 
670     @Test
671     public void testSTM5() throws Throwable {
672         testSTM();
673     }
674 
675     @Test
676     public void testSMT() throws Throwable {
677         double sigma = randomSigma();
678         verifyResults(sigma, randomMinimum(expectedSigma(sigma)), MIN_TOPSCORE, 0, 0);
679 
680         for (int i = 0; i < NUM_TESTS; i++) {
681             sigma = randomSigma();
682             double minimum = randomMinimum(expectedSigma(sigma));
683             int topScore = randomTopScore(expectedMinimum(minimum));
684 
685             verifyResults(sigma, minimum, topScore, expectedTopScore(topScore), 0);
686         }
687     }
688 
689     @Test
690     public void testSMT1() throws Throwable {
691         testSMT();
692     }
693 
694     @Test
695     public void testSMT2() throws Throwable {
696         testSMT();
697     }
698 
699     @Test
700     public void testSMT3() throws Throwable {
701         testSMT();
702     }
703 
704     @Test
705     public void testSMT4() throws Throwable {
706         testSMT();
707     }
708 
709     @Test
710     public void testSMT5() throws Throwable {
711         testSMT();
712     }
713 
714     @Test
715     // because of random failures during continuous integration,
716     // tests are disabled in regress mode until somebody figures out
717     // what the tests are supposed to measure and why they sometime fail
718     @Ignore("random failures during continuous integration")
719     public void testTSM() throws Throwable {
720         int topScore = randomTopScore();
721         verifyResults(randomSigma(expectedTopScore(topScore)), MAX_MINIMUM, topScore, 0, 0);
722 
723         for (int i = 0; i < NUM_TESTS; i++) {
724             topScore = randomTopScore();
725             double sigma = randomSigma(expectedTopScore(topScore));
726             double minimum = randomMinimum(expectedSigma(sigma));
727 
728             verifyResults(sigma, minimum, topScore, expectedMinimum(minimum), 0);
729         }
730     }
731 
732     @Test
733     @Ignore("random failures during continuous integration")
734     public void testTSM1() throws Throwable {
735         testTSM();
736     }
737 
738     @Test
739     @Ignore("random failures during continuous integration")
740     public void testTSM2() throws Throwable {
741         testTSM();
742     }
743 
744     @Test
745     @Ignore("random failures during continuous integration")
746     public void testTSM3() throws Throwable {
747         testTSM();
748     }
749 
750     @Test
751     @Ignore("random failures during continuous integration")
752     public void testTSM4() throws Throwable {
753         testTSM();
754     }
755 
756     @Test
757     @Ignore("random failures during continuous integration")
758     public void testTSM5() throws Throwable {
759         testTSM();
760     }
761 
762     @Test
763     public void testTMS() throws Throwable {
764         int topScore = randomTopScore();
765         verifyResults(MAX_SIGMA, randomMinimum(expectedTopScore(topScore)), topScore, 0, 0);
766 
767         for (int i = 0; i < NUM_TESTS; i++) {
768             topScore = randomTopScore();
769             double minimum = randomMinimum(expectedTopScore(topScore));
770             double sigma = randomSigma(expectedMinimum(minimum));
771 
772             verifyResults(sigma, minimum, topScore, expectedSigma(sigma), 2);
773         }
774     }
775 
776     @Test
777     public void testTMS1() throws Throwable {
778         testTMS();
779     }
780 
781     @Test
782     public void testTMS2() throws Throwable {
783         testTMS();
784     }
785 
786     @Test
787     public void testTMS3() throws Throwable {
788         testTMS();
789     }
790 
791     @Test
792     public void testTMS4() throws Throwable {
793         testTMS();
794     }
795 
796     @Test
797     public void testTMS5() throws Throwable {
798         testTMS();
799     }
800 
801     /**
802      * Verifies what happens when you pass these parameters
803      * into the thing.  DELTA is the amount of error allowed.
804      * Usually DELTA is only used for Sigma, as we really can't
805      * calculate it exactly.
806      */
807 
808     public void verifyResults(double sigma, double minimum, int topScore, int expected, int delta) {
809         try {
810             setUp();
811             if (sigma >= 0) {
812             	IUT.setProperty(SIGMA_DESCRIPTOR, sigma);
813             }
814 
815             if (minimum >= 0) {
816             	IUT.setProperty(MINIMUM_DESCRIPTOR, minimum);
817             }
818 
819             if (topScore >= 0) {
820                 IUT.setProperty(TOP_SCORE_DESCRIPTOR, topScore);
821             }
822 
823             Report report = makeReport(IUT);
824             if (delta == 0) {
825                 assertEquals("Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore), expected, report.size());
826             } else {
827                 String assertStr = "Unexpected number of results: sigma= " + Double.toString(sigma) + " min= " + Double.toString(minimum) + " topscore= " + Integer.toString(topScore) + " expected= " + Integer.toString(expected) + " +/- " + Integer.toString(delta) + " actual-result= " + report.size();
828 
829                 assertTrue(assertStr, report.size() >= (expected - delta));
830                 assertTrue(assertStr, report.size() <= (expected + delta));
831             }
832         } catch (AssertionFailedError afe) {
833             System.err.println("******** " + testName + " ***********");
834             if (sigma != NO_SIGMA) {
835                 System.err.println("SIGMA: " + Double.toString(sigma) + " EXPECT: " + Integer.toString(expectedSigma(sigma)));
836             }
837 
838             if (minimum != NO_MINIMUM) {
839                 System.err.println("MIN: " + Double.toString(minimum) + " EXPECT: " + Integer.toString(expectedMinimum(minimum)));
840             }
841 
842             if (topScore != NO_TOPSCORE) {
843                 System.err.println("TOP: " + Integer.toString(topScore) + " EXPECT: " + Integer.toString(expectedTopScore(topScore)));
844             }
845 
846             throw afe;
847 
848         }
849     }
850 
851     public Report makeReport(Rule IUT) {
852         List<Node> list = new ArrayList<Node>();
853         Report report = new Report();
854 
855         RuleContext ctx = new RuleContext();
856         ctx.setReport(report);
857         ctx.setSourceCodeFilename(testName);
858         ctx.setLanguageVersion(LanguageRegistry.getLanguage(DummyLanguageModule.NAME).getDefaultVersion());
859 
860         IUT.apply(list, ctx);
861 
862         return report;
863     }
864 
865     public static junit.framework.Test suite() {
866         return new junit.framework.JUnit4TestAdapter(StatisticalRuleTest.class);
867     }
868 }