Skip to content

Commit

Permalink
7903511: JMH: Add score stability performance tests
Browse files Browse the repository at this point in the history
  • Loading branch information
shipilev authored Jul 26, 2023
1 parent 6b09724 commit 8bc325b
Show file tree
Hide file tree
Showing 5 changed files with 163 additions and 8 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
@Measurement(iterations = 50)
@Threads(1)
@State(Scope.Thread)
public class ScoreStabilityBench {
public class BurstStabilityBench {

@Param("10")
private int delay;
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.jmh.benchmarks;

import org.openjdk.jmh.annotations.*;
import org.openjdk.jmh.infra.Blackhole;

import java.util.concurrent.TimeUnit;

@BenchmarkMode(Mode.Throughput)
@OutputTimeUnit(TimeUnit.SECONDS)
@Threads(1)
@State(Scope.Thread)
public class LongStabilityBench {

@Benchmark
public void test() {
Blackhole.consumeCPU(1_000_000);
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -158,8 +158,26 @@ public static void main(String[] args) throws RunnerException, CommandLineOption
case timing:
new TimingMeasurementsTest().runWith(pw, opts);
break;
case stability:
new ScoreStabilityTest().runWith(pw, opts);
case long_stability:
switch (mode) {
case flash:
new LongStabilityTest(3).runWith(pw, opts);
break;
case quick:
new LongStabilityTest(5).runWith(pw, opts);
break;
case normal:
new LongStabilityTest(18).runWith(pw, opts);
break;
case longer:
new LongStabilityTest(60).runWith(pw, opts);
break;
default:
throw new IllegalStateException();
}
break;
case burst_stability:
new BurstStabilityTest().runWith(pw, opts);
break;
case compiler_hints:
new CompilerHintsTest().runWith(pw, opts);
Expand Down Expand Up @@ -242,7 +260,8 @@ public enum Test {
timing,
compiler_hints,
thermal,
stability,
long_stability,
burst_stability,
thread_scale,
helpers,
blackhole_cpu,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@
package org.openjdk.jmh.validation.tests;

import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.benchmarks.ScoreStabilityBench;
import org.openjdk.jmh.benchmarks.BurstStabilityBench;
import org.openjdk.jmh.results.Result;
import org.openjdk.jmh.results.RunResult;
import org.openjdk.jmh.runner.Runner;
Expand All @@ -37,10 +37,10 @@

import java.io.PrintWriter;

public class ScoreStabilityTest extends ValidationTest {
public class BurstStabilityTest extends ValidationTest {
@Override
public void runWith(PrintWriter pw, Options parent) throws RunnerException {
pw.println("--------- SCORE STABILITY TEST");
pw.println("--------- BURST STABILITY TEST");
pw.println();

org.openjdk.jmh.util.Utils.reflow(pw,
Expand Down Expand Up @@ -74,7 +74,7 @@ public void runWith(PrintWriter pw, Options parent) throws RunnerException {
Options opts = new OptionsBuilder()
.parent(parent)
.mode(m)
.include(ScoreStabilityBench.class.getCanonicalName())
.include(BurstStabilityBench.class.getCanonicalName())
.verbosity(VerboseMode.SILENT)
.param("delay", String.valueOf(delay))
.build();
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
/*
* Copyright Amazon.com Inc. or its affiliates. All Rights Reserved.
* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
*
* This code is free software; you can redistribute it and/or modify it
* under the terms of the GNU General Public License version 2 only, as
* published by the Free Software Foundation. Oracle designates this
* particular file as subject to the "Classpath" exception as provided
* by Oracle in the LICENSE file that accompanied this code.
*
* This code is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
* version 2 for more details (a copy is included in the LICENSE file that
* accompanied this code).
*
* You should have received a copy of the GNU General Public License version
* 2 along with this work; if not, write to the Free Software Foundation,
* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
*
* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
* or visit www.oracle.com if you need additional information or have any
* questions.
*/
package org.openjdk.jmh.validation.tests;

import org.openjdk.jmh.benchmarks.LongStabilityBench;
import org.openjdk.jmh.results.Result;
import org.openjdk.jmh.results.RunResult;
import org.openjdk.jmh.runner.Runner;
import org.openjdk.jmh.runner.RunnerException;
import org.openjdk.jmh.runner.options.Options;
import org.openjdk.jmh.runner.options.OptionsBuilder;
import org.openjdk.jmh.runner.options.TimeValue;
import org.openjdk.jmh.runner.options.VerboseMode;
import org.openjdk.jmh.validation.ValidationTest;

import java.io.PrintWriter;
import java.util.concurrent.ThreadLocalRandom;

public class LongStabilityTest extends ValidationTest {
private final int tries;

public LongStabilityTest(int tries) {
this.tries = tries;
}

@Override
public void runWith(PrintWriter pw, Options parent) throws RunnerException {
pw.println("--------- LONG STABILITY TEST");
pw.println();

org.openjdk.jmh.util.Utils.reflow(pw,
"This test verifies the performance for a single test by running it several times with some " +
"delays between the runs. The performance should be the same across all runs. " +
"If there is a significant difference between the runs, this is usually " +
"indicative of noisy environment, e.g. a busy virtualized node, or background processes " +
"interfering with the run, making the benchmarks unreliable.",
80, 2);
pw.println();

for (int t = 0; t < tries; t++) {
int ms = (t == 0) ? 0 : ThreadLocalRandom.current().nextInt(5_000, 30_000);

pw.printf(" Sleeping for %6d ms...", ms);
pw.flush();
try {
Thread.sleep(ms);
} catch (InterruptedException e) {
// Do nothing.
}

pw.print(" Run: ");
pw.flush();

Options opts = new OptionsBuilder()
.parent(parent)
.include(LongStabilityBench.class.getCanonicalName())
.warmupIterations(5)
.warmupTime(TimeValue.seconds(1))
.measurementIterations(5)
.measurementTime(TimeValue.seconds(1))
.forks(1)
.verbosity(VerboseMode.SILENT)
.build();

RunResult result = new Runner(opts).runSingle();
Result r = result.getPrimaryResult();
pw.printf(" %16s", String.format("%.2f \u00b1 %.2f %s%n", r.getScore(), r.getScoreError(), r.getScoreUnit()));
pw.flush();
}
}
}

0 comments on commit 8bc325b

Please sign in to comment.