fix: unify sequential/parallel benchmarks

CodSpeedHQ · Sep 9, 2024 · 36274b4 · 36274b4
1 parent 9f3922f
commit 36274b4
Show file tree

Hide file tree

Showing 5 changed files with 32 additions and 75 deletions.
diff --git a/pingora-core/benches/tls_acceptor.rs b/pingora-core/benches/tls_acceptor.rs
@@ -1,26 +1,20 @@
 use ahash::{HashMap, HashMapExt};
 use iai_callgrind::{
     binary_benchmark, binary_benchmark_group, main, BinaryBenchmarkConfig, Command,
-    FlamegraphConfig, Stdio, Tool, ValgrindTool,
+    FlamegraphConfig, Tool, ValgrindTool,
 };
 use iai_callgrind::{Pipe, Stdin};
-use log::{debug, info, LevelFilter};
-use regex::Regex;
 use reqwest::blocking::Client;
-use reqwest::{Certificate, Error, Response, StatusCode, Version};
+use reqwest::{Certificate, StatusCode, Version};
 use std::env;
 use std::fs::File;
-use std::io::{Read, Stdout};
+use std::io::Read;
 use std::net::{IpAddr, Ipv4Addr, SocketAddr};
-use std::path::PathBuf;
 use tokio::task::JoinSet;
 
 mod utils;
 
-use utils::{
-    generate_random_ascii_data, http_version_to_port, wait_for_tcp_connect, CERT_PATH,
-    TLS_HTTP11_PORT, TLS_HTTP2_PORT,
-};
+use utils::{generate_random_ascii_data, http_version_to_port, wait_for_tcp_connect, CERT_PATH};
 
 main!(binary_benchmark_groups = tls_acceptor);
 
@@ -37,45 +31,32 @@ binary_benchmark_group!(
             // NOTE: for usage with callgrind::start_instrumentation() & stop_instrumentation()
             //"--instr-atstart=no"
     ]);
-    benchmarks = bench_server_sequential, bench_server_parallel
+    benchmarks = bench_server
 );
 
 static SEQUENTIAL_REQUEST_COUNT: i32 = 64;
 static SEQUENTIAL_REQUEST_SIZE: usize = 64;
+static PARALLEL_ACCEPTORS: u16 = 16;
+static PARALLEL_REQUEST_COUNT: i32 = SEQUENTIAL_REQUEST_COUNT / PARALLEL_ACCEPTORS as i32;
+static PARALLEL_REQUEST_SIZE: usize = 64;
 #[binary_benchmark]
-#[bench::http_11_handshake_always(args = (Version::HTTP_11),
+#[bench::seq_http_11_handshake_always(args = (1, Version::HTTP_11),
     setup = send_requests(1, false, Version::HTTP_11, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE))]
-#[bench::http_11_handshake_once(args = (Version::HTTP_11),
+#[bench::seq_http_11_handshake_once(args = (1, Version::HTTP_11),
     setup = send_requests(1, true, Version::HTTP_11, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE))]
-#[bench::http_2_handshake_always(args = (Version::HTTP_2),
+#[bench::seq_http_2_handshake_always(args = (1, Version::HTTP_2),
     setup = send_requests(1, false, Version::HTTP_2, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE))]
-#[bench::http_2_handshake_once(args = (Version::HTTP_2),
+#[bench::seq_http_2_handshake_once(args = (1, Version::HTTP_2),
     setup = send_requests(1, true, Version::HTTP_2, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE))]
-fn bench_server_sequential(http_version: Version) -> Command {
-    let path = format!(
-        "{}/../target/release/examples/bench_server",
-        env!("CARGO_MANIFEST_DIR")
-    );
-    Command::new(path)
-        // TODO: currently a workaround to keep the setup function running parallel with benchmark execution
-        .stdin(Stdin::Setup(Pipe::Stderr))
-        .args([format!("--http-version={:?}", http_version)])
-        .build()
-}
-
-static PARALLEL_ACCEPTORS: u16 = 16;
-static PARALLEL_REQUEST_COUNT: i32 = SEQUENTIAL_REQUEST_COUNT / PARALLEL_ACCEPTORS as i32;
-static PARALLEL_REQUEST_SIZE: usize = 64;
-#[binary_benchmark]
-#[bench::http_11_handshake_always(args = (PARALLEL_ACCEPTORS, Version::HTTP_11),
+#[bench::par_http_11_handshake_always(args = (PARALLEL_ACCEPTORS, Version::HTTP_11),
     setup = send_requests(PARALLEL_ACCEPTORS, false, Version::HTTP_11, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE))]
-#[bench::http_11_handshake_once(args = (PARALLEL_ACCEPTORS, Version::HTTP_11),
+#[bench::par_http_11_handshake_once(args = (PARALLEL_ACCEPTORS, Version::HTTP_11),
     setup = send_requests(PARALLEL_ACCEPTORS, true, Version::HTTP_11, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE))]
-#[bench::http_2_handshake_always(args = (PARALLEL_ACCEPTORS, Version::HTTP_2),
+#[bench::par_http_2_handshake_always(args = (PARALLEL_ACCEPTORS, Version::HTTP_2),
     setup = send_requests(PARALLEL_ACCEPTORS, false, Version::HTTP_2, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE))]
-#[bench::http_2_handshake_once(args = (PARALLEL_ACCEPTORS, Version::HTTP_2),
+#[bench::par_http_2_handshake_once(args = (PARALLEL_ACCEPTORS, Version::HTTP_2),
     setup = send_requests(PARALLEL_ACCEPTORS, true, Version::HTTP_2, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE))]
-fn bench_server_parallel(parallel_acceptors: u16, http_version: Version) -> Command {
+fn bench_server(parallel_acceptors: u16, http_version: Version) -> Command {
     let path = format!(
         "{}/../target/release/examples/bench_server",
         env!("CARGO_MANIFEST_DIR")

diff --git a/pingora-core/benches/tls_benchmarks.md b/pingora-core/benches/tls_benchmarks.md
@@ -131,6 +131,7 @@ target/iai/
 
 ### Parameters
 Server and client benchmark are parameterized with the following options:
+- number of parallel acceptors/connectors and servers/clients
 - client/session re-use
 - HTTP version `1.1|2.0`
 - number of requests

diff --git a/pingora-core/benches/tls_connector.rs b/pingora-core/benches/tls_connector.rs
@@ -32,55 +32,32 @@ binary_benchmark_group!(
             // NOTE: for usage with callgrind::start_instrumentation() & stop_instrumentation()
             //"--instr-atstart=no"
     ]);
-    benchmarks = bench_client_sequential, bench_client_parallel
+    benchmarks = bench_client
 );
 
 static SEQUENTIAL_REQUEST_COUNT: i32 = 64;
 static SEQUENTIAL_REQUEST_SIZE: usize = 64;
-#[binary_benchmark]
-#[bench::http_11_handshake_always(setup = start_servers(Version::HTTP_11, 1),
-    args = [false, Version::HTTP_11, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
-#[bench::http_11_handshake_once(setup = start_servers(Version::HTTP_11, 1),
-    args = [true, Version::HTTP_11, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
-#[bench::http_2_handshake_always(setup = start_servers(Version::HTTP_2, 1),
-    args = [false, Version::HTTP_2, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
-#[bench::http_2_handshake_once(setup = start_servers(Version::HTTP_2, 1),
-    args = [true, Version::HTTP_2, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
-fn bench_client_sequential(
-    stream_reuse: bool,
-    http_version: Version,
-    request_count: i32,
-    request_size: usize,
-) -> Command {
-    let path = format!(
-        "{}/../target/release/examples/bench_client",
-        env!("CARGO_MANIFEST_DIR")
-    );
-    Command::new(path)
-        // TODO: currently a workaround to keep the setup function running parallel with benchmark execution
-        .stdin(Stdin::Setup(Pipe::Stderr))
-        .args([
-            format!("--stream-reuse={}", stream_reuse),
-            format!("--http-version={:?}", http_version),
-            format!("--request-count={}", request_count),
-            format!("--request-size={}", request_size),
-        ])
-        .build()
-}
-
 static PARALLEL_CONNECTORS: u16 = 16;
 static PARALLEL_REQUEST_COUNT: i32 = SEQUENTIAL_REQUEST_COUNT / PARALLEL_CONNECTORS as i32;
 static PARALLEL_REQUEST_SIZE: usize = 64;
 #[binary_benchmark]
-#[bench::http_11_handshake_always(setup = start_servers(Version::HTTP_11, PARALLEL_CONNECTORS),
+#[bench::seq_http_11_handshake_always(setup = start_servers(Version::HTTP_11, 1),
+    args = [1, false, Version::HTTP_11, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
+#[bench::seq_http_11_handshake_once(setup = start_servers(Version::HTTP_11, 1),
+    args = [1, true, Version::HTTP_11, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
+#[bench::seq_http_2_handshake_always(setup = start_servers(Version::HTTP_2, 1),
+    args = [1, false, Version::HTTP_2, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
+#[bench::seq_http_2_handshake_once(setup = start_servers(Version::HTTP_2, 1),
+    args = [1, true, Version::HTTP_2, SEQUENTIAL_REQUEST_COUNT, SEQUENTIAL_REQUEST_SIZE])]
+#[bench::par_http_11_handshake_always(setup = start_servers(Version::HTTP_11, PARALLEL_CONNECTORS),
     args = [PARALLEL_CONNECTORS, false, Version::HTTP_11, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE])]
-#[bench::http_11_handshake_once(setup = start_servers(Version::HTTP_11, PARALLEL_CONNECTORS),
+#[bench::par_http_11_handshake_once(setup = start_servers(Version::HTTP_11, PARALLEL_CONNECTORS),
     args = [PARALLEL_CONNECTORS, true, Version::HTTP_11, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE])]
-#[bench::http_2_handshake_always(setup = start_servers(Version::HTTP_2, PARALLEL_CONNECTORS),
+#[bench::par_http_2_handshake_always(setup = start_servers(Version::HTTP_2, PARALLEL_CONNECTORS),
     args = [PARALLEL_CONNECTORS, false, Version::HTTP_2, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE])]
-#[bench::http_2_handshake_once(setup = start_servers(Version::HTTP_2, PARALLEL_CONNECTORS),
+#[bench::par_http_2_handshake_once(setup = start_servers(Version::HTTP_2, PARALLEL_CONNECTORS),
     args = [PARALLEL_CONNECTORS, true, Version::HTTP_2, PARALLEL_REQUEST_COUNT, PARALLEL_REQUEST_SIZE])]
-fn bench_client_parallel(
+fn bench_client(
     parallel_connectors: u16,
     stream_reuse: bool,
     http_version: Version,

diff --git a/pingora-core/examples/bench_client.rs b/pingora-core/examples/bench_client.rs
@@ -11,7 +11,6 @@ use pingora_core::protocols::http::v2::client::Http2Session;
 use pingora_http::RequestHeader;
 use reqwest::Version;
 
-#[allow(dead_code, unused_imports)]
 #[path = "../benches/utils/mod.rs"]
 mod bench_utils;
 use crate::bench_utils::TLS_HTTP11_PORT;

diff --git a/pingora-core/examples/bench_server.rs b/pingora-core/examples/bench_server.rs
@@ -12,7 +12,6 @@ use std::time::Duration;
 mod test_utils;
 use test_utils::EchoApp;
 
-#[allow(dead_code, unused_imports)]
 #[path = "../benches/utils/mod.rs"]
 mod bench_utils;
 use bench_utils::{http_version_parser, CERT_PATH, KEY_PATH, TLS_HTTP11_PORT, TLS_HTTP2_PORT};