benchmarks.conf

# -*- mode: yaml -*-
# Config file for ReBench
standard_experiment: all
standard_data_file: 'zero-overhead.data'

# settings and requirements for statistic evaluation
# Note, this is only used for explorative runs, for the final experiments, we
# rely soley on a fixed configuration of data points.
statistics:
    confidence_level: 0.95

runs:
    # Fixed minimum number of data points. Typically, the benchmarks are
    # configure to produce a specific number of datapoints independently of
    # this setting. ReBench will repeat the benchmarks until it got at least
    # this many data points.
    number_of_data_points: 3

# settings for quick runs, useful for fast feedback during experiments
quick_runs:
    number_of_data_points: 3
    max_time: 60   # time in seconds

# definition of benchmark suites
benchmark_suites:
    # Marcobenchmarks to be executed on one of the SOM implementations
    macro-steady:
        gauge_adapter: RebenchLog
        command: &MACRO_COMMAND " -cp Smalltalk::Smalltalk/Mate/:Smalltalk/Mate/MOP:Examples/Benchmarks/Richards:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody Examples/Benchmarks/BenchmarkHarness.som  %(benchmark)s "
        max_runtime: 60000
        benchmarks: &MACRO_STEADY
            - Richards:
                extra_args: "10 20 10"
#            - DeltaBlue:
#                extra_args: "1 0 1"
#            - Mandelbrot:
#                extra_args: "1 0 1"
#            - NBody:
#                extra_args: "1 0 1"

    # Microbenchmarks to be executed on one of the SOM implementations
    micro-steady:
        gauge_adapter: RebenchLog
        command: " -cp Smalltalk:Smalltalk/Mate/:Smalltalk/Mate/MOP:Examples/Benchmarks/LanguageFeatures Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks: &MICRO_STEADY
            - Bounce:
                extra_args: "1 0 1"
            - BubbleSort:
                extra_args: "1 0 1"
            - Dispatch:
                extra_args: "1 0 1"
            - Fannkuch:
                extra_args: "1 0 1"
            - Fibonacci:
                extra_args: "1 0 1"
            - FieldLoop:
                extra_args: "1 0 1"
            - IntegerLoop:
                extra_args: "1 0 1"
            - List:
                extra_args: "1 0 1"
            - Loop:
                extra_args: "1 0 1"
            - Permute:
                extra_args: "1 0 1"
            - Queens:
                extra_args: "1 0 1"
            - QuickSort:
                extra_args: "1 0 1"
            - Recurse:
                extra_args: "1 0 1"
            - Sieve:
                extra_args: "1 0 1"
            - Storage:
                extra_args: "1 0 1"
            - Sum:
                extra_args: "1 0 1"
            - Towers:
                extra_args: "1 0 1"
            - TreeSort:
                extra_args: "1 0 1"
            - WhileLoop:
                extra_args: "1 0 1"

    # Configuration of the Macrobenchmarks for SOM to be executed in the
    # context of the OMOP, or without it.
    # The results are shown in the paper in Figure 5.
    macro-steady-omop:
        gauge_adapter: RebenchLog
        command: " -cp Smalltalk:Examples/Benchmarks/Richards:Examples/Benchmarks/DeltaBlue:Examples/Benchmarks/NBody Examples/Benchmarks/OmopHarness.som %(benchmark)s %(variable)s "
        max_runtime: 60000
        benchmarks: *MACRO_STEADY
        variable_values:
            - "true"
            - "false"
    
    # Configuration of the SOM Reflection microbenchmarks
    # The results are shown in the paper in Figure 7.
    reflection:
        gauge_adapter: RebenchLog
        command: " -cp Smalltalk:Examples/Benchmarks/DoesNotUnderstand Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
            - DirectAdd:
                extra_args: 150 0 8000
            - DnuAdd:
                extra_args: 150 0 8000
            - DnuPerformAdd:
                extra_args: 150 0 8000
            - PerformAdd:
                extra_args: 150 0 8000
    
    # Configuration of the SOM Proxy microbenchmarks
    # The results are shown in the paper in Figure 7.
    proxy:
        gauge_adapter: RebenchLog
        command: " -cp Smalltalk:Examples/Benchmarks/DoesNotUnderstand Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
            - ProxyAdd:
                extra_args: 350 0 8000
            - IndirectAdd:
                extra_args: 350 0 8000
    
    # Configuration of the SOM Mate microbenchmarks
    mop-individual-activation:
        gauge_adapter: RebenchLog
        command: " -cp Smalltalk:Smalltalk/Mate/:Smalltalk/Mate/MOP:Examples/Benchmarks/Mate Examples/Benchmarks/BenchmarkHarness.som %(benchmark)s "
        max_runtime: 60000
        benchmarks:
            - FieldRead:
                extra_args: 50 700 500
            - FieldWrite:
                extra_args: 50 700 500
            - MessageSend:
                extra_args: 50 700 500
            - VMReflectiveFieldRead:
                extra_args: 50 700 500
            - VMReflectiveFieldWrite:
                extra_args: 50 700 500
            - VMReflectiveMethodActivation:
                extra_args: 50 700 500
            - VMReflectiveMessageSend:
                extra_args: 50 700 500
    
# VMs have a name and are specified by a path and the binary to be executed.
virtual_machines:
    # The configuration for TruffleSOM with and without the OMOP.
    # TruffleSOM is referred to as SOM_PE in the paper.
    # The different settings of Graal are used to see the impact of their
    # optimization settings, which have been changing in the time frame of these
    # experiments.
    TruffleMate-graal:
        path: .
        binary: som
#        args: " -G:+TruffleSplittingNew -G:-TraceTruffleInlining -G:+TraceTruffleCompilation"
        args: "--mate"
        
    TruffleSOM-graal:
        path: .
        binary: som
        #args: " -G:+TruffleSplittingNew -G:-TraceTruffleInlining -G:+TraceTruffleCompilation"
        args: ""

    TruffleMate-interpreter:
        path: .
        binary: som
        args: " -G:+TruffleSplittingNew -G:-TraceTruffleInlining -G:+TraceTruffleCompilation"
        args: "-G --mate"
    
        
# define the benchmarks to be executed for a re-executable benchmark run
experiments:
    # The results of this experiment are shown in the paper in Figure 5.
    Mate-Standard:
        actions: benchmark
        benchmark:
            - macro-steady
#            - micro-steady
        executions:
            - TruffleMate-graal
            - TruffleSOM-graal
            - TruffleMate-interpreter
    
    # The results of this experiment are shown in the paper in ...            
    MOP-Operations:
        actions: benchmark
        benchmark:
            - mop-individual-activation
        executions:
            - TruffleMate-graal