gem5-users@gem5.org

The gem5 Users mailing list

View all threads

Take and Restore checkpoints in FS mode

유예신
Fri, Apr 12, 2024 7:53 AM

We are currently implementing GAPS within FS, following the description
provided in 'Enabling Design Space Exploration of DRAM Caches in Emerging
Memory Systems.' We aim to replicate the approach depicted in Figure 3 of
the paper.

       "Beginning at the ROI, we simulate the workload for 100ms to

warm up the system including the DRAM cache. At the end of 100ms, we take a
checkpoint. This process is done once per workload. Later, we restore from
the checkpoint to run all our simulations with different DRAM cache
configurations."

So, the checkpoint extraction was performed as follows, and we have
confirmed the generation of the 'm5.cpt' file.

This is part of the Python code


max_ticks = 10**14

simulator.run(max_ticks=max_ticks)

end_tick = m5.curTick()

Since we simulated the ROI in details, therefore, simulation is over at

this

point.

Simulation is over at this point. We acknowledge that all the simulation

events were successful.

print("All simulation events were successful.")

We print the final simulation statistics.

print("Done with the simulation")

print("Taking a checkpoint at", args.checkpoint_path)

simulator.save_checkpoint(args.checkpoint_path)

print("Done taking a checkpoint")


1) I'm curious about how to restore the 'm5.cpt' file. I attempted the
following, but it didn't work.

2) If you have an example of using set_kernel_disk_workload while
taking/restoring a checkpoint, could you please share it?

3) Is it necessary to use a KVM CPU to extract a checkpoint in FS mode?
Can't it be done on an OOO CPU?

This is the error log.


build/X86/sim/simulate.cc:192: info: Entering event queue @ 0.  Starting
simulation...

gem5.opt: build/X86/sim/eventq.hh:766: void
gem5::EventQueue::schedule(gem5::Event*, gem5::Tick, bool): Assertion `when

= getCurTick()' failed.


This is the Python code.


#./build/X86/gem5.opt configs/example/gem5_library/x86-gabps-benchmarks.py
--benchmark bc --synthetic 1 --size 4

import argparse

import time

import sys

import m5

from m5.objects import Root

from gem5.utils.requires import requires

from gem5.components.boards.x86_board import X86Board

from gem5.components.memory import DualChannelDDR4_2400

from gem5.components.processors.simple_switchable_processor import (

SimpleSwitchableProcessor,

)

from gem5.components.processors.cpu_types import CPUTypes

from gem5.isas import ISA

from gem5.coherence_protocol import CoherenceProtocol

from gem5.resources.resource import Resource, CustomDiskImageResource,
CustomResource

from gem5.simulate.simulator import Simulator

from gem5.simulate.exit_event import ExitEvent

requires(

isa_required=ISA.X86,

coherence_protocol_required=CoherenceProtocol.MESI_TWO_LEVEL,

kvm_required=True,

)

benchmark_choices = ["cc", "bc", "tc", "pr", "bfs"]

synthetic_choices = ["0", "1"]

size_choices = [

"1",

"2",

"3",

"4",

"5",

"6",

"7",

"8",

"9",

"10",

"11",

"12",

"13",

"14",

"15",

"16",

"USA-road-d.NY.gr",

]

parser = argparse.ArgumentParser(

description="An example configuration script to run the gapbs

benchmarks."

)

parser.add_argument(

"--benchmark",

type=str,

required=True,

help="Input the benchmark program to execute.",

choices=benchmark_choices,

)

parser.add_argument(

"--synthetic",

type=str,

required=True,

help="Synthetic Graph:: 1: synthetic graph is True; 0: real graph",

choices=synthetic_choices,

)

parser.add_argument(

"--size",

type=str,

required=True,

help="Graph Size:: If synthetic is True, then specify a size [1 .. 15].

\

Otherwise, specify a graph name [USA-road-d.NY.gr]",

choices=size_choices,

)

parser.add_argument(

"--ticks",

type=int,

help="Optionally put the maximum number of ticks to execute during the "

"ROI. It accepts an integer value.",

)

parser.add_argument(

"--checkpoint-path",

type=str,

required=False,

default="riscv-hello-checkpoint/",

help="The directory to store the checkpoint.",

)

args = parser.parse_args()

from gem5.components.cachehierarchies.ruby.mesi_two_level_cache_hierarchy
import (

MESITwoLevelCacheHierarchy,

)

cache_hierarchy = MESITwoLevelCacheHierarchy(

l1d_size="32kB",

l1d_assoc=8,

l1i_size="32kB",

l1i_assoc=8,

l2_size="256kB",

l2_assoc=16,

num_l2_banks=2,

)

memory = DualChannelDDR4_2400(size="3GB")

processor = SimpleSwitchableProcessor(

starting_core_type=CPUTypes.KVM,

switch_core_type=CPUTypes.TIMING,

isa=ISA.X86,

num_cores=2,

)

Here we setup the board. The X86Board allows for Full-System X86

simulations

board = X86Board(

clk_freq="3GHz",

processor=processor,

memory=memory,

cache_hierarchy=cache_hierarchy,

)

if args.synthetic == "1":

if args.size == "USA-road-d.NY.gr":

    print(

        "fatal: cannot use a real graph with --synthetic 1",

        file=sys.stderr,

    )

    exit(-1)



command = "./{} -n 1 -u {}\n".format(args.benchmark, args.size)

else:

command = "./{} -sf ../{}".format(args.benchmark, args.size)

board.set_kernel_disk_workload(

# The x86 linux kernel will be automatically downloaded to the

# `~/.cache/gem5` directory if not already present.

# gapbs benchamarks was tested with kernel version 4.19.83

kernel=Resource("x86-linux-kernel-4.19.83"),

# The x86-gapbs image will be automatically downloaded to the

# `~/.cache/gem5` directory if not already present.

disk_image=Resource("x86-gapbs"),

readfile_contents=command,

checkpoint=CustomResource("/home/test/riscv-hello-checkpoint"),

)

def handle_workbegin():

print("Done booting Linux")

print("Resetting stats at the start of ROI!")

m5.stats.reset()

global start_tick

start_tick = m5.curTick()

# processor.switch()

# if args.ticks:

#     m5.scheduleTickExitFromCurrent(args.ticks)

yield False  # E.g., continue the simulation.

def handle_workend():

print("Dump stats at the end of the ROI!")

m5.stats.dump()

yield True  # Stop the simulation. We're done.

simulator = Simulator(

board=board,

)

We maintain the wall clock time.

globalStart = time.time()

print("Running the simulation")

print("Using KVM cpu")

simulator.run()

end_tick = m5.curTick()

print("All simulation events were successful.")

We print the final simulation statistics.

print("Done with the simulation")

print("Taking a checkpoint at", args.checkpoint_path)

simulator.save_checkpoint(args.checkpoint_path)

print("Done taking a checkpoint")

print()

print("Performance statistics:")

print("Simulated time in ROI: %.2fs" % ((end_tick - start_tick) / 1e12))

print(

"Ran a total of", simulator.get_current_tick() / 1e12, "simulated

seconds"

)

print(

"Total wallclock time: %.2fs, %.2f min"

% (time.time() - globalStart, (time.time() - globalStart) / 60)

)


We are currently implementing GAPS within FS, following the description provided in 'Enabling Design Space Exploration of DRAM Caches in Emerging Memory Systems.' We aim to replicate the approach depicted in Figure 3 of the paper. "Beginning at the ROI, we simulate the workload for 100ms to warm up the system including the DRAM cache. At the end of 100ms, we take a checkpoint. This process is done once per workload. Later, we restore from the checkpoint to run all our simulations with different DRAM cache configurations." So, the checkpoint extraction was performed as follows, and we have confirmed the generation of the 'm5.cpt' file. This is part of the Python code -------------------------------------------- max_ticks = 10**14 simulator.run(max_ticks=max_ticks) end_tick = m5.curTick() # Since we simulated the ROI in details, therefore, simulation is over at this # point. # Simulation is over at this point. We acknowledge that all the simulation # events were successful. print("All simulation events were successful.") # We print the final simulation statistics. print("Done with the simulation") print("Taking a checkpoint at", args.checkpoint_path) simulator.save_checkpoint(args.checkpoint_path) print("Done taking a checkpoint") -------------------------------------------- *1) I'm curious about how to restore the 'm5.cpt' file. I attempted the following, but it didn't work.* *2) If you have an example of using set_kernel_disk_workload while taking/restoring a checkpoint, could you please share it?* *3) Is it necessary to use a KVM CPU to extract a checkpoint in FS mode? Can't it be done on an OOO CPU?* This is the error log. -------------------------------------------- build/X86/sim/simulate.cc:192: info: Entering event queue @ 0. Starting simulation... gem5.opt: build/X86/sim/eventq.hh:766: void gem5::EventQueue::schedule(gem5::Event*, gem5::Tick, bool): Assertion `when >= getCurTick()' failed. -------------------------------------------- This is the Python code. -------------------------------------------- #./build/X86/gem5.opt configs/example/gem5_library/x86-gabps-benchmarks.py --benchmark bc --synthetic 1 --size 4 import argparse import time import sys import m5 from m5.objects import Root from gem5.utils.requires import requires from gem5.components.boards.x86_board import X86Board from gem5.components.memory import DualChannelDDR4_2400 from gem5.components.processors.simple_switchable_processor import ( SimpleSwitchableProcessor, ) from gem5.components.processors.cpu_types import CPUTypes from gem5.isas import ISA from gem5.coherence_protocol import CoherenceProtocol from gem5.resources.resource import Resource, CustomDiskImageResource, CustomResource from gem5.simulate.simulator import Simulator from gem5.simulate.exit_event import ExitEvent requires( isa_required=ISA.X86, coherence_protocol_required=CoherenceProtocol.MESI_TWO_LEVEL, kvm_required=True, ) benchmark_choices = ["cc", "bc", "tc", "pr", "bfs"] synthetic_choices = ["0", "1"] size_choices = [ "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15", "16", "USA-road-d.NY.gr", ] parser = argparse.ArgumentParser( description="An example configuration script to run the gapbs benchmarks." ) parser.add_argument( "--benchmark", type=str, required=True, help="Input the benchmark program to execute.", choices=benchmark_choices, ) parser.add_argument( "--synthetic", type=str, required=True, help="Synthetic Graph:: 1: synthetic graph is True; 0: real graph", choices=synthetic_choices, ) parser.add_argument( "--size", type=str, required=True, help="Graph Size:: If synthetic is True, then specify a size [1 .. 15]. \ Otherwise, specify a graph name [USA-road-d.NY.gr]", choices=size_choices, ) parser.add_argument( "--ticks", type=int, help="Optionally put the maximum number of ticks to execute during the " "ROI. It accepts an integer value.", ) parser.add_argument( "--checkpoint-path", type=str, required=False, default="riscv-hello-checkpoint/", help="The directory to store the checkpoint.", ) args = parser.parse_args() from gem5.components.cachehierarchies.ruby.mesi_two_level_cache_hierarchy import ( MESITwoLevelCacheHierarchy, ) cache_hierarchy = MESITwoLevelCacheHierarchy( l1d_size="32kB", l1d_assoc=8, l1i_size="32kB", l1i_assoc=8, l2_size="256kB", l2_assoc=16, num_l2_banks=2, ) memory = DualChannelDDR4_2400(size="3GB") processor = SimpleSwitchableProcessor( starting_core_type=CPUTypes.KVM, switch_core_type=CPUTypes.TIMING, isa=ISA.X86, num_cores=2, ) # Here we setup the board. The X86Board allows for Full-System X86 simulations board = X86Board( clk_freq="3GHz", processor=processor, memory=memory, cache_hierarchy=cache_hierarchy, ) if args.synthetic == "1": if args.size == "USA-road-d.NY.gr": print( "fatal: cannot use a real graph with --synthetic 1", file=sys.stderr, ) exit(-1) command = "./{} -n 1 -u {}\n".format(args.benchmark, args.size) else: command = "./{} -sf ../{}".format(args.benchmark, args.size) board.set_kernel_disk_workload( # The x86 linux kernel will be automatically downloaded to the # `~/.cache/gem5` directory if not already present. # gapbs benchamarks was tested with kernel version 4.19.83 kernel=Resource("x86-linux-kernel-4.19.83"), # The x86-gapbs image will be automatically downloaded to the # `~/.cache/gem5` directory if not already present. disk_image=Resource("x86-gapbs"), readfile_contents=command, checkpoint=CustomResource("/home/test/riscv-hello-checkpoint"), ) def handle_workbegin(): print("Done booting Linux") print("Resetting stats at the start of ROI!") m5.stats.reset() global start_tick start_tick = m5.curTick() # processor.switch() # if args.ticks: # m5.scheduleTickExitFromCurrent(args.ticks) yield False # E.g., continue the simulation. def handle_workend(): print("Dump stats at the end of the ROI!") m5.stats.dump() yield True # Stop the simulation. We're done. simulator = Simulator( board=board, ) # We maintain the wall clock time. globalStart = time.time() print("Running the simulation") print("Using KVM cpu") simulator.run() end_tick = m5.curTick() print("All simulation events were successful.") # We print the final simulation statistics. print("Done with the simulation") print("Taking a checkpoint at", args.checkpoint_path) simulator.save_checkpoint(args.checkpoint_path) print("Done taking a checkpoint") print() print("Performance statistics:") print("Simulated time in ROI: %.2fs" % ((end_tick - start_tick) / 1e12)) print( "Ran a total of", simulator.get_current_tick() / 1e12, "simulated seconds" ) print( "Total wallclock time: %.2fs, %.2f min" % (time.time() - globalStart, (time.time() - globalStart) / 60) ) --------------------------------------------