diff --git a/README.md b/README.md index ff85b17..c5ad945 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,16 @@ # GobExec -GobExec – because BenchExec isn't enough +GobExec – because BenchExec is not enough + +## Installing +1. Ensure that you are using [Python 3.10](https://www.python.org/downloads/release/python-3100/) or higher. +2. Install [Goblint](https://github.com/goblint/analyzer#installing). +3. Install [Goblint benchmark suite](https://github.com/goblint/bench#readme). + +## Running + +To ensure that you have installed everything correctly, run ``test.py`` within the GobExec directory +``` + python3 test.py +``` + +The generated output file ``GobExec/out.html`` can be viewed via your browser of choice diff --git a/gobexec/goblint/bench/yamlindex.py b/gobexec/goblint/bench/yamlindex.py index c79e15c..717d748 100644 --- a/gobexec/goblint/bench/yamlindex.py +++ b/gobexec/goblint/bench/yamlindex.py @@ -1,11 +1,57 @@ +import shlex from dataclasses import dataclass from pathlib import Path -from typing import Optional, List, Dict, Any + +from typing import Optional, List, Dict, Any, Protocol import yaml +from gobexec.goblint.tool import ARGS_TOOL_KEY +from gobexec.model.benchmark import Incremental, Group +from gobexec.model.scenario import Matrix +from gobexec.model.tool import Tool, R + + # TODO: inline and remove all dataclasses, construct Matrix directly +class ToolFactory(Protocol): + def __call__(self, name: str, args: List[str]) -> Tool[Incremental, R]: + ... + + +def load(def_path: Path, set_path: List[Path], tool_factory: ToolFactory) -> Matrix[Incremental, R]: + defsets_ = DefSets.from_paths(def_path, set_path) + groups: List[Group[Incremental]] = [] + tools: [Tool[Incremental, R]] = [] + for set_ in defsets_.sets: + groups.append(Group(name=set_.name, benchmarks=[])) + for bench in set_.benchmarks: + patch_path = Path("../bench") / bench.path.with_suffix("." + "patch") + parts = list(patch_path.parts) + temp = parts[3].split(".") + parts[3] = temp[0] + "01." + temp[1] + patch_path = Path(*parts) + groups[-1].benchmarks.append(Incremental( + name=bench.name, + description=bench.info, + files=Path("../bench") / bench.path, + patch=patch_path, + tool_data={ + ARGS_TOOL_KEY: shlex.split(bench.param) if bench.param else [] + })) + for conf in defsets_.def_.confs[1:]: + if conf.param is not None: + tools.append(tool_factory(name=conf.name,args= ["--conf", (Path("../bench") / Path(defsets_.def_.baseconf)).absolute()] + shlex.split(conf.param) )) + else: + tools.append(tool_factory(name=conf.name, args=["--conf", (Path("../bench") / Path(defsets_.def_.baseconf)).absolute()])) + + return Matrix( + name=def_path.stem, + tools=tools, + groups=groups, + ) + + @dataclass class Benchmark: name: str diff --git a/gobexec/goblint/extractor.py b/gobexec/goblint/extractor.py index c76c690..6509ed0 100644 --- a/gobexec/goblint/extractor.py +++ b/gobexec/goblint/extractor.py @@ -3,7 +3,7 @@ from typing import Any, Optional from gobexec.goblint.bench.tools import AssertCount -from gobexec.goblint.result import AssertSummary +from gobexec.goblint.result import AssertSummary, LineSummary from gobexec.model.context import ExecutionContext, CompletedSubprocess from gobexec.model.tool import Tool from gobexec.model.tools import ResultExtractor @@ -25,4 +25,4 @@ async def extract(self, ec: ExecutionContext[Any], cp: CompletedSubprocess) -> A unreachable = total - (success + warning + error) else: unreachable = None - return AssertSummary(success, warning, error, unreachable) + return AssertSummary(success, warning, error, unreachable) \ No newline at end of file diff --git a/gobexec/goblint/result.py b/gobexec/goblint/result.py index fb3c82a..4ef8405 100644 --- a/gobexec/goblint/result.py +++ b/gobexec/goblint/result.py @@ -12,6 +12,7 @@ class RaceSummary(Result): safe: int vulnerable: int unsafe: int + # total: int def __init__(self, @@ -91,3 +92,163 @@ def template(self, env): @staticmethod async def extract(ec: ExecutionContext[Any], cp: CompletedSubprocess) -> 'Rusage': return Rusage(cp.rusage) + + +@dataclass(init=False) +class LineSummary(Result): + live: int + dead: int + total: int + + def __init__(self, live: int, dead: int, total: int): + self.live = live + self.dead = dead + self.total = total + + def template(self, env): + return env.get_template("linesummary.jinja") + + @staticmethod + async def extract(ec: ExecutionContext[Any], cp: CompletedSubprocess) -> 'LineSummary': + stdout = cp.stdout.decode("utf-8") + live = re.search(r"live:[ ]*([0-9]*)", stdout) + dead = re.search(r"dead:[ ]*([0-9]*)", stdout) + if live is None and dead is None: + return LineSummary(-1, -1, -1) + else: + total = int(live.group(1)) + int(dead.group(1)) + return LineSummary(int(live.group(1)), int(dead.group(1)), total) + + +@dataclass(init=False) +class ThreadSummary(Result): + thread_id: int + unique_thread_id: int + max_protected: int + sum_protected: int + mutexes_count: int + + def __init__(self, thread_id: int, unique_thread_id: int, + max_protected: int, sum_protected: int, mutexes_count: int): + self.thread_id = thread_id + self.unique_thread_id = unique_thread_id + self.max_protected = max_protected + self.sum_protected = sum_protected + self.mutexes_count = mutexes_count + + def template(self, env): + return env.get_template("threadsummary.jinja") + + @staticmethod + async def extract(ec: ExecutionContext[Any], cp: CompletedSubprocess) -> 'ThreadSummary': + stdout = cp.stdout.decode("utf-8") + thread_id = re.search(r"Encountered number of thread IDs \(unique\): ([0-9]*)", stdout) + thread_id = 0 if thread_id is None else int(thread_id.group(1)) + unique_thread_id = re.search(r"Encountered number of thread IDs \(unique\): [0-9]* \(([0-9]*)\)", stdout) + unique_thread_id = 0 if unique_thread_id is None else int(unique_thread_id.group(1)) + max_protected = re.search(r"Max number variables of protected by a mutex: ([0-9]*)", stdout) + max_protected = 0 if max_protected is None else int(max_protected.group(1)) + sum_protected = re.search(r"Total number of protected variables \(including duplicates\): ([0-9]*)", stdout) + sum_protected = 0 if sum_protected is None else int(sum_protected.group(1)) + mutexes_count = re.search(r"Number of mutexes: ([0-9]*)", stdout) + mutexes_count = 0 if mutexes_count is None else int(mutexes_count.group(1)) + + return ThreadSummary(thread_id, unique_thread_id, max_protected, sum_protected, mutexes_count) + + +@dataclass(init=False) +class AssertTypeSummary(Result): + success: int + unknown: int + + def __init__(self, success: int, unknown: int): + self.success = success + self.unknown = unknown + + def template(self, env): + return env.get_template("asserttypesummary.jinja") + + @staticmethod + async def extract(ec: ExecutionContext[Any], cp: CompletedSubprocess) -> 'AssertTypeSummary': + stdout = cp.stdout.decode("utf-8") + success = len(re.findall(r"\[Success\]\[Assert\]", stdout)) + + unknown = len(re.findall(r"\[Warning\]\[Assert\]", stdout)) + return AssertTypeSummary(success, unknown) + + +@dataclass(init=False) +class YamlSummary(Result): + confirmed: int + unconfirmed: int + + def __init__(self, success: int, unknown: int): + self.success = success + self.unknown = unknown + + def template(self, env): + return env.get_template("yamlsummary.jinja") + + @staticmethod + async def extract(ec: ExecutionContext[Any], cp: CompletedSubprocess) -> 'AssertTypeSummary': + stdout = cp.stdout.decode("utf-8") + confirmed = re.search(r" confirmed:[ ]*([0-9]*)", stdout) + confirmed = 0 if confirmed is None else confirmed.group(1) + unconfirmed = re.search(r" unconfirmed:[ ]*([0-9]*)", stdout) + unconfirmed = 0 if unconfirmed is None else unconfirmed.group(1) + return AssertTypeSummary(int(confirmed), int(unconfirmed)) + + +dataclass(init=False) + + +class ConcratSummary(Result): + safe: int + vulnerable: int + unsafe: int + uncalled: int + + def __init__(self, safe: int, vulnerable: int, unsafe: int, uncalled: int) -> None: + self.safe = safe + self.vulnerable = vulnerable + self.unsafe = unsafe + self.uncalled = uncalled + + def template(self, env): + return env.get_template("concratresult.jinja") + + @staticmethod + async def extract(ec: ExecutionContext[Any], cp: CompletedSubprocess) -> "ConcratSummary": + stdout = cp.stdout.decode("utf-8") + safe = re.search(r"[^n]safe:[ ]*([0-9]*)", stdout) + safe = -1 if safe is None else safe.group(1) + vulnerable = re.search(r"vulnerable:[ ]*([0-9]*)", stdout) + vulnerable = -1 if vulnerable is None else vulnerable.group(1) + unsafe = re.search(r"unsafe:[ ]*([0-9]*)", stdout) + unsafe = -1 if unsafe is None else unsafe.group(1) + uncalled = re.findall(r"will never be called", stdout) + for elem in uncalled: + if bool(re.search(r"__check", elem)): + continue + else: + uncalled.remove(elem) + return ConcratSummary(safe, vulnerable, unsafe, len(uncalled)) + + +@dataclass(init=False) +class IncrementalSummary(Result): + vars: int + evals: int + + def __init__(self, vars: int, evals: int): + self.vars = vars + self.evals = evals + + def template(self, env): + return env.get_template("incremental.jinja") + + @staticmethod + async def extract(ec: ExecutionContext, cp: CompletedSubprocess): + stdout = cp.stdout.decode("utf-8") + data = re.search(r"\[\w+]\s\w+.{3}(\d+)\s+\w+.{3}(\d+)\s+\w+.{3}(\d+)", stdout) + return IncrementalSummary(data.group(1), data.group(2)) diff --git a/gobexec/goblint/tool.py b/gobexec/goblint/tool.py index bc676ce..1bc0ef3 100644 --- a/gobexec/goblint/tool.py +++ b/gobexec/goblint/tool.py @@ -1,29 +1,130 @@ import asyncio import tempfile -from typing import List - -from gobexec.model.benchmark import Single +from pathlib import Path +from typing import List, Optional +import shutil +import os +from glob import glob +from gobexec.model.benchmark import Single, Incremental from gobexec.model.context import ExecutionContext, CompletedSubprocess +from gobexec.model.result import PrivPrecResult, ApronPrecResult from gobexec.model.tool import Tool ARGS_TOOL_KEY = "goblint-args" +class GoblintToolFromScratch(Tool[Incremental, CompletedSubprocess]): + name: str + program: str + args: List[str] + + def __init__(self, + name: str = "Goblint", + program: str = "goblint", + args: List[str] = None, + ) -> None: + self.name = name + self.program = program + self.args = args if args else [] + + async def run_async(self, ec: ExecutionContext[Incremental], benchmark: Incremental) -> CompletedSubprocess: + data_path = ec.get_tool_data_path(self) + goblint_dir = data_path + goblint_dir.mkdir(parents=True, exist_ok=True) + shutil.copy(benchmark.files, data_path) + with (data_path / "out.txt").open("w+b") as out_file: + args = [self.program] + \ + ["--set", "goblint-dir", goblint_dir.absolute()] + \ + ["--conf", str(Path("../bench/index/conf/td3.json").absolute()), "--enable","incremental.save", "--set", "incremental.save-dir", + goblint_dir.absolute(), "-v", str(Path(data_path/benchmark.files.name).absolute())] + \ + self.args + \ + benchmark.tool_data.get(ARGS_TOOL_KEY, []) + + cp = await ec.subprocess_exec( + args[0], + *args[1:], + # capture_output=True, + stdout=out_file, + stderr=asyncio.subprocess.STDOUT, + cwd=benchmark.files.parent + ) + out_file.seek(0) + cp.stdout = out_file.read() # currently for extractors + + await ec.subprocess_exec("patch", str(Path(data_path/benchmark.files.name).absolute()), benchmark.patch) + return cp + + +class GoblintToolIncremental(Tool[Incremental, CompletedSubprocess]): + name: str + program: str + args: List[str] + from_scratch: Optional[GoblintToolFromScratch] + + def __init__(self, + name: str = "Goblint", + program: str = "goblint", + args: List[str] = None, + from_scratch: Optional[GoblintToolFromScratch] = None + ) -> None: + self.name = name + self.program = program + self.args = args if args else [] + self.from_scratch = from_scratch + + async def run_async(self, ec: ExecutionContext[Incremental], benchmark: Incremental) -> CompletedSubprocess: + await ec.get_tool_result(self.from_scratch) + data_path = ec.get_tool_data_path(self) + goblint_dir = data_path + goblint_dir.mkdir(parents=True, exist_ok=True) + + with (data_path / "out.txt").open("w+b") as out_file: + args = [self.program] + \ + ["--set", "goblint-dir", goblint_dir.absolute()] + \ + ["--conf", "../bench/index/conf/td3.json", "--enable","incremental.load", "--set", "incremental.load-dir", + str(Path(ec.get_tool_data_path(self.from_scratch)).absolute()), "-v", str(Path(ec.get_tool_data_path(self.from_scratch)/benchmark.files.name).absolute())] + \ + self.args + \ + benchmark.tool_data.get(ARGS_TOOL_KEY, []) + + cp = await ec.subprocess_exec( + args[0], + *args[1:], + # capture_output=True, + stdout=out_file, + stderr=asyncio.subprocess.STDOUT, + cwd=benchmark.files.parent + ) + out_file.seek(0) + cp.stdout = out_file.read() # currently for extractors + + return cp + + class GoblintTool(Tool[Single, CompletedSubprocess]): name: str program: str args: List[str] + dump: str + validate: Optional["GoblintTool"] + assertion: Optional["GoblintTool"] def __init__(self, name: str = "Goblint", program: str = "goblint", - args: List[str] = None + args: List[str] = None, + dump: str = '', + validate: Optional["GoblintTool"] = None, + assertion: Optional["GoblintTool"] = None ) -> None: self.name = name self.program = program self.args = args if args else [] + self.dump = dump + self.validate = validate + self.assertion = assertion + + # def run(self, benchmark: Single) -> str: - # def run(self, benchmark: Single) -> str: # bench = Path("/home/simmo/dev/goblint/sv-comp/goblint-bench") # args = ["/home/simmo/dev/goblint/sv-comp/goblint/goblint"] + self.args + benchmark.tool_data.get(ARGS_TOOL_KEY, []) + [str(bench / file) for file in benchmark.files] # print(args) @@ -43,16 +144,92 @@ async def run_async(self, ec: ExecutionContext[Single], benchmark: Single) -> Co args = [self.program] + \ ["--set", "goblint-dir", goblint_dir.absolute()] + \ self.args + \ - benchmark.tool_data.get(ARGS_TOOL_KEY, []) + \ - [str(file) for file in benchmark.files] - # print(args) + benchmark.tool_data.get(ARGS_TOOL_KEY, []) + if self.dump == "priv": + args += ["--set", "exp.priv-prec-dump", data_path.absolute() / "priv.txt"] + elif self.dump == "apron": + args += ["--set", "exp.relation.prec-dump", data_path.absolute() / "apron.txt"] + elif self.dump == "witness": + args += ["--set", "witness.yaml.path", data_path.absolute() / "witness.yaml"] + elif self.dump == "assert": + args += ["--set", "trans.output", data_path.absolute() / "out.c"] + if self.validate is not None: + await ec.get_tool_result(self.validate) + args += ["--set", "witness.yaml.validate", + ec.get_tool_data_path(self.validate).absolute() / "witness.yaml"] + if self.assertion is None: + args += [str(file) for file in benchmark.files] + if self.assertion is not None: + args += [ec.get_tool_data_path(self.assertion).absolute() / "out.c"] + args += ["--enable", "ana.sv-comp.functions"] + await ec.get_tool_result(self.assertion) + cp = await ec.subprocess_exec( args[0], *args[1:], # capture_output=True, stdout=out_file, - stderr=asyncio.subprocess.STDOUT + stderr=asyncio.subprocess.STDOUT, + cwd=benchmark.files[0].parent ) out_file.seek(0) cp.stdout = out_file.read() # currently for extractors return cp + + +class PrivPrecTool(Tool[Single, PrivPrecResult]): + name: str + program: str + args: List[GoblintTool] + + def __init__(self, + name: str = "privPrecCompare", + program: str = "../analyzer/privPrecCompare", + args: List[GoblintTool] = None, + ) -> None: + self.name = name + self.program = program + self.args = args if args else [] + + async def run_async(self, ec: ExecutionContext[Single], benchmark: Single) -> PrivPrecResult: + path = ec.get_tool_data_path(self) + for tool in self.args: + await ec.get_tool_result(tool) + with(path / 'priv_compare_out.txt').open("w") as out_file: + args = [self.program] + [str(ec.get_tool_data_path(tool) / "priv.txt") for tool in self.args] + await ec.subprocess_exec( + args[0], + stdout=out_file, + stderr=asyncio.subprocess.STDOUT, + + ) + return PrivPrecResult(str(path / 'out.txt')) + + +class ApronPrecTool(Tool[Single, ApronPrecResult]): + name: str + program: str + args: List[GoblintTool] + + def __init__(self, + name: str = "apronPrecCompare", + program: str = "../analyzer/apronPrecCompare", + args: List[GoblintTool] = None, + ) -> None: + self.name = name + self.program = program + self.args = args + + async def run_async(self, ec: ExecutionContext[Single], benchmark: Single) -> ApronPrecResult: + path = ec.get_tool_data_path(self) + for tool in self.args: + await ec.get_tool_result(tool) + with (path / 'out.txt').open('w') as out_file: + args = [self.program] + [str(ec.get_tool_data_path(tool).absolute() / 'apron.txt') for tool in self.args] + await ec.subprocess_exec( + args[0], + *args[1:], + stdout=out_file, + stderr=asyncio.subprocess.STDOUT, + ) + return ApronPrecResult(str(path / 'out.txt')) diff --git a/gobexec/model/benchmark.py b/gobexec/model/benchmark.py index 339789c..b14371e 100644 --- a/gobexec/model/benchmark.py +++ b/gobexec/model/benchmark.py @@ -13,6 +13,15 @@ class Single: tool_data: Dict[str, Any] +@dataclass +class Incremental: + name: str + description: str + files: Path + patch: Path + tool_data: Dict[str, Any] + + @dataclass class Group(Generic[B]): name: str diff --git a/gobexec/model/result.py b/gobexec/model/result.py index d670e9a..19a8d96 100644 --- a/gobexec/model/result.py +++ b/gobexec/model/result.py @@ -99,3 +99,25 @@ def kind(self): return self.primary.kind else: return ResultKind.DEFAULT + +@dataclass(init=False) +class PrivPrecResult(Result): + result_path: str + + def __init__(self, result_path: str) -> None: + self.result_path = result_path + + def template(self, env: Environment) -> Template: + return env.get_template('privprecresult.jinja') + + +@dataclass(init=False) +class ApronPrecResult(Result): + result_path: str + + def __init__(self,result_path: str) -> None: + self.result_path = result_path + + def template(self, env): + return env.get_template('apronprecresult.jinja') + diff --git a/gobexec/model/tools.py b/gobexec/model/tools.py index 269c6f4..79378b6 100644 --- a/gobexec/model/tools.py +++ b/gobexec/model/tools.py @@ -32,7 +32,7 @@ def name(self): return self.delegate.name async def run_async(self, ec: ExecutionContext[B], benchmark: B) -> R: - cp = await self.delegate.run_async(ec, benchmark) + cp = await ec.get_tool_result(self.delegate) results = await asyncio.gather(*[extractor.extract(ec, cp) for extractor in self.extractors]) if self.primary: primary_result = results[self.extractors.index(self.primary)] # TODO: something better than index-based diff --git a/gobexec/output/html/templates/apronprecresult.jinja b/gobexec/output/html/templates/apronprecresult.jinja new file mode 100644 index 0000000..7b0d2c6 --- /dev/null +++ b/gobexec/output/html/templates/apronprecresult.jinja @@ -0,0 +1 @@ +compare \ No newline at end of file diff --git a/gobexec/output/html/templates/asserttypesummary.jinja b/gobexec/output/html/templates/asserttypesummary.jinja new file mode 100644 index 0000000..b67b839 --- /dev/null +++ b/gobexec/output/html/templates/asserttypesummary.jinja @@ -0,0 +1,2 @@ +{{ this.success }}, +{{ this.unknown }} \ No newline at end of file diff --git a/gobexec/output/html/templates/concratresult.jinja b/gobexec/output/html/templates/concratresult.jinja new file mode 100644 index 0000000..853f726 --- /dev/null +++ b/gobexec/output/html/templates/concratresult.jinja @@ -0,0 +1,4 @@ +{{this.safe}} +{{this.vulnerable}} +{{this.unsafe}} +{{this.uncalled}} \ No newline at end of file diff --git a/gobexec/output/html/templates/incremental.jinja b/gobexec/output/html/templates/incremental.jinja new file mode 100644 index 0000000..869cdf8 --- /dev/null +++ b/gobexec/output/html/templates/incremental.jinja @@ -0,0 +1,2 @@ +{{this.vars}} +{{this.evals}} \ No newline at end of file diff --git a/gobexec/output/html/templates/linesummary.jinja b/gobexec/output/html/templates/linesummary.jinja new file mode 100644 index 0000000..331ae04 --- /dev/null +++ b/gobexec/output/html/templates/linesummary.jinja @@ -0,0 +1,3 @@ +{{this.live}} +{{this.dead}} +{{this.total}} diff --git a/gobexec/output/html/templates/matrix.jinja b/gobexec/output/html/templates/matrix.jinja index 1be603b..fe077c0 100644 --- a/gobexec/output/html/templates/matrix.jinja +++ b/gobexec/output/html/templates/matrix.jinja @@ -1,13 +1,14 @@ {% set result = this %}
Group | Benchmark | {% for tool in result.tools %} -{{ tool.name }} | +{{ tool.name }} | {% endfor %}{{ group_result.group.name }} | {% endif %} -{{ benchmark_results.benchmark.name }} | +{{ benchmark_results.benchmark.name }} | {% for benchmark_result in benchmark_results.results %} {% if benchmark_result.done() %} - {% if benchmark_result.result().result.kind == ResultKind.SUCCESS %} -- {% elif benchmark_result.result().result.kind == ResultKind.WARNING %} - | - {% elif benchmark_result.result().result.kind == ResultKind.FAILURE %} - | - {% elif benchmark_result.result().result.kind == ResultKind.ERROR %} - | + {% if benchmark_result.result().result.kind == ResultKind.DEFAULT %} + {% if "asserts" in result.tools[0].name%} + | + | + {% endif %} + + {% if loop.last %} + | + {% endif %} + + {% endif %} + {{ benchmark_result.result().result|template }} {% else %} - | - {% endif %} - {{ benchmark_result.result().result|template }} - | - {% else %} -+ {% endif %} {% endfor %} diff --git a/gobexec/output/html/templates/multi.jinja b/gobexec/output/html/templates/multi.jinja index 9d09830..b2bfc86 100644 --- a/gobexec/output/html/templates/multi.jinja +++ b/gobexec/output/html/templates/multi.jinja @@ -1,3 +1,15 @@ {% for result in this.results %} -{{ result|template }} + {% if result.kind == ResultKind.SUCCESS %} + | + {% elif result.kind == ResultKind.WARNING %} + | + {% elif result.kind == ResultKind.FAILURE %} + | + {% elif result.kind == ResultKind.ERROR %} + | + {% else %} + | + {% endif %} + {{ result|template }} + | {% endfor %} \ No newline at end of file diff --git a/gobexec/output/html/templates/privprecresult.jinja b/gobexec/output/html/templates/privprecresult.jinja new file mode 100644 index 0000000..7b0d2c6 --- /dev/null +++ b/gobexec/output/html/templates/privprecresult.jinja @@ -0,0 +1 @@ +compare \ No newline at end of file diff --git a/gobexec/output/html/templates/progress.jinja b/gobexec/output/html/templates/progress.jinja index ed9b70d..f479d14 100644 --- a/gobexec/output/html/templates/progress.jinja +++ b/gobexec/output/html/templates/progress.jinja @@ -2,7 +2,7 @@ {% block title %}Results ({{ progress.done }}/{{ progress.total }}){% endblock %} {% block head %} {{ super() }} - + {##} {% endblock %} {% block body %}
---|