Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Yet another btr update #4507

Merged
merged 5 commits into from
Oct 30, 2023
Merged

Yet another btr update #4507

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
d05da07
Moves skipping message to debug
jasonb5 Oct 27, 2023
a66c75e
Revert "Updates MEMCOMP and TPUTCOMP to DIFF if they're checked and f…
jasonb5 Oct 27, 2023
c6e3442
Updates performance bless to be separate from hist/nml
jasonb5 Oct 27, 2023
2f6d7c1
Updates MEMCOMP and TPUTCOMP to DIFF if they're checked and fail
jasonb5 Sep 20, 2023
9227c53
Updates bless perf flags
jasonb5 Oct 30, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions CIME/Tools/bless_test_results
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -116,12 +116,22 @@ def create_bless_options(parser):
"--hist-only", action="store_true", help="Only analyze history files."
)

mutual_bless_group.add_argument(
"--tput-only", action="store_true", help="Only analyze throughput."
mutual_perf_group = bless_group.add_mutually_exclusive_group()

mutual_perf_group.add_argument(
"--bless-tput",
action="store_true",
help="Bless throughput, use `--bless-perf` to bless throughput and memory",
)

mutual_bless_group.add_argument(
"--mem-only", action="store_true", help="Only analyze memory."
mutual_perf_group.add_argument(
"--bless-mem",
action="store_true",
help="Bless memory, use `--bless-perf` to bless throughput and memory",
)

bless_group.add_argument(
"--bless-perf", action="store_true", help="Bless both throughput and memory"
)


Expand Down
4 changes: 4 additions & 0 deletions CIME/baselines/performance.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -129,6 +129,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True):

write_baseline_file(baseline_file, tput)

logger.info("Updated throughput baseline to {!s}".format(tput))

if memory:
try:
mem = perf_get_memory(case, config)
Expand All @@ -139,6 +141,8 @@ def perf_write_baseline(case, basegen_dir, throughput=True, memory=True):

write_baseline_file(baseline_file, mem)

logger.info("Updated memory usage baseline to {!s}".format(mem))


def load_coupler_customization(case):
"""
Expand Down
56 changes: 23 additions & 33 deletions CIME/bless_test_results.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
logger = logging.getLogger(__name__)


def bless_throughput(
def _bless_throughput(
case,
test_name,
baseline_root,
Expand All @@ -32,6 +32,7 @@ def bless_throughput(
):
success = True
reason = None
below_threshold = False

baseline_dir = os.path.join(
baseline_root, baseline_name, case.get_value("CASEBASEID")
Expand All @@ -42,20 +43,10 @@ def bless_throughput(
case, baseline_dir=baseline_dir
)
except FileNotFoundError as e:
success = False

comment = f"Could not read throughput file: {e!s}"
except Exception as e:
success = False

comment = f"Error comparing throughput baseline: {e!s}"

# fail early
if not success:
logger.info(comment)

return success, comment

if below_threshold:
logger.info("Throughput diff appears to have been already resolved.")
else:
Expand All @@ -74,7 +65,7 @@ def bless_throughput(
return success, reason


def bless_memory(
def _bless_memory(
case,
test_name,
baseline_root,
Expand All @@ -84,6 +75,7 @@ def bless_memory(
):
success = True
reason = None
below_threshold = False

baseline_dir = os.path.join(
baseline_root, baseline_name, case.get_value("CASEBASEID")
Expand All @@ -94,20 +86,10 @@ def bless_memory(
case, baseline_dir=baseline_dir
)
except FileNotFoundError as e:
success = False

comment = f"Could not read memory usage file: {e!s}"
except Exception as e:
success = False

comment = f"Error comparing memory baseline: {e!s}"

# fail early
if not success:
logger.info(comment)

return success, comment

if below_threshold:
logger.info("Memory usage diff appears to have been already resolved.")
else:
Expand Down Expand Up @@ -221,8 +203,6 @@ def bless_test_results(
test_id=None,
namelists_only=False,
hist_only=False,
tput_only=False,
mem_only=False,
report_only=False,
force=False,
pes_file=None,
Expand All @@ -231,9 +211,12 @@ def bless_test_results(
new_test_root=None,
new_test_id=None,
exclude=None,
bless_tput=False,
bless_mem=False,
bless_perf=False,
**_, # Capture all for extra
):
bless_all = not (namelists_only | hist_only | tput_only | mem_only)
bless_all = not (namelists_only | hist_only)

test_status_files = get_test_status_files(test_root, compiler, test_id=test_id)

Expand Down Expand Up @@ -294,15 +277,15 @@ def bless_test_results(
excluded = exclude.match(test_name) if exclude else False

if (not has_no_tests and not match_test_name) or excluded:
logger.info("Skipping {!r}".format(test_name))
logger.debug("Skipping {!r}".format(test_name))

continue

overall_result, phase = ts.get_overall_test_status(
ignore_namelists=True,
ignore_memleak=True,
check_throughput=True,
check_memory=True,
check_throughput=False,
check_memory=False,
)

# See if we need to bless namelist
Expand All @@ -326,13 +309,13 @@ def bless_test_results(
if hist_only or bless_all:
hist_bless = bless_needed

if tput_only or bless_all:
if bless_tput or bless_perf:
tput_bless = bless_needed

if not tput_bless:
tput_bless = ts.get_status(THROUGHPUT_PHASE) != TEST_PASS_STATUS

if mem_only or bless_all:
if bless_mem or bless_perf:
mem_bless = bless_needed

if not mem_bless:
Expand All @@ -346,6 +329,12 @@ def bless_test_results(
)
)
else:
logger.debug("Determined blesses for {!r}".format(test_name))
logger.debug("nl_bless = {}".format(nl_bless))
logger.debug("hist_bless = {}".format(hist_bless))
logger.debug("tput_bless = {}".format(tput_bless))
logger.debug("mem_bless = {}".format(mem_bless))

logger.info(
"###############################################################################"
)
Expand All @@ -366,8 +355,9 @@ def bless_test_results(
if baseline_name is None:
baseline_name_resolved = case.get_value("BASELINE_NAME_CMP")
if not baseline_name_resolved:
cime_root = CIME.utils.get_cime_root()
baseline_name_resolved = CIME.utils.get_current_branch(
repo=CIME.utils.get_cime_root()
repo=cime_root
)
else:
baseline_name_resolved = baseline_name
Expand Down Expand Up @@ -423,7 +413,7 @@ def bless_test_results(
broken_blesses.append((test_name, reason))

if tput_bless:
success, reason = bless_throughput(
success, reason = _bless_throughput(
case,
test_name,
baseline_root_resolved,
Expand All @@ -436,7 +426,7 @@ def bless_test_results(
broken_blesses.append((test_name, reason))

if mem_bless:
success, reason = bless_memory(
success, reason = _bless_memory(
case,
test_name,
baseline_root_resolved,
Expand Down
Loading
Loading