- {{ heading_3(a.name) }}
- {% elif a.name == None %}
-
- {% endif %}
- {% if a.type == "table" %}
- {{ table(a.data) }}
- {% elif a.type == "wide_table" %}
- {{ wide_table(a.data) }}
- {% elif a.type == "image_links" %}
- {{ draw_images(a.data) }}
- {% elif a.type == "image_links_horizontal" %}
- {{ draw_images_horizontal(a.data) }}
- {% elif a.type == "string" %}
- {{ string_data(a.data) }}
- {% elif a.type == "para_and_image" %}
-
#}
-{# {% endif %}#}
{% endfor %}
{% endmacro %}
@@ -488,8 +492,14 @@
{{ section_title('Privacy Evaluation') }}
- {% for item in data['privacy'] %}
+ {% for i, item in enumerate(data['privacy']) %}
+ {% if i > 0 %}
+
+
+
+
+ {% endif %}
{{ evaluation(item) }}
{% endfor %}
diff --git a/sdnist/report/score/paragraphs.py b/sdnist/report/score/paragraphs.py
index 8dd7467..3783223 100644
--- a/sdnist/report/score/paragraphs.py
+++ b/sdnist/report/score/paragraphs.py
@@ -43,7 +43,7 @@
"Joshua Snoke and Gillian Raab and
Claire Bowen" \
", all of whom have participated on the NIST Synthetic Data Challenges SME panels."
-k_marg_break_para = "In the metrics above we’ve considered all of the data together; " \
+k_marg_break_para = "In the metrics above we've considered all of the data together; " \
"however we know that algorithms may behave differently on different " \
"subgroups in the population. Below we look in more detail at deidentification " \
"performance just in the worst performing PUMA, based on k-marginal score."
@@ -78,7 +78,39 @@
"If the distribution is centered below 50% that means the deidentified records are very " \
"different from the target records, and the apparent matches are not real matches."
-unique_exact_match_para = "This is a count of unique records in the target data that were exactly reproduced " \
- "in the deidentified data. Because these records were unique outliers in the " \
- "target data, and they still appear unchanged in the deidentified data, " \
- "they are potentially vulnerable to reidentification."
\ No newline at end of file
+unique_exact_match_para_1 = "Unique Exact Match (UEM) is a simple privacy metric that counts the " \
+ "percentage of singleton records in the target that are also present in " \
+ "the deidentified data; these uniquely identifiable individuals leaked " \
+ "through the deidentification process."
+
+unique_exact_match_para_2 = "Below we also include an estimate of the feature space size. The feature " \
+ "space is the set of all possible record values given the selected target " \
+ "data and feature subset. For instance, if we had two features, Hat" \
+ " [cap, bonnet] and Color [green, blue, purple], our feature space would " \
+ "consist of 2 x 3 = 6 possible combinations (e.g. 'green cap', " \
+ "'blue bonnet'). Note that feature spaces are based on the " \
+ "feature set, not on what records actually exist in the data. " \
+ "Purple bonnet is a possible combination in this feature space, " \
+ "but it's likely no one in the hypothetical data owns a purple " \
+ "bonnet (and the count of that record value would be 0)."
+
+unique_exact_match_para_3 = "As we add features to the feature set, we increase the size of the " \
+ "feature space, but we don't change the actual number of records in " \
+ "the data-- it's the same people, but now they're spread out more thinly " \
+ "across a wider set of possible record values. Large feature spaces will " \
+ "disperse populations very sparsely (most possible record values will have " \
+ "count 0 or 1) and as a result the data will contain very many uniquely " \
+ "identifiable records. Intuitively, once you know enough pieces of " \
+ "information about someone, everyone becomes very distinct from everyone else. " \
+ "This can pose a challenge for privacy. "
+
+unique_exact_match_para_4 = "The Target Data Properties below provides an estimate of the feature " \
+ "space size (100 is used for continuous feature), " \
+ "along with the portion of the records in the ground truth " \
+ "target data that are unique (ie, they are the only person " \
+ "with that record value, they have a count of 1). " \
+ "The Deidentified Data Properties reports the percentage " \
+ "of those uniquely identifiable individuals that are still " \
+ "present in the deidentified data. " \
+ "Because they are unique, real records, they are " \
+ "potentially vulnerable to reidentification."
\ No newline at end of file
diff --git a/sdnist/report/score/privacy.py b/sdnist/report/score/privacy.py
index 889b4f1..69279f6 100644
--- a/sdnist/report/score/privacy.py
+++ b/sdnist/report/score/privacy.py
@@ -22,13 +22,23 @@ def privacy_score(dataset: Dataset, ui_data: ReportUIData, report_data, log: Sim
unique_target_records, perc_unique_target_records = \
unique_exact_matches(ds.c_target_data, ds.c_synthetic_data)
perc_t_rec_matched = perc_t_rec_matched
- uem_para_a = Attachment(name=None,
- _data=unique_exact_match_para,
+ uem_para1_a = Attachment(name=None,
+ _data=unique_exact_match_para_1,
+ _type=AttachmentType.String)
+ uem_para2_a = Attachment(name=None,
+ _data=unique_exact_match_para_2,
+ _type=AttachmentType.String)
+ uem_para3_a = Attachment(name=None,
+ _data=unique_exact_match_para_3,
+ _type=AttachmentType.String)
+ uem_para4_a = Attachment(name=None,
+ _data=unique_exact_match_para_4,
_type=AttachmentType.String)
+ feat_space_str = "{:0.3e}".format(ds.feature_space)
target_matched_a = Attachment(name="Target Data Properties",
_data=f"Feature space size (possible combinations): "
- f"-Highlight-{f'{dataset.feature_space:,}'}-Highlight-
"
+ f"-Highlight-{feat_space_str}-Highlight-
"
f"Number of unique records in Target Data: "
f"-Highlight-{unique_target_records} "
f"({perc_unique_target_records}%-Highlight-)",
@@ -41,7 +51,7 @@ def privacy_score(dataset: Dataset, ui_data: ReportUIData, report_data, log: Sim
_type=AttachmentType.String)
r_ui_d.add(PrivacyScorePacket("Unique Exact Matches",
None,
- [uem_para_a,
+ [uem_para1_a, uem_para2_a, uem_para3_a, uem_para4_a,
target_matched_a,
deid_matched_a]))
rd.add('unique_exact_matches', {
diff --git a/sdnist/report/score/utility/__init__.py b/sdnist/report/score/utility/__init__.py
index 7079341..149362c 100644
--- a/sdnist/report/score/utility/__init__.py
+++ b/sdnist/report/score/utility/__init__.py
@@ -125,12 +125,13 @@ def worst_score_breakdown(worst_scores: List,
up = UnivariatePlots(s, t,
ds, out_dir, ds.challenge, worst_univariates_to_display=3)
- u_feature_data = up.save()
- k_marg_break_rd[f'worst_{len(wpf)}_puma_univariate'] = up.report_data()
+ u_feature_data = up.save(level=3)
+ k_marg_break_rd[f'worst_{len(wpf)}_puma_univariate'] = up.report_data(level=3)
k_marg_break_rd[f'worst_{len(wpf)}_puma_k_marginal_scores'] = \
relative_path(save_data_frame(wsh,
out_dir,
- f'worst_{len(wpf)}_puma_k_marginal_scores'))
+ f'worst_{len(wpf)}_puma_k_marginal_scores'),
+ level=2)
u_as = []
u_as.append(Attachment(name=None,
_data=f"h3Univariate Distribution of Worst "
@@ -189,7 +190,7 @@ def worst_score_breakdown(worst_scores: List,
corr_features)
pcd.compute()
pcp = PearsonCorrelationPlot(pcd.pp_corr_diff, out_dir)
- pcp_saved_file_paths = pcp.save()
+ pcp_saved_file_paths = pcp.save(path_level=3)
k_marg_break_rd['correlation_difference'] = {
"pearson_correlation_difference": pcp.report_data
}
@@ -246,16 +247,17 @@ def create_subsample(frac: float):
# mapping of sub sample frac to k-marginal score of fraction
ssample_score = dict() # subsample scores dictionary
- # find k-marginal of 10%, 20% ... 90% of sub-sample of target data
- for i in range(1, 11):
+ # find k-marginal of 1%, 5%, 10%, 20% ... 90% of sub-sample of target data
+ sample_sizes = [1, 5] + [i*10 for i in range(1, 10)]
+ for i in sample_sizes:
# using subsample of target data as synthetic data
- s_sd = create_subsample(frac=i * 0.1)
+ s_sd = create_subsample(frac=i * 0.01)
s_kmarg = k_marginal_cls(dataset.d_target_data,
s_sd,
group_features)
s_kmarg.compute_score()
s_score = int(s_kmarg.score)
- ssample_score[i * 0.1] = s_score
+ ssample_score[i * 0.01] = s_score
puma_scores = None
if len(group_features):
@@ -344,6 +346,7 @@ def min_index(data_list: List[float]):
# add k-marginal subsample and deidentified data scores to json report
k_marg_synop_rd['subsample_error_comparison'] = \
relative_path(save_data_frame(sedf_df, k_marg_synopsys_path, 'subsample_error_comparison'))
+ k_marg_synop_rd['sub_sampling_equivalent'] = int(min_frac * 100)
k_marg_synop_rd['k_marginal_score'] = k_marginal_score
report_data.add('k_marginal', {
@@ -420,10 +423,7 @@ def min_index(data_list: List[float]):
bs_a = Attachment(name=f"{len(best_scores)} Best Performing " + '-'.join(group_features),
_data=best_scores)
- report_data.add('k_marginal', {
- "k_marginal_breakdown": k_marg_break_rd
- })
-
+ report_data.add('worst_PUMA_breakdown', k_marg_break_rd)
attachments.extend([as_para_a, as_a])
metric_attachments = [k_marg_break_para_a, ws_para_a, ws_a]
@@ -436,7 +436,7 @@ def min_index(data_list: List[float]):
metric_attachments.append(gp_a)
metric_attachments.extend(worst_break_down)
- kmarg_det_pkt = UtilityScorePacket('K-Marginal Score Breakdown',
+ kmarg_det_pkt = UtilityScorePacket('Worst Performing PUMAs Breakdown',
None,
metric_attachments)
@@ -609,10 +609,10 @@ def utility_score(dataset: Dataset, ui_data: ReportUIData, report_data: ReportDa
p_dist_plot = PropensityDistribution(s.prob_dist, r_ui_d.output_directory)
# pps = PropensityPairPlot(s.std_two_way_scores, rd.output_directory)
#
- prop_rep_data = {**s.report_data, **p_dist_plot.report_data}
- rd.add('propensity mean square error', prop_rep_data)
p_dist_paths = p_dist_plot.save()
+ prop_rep_data = {**s.report_data, **p_dist_plot.report_data}
+ rd.add('propensity mean square error', prop_rep_data)
# pps_paths = pps.save('spmse',
# 'Two-Way Standardized Propensity Mean Square Error')
rel_pd_path = ["/".join(list(p.parts)[-2:])
diff --git a/sdnist/report/score/utility/inconsistency.py b/sdnist/report/score/utility/inconsistency.py
index 612b221..1f2eed1 100644
--- a/sdnist/report/score/utility/inconsistency.py
+++ b/sdnist/report/score/utility/inconsistency.py
@@ -9,6 +9,22 @@
from sdnist.utils import *
+ic_paragraphs = [
+ "In real world tabular data, it's common for record features to have "
+ "some deterministic, publicly known dependencies on each other: "
+ "knowing someone's AGEP= 3 necessarily tells you something about "
+ "their marital status, income and educational attainment. "
+ "Different deidentification methods may be better or worse at "
+ "automatically preserving these relationships. "
+ "When they fail (ex: producing toddlers with PhDs) we say "
+ "those records contain \"inconsistencies\". Our consistency "
+ "check metric below is not exhaustive, we don't catch everything "
+ "the way a production-grade system should, but this will give you "
+ "a sense of how consistency is preserved for age, work and household "
+ "features. Note that different deidentification approaches may do "
+ "better or worse with different types of inconsistencies."
+]
+
class InconsistenciesReport:
"""
@@ -42,6 +58,13 @@ def _create(self):
# add inconsistencies stats and data to json report data
self.rd.add('inconsistencies', self.ic.report_data)
+ # create report attachments
+ for p in ic_paragraphs:
+ para_a = Attachment(name=None,
+ _data=p,
+ _type=AttachmentType.String)
+ self.attachments.append(para_a)
+
# --------- Add inconsistencies stats and dat to ui report data
# UI attachment for summary of inconsistencies found in the deidentified data
a_sum_h = Attachment(name='Summary',
diff --git a/sdnist/report/score/utility/pca.py b/sdnist/report/score/utility/pca.py
index e349c89..4f5eb38 100644
--- a/sdnist/report/score/utility/pca.py
+++ b/sdnist/report/score/utility/pca.py
@@ -19,10 +19,10 @@
"dimensions of the original feature space). Descriptions " \
"of these new five dimensions (components) are " \
"given in the components table; the components will change " \
- "depending on which target data set you’re using. " \
+ "depending on which target data set you're using. " \
"Five dimensions are better than 22, but we actually want to " \
"get down to two dimensions so we can plot the data " \
- "on simple (x,y) axes– the plots below show the data " \
+ "on simple (x,y) axes the plots below show the data " \
"across each possible pair combination of our five components. " \
"You can compare how the shapes change between the target data " \
"and the deidentified data, and consider what that might mean in light " \
@@ -40,7 +40,7 @@
pca_highlight_para = "The queries below explore the PCA metric results in more detail " \
"by zooming in on a single component-pair panel and highlighting " \
- "all individuals that satisfy a given constraint (such as MSP = “N”, " \
+ "all individuals that satisfy a given constraint (such as MSP='N', " \
"individuals who are unmarried because they are children). " \
"If the deidentified data preserves the structure and feature " \
"correlations of the target data, the highlighted areas should have " \
diff --git a/sdnist/version.py b/sdnist/version.py
index 8a124bf..55e4709 100644
--- a/sdnist/version.py
+++ b/sdnist/version.py
@@ -1 +1 @@
-__version__ = "2.2.0"
+__version__ = "2.3.0"