From 83b4cc1ded4aeb85f68173ecf6f51fae7c38ff09 Mon Sep 17 00:00:00 2001 From: huangjg Date: Sun, 24 Dec 2023 22:40:51 +0800 Subject: [PATCH 01/16] update examples --- examples/conformal_training.py | 1 - pytest.ini | 3 + tests/dataset.py | 70 +++++++++++++++++++ .../imagenet_example_logits.py | 0 {examples => tests}/run_test_logits.sh | 0 torchcp/VERSION | 2 +- torchcp/classification/scores/raps.py | 1 - 7 files changed, 74 insertions(+), 3 deletions(-) create mode 100644 pytest.ini create mode 100644 tests/dataset.py rename {examples => tests}/imagenet_example_logits.py (100%) rename {examples => tests}/run_test_logits.sh (100%) diff --git a/examples/conformal_training.py b/examples/conformal_training.py index d06c0c78..8b4e2452 100644 --- a/examples/conformal_training.py +++ b/examples/conformal_training.py @@ -71,7 +71,6 @@ def forward(self, x): criterion = ConfTr(weights=0.01, predictor=predictor, alpha=0.05, - device=device, fraction=0.5, loss_types="valid", base_loss_fn=nn.CrossEntropyLoss()) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 00000000..316f4fd7 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +testpaths = + tests diff --git a/tests/dataset.py b/tests/dataset.py new file mode 100644 index 00000000..3706a513 --- /dev/null +++ b/tests/dataset.py @@ -0,0 +1,70 @@ +import os +import pathlib + +import torchvision.datasets as dset +import torchvision.transforms as trn +from PIL import Image +from torch.utils.data import Dataset + + +def build_dataset(dataset_name, transform=None, mode="train"): + # path of usr + usr_dir = os.path.expanduser('~') + data_dir = os.path.join(usr_dir, "data") + + if dataset_name == 'imagenet': + if transform == None: + transform = trn.Compose([ + trn.Resize(256), + trn.CenterCrop(224), + trn.ToTensor(), + trn.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) + + dataset = dset.ImageFolder(data_dir + "/imagenet/val", + transform) + elif dataset_name == 'imagenetv2': + if transform == None: + transform = trn.Compose([ + trn.Resize(256), + trn.CenterCrop(224), + trn.ToTensor(), + trn.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) + + dataset = ImageNetV2Dataset(os.path.join(data_dir, "imagenetv2/imagenetv2-matched-frequency-format-val"), + transform) + + elif dataset_name == 'mnist': + if transform == None: + transform = trn.Compose([ + trn.ToTensor(), + trn.Normalize((0.1307,), (0.3081,)) + ]) + if mode == "train": + dataset = dset.MNIST(data_dir, train=True, download=True, transform=transform) + elif mode == "test": + dataset = dset.MNIST(data_dir, train=False, download=True, transform=transform) + + else: + raise NotImplementedError + + return dataset + + +class ImageNetV2Dataset(Dataset): + def __init__(self, root, transform=None): + self.dataset_root = pathlib.Path(root) + self.fnames = list(self.dataset_root.glob("**/*.jpeg")) + self.transform = transform + + def __len__(self): + return len(self.fnames) + + def __getitem__(self, i): + img, label = Image.open(self.fnames[i]), int(self.fnames[i].parent.name) + if self.transform is not None: + img = self.transform(img) + return img, label diff --git a/examples/imagenet_example_logits.py b/tests/imagenet_example_logits.py similarity index 100% rename from examples/imagenet_example_logits.py rename to tests/imagenet_example_logits.py diff --git a/examples/run_test_logits.sh b/tests/run_test_logits.sh similarity index 100% rename from examples/run_test_logits.sh rename to tests/run_test_logits.sh diff --git a/torchcp/VERSION b/torchcp/VERSION index 6c6aa7cb..6da28dde 100644 --- a/torchcp/VERSION +++ b/torchcp/VERSION @@ -1 +1 @@ -0.1.0 \ No newline at end of file +0.1.1 \ No newline at end of file diff --git a/torchcp/classification/scores/raps.py b/torchcp/classification/scores/raps.py index e2d5db22..a2679b56 100644 --- a/torchcp/classification/scores/raps.py +++ b/torchcp/classification/scores/raps.py @@ -45,7 +45,6 @@ def _calculate_all_label(self, probs): scores = ordered_scores.gather(dim=-1, index=sorted_indices) return scores - def _calculate_single_label(self, probs, y): indices, ordered, cumsum = self._sort_sum(probs) From 99bebb93d9be11c2f3470bbfc429c86b58eb43c3 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 15:06:18 +0800 Subject: [PATCH 02/16] update confTr and docs --- docs/source/installation.rst | 2 +- docs/source/torchcp.classification.rst | 11 ++ examples/conformal_training.py | 153 ++++++++++------------ examples/imagenet_example.py | 28 ++-- tests/test_calssification_logits.py | 100 ++++++++++++++ torchcp/classification/loss/conftr.py | 69 +++++----- torchcp/classification/predictors/base.py | 2 + 7 files changed, 221 insertions(+), 144 deletions(-) create mode 100644 tests/test_calssification_logits.py diff --git a/docs/source/installation.rst b/docs/source/installation.rst index 586374df..80c41575 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -9,7 +9,7 @@ We developed TorchCP under Python 3.9 and PyTorch 2.0.1. To install TorchCP, sim .. code-block:: bash - pip install --index-url https://test.pypi.org/simple/ --no-deps torchcp + pip install torchcp or clone the repo and run diff --git a/docs/source/torchcp.classification.rst b/docs/source/torchcp.classification.rst index e06fd95b..0b332249 100644 --- a/docs/source/torchcp.classification.rst +++ b/docs/source/torchcp.classification.rst @@ -26,6 +26,14 @@ Predictors ClusterPredictor WeightedPredictor +.. automodule:: torchcp.classification.loss +Predictors +------- + +.. autosummary:: + :nosignatures: + + ConfTr Detailed description -------------------- @@ -57,4 +65,7 @@ Detailed description :members: .. autoclass:: WeightedPredictor + :members: + +.. autoclass:: ConfTr :members: \ No newline at end of file diff --git a/examples/conformal_training.py b/examples/conformal_training.py index 8b4e2452..99c367d9 100644 --- a/examples/conformal_training.py +++ b/examples/conformal_training.py @@ -17,6 +17,7 @@ import argparse +import itertools import torch import torch.nn as nn @@ -29,27 +30,9 @@ from torchcp.classification.scores import THR, APS, SAPS, RAPS from torchcp.utils import fix_randomness -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Covariate shift') - parser.add_argument('--seed', default=0, type=int) - parser.add_argument('--predictor', default="Standard", help="Standard") - parser.add_argument('--score', default="THR", help="THR") - parser.add_argument('--loss', default="CE", help="CE | ConfTr") - args = parser.parse_args() - res = {'Coverage_rate': 0, 'Average_size': 0} - num_trials = 1 - for seed in range(num_trials): - fix_randomness(seed=seed) - - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - ################################## - # Invalid prediction sets - ################################## - train_dataset = build_dataset("mnist") - train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) - class Net(nn.Module): +class Net(nn.Module): def __init__(self): super(Net, self).__init__() self.fc1 = nn.Linear(28 * 28, 500) @@ -60,76 +43,74 @@ def forward(self, x): x = F.relu(self.fc1(x)) x = self.fc2(x) return x + +def train(model, device, train_loader, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + - - model = Net().to(device) - - if args.loss == "CE": +if __name__ == '__main__': + alpha = 0.01 + num_trials = 5 + + result = {} + for loss in ["CE", "ConfTr"]: + print(f"############################## {loss} #########################") + result[loss] = {} + if loss == "CE": criterion = nn.CrossEntropyLoss() - elif args.loss == "ConfTr": + elif loss == "ConfTr": predictor = SplitPredictor(score_function=THR(score_type="log_softmax")) criterion = ConfTr(weights=0.01, - predictor=predictor, - alpha=0.05, - fraction=0.5, - loss_types="valid", - base_loss_fn=nn.CrossEntropyLoss()) + predictor=predictor, + alpha=0.05, + fraction=0.5, + loss_types="valid", + base_loss_fn=nn.CrossEntropyLoss()) else: raise NotImplementedError - - optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) - - - def train(model, device, train_loader, optimizer, epoch): - model.train() - for batch_idx, (data, target) in enumerate(train_loader): - data, target = data.to(device), target.to(device) - optimizer.zero_grad() - output = model(data) - loss = criterion(output, target) - loss.backward() - optimizer.step() - if batch_idx % 10 == 0: - print( - f'Train Epoch: {epoch} [{batch_idx * len(data)}/{len(train_loader.dataset)} ({100. * batch_idx / len(train_loader):.0f}%)]\tLoss: {loss.item():.6f}') - - - checkpoint_path = f'.cache/conformal_training_model_checkpoint_{args.loss}_seed={seed}.pth' - # if os.path.exists(checkpoint_path): - # checkpoint = torch.load(checkpoint_path) - # model.load_state_dict(checkpoint['model_state_dict']) - # else: - for epoch in range(1, 10): - train(model, device, train_data_loader, optimizer, epoch) - - torch.save({'model_state_dict': model.state_dict(), }, checkpoint_path) - - test_dataset = build_dataset("mnist", mode='test') - cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) - cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) - test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) - - if args.score == "THR": - score_function = THR() - elif args.score == "APS": - score_function = APS() - elif args.score == "RAPS": - score_function = RAPS(args.penalty, args.kreg) - elif args.score == "SAPS": - score_function = SAPS(weight=args.weight) - - alpha = 0.01 - if args.predictor == "Standard": - predictor = SplitPredictor(score_function, model) - elif args.predictor == "ClassWise": - predictor = ClassWisePredictor(score_function, model) - elif args.predictor == "Cluster": - predictor = ClusterPredictor(score_function, model, args.seed) - predictor.calibrate(cal_data_loader, alpha) - - # test examples - tmp_res = predictor.evaluate(test_data_loader) - res['Coverage_rate'] += tmp_res['Coverage_rate'] / num_trials - res['Average_size'] += tmp_res['Average_size'] / num_trials - - print(res) + for seed in range(num_trials): + fix_randomness(seed=seed) + ################################## + # Training a pyotrch model + ################################## + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_dataset = build_dataset("mnist") + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) + test_dataset = build_dataset("mnist", mode='test') + cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) + + model = Net().to(device) + optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) + for epoch in range(1, 10): + train(model, device, train_data_loader, optimizer, epoch) + + for score in ["THR", "APS", "RAPS", "SAPS"]: + if score == "THR": + score_function = THR() + elif score == "APS": + score_function = APS() + elif score == "RAPS": + score_function = RAPS(1, 0) + elif score == "SAPS": + score_function = SAPS(weight=0.2) + if score not in result[loss]: + result[loss][score] = {} + result[loss][score]['Coverage_rate'] = 0 + result[loss][score]['Average_size'] = 0 + predictor = SplitPredictor(score_function, model) + predictor.calibrate(cal_data_loader, alpha) + tmp_res = predictor.evaluate(test_data_loader) + result[loss][score]['Coverage_rate'] += tmp_res['Coverage_rate'] / num_trials + result[loss][score]['Average_size'] += tmp_res['Average_size'] / num_trials + + for score in ["THR", "APS", "RAPS", "SAPS"]: + print(f"Score: {score}. Result is {result[loss][score]}") diff --git a/examples/imagenet_example.py b/examples/imagenet_example.py index 1e911b06..16529330 100644 --- a/examples/imagenet_example.py +++ b/examples/imagenet_example.py @@ -34,8 +34,10 @@ fix_randomness(seed=args.seed) + ####################################### + # Loading ImageNet dataset and a pytorch model + ####################################### model_name = 'ResNet101' - # load model model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) model_device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") model.to(model_device) @@ -55,6 +57,10 @@ cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1024, shuffle=False, pin_memory=True) test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1024, shuffle=False, pin_memory=True) + + ####################################### + # A standard process of conformal prediction + ####################################### alpha = args.alpha print( f"Experiment--Data : ImageNet, Model : {model_name}, Score : {args.score}, Predictor : {args.predictor}, Alpha : {alpha}") @@ -80,22 +86,4 @@ raise NotImplementedError print(f"The size of calibration set is {len(cal_dataset)}.") predictor.calibrate(cal_data_loader, alpha) - # predictor.evaluate(test_data_loader) - - # test examples - print("Testing examples...") - prediction_sets = [] - labels_list = [] - with torch.no_grad(): - for examples in tqdm(test_data_loader): - tmp_x, tmp_label = examples[0], examples[1] - prediction_sets_batch = predictor.predict(tmp_x) - prediction_sets.extend(prediction_sets_batch) - labels_list.append(tmp_label) - test_labels = torch.cat(labels_list) - - metrics = Metrics() - print("Etestuating prediction sets...") - print(f"Coverage_rate: {metrics('coverage_rate')(prediction_sets, test_labels)}.") - print(f"Average_size: {metrics('average_size')(prediction_sets, test_labels)}.") - print(f"CovGap: {metrics('CovGap')(prediction_sets, test_labels, alpha, num_classes)}.") + predictor.evaluate(test_data_loader) diff --git a/tests/test_calssification_logits.py b/tests/test_calssification_logits.py new file mode 100644 index 00000000..6648e135 --- /dev/null +++ b/tests/test_calssification_logits.py @@ -0,0 +1,100 @@ +# Copyright (c) 2023-present, SUSTech-ML. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + + +import argparse +import os +import pickle + +import torch +import torchvision +import torchvision.datasets as dset +import torchvision.transforms as trn +from tqdm import tqdm + +from torchcp.classification.predictors import SplitPredictor, ClusterPredictor, ClassWisePredictor +from torchcp.classification.scores import THR, APS, SAPS, RAPS, Margin +from torchcp.classification.utils.metrics import Metrics +from torchcp.utils import fix_randomness + + + + +def test_imagenet(): + ####################################### + # Loading ImageNet dataset and a pytorch model + ####################################### + fix_randomness(seed=0) + model_name = 'ResNet101' + fname = ".cache/" + model_name + ".pkl" + if os.path.exists(fname): + with open(fname, 'rb') as handle: + dataset = pickle.load(handle) + + else: + # load dataset + transform = trn.Compose([trn.Resize(256), + trn.CenterCrop(224), + trn.ToTensor(), + trn.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) + usr_dir = os.path.expanduser('~') + data_dir = os.path.join(usr_dir, "data") + dataset = dset.ImageFolder(data_dir + "/imagenet/val", + transform) + data_loader = torch.utils.data.DataLoader(dataset, batch_size=320, shuffle=False, pin_memory=True) + + # load model + model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) + + logits_list = [] + labels_list = [] + with torch.no_grad(): + for examples in tqdm(data_loader): + tmp_x, tmp_label = examples[0], examples[1] + tmp_logits = model(tmp_x) + logits_list.append(tmp_logits) + labels_list.append(tmp_label) + logits = torch.cat(logits_list) + labels = torch.cat(labels_list) + dataset = torch.utils.data.TensorDataset(logits, labels.long()) + with open(fname, 'wb') as handle: + pickle.dump(dataset, handle, protocol=pickle.HIGHEST_PROTOCOL) + + cal_data, val_data = torch.utils.data.random_split(dataset, [25000, 25000]) + cal_logits = torch.stack([sample[0] for sample in cal_data]) + cal_labels = torch.stack([sample[1] for sample in cal_data]) + + test_logits = torch.stack([sample[0] for sample in val_data]) + test_labels = torch.stack([sample[1] for sample in val_data]) + + num_classes = 1000 + + ####################################### + # A standard process of conformal prediction + ####################################### + alpha = 0.1 + predictors = [SplitPredictor, ClassWisePredictor, ClusterPredictor] + score_functions = [THR(), APS(), RAPS(1, 0), SAPS(0.2), Margin()] + for score in score_functions: + for class_predictor in predictors: + predictor = class_predictor(score) + predictor.calculate_threshold(cal_logits, cal_labels, alpha) + print(f"Experiment--Data : ImageNet, Model : {model_name}, Score : {score.__class__.__name__}, Predictor : {predictor.__class__.__name__}, Alpha : {alpha}") + # print("Testing examples...") + # prediction_sets = [] + # for index, ele in enumerate(test_logits): + # prediction_set = predictor.predict_with_logits(ele) + # prediction_sets.append(prediction_set) + prediction_sets = predictor.predict_with_logits(test_logits) + + metrics = Metrics() + print("Evaluating prediction sets...") + print(f"Coverage_rate: {metrics('coverage_rate')(prediction_sets, test_labels)}.") + print(f"Average_size: {metrics('average_size')(prediction_sets, test_labels)}.") + print(f"CovGap: {metrics('CovGap')(prediction_sets, test_labels, alpha, num_classes)}.") diff --git a/torchcp/classification/loss/conftr.py b/torchcp/classification/loss/conftr.py index 9978302d..e4d8fb6b 100644 --- a/torchcp/classification/loss/conftr.py +++ b/torchcp/classification/loss/conftr.py @@ -12,48 +12,48 @@ class ConfTr(nn.Module): - def __init__(self, weights, predictor, alpha, fraction, loss_types="valid", target_size=1, + """ + Conformal Training (Stutz et al., 2021). + Paper: https://arxiv.org/abs/2110.09192. + + + :param weights: the weight of each loss function + :param predictor: the CP predictors + :param alpha: the significance level for each training batch + :param fraction: the fraction of the calibration set in each training batch + :param loss_types: the selected (multi-selected) loss functions, which can be "valid", "classification", "probs", "coverage". + :param target_size: Optional: 0 | 1. + :param loss_transform: a transform for loss + :param base_loss_fn: a base loss function. For example, cross entropy in classification. + """ + def __init__(self, weight, predictor, alpha, fraction, loss_type="valid", target_size=1, loss_transform="square", base_loss_fn=None): - """ - :param weights: the weight of each loss function - :param predictor: the CP predictors - :param alpha: the significance level for each training batch - :param fraction: the fraction of the calibration set in each training batch - :param loss_types: the selected (multi-selected) loss functions, which can be "valid", "classification", "probs", "coverage". - :param target_size: - :param loss_transform: a transform for loss - :param base_loss_fn: a base loss function, such as cross entropy for classification - """ + super(ConfTr, self).__init__() - self.weight = weights + assert weight>0, "weight must be greater than 0." + assert (fraction > 0 and fraction<1), "fraction should be a value in (0,1)." + assert loss_type in ["valid", "classification", "probs", "coverage"], 'loss_type should be a value in ["valid", "classification", "probs", "coverage"].' + assert target_size==0 or target_size ==1, "target_size should be 0 or 1." + assert loss_transform in ["square", "abs", "log"], 'loss_transform should be a value in ["square", "abs", "log"].' + self.weight = weight self.predictor = predictor self.alpha = alpha self.fraction = fraction - self.base_loss_fn = base_loss_fn - + self.loss_type = loss_type self.target_size = target_size + self.base_loss_fn = base_loss_fn + if loss_transform == "square": self.transform = torch.square elif loss_transform == "abs": self.transform = torch.abs elif loss_transform == "log": self.transform = torch.log - else: - raise NotImplementedError self.loss_functions_dict = {"valid": self.__compute_hinge_size_loss, "probs": self.__compute_probabilistic_size_loss, "coverage": self.__compute_coverage_loss, - "classification": self.__compute_classification_loss} - - if type(loss_types) == set: - if type(weights) != set: - raise TypeError("weights must be a set.") - elif type(loss_types) == str: - if type(weights) != float and type(weights) != int: - raise TypeError("weights must be a float or a int.") - else: - raise TypeError("types must be a set or a string.") - self.loss_types = loss_types + "classification": self.__compute_classification_loss + } def forward(self, logits, labels): # Compute Size Loss @@ -65,15 +65,10 @@ def forward(self, logits, labels): self.predictor.calculate_threshold(cal_logits.detach(), cal_labels.detach(), self.alpha) tau = self.predictor.q_hat - test_scores = self.predictor.score_function.predict(test_logits) + test_scores = self.predictor.score_function(test_logits) + # Computing the probability of each label contained in the prediction set. pred_sets = torch.sigmoid(tau - test_scores) - - if type(self.loss_types) == set: - loss = torch.tensor(0).to(logits.device) - for i in range(len(self.loss_types)): - loss += self.weight[i] * self.loss_functions_dict[self.loss_types[i]](pred_sets, test_labels) - else: - loss = self.weight * self.loss_functions_dict[self.loss_types](pred_sets, test_labels) + loss = self.weight * self.loss_functions_dict[self.loss_type](pred_sets, test_labels) if self.base_loss_fn is not None: loss += self.base_loss_fn(logits, labels).float() @@ -109,13 +104,13 @@ def __compute_coverage_loss(self, pred_sets, labels): def __compute_classification_loss(self, pred_sets, labels): # Convert labels to one-hot encoding one_hot_labels = F.one_hot(labels, num_classes=pred_sets.shape[1]).float() - loss_matrix = torch.eye(pred_sets.shape[1]).to(pred_sets.device) + loss_matrix = torch.eye(pred_sets.shape[1], device=pred_sets.device) # Calculate l1 and l2 losses l1 = (1 - pred_sets) * one_hot_labels * loss_matrix[labels] l2 = pred_sets * (1 - one_hot_labels) * loss_matrix[labels] # Calculate the total loss - loss = torch.sum(torch.maximum(l1 + l2, torch.zeros_like(l1).to(pred_sets.device)), dim=1) + loss = torch.sum(torch.maximum(l1 + l2, torch.zeros_like(l1, device=pred_sets.device)), dim=1) # Return the mean loss return torch.mean(loss) diff --git a/torchcp/classification/predictors/base.py b/torchcp/classification/predictors/base.py index ae954c01..528efd21 100644 --- a/torchcp/classification/predictors/base.py +++ b/torchcp/classification/predictors/base.py @@ -24,6 +24,8 @@ class BasePredictor(object): def __init__(self, score_function, model=None, temperature=1): """ + Abstract base class for all conformal predictors. + :param score_function: non-conformity score function. :param model: a deep learning model. """ From 98161c31fde8f7e58fea62b214a716440a9b14ed Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 17:11:00 +0800 Subject: [PATCH 03/16] update comments and Folder 'examples' --- docs/source/torchcp.classification.rst | 2 +- docs/source/torchcp.regression.rst | 12 ++ examples/conformal_training.py | 92 ++++++-------- examples/reg_CQR.py | 74 ----------- examples/reg_aci.py | 56 -------- examples/reg_splitCP.py | 78 ------------ examples/regression.py | 81 ++++++++++++ examples/time_series.py | 77 +++++++++++ tests/imagenet_example_logits.py | 120 ------------------ tests/run_test_logits.sh | 15 --- ...ation_logits.py => test_classification.py} | 52 ++++++-- tests/test_conformal_training.py | 115 +++++++++++++++++ torchcp/classification/predictors/base.py | 14 +- .../classification/predictors/classwise.py | 4 + torchcp/classification/predictors/cluster.py | 16 +-- torchcp/classification/predictors/split.py | 8 ++ torchcp/classification/predictors/weight.py | 6 + torchcp/classification/scores/raps.py | 9 +- torchcp/classification/scores/saps.py | 6 +- torchcp/classification/scores/thr.py | 10 +- torchcp/regression/loss/quantile.py | 17 ++- torchcp/regression/predictors/aci.py | 3 +- torchcp/regression/predictors/cqr.py | 2 +- torchcp/regression/predictors/split.py | 2 + 24 files changed, 423 insertions(+), 448 deletions(-) delete mode 100644 examples/reg_CQR.py delete mode 100644 examples/reg_aci.py delete mode 100644 examples/reg_splitCP.py create mode 100644 examples/regression.py create mode 100644 examples/time_series.py delete mode 100644 tests/imagenet_example_logits.py delete mode 100644 tests/run_test_logits.sh rename tests/{test_calssification_logits.py => test_classification.py} (66%) create mode 100644 tests/test_conformal_training.py diff --git a/docs/source/torchcp.classification.rst b/docs/source/torchcp.classification.rst index 0b332249..c30d5fdd 100644 --- a/docs/source/torchcp.classification.rst +++ b/docs/source/torchcp.classification.rst @@ -27,7 +27,7 @@ Predictors WeightedPredictor .. automodule:: torchcp.classification.loss -Predictors +Loss functions ------- .. autosummary:: diff --git a/docs/source/torchcp.regression.rst b/docs/source/torchcp.regression.rst index 39b08d3e..24b74b66 100644 --- a/docs/source/torchcp.regression.rst +++ b/docs/source/torchcp.regression.rst @@ -12,6 +12,14 @@ Predictors cqr ACI +Loss functions +------- + +.. autosummary:: + :nosignatures: + + QuantileLoss + Detailed description -------------------- @@ -24,3 +32,7 @@ Detailed description .. autoclass:: ACI :members: + +.. autoclass:: QuantileLoss + :members: + diff --git a/examples/conformal_training.py b/examples/conformal_training.py index 99c367d9..c875b668 100644 --- a/examples/conformal_training.py +++ b/examples/conformal_training.py @@ -44,7 +44,7 @@ def forward(self, x): x = self.fc2(x) return x -def train(model, device, train_loader, optimizer, epoch): +def train(model, device, train_loader,criterion, optimizer, epoch): model.train() for batch_idx, (data, target) in enumerate(train_loader): data, target = data.to(device), target.to(device) @@ -58,59 +58,39 @@ def train(model, device, train_loader, optimizer, epoch): if __name__ == '__main__': alpha = 0.01 num_trials = 5 - + loss = "ConfTr" result = {} - for loss in ["CE", "ConfTr"]: - print(f"############################## {loss} #########################") - result[loss] = {} - if loss == "CE": - criterion = nn.CrossEntropyLoss() - elif loss == "ConfTr": - predictor = SplitPredictor(score_function=THR(score_type="log_softmax")) - criterion = ConfTr(weights=0.01, - predictor=predictor, - alpha=0.05, - fraction=0.5, - loss_types="valid", - base_loss_fn=nn.CrossEntropyLoss()) - else: - raise NotImplementedError - for seed in range(num_trials): - fix_randomness(seed=seed) - ################################## - # Training a pyotrch model - ################################## - device = torch.device("cuda" if torch.cuda.is_available() else "cpu") - train_dataset = build_dataset("mnist") - train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) - test_dataset = build_dataset("mnist", mode='test') - cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) - cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) - test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) - - model = Net().to(device) - optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) - for epoch in range(1, 10): - train(model, device, train_data_loader, optimizer, epoch) - - for score in ["THR", "APS", "RAPS", "SAPS"]: - if score == "THR": - score_function = THR() - elif score == "APS": - score_function = APS() - elif score == "RAPS": - score_function = RAPS(1, 0) - elif score == "SAPS": - score_function = SAPS(weight=0.2) - if score not in result[loss]: - result[loss][score] = {} - result[loss][score]['Coverage_rate'] = 0 - result[loss][score]['Average_size'] = 0 - predictor = SplitPredictor(score_function, model) - predictor.calibrate(cal_data_loader, alpha) - tmp_res = predictor.evaluate(test_data_loader) - result[loss][score]['Coverage_rate'] += tmp_res['Coverage_rate'] / num_trials - result[loss][score]['Average_size'] += tmp_res['Average_size'] / num_trials - - for score in ["THR", "APS", "RAPS", "SAPS"]: - print(f"Score: {score}. Result is {result[loss][score]}") + print(f"############################## {loss} #########################") + + predictor = SplitPredictor(score_function=THR(score_type="log_softmax")) + criterion = ConfTr(weight=0.01, + predictor=predictor, + alpha=0.05, + fraction=0.5, + loss_type="valid", + base_loss_fn=nn.CrossEntropyLoss()) + + fix_randomness(seed=0) + ################################## + # Training a pyotrch model + ################################## + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_dataset = build_dataset("mnist") + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) + test_dataset = build_dataset("mnist", mode='test') + cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) + + model = Net().to(device) + optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) + for epoch in range(1, 10): + train(model, device, train_data_loader, criterion, optimizer, epoch) + + + score_function = THR() + + predictor = SplitPredictor(score_function, model) + predictor.calibrate(cal_data_loader, alpha) + result = predictor.evaluate(test_data_loader) + print(f"Result--Coverage_rate: {result['Coverage_rate']}, Average_size: {result['Average_size']}") \ No newline at end of file diff --git a/examples/reg_CQR.py b/examples/reg_CQR.py deleted file mode 100644 index 44af2b5d..00000000 --- a/examples/reg_CQR.py +++ /dev/null @@ -1,74 +0,0 @@ -import numpy as np -import torch -from torch.utils.data import TensorDataset -from tqdm import tqdm - -from torchcp.regression.loss import QuantileLoss -from torchcp.regression.predictors import CQR -from torchcp.regression.utils.metrics import Metrics -from torchcp.utils import fix_randomness -from utils import build_reg_data, build_regression_model - -device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") -fix_randomness(seed=2) - -X, y = build_reg_data() -indices = np.arange(X.shape[0]) -np.random.shuffle(indices) -split_index1 = int(len(indices) * 0.4) -split_index2 = int(len(indices) * 0.6) -part1, part2, part3 = np.split(indices, [split_index1, split_index2]) - -from sklearn.preprocessing import StandardScaler - -scalerX = StandardScaler() -scalerX = scalerX.fit(X[part1, :]) - -train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) -cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) -test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) - -train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) -cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) -test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) - -alpha = 0.1 -quantiles = [alpha / 2, 1 - alpha / 2] -model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) -criterion = QuantileLoss(quantiles) -optimizer = torch.optim.Adam(model.parameters(), lr=0.01) - -# Train Model -epochs = 100 -for epoch in tqdm(range(epochs)): - for index, (tmp_x, tmp_y) in enumerate(train_data_loader): - outputs = model(tmp_x.to(device)) - loss = criterion(outputs, tmp_y.to(device)) - optimizer.zero_grad() - loss.backward() - optimizer.step() - -predictor = CQR(model) -predictor.calibrate(cal_data_loader, alpha) - -y_list = [] -x_list = [] -predict_list = [] -with torch.no_grad(): - for examples in test_data_loader: - tmp_x, tmp_y = examples[0].to(device), examples[1] - tmp_prediction_intervals = predictor.predict(tmp_x) - y_list.append(tmp_y) - x_list.append(tmp_x) - predict_list.append(tmp_prediction_intervals) - -predicts = torch.cat(predict_list).float().cpu() -test_y = torch.cat(y_list) -x = torch.cat(x_list).float() - -metrics = Metrics() -print("Etestuating prediction sets...") -print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}.") -print(f"Average_size: {metrics('average_size')(predicts)}.") - -# print(predictors.evaluate(test_data_loader)) diff --git a/examples/reg_aci.py b/examples/reg_aci.py deleted file mode 100644 index d1030790..00000000 --- a/examples/reg_aci.py +++ /dev/null @@ -1,56 +0,0 @@ -import torch -from torch.utils.data import TensorDataset -from tqdm import tqdm - -from torchcp.regression import Metrics -from torchcp.regression.loss import QuantileLoss -from torchcp.regression.predictors import ACI -from torchcp.utils import fix_randomness -from utils import build_reg_data, build_regression_model - -device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") -fix_randomness(seed=2) - -X, y = build_reg_data(data_name="synthetic") -num_examples = X.shape[0] - -T0 = int(num_examples * 0.4) - -train_dataset = TensorDataset(torch.from_numpy(X[:T0, :]), torch.from_numpy(y[:T0])) -train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) - -alpha = 0.1 -quantiles = [alpha / 2, 1 - alpha / 2] -model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) -criterion = QuantileLoss(quantiles) -optimizer = torch.optim.Adam(model.parameters(), lr=0.01) - -epochs = 10 -for epoch in tqdm(range(epochs)): - for index, (tmp_x, tmp_y) in enumerate(train_data_loader): - outputs = model(tmp_x.to(device)) - loss = criterion(outputs, tmp_y.to(device)) - optimizer.zero_grad() - loss.backward() - optimizer.step() - -predictor = ACI(model, 0.0001) - -test_y = torch.from_numpy(y[T0:num_examples]).to(device) -predicts = torch.zeros((num_examples - T0, 2)).to(device) -for i in range(num_examples - T0): - with torch.no_grad(): - cal_dataset = TensorDataset(torch.from_numpy(X[i:(T0 + i), :]), torch.from_numpy(y[i:(T0 + i)])) - cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) - predictor.calibrate(cal_data_loader, alpha) - tmp_x = torch.from_numpy(X[(T0 + i), :]) - if i == 0: - tmp_prediction_intervals = predictor.predict(tmp_x) - else: - tmp_prediction_intervals = predictor.predict(tmp_x, test_y[i - 1], predicts[i - 1]) - predicts[i, :] = tmp_prediction_intervals - -metrics = Metrics() -print("Etestuating prediction sets...") -print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}.") -print(f"Average_size: {metrics('average_size')(predicts)}.") diff --git a/examples/reg_splitCP.py b/examples/reg_splitCP.py deleted file mode 100644 index 0cb26cff..00000000 --- a/examples/reg_splitCP.py +++ /dev/null @@ -1,78 +0,0 @@ -import numpy as np -import torch -import torch.nn as nn -from torch.utils.data import TensorDataset -from tqdm import tqdm - -from torchcp.regression.predictors import SplitPredictor -from torchcp.utils import fix_randomness -from utils import build_reg_data, build_regression_model - -device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") -fix_randomness(seed=2) - -X, y = build_reg_data() -indices = np.arange(X.shape[0]) -np.random.shuffle(indices) -split_index1 = int(len(indices) * 0.4) -split_index2 = int(len(indices) * 0.6) -part1, part2, part3 = np.split(indices, [split_index1, split_index2]) - -from sklearn.preprocessing import StandardScaler - -scalerX = StandardScaler() -scalerX = scalerX.fit(X[part1, :]) -train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) -cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) -test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) - -train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) -cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) -test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) - -model = build_regression_model("NonLinearNet")(X.shape[1], 1, 64, 0.5).to(device) -criterion = nn.MSELoss() -optimizer = torch.optim.Adam(model.parameters(), lr=0.01) - -# Train Model -epochs = 200 -for epoch in tqdm(range(epochs)): - for index, (tmp_x, tmp_y) in enumerate(train_data_loader): - outputs = model(tmp_x.to(device)) - loss = criterion(outputs.reshape(-1), tmp_y.to(device)) - optimizer.zero_grad() - loss.backward() - optimizer.step() - -alpha = 0.1 -model.eval() -predictor = SplitPredictor(model) -predictor.calibrate(cal_data_loader, alpha) - -############################################ -# First method to evaluate test instances -############################################ -print(predictor.evaluate(test_data_loader)) - -############################################ -# Second method to evaluate test instances -############################################ -# y_list = [] -# x_list = [] -# predict_list = [] -# with torch.no_grad(): -# for examples in test_data_loader: -# tmp_x, tmp_y = examples[0].to(device), examples[1] -# tmp_prediction_intervals = predictors.predict(tmp_x) -# y_list.append(tmp_y) -# x_list.append(tmp_x) -# predict_list.append(tmp_prediction_intervals) - -# predicts = torch.cat(predict_list).float().cpu() -# test_y = torch.cat(y_list) -# x = torch.cat(x_list).float() - -# metrics = Metrics() -# print("Etestuating prediction sets...") -# print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}.") -# print(f"Average_size: {metrics('average_size')(predicts)}.") diff --git a/examples/regression.py b/examples/regression.py new file mode 100644 index 00000000..592c80dc --- /dev/null +++ b/examples/regression.py @@ -0,0 +1,81 @@ +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import TensorDataset +from tqdm import tqdm +from sklearn.preprocessing import StandardScaler + + +from torchcp.regression.predictors import SplitPredictor,CQR +from torchcp.regression.loss import QuantileLoss +from torchcp.utils import fix_randomness +from utils import build_reg_data, build_regression_model + + +def train(model, device, epoch, train_data_loader, criterion, optimizer): + for index, (tmp_x, tmp_y) in enumerate(train_data_loader): + outputs = model(tmp_x.to(device)) + loss = criterion(outputs, tmp_y.unsqueeze(dim=1).to(device)) + optimizer.zero_grad() + loss.backward() + optimizer.step() + +if __name__ == '__main__': + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=1) + X, y = build_reg_data() + indices = np.arange(X.shape[0]) + np.random.shuffle(indices) + split_index1 = int(len(indices) * 0.4) + split_index2 = int(len(indices) * 0.6) + part1, part2, part3 = np.split(indices, [split_index1, split_index2]) + scalerX = StandardScaler() + scalerX = scalerX.fit(X[part1, :]) + train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) + cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) + test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) + + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + + epochs = 100 + alpha = 0.1 + + ################################## + # Split Conformal Prediction + ################################## + print("########################## SplitPredictor ###########################") + model = build_regression_model("NonLinearNet")(X.shape[1], 1, 64, 0.5).to(device) + criterion = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + predictor = SplitPredictor(model) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + ################################## + # Conformal Quantile Regression + ################################## + print("########################## CQR ###########################") + + + quantiles = [alpha / 2, 1 - alpha / 2] + model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) + criterion = QuantileLoss(quantiles) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + predictor = CQR(model) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) diff --git a/examples/time_series.py b/examples/time_series.py new file mode 100644 index 00000000..76b9ff2d --- /dev/null +++ b/examples/time_series.py @@ -0,0 +1,77 @@ +import torch +from torch.utils.data import TensorDataset +from tqdm import tqdm + +from torchcp.regression import Metrics +from torchcp.regression.loss import QuantileLoss +from torchcp.regression.predictors import ACI, CQR +from torchcp.utils import fix_randomness +from utils import build_reg_data, build_regression_model +from regression import train + + + +if __name__ == '__main__': + + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=2) + X, y = build_reg_data(data_name="synthetic") + num_examples = X.shape[0] + T0 = int(num_examples * 0.4) + train_dataset = TensorDataset(torch.from_numpy(X[:T0, :]), torch.from_numpy(y[:T0])) + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + + + + + + alpha = 0.1 + quantiles = [alpha / 2, 1 - alpha / 2] + model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) + criterion = QuantileLoss(quantiles) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + epochs = 10 + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + ################################## + # Conformal Quantile Regression + ################################## + print("########################## CQR ###########################") + + predictor = CQR(model) + cal_dataset = TensorDataset(torch.from_numpy(X[0:T0, :]), torch.from_numpy(y[0:T0])) + test_dataset = TensorDataset(torch.from_numpy(X[T0:, :]), torch.from_numpy(y[T0:])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + ################################## + # Adaptive Conformal Inference, + ################################## + print("########################## ACI ###########################") + predictor = ACI(model, 0.0001) + test_y = torch.from_numpy(y[T0:num_examples]).to(device) + predicts = torch.zeros((num_examples - T0, 2)).to(device) + for i in range(num_examples - T0): + with torch.no_grad(): + cal_dataset = TensorDataset(torch.from_numpy(X[i:(T0 + i), :]), torch.from_numpy(y[i:(T0 + i)])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + tmp_x = torch.from_numpy(X[(T0 + i), :]) + if i == 0: + tmp_prediction_intervals = predictor.predict(tmp_x) + else: + tmp_prediction_intervals = predictor.predict(tmp_x, test_y[i - 1], predicts[i - 1]) + predicts[i, :] = tmp_prediction_intervals + + metrics = Metrics() + print("Etestuating prediction sets...") + print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}") + print(f"Average_size: {metrics('average_size')(predicts)}") diff --git a/tests/imagenet_example_logits.py b/tests/imagenet_example_logits.py deleted file mode 100644 index ab920162..00000000 --- a/tests/imagenet_example_logits.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2023-present, SUSTech-ML. -# All rights reserved. -# -# This source code is licensed under the license found in the -# LICENSE file in the root directory of this source tree. -# - - -import argparse -import os -import pickle - -import torch -import torchvision -import torchvision.datasets as dset -import torchvision.transforms as trn -from tqdm import tqdm - -from torchcp.classification.predictors import SplitPredictor, ClusterPredictor, ClassWisePredictor -from torchcp.classification.scores import THR, APS, SAPS, RAPS, Margin -from torchcp.classification.utils.metrics import Metrics -from torchcp.utils import fix_randomness - -if __name__ == '__main__': - parser = argparse.ArgumentParser(description='') - parser.add_argument('--seed', default=0, type=int) - parser.add_argument('--alpha', default=0.1, type=float) - parser.add_argument('--predictor', default="Standard", help="Standard | ClassWise | Cluster") - parser.add_argument('--score', default="THR", help="THR | APS | SAPS") - parser.add_argument('--penalty', default=1, type=float) - parser.add_argument('--kreg', default=0, type=int) - parser.add_argument('--weight', default=0.2, type=int) - parser.add_argument('--split', default="random", type=str, help="proportional | doubledip | random") - args = parser.parse_args() - - fix_randomness(seed=args.seed) - - model_name = 'ResNet101' - fname = ".cache/" + model_name + ".pkl" - if os.path.exists(fname): - with open(fname, 'rb') as handle: - dataset = pickle.load(handle) - - else: - # load dataset - transform = trn.Compose([trn.Resize(256), - trn.CenterCrop(224), - trn.ToTensor(), - trn.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) - ]) - usr_dir = os.path.expanduser('~') - data_dir = os.path.join(usr_dir, "data") - dataset = dset.ImageFolder(data_dir + "/imagenet/val", - transform) - data_loader = torch.utils.data.DataLoader(dataset, batch_size=320, shuffle=False, pin_memory=True) - - # load model - model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) - - logits_list = [] - labels_list = [] - with torch.no_grad(): - for examples in tqdm(data_loader): - tmp_x, tmp_label = examples[0], examples[1] - tmp_logits = model(tmp_x) - logits_list.append(tmp_logits) - labels_list.append(tmp_label) - logits = torch.cat(logits_list) - labels = torch.cat(labels_list) - dataset = torch.utils.data.TensorDataset(logits, labels.long()) - with open(fname, 'wb') as handle: - pickle.dump(dataset, handle, protocol=pickle.HIGHEST_PROTOCOL) - - cal_data, val_data = torch.utils.data.random_split(dataset, [25000, 25000]) - cal_logits = torch.stack([sample[0] for sample in cal_data]) - cal_labels = torch.stack([sample[1] for sample in cal_data]) - - test_logits = torch.stack([sample[0] for sample in val_data]) - test_labels = torch.stack([sample[1] for sample in val_data]) - - num_classes = 1000 - alpha = args.alpha - print( - f"Experiment--Data : ImageNet, Model : {model_name}, Score : {args.score}, Predictor : {args.predictor}, Alpha : {alpha}") - if args.score == "THR": - score_function = THR() - elif args.score == "APS": - score_function = APS() - elif args.score == "RAPS": - score_function = RAPS(args.penalty, args.kreg) - elif args.score == "SAPS": - score_function = SAPS(weight=args.weight) - elif args.score == "Margin": - score_function = Margin() - else: - raise NotImplementedError - - if args.predictor == "Standard": - predictor = SplitPredictor(score_function, model=None) - elif args.predictor == "ClassWise": - predictor = ClassWisePredictor(score_function, model=None) - elif args.predictor == "Cluster": - predictor = ClusterPredictor(score_function, split=args.split, model=None) - print(f"The size of calibration set is {cal_labels.shape[0]}.") - predictor.calculate_threshold(cal_logits, cal_labels, alpha) - - # print("Testing examples...") - # prediction_sets = [] - # for index, ele in enumerate(test_logits): - # prediction_set = predictor.predict_with_logits(ele) - # prediction_sets.append(prediction_set) - - prediction_sets = predictor.predict_with_logits(test_logits) - - metrics = Metrics() - print("Evaluating prediction sets...") - print(f"Coverage_rate: {metrics('coverage_rate')(prediction_sets, test_labels)}.") - print(f"Average_size: {metrics('average_size')(prediction_sets, test_labels)}.") - print(f"CovGap: {metrics('CovGap')(prediction_sets, test_labels, alpha, num_classes)}.") diff --git a/tests/run_test_logits.sh b/tests/run_test_logits.sh deleted file mode 100644 index 0bf3cb0d..00000000 --- a/tests/run_test_logits.sh +++ /dev/null @@ -1,15 +0,0 @@ -#!/bin/bash - -predictors=("Standard" "ClassWise" "Cluster") -scores=("THR" "APS" "RAPS" "SAPS") - -for predictor in "${predictors[@]}" -do - for score in "${scores[@]}" - do - python examples/imagenet_example_logits.py \ - --predictor ${predictor} \ - --score ${score} \ - --seed 0 - done -done \ No newline at end of file diff --git a/tests/test_calssification_logits.py b/tests/test_classification.py similarity index 66% rename from tests/test_calssification_logits.py rename to tests/test_classification.py index 6648e135..096a85d3 100644 --- a/tests/test_calssification_logits.py +++ b/tests/test_classification.py @@ -22,9 +22,14 @@ from torchcp.utils import fix_randomness +transform = trn.Compose([trn.Resize(256), + trn.CenterCrop(224), + trn.ToTensor(), + trn.Normalize(mean=[0.485, 0.456, 0.406], + std=[0.229, 0.224, 0.225]) + ]) - -def test_imagenet(): +def test_imagenet_logits(): ####################################### # Loading ImageNet dataset and a pytorch model ####################################### @@ -36,13 +41,6 @@ def test_imagenet(): dataset = pickle.load(handle) else: - # load dataset - transform = trn.Compose([trn.Resize(256), - trn.CenterCrop(224), - trn.ToTensor(), - trn.Normalize(mean=[0.485, 0.456, 0.406], - std=[0.229, 0.224, 0.225]) - ]) usr_dir = os.path.expanduser('~') data_dir = os.path.join(usr_dir, "data") dataset = dset.ImageFolder(data_dir + "/imagenet/val", @@ -98,3 +96,39 @@ def test_imagenet(): print(f"Coverage_rate: {metrics('coverage_rate')(prediction_sets, test_labels)}.") print(f"Average_size: {metrics('average_size')(prediction_sets, test_labels)}.") print(f"CovGap: {metrics('CovGap')(prediction_sets, test_labels, alpha, num_classes)}.") + + + + + +def test_imagenet(): + fix_randomness(seed=0) + ####################################### + # Loading ImageNet dataset and a pytorch model + ####################################### + model_name = 'ResNet101' + model = torchvision.models.resnet101(weights="IMAGENET1K_V1", progress=True) + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + model.to(device) + usr_dir = os.path.expanduser('~') + data_dir = os.path.join(usr_dir, "data") + dataset = dset.ImageFolder(data_dir + "/imagenet/val", transform) + + cal_dataset, test_dataset = torch.utils.data.random_split(dataset, [25000, 25000]) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1024, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1024, shuffle=False, pin_memory=True) + + ####################################### + # A standard process of conformal prediction + ####################################### + alpha = 0.1 + predictors = [SplitPredictor, ClassWisePredictor, ClusterPredictor] + score_functions = [THR(), APS(), RAPS(1, 0), SAPS(0.2), Margin()] + for score in score_functions: + for class_predictor in predictors: + predictor = class_predictor(score, model) + predictor.calibrate(cal_data_loader, alpha) + print(f"Experiment--Data : ImageNet, Model : {model_name}, Score : {score.__class__.__name__}, Predictor : {predictor.__class__.__name__}, Alpha : {alpha}") + print(predictor.evaluate(test_data_loader)) + + diff --git a/tests/test_conformal_training.py b/tests/test_conformal_training.py new file mode 100644 index 00000000..3674a587 --- /dev/null +++ b/tests/test_conformal_training.py @@ -0,0 +1,115 @@ +# Copyright (c) 2023-present, SUSTech-ML. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + +# @Time : 13/12/2023 21:13 + + +# Copyright (c) 2023-present, SUSTech-ML. +# All rights reserved. +# +# This source code is licensed under the license found in the +# LICENSE file in the root directory of this source tree. +# + + +import argparse +import itertools + +import torch +import torch.nn as nn +import torch.nn.functional as F +import torch.optim as optim + +from dataset import build_dataset +from torchcp.classification.loss import ConfTr +from torchcp.classification.predictors import SplitPredictor, ClusterPredictor, ClassWisePredictor +from torchcp.classification.scores import THR, APS, SAPS, RAPS +from torchcp.utils import fix_randomness + + + +class Net(nn.Module): + def __init__(self): + super(Net, self).__init__() + self.fc1 = nn.Linear(28 * 28, 500) + self.fc2 = nn.Linear(500, 10) + + def forward(self, x): + x = x.view(-1, 28 * 28) + x = F.relu(self.fc1(x)) + x = self.fc2(x) + return x + +def train(model, device, train_loader,criterion, optimizer, epoch): + model.train() + for batch_idx, (data, target) in enumerate(train_loader): + data, target = data.to(device), target.to(device) + optimizer.zero_grad() + output = model(data) + loss = criterion(output, target) + loss.backward() + optimizer.step() + + +def test_training(): + alpha = 0.01 + num_trials = 5 + result = {} + for loss in ["CE", "ConfTr"]: + print(f"############################## {loss} #########################") + result[loss] = {} + if loss == "CE": + criterion = nn.CrossEntropyLoss() + elif loss == "ConfTr": + predictor = SplitPredictor(score_function=THR(score_type="log_softmax")) + criterion = ConfTr(weight=0.01, + predictor=predictor, + alpha=0.05, + fraction=0.5, + loss_type="valid", + base_loss_fn=nn.CrossEntropyLoss()) + else: + raise NotImplementedError + for seed in range(num_trials): + fix_randomness(seed=seed) + ################################## + # Training a pyotrch model + ################################## + device = torch.device("cuda" if torch.cuda.is_available() else "cpu") + train_dataset = build_dataset("mnist") + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=512, shuffle=True, pin_memory=True) + test_dataset = build_dataset("mnist", mode='test') + cal_dataset, test_dataset = torch.utils.data.random_split(test_dataset, [5000, 5000]) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=1600, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=1600, shuffle=False, pin_memory=True) + + model = Net().to(device) + optimizer = optim.SGD(model.parameters(), lr=0.01, momentum=0.5) + for epoch in range(1, 10): + train(model, device, train_data_loader, criterion, optimizer, epoch) + + for score in ["THR", "APS", "RAPS", "SAPS"]: + if score == "THR": + score_function = THR() + elif score == "APS": + score_function = APS() + elif score == "RAPS": + score_function = RAPS(1, 0) + elif score == "SAPS": + score_function = SAPS(weight=0.2) + if score not in result[loss]: + result[loss][score] = {} + result[loss][score]['Coverage_rate'] = 0 + result[loss][score]['Average_size'] = 0 + predictor = SplitPredictor(score_function, model) + predictor.calibrate(cal_data_loader, alpha) + tmp_res = predictor.evaluate(test_data_loader) + result[loss][score]['Coverage_rate'] += tmp_res['Coverage_rate'] / num_trials + result[loss][score]['Average_size'] += tmp_res['Average_size'] / num_trials + + for score in ["THR", "APS", "RAPS", "SAPS"]: + print(f"Score: {score}. Result is {result[loss][score]}") diff --git a/torchcp/classification/predictors/base.py b/torchcp/classification/predictors/base.py index 528efd21..378a1f7b 100644 --- a/torchcp/classification/predictors/base.py +++ b/torchcp/classification/predictors/base.py @@ -17,18 +17,16 @@ class BasePredictor(object): """ - Abstract base class for all predictors classes. + Abstract base class for all conformal predictors. + + :param score_function: non-conformity score function. + :param model: a pytorch model. + :param temperature: the temperature of Temperature Scaling. """ __metaclass__ = ABCMeta def __init__(self, score_function, model=None, temperature=1): - """ - Abstract base class for all conformal predictors. - - :param score_function: non-conformity score function. - :param model: a deep learning model. - """ self.score_function = score_function self._model = model @@ -49,7 +47,7 @@ def calibrate(self, cal_dataloader, alpha): @abstractmethod def predict(self, x_batch): """ - Generate prediction sets for test examples. + Generate prediction sets for the test examples. :param x_batch: a batch of input. """ diff --git a/torchcp/classification/predictors/classwise.py b/torchcp/classification/predictors/classwise.py index f2b90265..e415acfc 100644 --- a/torchcp/classification/predictors/classwise.py +++ b/torchcp/classification/predictors/classwise.py @@ -14,6 +14,10 @@ class ClassWisePredictor(SplitPredictor): Applications of Class-Conditional Conformal Predictor in Multi-Class Classification (Shi et al., 2013) paper: https://ieeexplore.ieee.org/document/6784618 + + + :param score_function: non-conformity score function. + :param model: a pytorch model. """ def __init__(self, score_function, model=None): diff --git a/torchcp/classification/predictors/cluster.py b/torchcp/classification/predictors/cluster.py index df6b884b..4d36dafa 100644 --- a/torchcp/classification/predictors/cluster.py +++ b/torchcp/classification/predictors/cluster.py @@ -17,19 +17,17 @@ class ClusterPredictor(SplitPredictor): """ Class-Conditional Conformal Prediction with Many Classes (Ding et al., 2023). - paper: https://arxiv.org/abs/2306.09335 + paper: https://arxiv.org/abs/2306.09335. + + :param score_function: a non-conformity score function. + :param model: a pytorch model. + :param ratio_clustering: the ratio of examples in the calibration dataset used to cluster classes. + :param num_clusters: the number of clusters. If ratio_clustering is "auto", the number of clusters is automatically computed. + :param split: the method to split the dataset into clustering dataset and calibration set. Options are 'proportional' (sample proportional to distribution such that rarest class has n_clustering example), 'doubledip' (don't split and use all data for both steps, or 'random' (each example is assigned to clustering step with some fixed probability). """ def __init__(self, score_function, model=None, ratio_clustering="auto", num_clusters="auto", split='random', temperature=1): - """ - - :param score_function: score functions of CP - :param model: a deep learning model - :param ratio_clustering: The ratio of examples in the calibration dataset used to cluster classes - :param num_clusters: The number of clusters. If cluster_ratio is "auto", the number of clusters is automatically computed. - :param split: The method to split the dataset into clustering dataset and calibration set. split: How to split data between clustering step and calibration step. Options are 'proportional' (sample proportional to distribution such that rarest class has n_clustering example), 'doubledip' (don't split and use all data for both steps, or 'random' (each example is assigned to clustering step with some fixed probability) - """ super(ClusterPredictor, self).__init__(score_function, model, temperature) self.__ratio_clustering = ratio_clustering diff --git a/torchcp/classification/predictors/split.py b/torchcp/classification/predictors/split.py index 2ecd529c..09b2af26 100644 --- a/torchcp/classification/predictors/split.py +++ b/torchcp/classification/predictors/split.py @@ -12,6 +12,14 @@ class SplitPredictor(BasePredictor): + """ + Split Conformal Prediction (Vovk et a., 2005). + Book: https://link.springer.com/book/10.1007/978-3-031-06649-8. + + :param score_function: non-conformity score function. + :param model: a pytorch model. + :param temperature: the temperature of Temperature Scaling. + """ def __init__(self, score_function, model=None, temperature=1): super().__init__(score_function, model, temperature) diff --git a/torchcp/classification/predictors/weight.py b/torchcp/classification/predictors/weight.py index 7a3c0f64..6ce4a279 100644 --- a/torchcp/classification/predictors/weight.py +++ b/torchcp/classification/predictors/weight.py @@ -14,6 +14,12 @@ class WeightedPredictor(SplitPredictor): """ Conformal Prediction Under Covariate Shift (Tibshirani et al., 2019) paper : https://arxiv.org/abs/1904.06019 + + :param score_function: non-conformity score function. + :param model: a pytorch model. + :param image_encoder: a pytorch model to generate the embedding feature of a input image. + :param domain_classifier: a pytorch model (a binary classifier ) to predict the probability that a embedding feature comes from the source domain. + :param temperature: the temperature of Temperature Scaling. """ def __init__(self, score_function, model, image_encoder, domain_classifier=None, temperature=1): diff --git a/torchcp/classification/scores/raps.py b/torchcp/classification/scores/raps.py index b887c542..5ba79419 100644 --- a/torchcp/classification/scores/raps.py +++ b/torchcp/classification/scores/raps.py @@ -17,14 +17,13 @@ class RAPS(APS): """ Regularized Adaptive Prediction Sets (Angelopoulos et al., 2020) paper : https://arxiv.org/abs/2009.14193 + + :param penalty: the weight of regularization. When penalty = 0, RAPS=APS. + :param kreg : the rank of regularization which is an integer in [0,labels_num]. """ def __init__(self, penalty, kreg=0): - """ - when penalty = 0, RAPS=APS. - - :param kreg : the rank of regularization which is an integer in [0,labels_num]. - """ + if penalty <= 0: raise ValueError("The parameter 'penalty' must be a positive value.") if kreg < 0: diff --git a/torchcp/classification/scores/saps.py b/torchcp/classification/scores/saps.py index 5983fbfa..0b36eac0 100644 --- a/torchcp/classification/scores/saps.py +++ b/torchcp/classification/scores/saps.py @@ -15,12 +15,12 @@ class SAPS(APS): """ Sorted Adaptive Prediction Sets (Huang et al., 2023) paper: https://arxiv.org/abs/2310.06430 + + :param weight: the weight of label ranking. """ def __init__(self, weight): - """ - :param weight: the weigth of label ranking. - """ + super(SAPS, self).__init__() if weight <= 0: raise ValueError("The parameter 'weight' must be a positive value.") diff --git a/torchcp/classification/scores/thr.py b/torchcp/classification/scores/thr.py index ae4a87f5..5c4aa692 100644 --- a/torchcp/classification/scores/thr.py +++ b/torchcp/classification/scores/thr.py @@ -11,14 +11,14 @@ class THR(BaseScore): """ - Threshold conformal predictors (Sadinle et al., 2016) - paper : https://arxiv.org/abs/1609.00451 + Threshold conformal predictors (Sadinle et al., 2016). + paper : https://arxiv.org/abs/1609.00451. + + param score_type: a transformation on logits. Default: "softmax". Optional: "softmax", "Identity", "log_softmax" or "log". """ def __init__(self, score_type="softmax") -> None: - """ - param score_type: either "softmax" "Identity", "log_softmax" or "log". Default: "softmax". A transformation for logits. - """ + super().__init__() self.score_type = score_type if score_type == "Identity": diff --git a/torchcp/regression/loss/quantile.py b/torchcp/regression/loss/quantile.py index 76bd2e47..934271a8 100644 --- a/torchcp/regression/loss/quantile.py +++ b/torchcp/regression/loss/quantile.py @@ -1,6 +1,7 @@ import torch import torch.nn as nn +__all__ = ["QuantileLoss"] class QuantileLoss(nn.Module): """ Pinball loss function @@ -8,18 +9,20 @@ class QuantileLoss(nn.Module): def __init__(self, quantiles): """ - - :param quantiles: quantile levels of predictions, each in the range (0,1) + A loss to training a quantile-regression model (Romano et al., 2019). + Paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/5103c3584b063c431bd1268e9b5e76fb-Paper.pdf. + + :param quantiles: a list of quantiles, such as $[\frac{alpha}{2}, 1-\frac{alpha}{2}]$. """ super().__init__() self.quantiles = quantiles def forward(self, preds, target): """ - Compute the pinball loss + Compute the pinball loss. - :param preds: the alpha/2 and 1-alpha/2 predictions of the model - :param target: the truth values + :param preds: the alpha/2 and 1-alpha/2 predictions of the model. The shape is batch x 2. + :param target: the truth values. The shape is batch x 1. """ assert not target.requires_grad if preds.size(0) != target.size(0): @@ -27,7 +30,7 @@ def forward(self, preds, target): losses = preds.new_zeros(len(self.quantiles)) for i, q in enumerate(self.quantiles): - errors = target - preds[:, i] - losses[i] = torch.sum(torch.max((q - 1) * errors, q * errors).unsqueeze(1)) + errors = target - preds[:, i:i+1] + losses[i] = torch.sum(torch.max((q - 1) * errors, q * errors).squeeze(1)) loss = torch.mean(losses) return loss diff --git a/torchcp/regression/predictors/aci.py b/torchcp/regression/predictors/aci.py index c1ddf2e7..df1b3aef 100644 --- a/torchcp/regression/predictors/aci.py +++ b/torchcp/regression/predictors/aci.py @@ -15,7 +15,8 @@ class ACI(SplitPredictor): Adaptive conformal inference (Gibbs et al., 2021) paper: https://arxiv.org/abs/2106.00170 - :param model: a deep learning model that can output alpha/2 and 1-alpha/2 quantile regression. + :param model: a pytorch model that can output the values of different quantiles. + :param gamma: a step size parameter. """ def __init__(self, model, gamma): diff --git a/torchcp/regression/predictors/cqr.py b/torchcp/regression/predictors/cqr.py index c7c7084f..3b55d208 100644 --- a/torchcp/regression/predictors/cqr.py +++ b/torchcp/regression/predictors/cqr.py @@ -16,7 +16,7 @@ class CQR(SplitPredictor): Conformalized Quantile Regression (Romano et al., 2019) paper: https://arxiv.org/abs/1905.03222 - :param model: a deep learning model that can output alpha/2 and 1-alpha/2 quantile regression. + :param model: a pytorch model that can output alpha/2 and 1-alpha/2 quantile regression. """ def __init__(self, model): diff --git a/torchcp/regression/predictors/split.py b/torchcp/regression/predictors/split.py index 94b7a119..017181e5 100644 --- a/torchcp/regression/predictors/split.py +++ b/torchcp/regression/predictors/split.py @@ -17,6 +17,8 @@ class SplitPredictor(object): """ Distribution-Free Predictive Inference For Regression (Lei et al., 2017) paper: https://arxiv.org/abs/1604.04173 + + :param model: a pytorch model for regression. """ def __init__(self, model): From 31475279eafd35589302822d7721d9e7fdf608f7 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 18:23:02 +0800 Subject: [PATCH 04/16] update tests --- docs/source/torchcp.regression.rst | 1 + tests/test_regression.py | 128 +++++++++++++++++++++++++++++ tests/utils.py | 85 +++++++++++++++++++ 3 files changed, 214 insertions(+) create mode 100644 tests/test_regression.py create mode 100644 tests/utils.py diff --git a/docs/source/torchcp.regression.rst b/docs/source/torchcp.regression.rst index 24b74b66..d03e0637 100644 --- a/docs/source/torchcp.regression.rst +++ b/docs/source/torchcp.regression.rst @@ -12,6 +12,7 @@ Predictors cqr ACI +.. automodule:: torchcp.regression.loss Loss functions ------- diff --git a/tests/test_regression.py b/tests/test_regression.py new file mode 100644 index 00000000..8b8f68df --- /dev/null +++ b/tests/test_regression.py @@ -0,0 +1,128 @@ +import numpy as np +import torch +import torch.nn as nn +from torch.utils.data import TensorDataset +from tqdm import tqdm +from sklearn.preprocessing import StandardScaler + + +from torchcp.regression.predictors import SplitPredictor,CQR,ACI +from torchcp.regression.loss import QuantileLoss +from torchcp.regression import Metrics +from torchcp.utils import fix_randomness +from utils import build_reg_data, build_regression_model + + +def train(model, device, epoch, train_data_loader, criterion, optimizer): + for index, (tmp_x, tmp_y) in enumerate(train_data_loader): + outputs = model(tmp_x.to(device)) + loss = criterion(outputs, tmp_y.unsqueeze(dim=1).to(device)) + optimizer.zero_grad() + loss.backward() + optimizer.step() + +def test_SplitPredictor(): + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:0" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=1) + X, y = build_reg_data() + indices = np.arange(X.shape[0]) + np.random.shuffle(indices) + split_index1 = int(len(indices) * 0.4) + split_index2 = int(len(indices) * 0.6) + part1, part2, part3 = np.split(indices, [split_index1, split_index2]) + scalerX = StandardScaler() + scalerX = scalerX.fit(X[part1, :]) + train_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part1, :])), torch.from_numpy(y[part1])) + cal_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part2, :])), torch.from_numpy(y[part2])) + test_dataset = TensorDataset(torch.from_numpy(scalerX.transform(X[part3, :])), torch.from_numpy(y[part3])) + + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + + epochs = 100 + alpha = 0.1 + + ################################## + # Split Conformal Prediction + ################################## + print("########################## SplitPredictor ###########################") + model = build_regression_model("NonLinearNet")(X.shape[1], 1, 64, 0.5).to(device) + criterion = nn.MSELoss() + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + predictor = SplitPredictor(model) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + +def test_time_series(): + ################################## + # Preparing dataset + ################################## + device = torch.device("cuda:2" if torch.cuda.is_available() else "cpu") + fix_randomness(seed=2) + X, y = build_reg_data(data_name="synthetic") + num_examples = X.shape[0] + T0 = int(num_examples * 0.4) + train_dataset = TensorDataset(torch.from_numpy(X[:T0, :]), torch.from_numpy(y[:T0])) + train_data_loader = torch.utils.data.DataLoader(train_dataset, batch_size=100, shuffle=True, pin_memory=True) + + + + + + alpha = 0.1 + quantiles = [alpha / 2, 1 - alpha / 2] + model = build_regression_model("NonLinearNet")(X.shape[1], 2, 64, 0.5).to(device) + criterion = QuantileLoss(quantiles) + optimizer = torch.optim.Adam(model.parameters(), lr=0.01) + + epochs = 10 + for epoch in range(epochs): + train(model, device, epoch, train_data_loader, criterion, optimizer) + + model.eval() + ################################## + # Conformal Quantile Regression + ################################## + print("########################## CQR ###########################") + + predictor = CQR(model) + cal_dataset = TensorDataset(torch.from_numpy(X[0:T0, :]), torch.from_numpy(y[0:T0])) + test_dataset = TensorDataset(torch.from_numpy(X[T0:, :]), torch.from_numpy(y[T0:])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + test_data_loader = torch.utils.data.DataLoader(test_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + print(predictor.evaluate(test_data_loader)) + + ################################## + # Adaptive Conformal Inference, + ################################## + print("########################## ACI ###########################") + predictor = ACI(model, 0.0001) + test_y = torch.from_numpy(y[T0:num_examples]).to(device) + predicts = torch.zeros((num_examples - T0, 2)).to(device) + for i in range(num_examples - T0): + with torch.no_grad(): + cal_dataset = TensorDataset(torch.from_numpy(X[i:(T0 + i), :]), torch.from_numpy(y[i:(T0 + i)])) + cal_data_loader = torch.utils.data.DataLoader(cal_dataset, batch_size=100, shuffle=False, pin_memory=True) + predictor.calibrate(cal_data_loader, alpha) + tmp_x = torch.from_numpy(X[(T0 + i), :]) + if i == 0: + tmp_prediction_intervals = predictor.predict(tmp_x) + else: + tmp_prediction_intervals = predictor.predict(tmp_x, test_y[i - 1], predicts[i - 1]) + predicts[i, :] = tmp_prediction_intervals + + metrics = Metrics() + print("Etestuating prediction sets...") + print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}") + print(f"Average_size: {metrics('average_size')(predicts)}") diff --git a/tests/utils.py b/tests/utils.py new file mode 100644 index 00000000..104787d0 --- /dev/null +++ b/tests/utils.py @@ -0,0 +1,85 @@ +import numpy as np +import pandas as pd +import torch.nn as nn + +base_path = ".cache/data/" + + +def build_reg_data(data_name="community"): + if data_name == "community": + # https://github.com/vbordalo/Communities-Crime/blob/master/Crime_v1.ipynb + attrib = pd.read_csv(base_path + 'communities_attributes.csv', delim_whitespace=True) + data = pd.read_csv(base_path + 'communities.data', names=attrib['attributes']) + data = data.drop(columns=['state', 'county', + 'community', 'communityname', + 'fold'], axis=1) + + data = data.replace('?', np.nan) + + # Impute mean values for samples with missing values + + # imputer = SimpleImputer(missing_values = 'NaN', strategy = 'mean') + + # imputer = imputer.fit(data[['OtherPerCap']]) + # data[['OtherPerCap']] = imputer.transform(data[['OtherPerCap']]) + data['OtherPerCap'] = data['OtherPerCap'].astype("float") + mean_value = data['OtherPerCap'].mean() + data['OtherPerCap'].fillna(value=mean_value, inplace=True) + data = data.dropna(axis=1) + X = data.iloc[:, 0:100].values + y = data.iloc[:, 100].values + + # imputer = SimpleImputer(missing_values = 'NaN', strategy = 'mean') + + # imputer = imputer.fit(data[['OtherPerCap']]) + # data[['OtherPerCap']] = imputer.transform(data[['OtherPerCap']]) + data['OtherPerCap'] = data['OtherPerCap'].astype("float") + mean_value = data['OtherPerCap'].mean() + data['OtherPerCap'].fillna(value=mean_value, inplace=True) + data = data.dropna(axis=1) + X = data.iloc[:, 0:100].values + y = data.iloc[:, 100].values + elif data_name == "synthetic": + X = np.random.rand(500, 5) + y_wo_noise = 10 * np.sin(X[:, 0] * X[:, 1] * np.pi) + 20 * (X[:, 2] - 0.5) ** 2 + 10 * X[:, 3] + 5 * X[:, 4] + eplison = np.zeros(500) + phi = theta = 0.8 + delta_t_1 = np.random.randn() + for i in range(1, 500): + delta_t = np.random.randn() + eplison[i] = phi * eplison[i - 1] + delta_t_1 + theta * delta_t + delta_t_1 = delta_t + + y = y_wo_noise + eplison + + X = X.astype(np.float32) + y = y.astype(np.float32) + + return X, y + + +def build_regression_model(model_name="NonLinearNet"): + if model_name == "NonLinearNet": + class NonLinearNet(nn.Module): + def __init__(self, in_shape, out_shape, hidden_size, dropout): + super(NonLinearNet, self).__init__() + self.hidden_size = hidden_size + self.in_shape = in_shape + self.out_shape = out_shape + self.dropout = dropout + self.base_model = nn.Sequential( + nn.Linear(self.in_shape, self.hidden_size), + nn.ReLU(), + nn.Dropout(self.dropout), + nn.Linear(self.hidden_size, self.hidden_size), + nn.ReLU(), + nn.Dropout(self.dropout), + nn.Linear(self.hidden_size, self.out_shape), + ) + + def forward(self, x): + return self.base_model(x) + + return NonLinearNet + else: + raise NotImplementedError From d4d23764df4fcdbf2e09c8a2d6b707d0e8839f3c Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 18:34:12 +0800 Subject: [PATCH 05/16] update comments --- torchcp/classification/loss/conftr.py | 4 ++-- torchcp/classification/scores/thr.py | 2 +- torchcp/regression/loss/quantile.py | 14 ++++++++------ 3 files changed, 11 insertions(+), 9 deletions(-) diff --git a/torchcp/classification/loss/conftr.py b/torchcp/classification/loss/conftr.py index e4d8fb6b..5ea8bd5e 100644 --- a/torchcp/classification/loss/conftr.py +++ b/torchcp/classification/loss/conftr.py @@ -17,11 +17,11 @@ class ConfTr(nn.Module): Paper: https://arxiv.org/abs/2110.09192. - :param weights: the weight of each loss function + :param weight: the weight of each loss function :param predictor: the CP predictors :param alpha: the significance level for each training batch :param fraction: the fraction of the calibration set in each training batch - :param loss_types: the selected (multi-selected) loss functions, which can be "valid", "classification", "probs", "coverage". + :param loss_type: the selected (multi-selected) loss functions, which can be "valid", "classification", "probs", "coverage". :param target_size: Optional: 0 | 1. :param loss_transform: a transform for loss :param base_loss_fn: a base loss function. For example, cross entropy in classification. diff --git a/torchcp/classification/scores/thr.py b/torchcp/classification/scores/thr.py index 5c4aa692..015e6f69 100644 --- a/torchcp/classification/scores/thr.py +++ b/torchcp/classification/scores/thr.py @@ -14,7 +14,7 @@ class THR(BaseScore): Threshold conformal predictors (Sadinle et al., 2016). paper : https://arxiv.org/abs/1609.00451. - param score_type: a transformation on logits. Default: "softmax". Optional: "softmax", "Identity", "log_softmax" or "log". + :param score_type: a transformation on logits. Default: "softmax". Optional: "softmax", "Identity", "log_softmax" or "log". """ def __init__(self, score_type="softmax") -> None: diff --git a/torchcp/regression/loss/quantile.py b/torchcp/regression/loss/quantile.py index 934271a8..af50df90 100644 --- a/torchcp/regression/loss/quantile.py +++ b/torchcp/regression/loss/quantile.py @@ -3,16 +3,18 @@ __all__ = ["QuantileLoss"] + class QuantileLoss(nn.Module): - """ Pinball loss function + """ + Pinball loss function (Romano et al., 2019) + Paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/5103c3584b063c431bd1268e9b5e76fb-Paper.pdf + + :param quantiles: a list of quantiles, such as $[\frac{alpha}{2}, 1-\frac{alpha}{2}]$. """ def __init__(self, quantiles): """ - A loss to training a quantile-regression model (Romano et al., 2019). - Paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/5103c3584b063c431bd1268e9b5e76fb-Paper.pdf. - - :param quantiles: a list of quantiles, such as $[\frac{alpha}{2}, 1-\frac{alpha}{2}]$. + """ super().__init__() self.quantiles = quantiles @@ -30,7 +32,7 @@ def forward(self, preds, target): losses = preds.new_zeros(len(self.quantiles)) for i, q in enumerate(self.quantiles): - errors = target - preds[:, i:i+1] + errors = target - preds[:, i:i + 1] losses[i] = torch.sum(torch.max((q - 1) * errors, q * errors).squeeze(1)) loss = torch.mean(losses) return loss From a51dd15abe8e99c06cbc93b6b0ddf13093626376 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 18:43:43 +0800 Subject: [PATCH 06/16] update docs --- docs/source/conf.py | 4 ++++ docs/source/custom.css | 5 +++++ torchcp/regression/loss/quantile.py | 4 ++-- 3 files changed, 11 insertions(+), 2 deletions(-) create mode 100644 docs/source/custom.css diff --git a/docs/source/conf.py b/docs/source/conf.py index 5036df61..6ae95375 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -95,3 +95,7 @@ # If true, `todo` and `todoList` produce output, else they produce nothing. todo_include_todos = True + + +def setup(app): + app.add_stylesheet('custom.css') diff --git a/docs/source/custom.css b/docs/source/custom.css new file mode 100644 index 00000000..7dfbdd37 --- /dev/null +++ b/docs/source/custom.css @@ -0,0 +1,5 @@ +.wy-nav-side a { + font-family: 'Arial', sans-serif; + font-size: 16px; + +} diff --git a/torchcp/regression/loss/quantile.py b/torchcp/regression/loss/quantile.py index af50df90..1e1678a9 100644 --- a/torchcp/regression/loss/quantile.py +++ b/torchcp/regression/loss/quantile.py @@ -6,10 +6,10 @@ class QuantileLoss(nn.Module): """ - Pinball loss function (Romano et al., 2019) + Pinball loss function (Romano et al., 2019). Paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/5103c3584b063c431bd1268e9b5e76fb-Paper.pdf - :param quantiles: a list of quantiles, such as $[\frac{alpha}{2}, 1-\frac{alpha}{2}]$. + :param quantiles: a list of quantiles, such as $[\\frac{alpha}{2}, 1-\\frac{alpha}{2}]$. """ def __init__(self, quantiles): From 63ddae11c11ce1b448112da3f6aa9a83593dc772 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 18:52:53 +0800 Subject: [PATCH 07/16] update docs --- docs/source/conf.py | 2 -- docs/source/custom.css | 5 ----- 2 files changed, 7 deletions(-) delete mode 100644 docs/source/custom.css diff --git a/docs/source/conf.py b/docs/source/conf.py index 6ae95375..edaae0b6 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -97,5 +97,3 @@ todo_include_todos = True -def setup(app): - app.add_stylesheet('custom.css') diff --git a/docs/source/custom.css b/docs/source/custom.css deleted file mode 100644 index 7dfbdd37..00000000 --- a/docs/source/custom.css +++ /dev/null @@ -1,5 +0,0 @@ -.wy-nav-side a { - font-family: 'Arial', sans-serif; - font-size: 16px; - -} From 2fa097976ec1ad007e5f81506f210f6725384da6 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 18:59:23 +0800 Subject: [PATCH 08/16] update docs --- docs/source/torchcp.classification.rst | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/source/torchcp.classification.rst b/docs/source/torchcp.classification.rst index c30d5fdd..2189d4e3 100644 --- a/docs/source/torchcp.classification.rst +++ b/docs/source/torchcp.classification.rst @@ -68,4 +68,5 @@ Detailed description :members: .. autoclass:: ConfTr - :members: \ No newline at end of file + :members: + From 1a1869ba857e1992fc6ef8ecc20e4d4112d600e3 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 19:07:23 +0800 Subject: [PATCH 09/16] update docs --- torchcp/classification/loss/conftr.py | 16 ++++++++++------ 1 file changed, 10 insertions(+), 6 deletions(-) diff --git a/torchcp/classification/loss/conftr.py b/torchcp/classification/loss/conftr.py index 5ea8bd5e..03806995 100644 --- a/torchcp/classification/loss/conftr.py +++ b/torchcp/classification/loss/conftr.py @@ -4,6 +4,8 @@ # This source code is licensed under the license found in the # LICENSE file in the root directory of this source tree. # +__all__ = ["ConfTr"] + import torch @@ -14,9 +16,8 @@ class ConfTr(nn.Module): """ Conformal Training (Stutz et al., 2021). - Paper: https://arxiv.org/abs/2110.09192. - - + Paper: https://arxiv.org/abs/2110.09192 + :param weight: the weight of each loss function :param predictor: the CP predictors :param alpha: the significance level for each training batch @@ -31,10 +32,13 @@ def __init__(self, weight, predictor, alpha, fraction, loss_type="valid", target super(ConfTr, self).__init__() assert weight>0, "weight must be greater than 0." - assert (fraction > 0 and fraction<1), "fraction should be a value in (0,1)." - assert loss_type in ["valid", "classification", "probs", "coverage"], 'loss_type should be a value in ["valid", "classification", "probs", "coverage"].' + assert (0 < fraction < 1), "fraction should be a value in (0,1)." + assert loss_type in ["valid", "classification", "probs", "coverage"], ('loss_type should be a value in [' + '"valid", "classification", "probs", ' + '"coverage"].') assert target_size==0 or target_size ==1, "target_size should be 0 or 1." - assert loss_transform in ["square", "abs", "log"], 'loss_transform should be a value in ["square", "abs", "log"].' + assert loss_transform in ["square", "abs", "log"], ('loss_transform should be a value in ["square", "abs", ' + '"log"].') self.weight = weight self.predictor = predictor self.alpha = alpha From 0f84455b207b791d1fdfe79678c73e986a1c00f1 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 19:21:24 +0800 Subject: [PATCH 10/16] fix some code issues --- examples/conformal_training.py | 2 +- examples/covariate_shift.py | 2 +- examples/dataset.py | 6 +++--- examples/time_series.py | 2 +- tests/test_conformal_training.py | 2 +- tests/test_regression.py | 2 +- torchcp/classification/predictors/split.py | 5 ++--- torchcp/classification/predictors/weight.py | 7 +++---- torchcp/classification/scores/thr.py | 2 +- 9 files changed, 14 insertions(+), 16 deletions(-) diff --git a/examples/conformal_training.py b/examples/conformal_training.py index c875b668..f3840c60 100644 --- a/examples/conformal_training.py +++ b/examples/conformal_training.py @@ -72,7 +72,7 @@ def train(model, device, train_loader,criterion, optimizer, epoch): fix_randomness(seed=0) ################################## - # Training a pyotrch model + # Training a pytorch model ################################## device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_dataset = build_dataset("mnist") diff --git a/examples/covariate_shift.py b/examples/covariate_shift.py index 6ce746ca..ecb6e189 100644 --- a/examples/covariate_shift.py +++ b/examples/covariate_shift.py @@ -19,7 +19,7 @@ from torchcp.utils import fix_randomness if __name__ == '__main__': - parser = argparse.ArgumentParser(description='Covariate shift') + parser = argparse.ArgumentParser(description='Coveriate shift') parser.add_argument('--seed', default=0, type=int) args = parser.parse_args() diff --git a/examples/dataset.py b/examples/dataset.py index 3706a513..8fd83d84 100644 --- a/examples/dataset.py +++ b/examples/dataset.py @@ -13,7 +13,7 @@ def build_dataset(dataset_name, transform=None, mode="train"): data_dir = os.path.join(usr_dir, "data") if dataset_name == 'imagenet': - if transform == None: + if transform is None: transform = trn.Compose([ trn.Resize(256), trn.CenterCrop(224), @@ -25,7 +25,7 @@ def build_dataset(dataset_name, transform=None, mode="train"): dataset = dset.ImageFolder(data_dir + "/imagenet/val", transform) elif dataset_name == 'imagenetv2': - if transform == None: + if transform is None: transform = trn.Compose([ trn.Resize(256), trn.CenterCrop(224), @@ -38,7 +38,7 @@ def build_dataset(dataset_name, transform=None, mode="train"): transform) elif dataset_name == 'mnist': - if transform == None: + if transform is None: transform = trn.Compose([ trn.ToTensor(), trn.Normalize((0.1307,), (0.3081,)) diff --git a/examples/time_series.py b/examples/time_series.py index 76b9ff2d..6856d9f8 100644 --- a/examples/time_series.py +++ b/examples/time_series.py @@ -72,6 +72,6 @@ predicts[i, :] = tmp_prediction_intervals metrics = Metrics() - print("Etestuating prediction sets...") + print("Evaluating prediction sets...") print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}") print(f"Average_size: {metrics('average_size')(predicts)}") diff --git a/tests/test_conformal_training.py b/tests/test_conformal_training.py index 3674a587..3ff5de75 100644 --- a/tests/test_conformal_training.py +++ b/tests/test_conformal_training.py @@ -77,7 +77,7 @@ def test_training(): for seed in range(num_trials): fix_randomness(seed=seed) ################################## - # Training a pyotrch model + # Training a pytorch model ################################## device = torch.device("cuda" if torch.cuda.is_available() else "cpu") train_dataset = build_dataset("mnist") diff --git a/tests/test_regression.py b/tests/test_regression.py index 8b8f68df..1d990a1c 100644 --- a/tests/test_regression.py +++ b/tests/test_regression.py @@ -123,6 +123,6 @@ def test_time_series(): predicts[i, :] = tmp_prediction_intervals metrics = Metrics() - print("Etestuating prediction sets...") + print("Evaluating prediction sets...") print(f"Coverage_rate: {metrics('coverage_rate')(predicts, test_y)}") print(f"Average_size: {metrics('average_size')(predicts)}") diff --git a/torchcp/classification/predictors/split.py b/torchcp/classification/predictors/split.py index 09b2af26..b2738532 100644 --- a/torchcp/classification/predictors/split.py +++ b/torchcp/classification/predictors/split.py @@ -118,7 +118,6 @@ def evaluate(self, val_dataloader): labels_list.append(tmp_label) val_labels = torch.cat(labels_list) - res_dict = {} - res_dict["Coverage_rate"] = self._metric('coverage_rate')(prediction_sets, val_labels) - res_dict["Average_size"] = self._metric('average_size')(prediction_sets, val_labels) + res_dict = {"Coverage_rate": self._metric('coverage_rate')(prediction_sets, val_labels), + "Average_size": self._metric('average_size')(prediction_sets, val_labels)} return res_dict diff --git a/torchcp/classification/predictors/weight.py b/torchcp/classification/predictors/weight.py index 6ce4a279..b6284833 100644 --- a/torchcp/classification/predictors/weight.py +++ b/torchcp/classification/predictors/weight.py @@ -139,7 +139,6 @@ def evaluate(self, val_dataloader): labels_list.append(tmp_label) val_labels = torch.cat(labels_list) - res_dict = {} - res_dict["Coverage_rate"] = self._metric('coverage_rate')(prediction_sets, val_labels) - res_dict["Average_size"] = self._metric('average_size')(prediction_sets, val_labels) - return res_dict + result_dict = {"Coverage_rate": self._metric('coverage_rate')(prediction_sets, val_labels), + "Average_size": self._metric('average_size')(prediction_sets, val_labels)} + return result_dict diff --git a/torchcp/classification/scores/thr.py b/torchcp/classification/scores/thr.py index 015e6f69..705c3ee4 100644 --- a/torchcp/classification/scores/thr.py +++ b/torchcp/classification/scores/thr.py @@ -28,7 +28,7 @@ def __init__(self, score_type="softmax") -> None: elif score_type == "log_softmax": self.transform = lambda x: torch.log_softmax(x, dim=-1) elif score_type == "log": - self.transform = lambda x: torch.log(x, dim=-1) + self.transform = lambda x: torch.log(x) else: raise NotImplementedError From 05b1be1b6d58f19f7bfa324d0f0c31903580716a Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 19:39:40 +0800 Subject: [PATCH 11/16] update docs --- docs/source/conf.py | 27 +++++++++++++++++++++++++++ torchcp/classification/scores/raps.py | 2 +- torchcp/regression/loss/quantile.py | 2 +- 3 files changed, 29 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index edaae0b6..e4502be0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -78,6 +78,33 @@ # The master toctree document. master_doc = 'index' + + +# Resolve function for the linkcode extension. +def linkcode_resolve(domain, info): + def find_source(): + # try to find the file and line number, based on code from numpy: + # https://github.com/numpy/numpy/blob/master/doc/source/conf.py#L286 + obj = sys.modules[info['module']] + for part in info['fullname'].split('.'): + obj = getattr(obj, part) + import inspect + import os + fn = inspect.getsourcefile(obj) + fn = os.path.relpath(fn, start=os.path.dirname(torchcp.__file__)) + source, lineno = inspect.getsourcelines(obj) + return fn, lineno, lineno + len(source) - 1 + + if domain != 'py' or not info['module']: + return None + try: + filename = 'advertorch/%s#L%d-L%d' % find_source() + except Exception: + filename = info['module'].replace('.', '/') + '.py' + tag = 'master' + url = "https://github.com/ml-stat-Sustech/TorchCP/blob/%s/%s" + return url % (tag, filename) + # -- Options for HTML output ------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output diff --git a/torchcp/classification/scores/raps.py b/torchcp/classification/scores/raps.py index 5ba79419..814452a5 100644 --- a/torchcp/classification/scores/raps.py +++ b/torchcp/classification/scores/raps.py @@ -19,7 +19,7 @@ class RAPS(APS): paper : https://arxiv.org/abs/2009.14193 :param penalty: the weight of regularization. When penalty = 0, RAPS=APS. - :param kreg : the rank of regularization which is an integer in [0,labels_num]. + :param kreg: the rank of regularization which is an integer in [0,labels_num]. """ def __init__(self, penalty, kreg=0): diff --git a/torchcp/regression/loss/quantile.py b/torchcp/regression/loss/quantile.py index 1e1678a9..31140a75 100644 --- a/torchcp/regression/loss/quantile.py +++ b/torchcp/regression/loss/quantile.py @@ -9,7 +9,7 @@ class QuantileLoss(nn.Module): Pinball loss function (Romano et al., 2019). Paper: https://proceedings.neurips.cc/paper_files/paper/2019/file/5103c3584b063c431bd1268e9b5e76fb-Paper.pdf - :param quantiles: a list of quantiles, such as $[\\frac{alpha}{2}, 1-\\frac{alpha}{2}]$. + :param quantiles: a list of quantiles, such as :math: [alpha/2, 1-alpha/2]. """ def __init__(self, quantiles): From 2a3a16c3fedfb6762924a9dc03fd7c63a016fc29 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 19:43:28 +0800 Subject: [PATCH 12/16] update docs --- docs/source/conf.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index e4502be0..a8ae57f9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -78,7 +78,15 @@ # The master toctree document. master_doc = 'index' +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +# This pattern also affects html_static_path and html_extra_path. +exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +todo_include_todos = False # Resolve function for the linkcode extension. def linkcode_resolve(domain, info): @@ -98,7 +106,7 @@ def find_source(): if domain != 'py' or not info['module']: return None try: - filename = 'advertorch/%s#L%d-L%d' % find_source() + filename = 'torchcp/%s#L%d-L%d' % find_source() except Exception: filename = info['module'].replace('.', '/') + '.py' tag = 'master' From f2b03ec8ba8c226bae4e51a31b689292726c25de Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 19:45:58 +0800 Subject: [PATCH 13/16] update docs --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index a8ae57f9..2f30a3ae 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -109,7 +109,7 @@ def find_source(): filename = 'torchcp/%s#L%d-L%d' % find_source() except Exception: filename = info['module'].replace('.', '/') + '.py' - tag = 'master' + tag = 'development' url = "https://github.com/ml-stat-Sustech/TorchCP/blob/%s/%s" return url % (tag, filename) From 0feeae42bdbfe9d5444bb6c29d1e1d51dbd17632 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 19:48:35 +0800 Subject: [PATCH 14/16] update docs --- docs/source/conf.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 2f30a3ae..9c47f153 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -110,7 +110,7 @@ def find_source(): except Exception: filename = info['module'].replace('.', '/') + '.py' tag = 'development' - url = "https://github.com/ml-stat-Sustech/TorchCP/blob/%s/%s" + url = "https://github.com/ml-stat-Sustech/TorchCP/%s/%s" return url % (tag, filename) # -- Options for HTML output ------------------------------------------------- From cbdb41a49e02a6140dc9919d328c889ac93b867c Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 19:52:10 +0800 Subject: [PATCH 15/16] update docs --- docs/source/conf.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/conf.py b/docs/source/conf.py index 9c47f153..a8ae57f9 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -109,8 +109,8 @@ def find_source(): filename = 'torchcp/%s#L%d-L%d' % find_source() except Exception: filename = info['module'].replace('.', '/') + '.py' - tag = 'development' - url = "https://github.com/ml-stat-Sustech/TorchCP/%s/%s" + tag = 'master' + url = "https://github.com/ml-stat-Sustech/TorchCP/blob/%s/%s" return url % (tag, filename) # -- Options for HTML output ------------------------------------------------- From 3e3db8f1f988531207466434523b26686b4c2628 Mon Sep 17 00:00:00 2001 From: huangjg Date: Mon, 25 Dec 2023 20:00:07 +0800 Subject: [PATCH 16/16] update imagenet_example.py --- examples/imagenet_example.py | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/examples/imagenet_example.py b/examples/imagenet_example.py index 16529330..4d63a47b 100644 --- a/examples/imagenet_example.py +++ b/examples/imagenet_example.py @@ -24,12 +24,6 @@ parser = argparse.ArgumentParser(description='') parser.add_argument('--seed', default=0, type=int) parser.add_argument('--alpha', default=0.1, type=float) - parser.add_argument('--predictor', default="Standard", help="Standard | ClassWise | Cluster") - parser.add_argument('--score', default="THR", help="THR | APS | SAPS") - parser.add_argument('--penalty', default=1, type=float) - parser.add_argument('--kreg', default=0, type=int) - parser.add_argument('--weight', default=0.2, type=int) - parser.add_argument('--split', default="random", type=str, help="proportional | doubledip | random") args = parser.parse_args() fix_randomness(seed=args.seed) @@ -62,28 +56,9 @@ # A standard process of conformal prediction ####################################### alpha = args.alpha - print( - f"Experiment--Data : ImageNet, Model : {model_name}, Score : {args.score}, Predictor : {args.predictor}, Alpha : {alpha}") - num_classes = 1000 - if args.score == "THR": - score_function = THR() - elif args.score == "APS": - score_function = APS() - elif args.score == "RAPS": - score_function = RAPS(args.penalty, args.kreg) - elif args.score == "SAPS": - score_function = SAPS(weight=args.weight) - else: - raise NotImplementedError - - if args.predictor == "Standard": - predictor = SplitPredictor(score_function, model) - elif args.predictor == "ClassWise": - predictor = ClassWisePredictor(score_function, model) - elif args.predictor == "Cluster": - predictor = ClusterPredictor(score_function, model, args.seed) - else: - raise NotImplementedError + print(f"Experiment--Data : ImageNet, Model : {model_name}, Score : THR, Predictor : SplitPredictor, Alpha : {alpha}") + score_function = THR() + predictor = SplitPredictor(score_function, model) print(f"The size of calibration set is {len(cal_dataset)}.") predictor.calibrate(cal_data_loader, alpha) predictor.evaluate(test_data_loader)