Merge pull request #23 from ispras/Jaccard_defense

Jaccard defense

experiments/attack_defense_test.py

@@ -175,7 +175,7 @@ def test_attack_defense():
     print(metric_loc)
 
 def test_meta():
-    from attacks.poison_attacks_collection.metattack import meta_gradient_attack
+    from attacks.metattack import meta_gradient_attack
     my_device = device('cpu')
     full_name = ("single-graph", "Planetoid", 'Cora')
 
@@ -459,10 +459,161 @@ def test_qattack():
     # print(f"info_before_evasion_attack: {info_before_evasion_attack}")
     # print(f"info_after_evasion_attack: {info_after_evasion_attack}")
 
+def test_jaccard():
+    from defense.JaccardDefense import jaccard_def
+    # my_device = device('cuda' if is_available() else 'cpu')
+    my_device = device('cpu')
+
+    full_name = None
+
+    # full_name = ("multiple-graphs", "TUDataset", 'MUTAG')
+    # full_name = ("single-graph", "custom", 'karate')
+    full_name = ("single-graph", "Planetoid", 'Cora')
+    # full_name = ("multiple-graphs", "TUDataset", 'PROTEINS')
+
+    dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+        full_name=full_name,
+        dataset_ver_ind=0
+    )
+
+    # dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+    #     full_name=("single-graph", "custom", "example",),
+    #     features={'attr': {'a': 'as_is', 'b': 'as_is'}},
+    #     labeling='threeClasses',
+    #     dataset_ver_ind=0
+    # )
+
+    # dataset, data, results_dataset_path = DatasetManager.get_by_full_name(
+    #     # full_name=("single-graph", "vk_samples", "vk2-ff40-N100000-A.1612175945",),
+    #     full_name=("single-graph", "vk_samples", "vk2-ff20-N10000-A.1611943634",),
+    #     # full_name=("single-graph", "vk_samples", "vk2-ff20-N1000-U.1612273925",),
+    #     # features=('sex',),
+    #     features={'str_f': tuple(), 'str_g': None, 'attr': {
+    #         # "('personal', 'political')": 'one_hot',
+    #         # "('occupation', 'type')": 'one_hot', # Don't work now
+    #         # "('relation',)": 'one_hot',
+    #         # "('age',)": 'one_hot',
+    #         "('sex',)": 'one_hot',
+    #     }},
+    #     # features={'str_f': tuple(), 'str_g': None, 'attr': {'sex': 'one_hot', }},
+    #     labeling='sex1',
+    #     dataset_ver_ind=0
+    # )
+
+    # print(data.train_mask)
+
+    gnn = model_configs_zoo(dataset=dataset, model_name='gcn_gcn')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gcn_gcn_lin')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='test_gnn')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gin_gin_gin_lin_lin')
+    # gnn = model_configs_zoo(dataset=dataset, model_name='gin_gin_gin_lin_lin_prot')
+
+    manager_config = ConfigPattern(
+        _config_class="ModelManagerConfig",
+        _config_kwargs={
+            "mask_features": [],
+            "optimizer": {
+                # "_config_class": "Config",
+                "_class_name": "Adam",
+                # "_import_path": OPTIMIZERS_PARAMETERS_PATH,
+                # "_class_import_info": ["torch.optim"],
+                "_config_kwargs": {},
+            }
+        }
+    )
+    # manager_config = ModelManagerConfig(**{
+    #         "mask_features": [],
+    #         "optimizer": {
+    #             # "_config_class": "Config",
+    #             "_class_name": "Adam",
+    #             # "_import_path": OPTIMIZERS_PARAMETERS_PATH,
+    #             # "_class_import_info": ["torch.optim"],
+    #             "_config_kwargs": {},
+    #         }
+    #     }
+    # )
+
+    # train_test_split = [0.8, 0.2]
+    # train_test_split = [0.6, 0.4]
+    steps_epochs = 200
+    gnn_model_manager = FrameworkGNNModelManager(
+        gnn=gnn,
+        dataset_path=results_dataset_path,
+        manager_config=manager_config,
+        modification=ModelModificationConfig(model_ver_ind=0, epochs=steps_epochs)
+    )
+
+    save_model_flag = False
+    # save_model_flag = True
+
+    # data.x = data.x.float()
+    gnn_model_manager.gnn.to(my_device)
+    data = data.to(my_device)
+
+    evasion_attack_config = ConfigPattern(
+        _class_name="FGSM",
+        _import_path=EVASION_ATTACK_PARAMETERS_PATH,
+        _config_class="EvasionAttackConfig",
+        _config_kwargs={
+            "epsilon": 0.007 * 1,
+        }
+    )
+    # evasion_defense_config = ConfigPattern(
+    #     _class_name="JaccardDefender",
+    #     _import_path=EVASION_DEFENSE_PARAMETERS_PATH,
+    #     _config_class="EvasionDefenseConfig",
+    #     _config_kwargs={
+    #     }
+    # )
+    poison_defense_config = ConfigPattern(
+        _class_name="JaccardDefender",
+        _import_path=POISON_DEFENSE_PARAMETERS_PATH,
+        _config_class="PoisonDefenseConfig",
+        _config_kwargs={
+        }
+    )
+
+    # gnn_model_manager.set_poison_attacker(poison_attack_config=poison_attack_config)
+    gnn_model_manager.set_poison_defender(poison_defense_config=poison_defense_config)
+    gnn_model_manager.set_evasion_attacker(evasion_attack_config=evasion_attack_config)
+    # gnn_model_manager.set_evasion_defender(evasion_defense_config=evasion_defense_config)
+
+    warnings.warn("Start training")
+    dataset.train_test_split()
+
+    try:
+        raise FileNotFoundError()
+        # gnn_model_manager.load_model_executor()
+    except FileNotFoundError:
+        gnn_model_manager.epochs = gnn_model_manager.modification.epochs = 0
+        train_test_split_path = gnn_model_manager.train_model(gen_dataset=dataset, steps=steps_epochs,
+                                                              save_model_flag=save_model_flag,
+                                                              metrics=[Metric("F1", mask='train', average=None),
+                                                                       Metric("Accuracy", mask="train")])
+
+        if train_test_split_path is not None:
+            dataset.save_train_test_mask(train_test_split_path)
+            train_mask, val_mask, test_mask, train_test_sizes = torch.load(train_test_split_path / 'train_test_split')[
+                                                                :]
+            dataset.train_mask, dataset.val_mask, dataset.test_mask = train_mask, val_mask, test_mask
+            data.percent_train_class, data.percent_test_class = train_test_sizes
+
+    warnings.warn("Training was successful")
+
+    metric_loc = gnn_model_manager.evaluate_model(
+        gen_dataset=dataset, metrics=[Metric("F1", mask='train', average='macro'),
+                                      Metric("Accuracy", mask='train')])
+    print("TRAIN", metric_loc)
+
+    metric_loc = gnn_model_manager.evaluate_model(
+        gen_dataset=dataset, metrics=[Metric("F1", mask='test', average='macro'),
+                                      Metric("Accuracy", mask='test')])
+    print("TEST", metric_loc)
+
 
 if __name__ == '__main__':
+    torch.manual_seed(5000)
+    #test_meta()
+    #test_qattack()
     #test_attack_defense()
-    # torch.manual_seed(5000)
-    # test_meta()
-    # test_qattack()
-    test_nettack_evasion()
+    test_jaccard()

metainfo/evasion_defense_parameters.json

@@ -7,6 +7,5 @@
  "QuantizationDefender": {
   "qbit": ["qbit", "int", 8, {"min": 1, "step": 1}, "?"]
  }
-
 }
 

metainfo/poison_defense_parameters.json

@@ -3,7 +3,9 @@
  },
  "BadRandomPoisonDefender": {
   "n_edges_percent": ["n_edges_percent", "float", 0.1, {"min": 0.0001, "step": 0.01}, "?"]
+ },
+ "JaccardDefender": {
+  "threshold": ["Edge Threshold", "float", 0.35, {"min": 0, "max": 1, "step":  0.01}, "Jaccard index threshold for dropping edges"]
  }
-
 }
 

src/attacks/poison_attacks_collection/metattack/meta_gradient_attack.py → src/attacks/metattack/meta_gradient_attack.py

@@ -2,16 +2,16 @@
 import torch
 import numpy as np
 import scipy.sparse as sp
-import attacks.poison_attacks_collection.metattack.utils as utils
+import attacks.metattack.utils as utils
 from torch.nn import functional as F
 from torch.nn.parameter import Parameter
 from torch import optim
 from tqdm import tqdm
 from models_builder.gnn_models import FrameworkGNNModelManager
 from models_builder.models_zoo import model_configs_zoo
-from aux.configs import ModelManagerConfig, ModelModificationConfig, DatasetConfig, DatasetVarConfig, ConfigPattern
+from aux.configs import ModelModificationConfig, ConfigPattern
 from aux.utils import OPTIMIZERS_PARAMETERS_PATH
-from torch_geometric.utils import to_dense_adj, to_torch_csr_tensor, to_torch_coo_tensor, dense_to_sparse, to_edge_index
+from torch_geometric.utils import dense_to_sparse
 
 from attacks.poison_attacks import PoisonAttacker
 
@@ -380,7 +380,8 @@ def get_meta_grad(self, features, adj_norm, idx_train, idx_unlabeled, labels, la
             attack_loss = self.lambda_ * loss_labeled + (1 - self.lambda_) * loss_unlabeled
 
         print('GCN loss on unlabled data: {}'.format(loss_test_val.item()))
-        print('GCN acc on unlabled data: {}'.format(utils.accuracy(output[idx_unlabeled], labels[idx_unlabeled]).item()))
+        print('GCN acc on unlabled data: {}'.format(
+            utils.accuracy(output[idx_unlabeled], labels[idx_unlabeled]).item()))
         print('attack loss: {}'.format(attack_loss.item()))
 
         adj_grad, feature_grad = None, None
@@ -553,4 +554,5 @@ def inner_train(self, features, modified_adj, idx_train, idx_unlabeled, labels,
 
         loss_test_val = F.nll_loss(output[idx_unlabeled], labels[idx_unlabeled])
         print('GCN loss on unlabled data: {}'.format(loss_test_val.item()))
-        print('GCN acc on unlabled data: {}'.format(utils.accuracy(output[idx_unlabeled], labels[idx_unlabeled]).item()))
+        print('GCN acc on unlabled data: {}'.format(
+            utils.accuracy(output[idx_unlabeled], labels[idx_unlabeled]).item()))

src/defense/JaccardDefense/jaccard_def.py

@@ -0,0 +1,67 @@
+import torch
+import numpy as np
+
+from defense.evasion_defense import EvasionDefender
+
+from defense.poison_defense import PoisonDefender
+
+# class JaccardDefender(EvasionDefender):
+#     name = 'JaccardDefender'
+#
+#     def __init__(self, threshold, **kwargs):
+#         super().__init__()
+#         self.thrsh = threshold
+#
+#     def pre_batch(self, model_manager, batch, **kwargs):
+#         # TODO need to check whether features binary or not. Consistency required - Cora has 'unknown' features e.g.
+#         #self.drop_edges(batch)
+#         edge_index = batch.edge_index.tolist()
+#         new_edge_mask = torch.zeros_like(batch.edge_index)
+#         for i in range(len(edge_index)):
+#             if self.jaccard_index(batch.x, edge_index[0][i], edge_index[1][i]) < self.thrsh:
+#                 new_edge_mask[0,i] = True
+#                 new_edge_mask[1,i] = True
+#         batch.edge_index = batch.edge_index[new_edge_mask]
+#
+#     def jaccard_index(self, x, u, v):
+#         im1 = x[u,:].numpy().astype(bool)
+#         im2 = x[v,:].numpy().astype(bool)
+#         intersection = np.logical_and(im1, im2)
+#         union = np.logical_or(im1, im2)
+#         return intersection.sum() / float(union.sum())
+#
+#     def post_batch(self, **kwargs):
+#         pass
+
+    # def drop_edges(self, batch):
+    #     print("KEK")
+
+class JaccardDefender(PoisonDefender):
+    name = 'JaccardDefender'
+
+    def __init__(self, threshold, **kwargs):
+        super().__init__()
+        self.thrsh = threshold
+
+    def defense(self, gen_dataset, **kwargs):
+        # TODO need to check whether features binary or not. Consistency required - Cora has 'unknown' features e.g.
+        #self.drop_edges(batch)
+        edge_index = gen_dataset.dataset.data.edge_index.tolist()
+        #new_edge_mask = torch.zeros_like(gen_dataset.dataset.data.edge_index).bool()
+        new_edge_index = [[],[]]
+        for i in range(len(edge_index[0])):
+            if self.jaccard_index(gen_dataset.dataset.data.x, edge_index[0][i], edge_index[1][i]) > self.thrsh:
+                # new_edge_mask[0,i] = True
+                # new_edge_mask[1,i] = True
+                new_edge_index[0].append(edge_index[0][i])
+                new_edge_index[1].append(edge_index[1][i])
+        # gen_dataset.dataset.data.edge_index *= new_edge_mask.float()
+        gen_dataset.dataset.data.edge_index = torch.tensor(new_edge_index).long()
+        return gen_dataset
+
+    def jaccard_index(self, x, u, v):
+        im1 = x[u,:].numpy().astype(bool)
+        im2 = x[v,:].numpy().astype(bool)
+        intersection = np.logical_and(im1, im2)
+        union = np.logical_or(im1, im2)
+        return intersection.sum() / float(union.sum())

tests/attacks_test.py

@@ -4,16 +4,15 @@
 
 from base.datasets_processing import DatasetManager
 from models_builder.gnn_models import FrameworkGNNModelManager, Metric
-from aux.configs import ModelManagerConfig, ModelModificationConfig, DatasetConfig, DatasetVarConfig, ConfigPattern
+from aux.configs import ModelModificationConfig, DatasetConfig, DatasetVarConfig, ConfigPattern
 from models_builder.models_zoo import model_configs_zoo
 
-from aux.utils import POISON_ATTACK_PARAMETERS_PATH, POISON_DEFENSE_PARAMETERS_PATH, EVASION_ATTACK_PARAMETERS_PATH, \
-    EVASION_DEFENSE_PARAMETERS_PATH, OPTIMIZERS_PARAMETERS_PATH
+from aux.utils import POISON_ATTACK_PARAMETERS_PATH, EVASION_ATTACK_PARAMETERS_PATH, \
+    OPTIMIZERS_PARAMETERS_PATH
 
 class AttacksTest(unittest.TestCase):
     def setUp(self):
         print('setup')
-        from attacks.poison_attacks_collection.metattack import meta_gradient_attack
 
         # Init datasets
         # Single-Graph - Example
@@ -58,6 +57,7 @@ def setUp(self):
 
 
     def test_metattack_full(self):
+        from attacks.metattack import meta_gradient_attack
         poison_attack_config = ConfigPattern(
             _class_name="MetaAttackFull",
             _import_path=POISON_ATTACK_PARAMETERS_PATH,