Merge branch 'master' of https://github.com/pingcap/kvproto into comp…

…act-log-backup

Signed-off-by: hillium <[email protected]>

OWNERS_ALIASES

@@ -20,9 +20,9 @@ aliases:
   sig-approvers-pb-coprocessor: [cfzjywxk]
   sig-approvers-pb-deadlock: [MyonKeminta, cfzjywxk]
   sig-approvers-pb-debug: [cfzjywxk]
-  sig-approvers-pb-pd: [niubell, rleungx]
-  sig-approvers-pb-raftstore: [zhangjinpeng1987, overvenus]
-  sig-approvers-pb-tikv: [zhangjinpeng1987]
+  sig-approvers-pb-pd: [niubell, rleungx, nolouch]
+  sig-approvers-pb-raftstore: [zhangjinpeng87, overvenus]
+  sig-approvers-pb-tikv: [zhangjinpeng87]
   sig-approvers-pb-error: [cfzjywxk]
   sig-approvers-pb-kvrpc: [cfzjywxk]
   sig-approvers-pb-trace: [you06]

proto/OWNERS

@@ -40,12 +40,12 @@ filters:
     approvers: [sig-approvers-pb-raftstore]
   "^recoverdatapb\\.proto$":
     approvers: [sig-approvers-pb-br]
-  "^resource_manager\\.proto$":
-    approvers: [sig-approvers-pb-raftstore]
   "^tikvpb\\.proto$":
     approvers: [sig-approvers-pb-tikv, sig-approvers-pb-kvrpc]
   "^tracepb\\.proto$":
     approvers: [sig-approvers-pb-trace]
+  "^resource_manager\\.proto$":
+    approvers: [sig-approvers-pb-pd]
   "^pdpb\\.proto$":
     approvers: [sig-approvers-pb-pd]
   "^keyspacepb\\.proto$":

proto/brpb.proto

@@ -277,6 +277,14 @@ enum CompressionType {
     ZSTD = 3;
 }
 
+message StreamBackupTaskSecurityConfig {
+    oneof encryption {
+        // not recommended in production. configure KMS based or locally managed master key instead in TiKV StreamBackupConfig
+        CipherInfo plaintext_data_key = 1;
+        MasterKeyConfig master_key_config = 2;
+    }
+}
+
 // BackupMpde represents the mode of this whole backup request to the cluster.
 // and we need to store it in `backupmeta`.
 enum BackupMode {
@@ -295,6 +303,11 @@ message CipherInfo {
     bytes cipher_key = 2;
 }
 
+message MasterKeyConfig {
+    encryptionpb.EncryptionMethod encryption_type = 1;
+    repeated encryptionpb.MasterKey master_keys = 2;
+}
+
 message BackupRequest {
     uint64 cluster_id = 1;
 
@@ -356,7 +369,7 @@ message StreamBackupTaskInfo {
     uint64 start_ts = 2;
     uint64 end_ts = 3;
 
-    // Misc meta datas.
+    // Misc meta data.
     // The name of the task, also the ID of the task.
     string name = 4;
     // The table filter of the task.
@@ -366,7 +379,10 @@ message StreamBackupTaskInfo {
     // compression type
     CompressionType compression_type = 6;
 
-    // The last timestamp of the task has been updated. 
+    // security config for backup files
+    StreamBackupTaskSecurityConfig security_config = 7;
+
+    // The last timestamp of the task has been updated.
     // This is a simple solution for unfrequent config changing: 
     // When we watched a config change(via polling or etcd watching),
     // We perform a incremental scan between [last_update_ts, now), 
@@ -665,7 +681,7 @@ message DataFileGroup {
 }
 
 message DataFileInfo {
-    // SHA256 of the file.
+    // Checksum of the plaintext file, i.e., pre-compression, pre-encryption.
     bytes sha256 = 1;
     // Path of the file.
     string path = 2;
@@ -723,13 +739,13 @@ message DataFileInfo {
     bytes region_end_key = 21;
     // The region epoch that the log file belongs to.
     // In older versions, this might be empty.
-    // If a region get split or merged during observing, 
+    // If a region get split or merged during observing,
     // the file may contain multi epoches.
     // This may not be complete: file may contain records from other versions.
     repeated metapb.RegionEpoch region_epoch = 22;
 
-    // It may support encrypting at future.
-    reserved "iv";
+    // Encryption information of this data file, not set if plaintext.
+    encryptionpb.FileEncryptionInfo file_encryption_info = 23;
 }
 
 message StreamBackupError {
@@ -829,12 +845,12 @@ message LogFileCompaction {
 }
 
 message MetaEdit {
-    // Path to the meta file. 
+    // Path to the meta file.
     string path = 1;
     // Delete the physical files (MetaFileGroup) in the meta file.
     repeated string delete_physical_files = 2;
     // Delete the logical files (MetaFileInfo) in the meta file.
-    // Note: Even the operation have been performed in the meta, 
+    // Note: Even the operation have been performed in the meta,
     // this modification should be kept as long as the corresponding physical
     // file not deleted. Or we may cannot know when to delete the physical file.
     // Then the file will be leak until truncated.
@@ -843,7 +859,7 @@ message MetaEdit {
     bool destruct_self = 4;
 }
 
-// An extended version of `SpansOfFile`, added more metadata for the 
+// An extended version of `SpansOfFile`, added more metadata for the
 // execution of delayed deletion.
 message DeleteSpansOfFile {
     string path = 1;

proto/cdcpb.proto

@@ -38,6 +38,13 @@ message ClusterIDMismatch {
     uint64 request = 2;
 }
 
+// Congested is an error variable that
+// tells people that the TiKV-CDC is congested.
+message Congested {
+    // The region ID that triggers the congestion.
+    uint64 region_id = 1;
+}
+
 message Error {
     errorpb.NotLeader not_leader = 1;
     errorpb.RegionNotFound region_not_found = 2;
@@ -46,6 +53,7 @@ message Error {
     Compatibility compatibility = 5;
     ClusterIDMismatch cluster_id_mismatch = 6;
     errorpb.ServerIsBusy server_is_busy = 7;
+    Congested congested = 8;
 }
 
 message TxnInfo {

proto/deadlock.proto

@@ -33,19 +33,33 @@ enum DeadlockRequestType {
 	CleanUp = 2;
 }
 
+message ReplaceLockByKeyItem {
+	uint64 key_hash = 1;
+	bytes key = 2;
+	uint64 old_lock_ts = 3;
+	uint64 new_lock_ts = 4;
+}
+
+message ReplaceLocksByKeysRequest {
+	repeated ReplaceLockByKeyItem items = 1;
+}
+
 message DeadlockRequest {
 	DeadlockRequestType tp = 1;
 	WaitForEntry entry = 2 [(gogoproto.nullable) = false];
+	ReplaceLocksByKeysRequest replace_locks_by_keys = 3;
 }
 
 message DeadlockResponse {
 	// The same entry sent by DeadlockRequest, identifies the sender.
 	WaitForEntry entry = 1 [(gogoproto.nullable) = false];
-	// The key hash of the lock that is hold by the waiting transaction.
+	// The key hash of the lock that is hold by the waiting transaction. The hash of the `deadlock_key` field.
 	uint64 deadlock_key_hash = 2;
 	// The other entries of the dead lock circle. The current entry is in `entry` field and  not
 	// included in this field.
 	repeated WaitForEntry wait_chain = 3;
+	// The key of the lock that is hold by the waiting transaction.
+	bytes deadlock_key = 4;
 }
 
 service Deadlock {

proto/encryptionpb.proto

@@ -96,6 +96,36 @@ message MasterKeyKms {
     string region = 3;
     // KMS endpoint. Normally not needed.
     string endpoint = 4;
+    // optional, used to set up azure master key backend
+    AzureKms azure_kms = 5;
+    // optional, used to set up gcp master key backend
+    GcpKms gcp_kms = 6;
+    // optional, used to set up aws master key backend
+    AwsKms aws_kms = 7;
+}
+
+message AzureKms {
+    string tenant_id = 1;
+    string client_id = 2;
+    string client_secret = 3;
+    // Key vault to encrypt/decrypt data key.
+    string key_vault_url = 4;
+    // optional hsm used to generate data key
+    string hsm_name = 5;
+    string hsm_url = 6;
+    string client_certificate = 7;
+    string client_certificate_path = 8;
+    string client_certificate_password = 9;
+
+}
+
+message GcpKms {
+    string credential = 1;
+}
+
+message AwsKms {
+    string access_key = 1;
+    string secret_access_key = 2;
 }
 
 message EncryptedContent {
@@ -113,3 +143,25 @@ message EncryptedContent {
     // Valid only when KMS is used.
     bytes ciphertext_key = 5;
 }
+
+message FileEncryptionInfo {
+    oneof mode {
+        PlainTextDataKey plain_text_data_key = 1;
+        MasterKeyBased master_key_based = 2;
+    }
+    // file encryption method
+    encryptionpb.EncryptionMethod encryption_method = 3;
+    // iv to encrypt the file by data key
+    bytes file_iv = 4;
+    // file checksum after encryption, optional if using GCM
+    bytes checksum = 5;
+}
+
+// not recommended in production.
+// user needs to pass back the same data key for restore.
+message PlainTextDataKey {}
+
+message MasterKeyBased {
+    // encrypted data key with metadata
+    repeated encryptionpb.EncryptedContent data_key_encrypted_content = 1;
+}

proto/import_sstpb.proto

@@ -8,6 +8,7 @@ import "kvrpcpb.proto";
 import "gogoproto/gogo.proto";
 import "rustproto.proto";
 import "brpb.proto";
+import "encryptionpb.proto";
 
 option (gogoproto.sizer_all) = true;
 option (gogoproto.marshaler_all) = true;
@@ -394,6 +395,9 @@ message KVMeta {
 
     // the compression type for the file.
     backup.CompressionType compression_type = 13;
+
+    // encryption information of the kv file, not set if encryption is not enabled.
+    encryptionpb.FileEncryptionInfo file_encryption_info = 14;
 }
 
 
@@ -425,8 +429,11 @@ message ApplyRequest {
     // context represents region info and it used to build raft commands.
     kvrpcpb.Context context = 4;
 
-    // cipher_info is used to decrypt kv file when download file.
+    // plaintext data key to decrypt kv file if configured during log backup.
     backup.CipherInfo cipher_info = 11;
+
+    // master keys config used to decrypt data keys in restore if configured during log backup.
+    repeated encryptionpb.MasterKey master_keys = 14;
 }
 
 message ApplyResponse {

proto/kvrpcpb.proto

@@ -1039,8 +1039,13 @@ message AlreadyExist {
 message Deadlock {
     uint64 lock_ts = 1;
     bytes lock_key = 2;
+
+    // The hash of `deadlock_key` field.
     uint64 deadlock_key_hash = 3;
     repeated deadlock.WaitForEntry wait_chain = 4;
+
+    // The key that the current transaction has already acquired and blocks another transaction to form the deadlock.
+    bytes deadlock_key = 5;
 }
 
 message CommitTsExpired {

proto/raft_serverpb.proto

@@ -255,6 +255,12 @@ enum ExtraMessageType {
     // Messages for the snapshot gen precheck process.
     MsgSnapGenPrecheckRequest = 15;
     MsgSnapGenPrecheckResponse = 16;
+    // Used in transfer leader process for leader to inform follower to load the 
+    // region into in-memory engine if the relevant region is cached.
+    MsgPreLoadRegionRequest = 17;
+    // Used in transfer leader process for follower to inform leader the completes 
+    // of the region cache
+    MsgPreLoadRegionResponse = 18;
 }
 
 message FlushMemtable {

proto/resource_manager.proto

@@ -196,9 +196,10 @@ message Error {
 
 enum RunawayAction {
   NoneAction = 0;
-  DryRun = 1;    // do nothing
-  CoolDown = 2;  // deprioritize the task 
-  Kill = 3;      // kill the task
+  DryRun = 1;      // do nothing
+  CoolDown = 2;    // deprioritize the task 
+  Kill = 3;        // kill the task
+  SwitchGroup = 4; // switch the task to another group
 }
 
 enum RunawayWatchType {
@@ -210,6 +211,8 @@ enum RunawayWatchType {
 
 message RunawayRule {
   uint64 exec_elapsed_time_ms = 1;
+  int64 processed_keys = 2;
+  int64 request_unit = 3;
 }
 
 message RunawayWatch {
@@ -219,13 +222,19 @@ message RunawayWatch {
 }
 
 message RunawaySettings {
-  RunawayRule rule = 1; 
+  RunawayRule rule = 1;
   RunawayAction action = 2;
   RunawayWatch watch = 3;
+  // When the runaway action is `SwitchGroup`,
+  // this field will be used to indicate which group to switch.
+  string switch_group_name = 4;
 }
 
 message BackgroundSettings {
-  repeated string job_types = 1; 
+  // background task types.
+  repeated string job_types = 1;
+  // the percentage limit of total resource(cpu/io) that background tasks can use.
+  uint64 utilization_limit = 2;
 }
 
 message Participant {

scripts/check.sh

@@ -1,12 +1,17 @@
 #!/usr/bin/env bash
 
 check_protoc_version() {
-    version=$(protoc --version)
-    major=$(echo ${version} | sed -n -e 's/.*\([0-9]\{1,\}\)\.[0-9]\{1,\}\.[0-9]\{1,\}.*/\1/p')
-    minor=$(echo ${version} | sed -n -e 's/.*[0-9]\{1,\}\.\([0-9]\{1,\}\)\.[0-9]\{1,\}.*/\1/p')
+    version=$(protoc --version | awk '{print $NF}')
+    major=$(echo ${version} | cut -d '.' -f 1)
+    minor=$(echo ${version} | cut -d '.' -f 2)
     if [ "$major" -eq 3 ] && [ "$minor" -ge 8 ]; then
         return 0
     fi
+    # protobuf bumps the major version to 21 after 3.
+    # https://github.com/protocolbuffers/protobuf/releases/tag/v21.7
+    if [ "$major" -ge 21 ]; then
+        return 0
+    fi
     echo "protoc version not match, version 3.8.x+ is needed, current version: ${version}"
     return 1
 }