From b5df220131969027ca8838cd8556622d8f449e0d Mon Sep 17 00:00:00 2001 From: huanghaoyuanhhy Date: Fri, 27 Dec 2024 16:27:46 +0800 Subject: [PATCH] backup: reduce memory usage Signed-off-by: huanghaoyuanhhy --- core/backup_impl_create_backup.go | 11 +++++------ core/backup_meta.go | 15 +++++++++++++++ 2 files changed, 20 insertions(+), 6 deletions(-) diff --git a/core/backup_impl_create_backup.go b/core/backup_impl_create_backup.go index 448e2a73..0568fd1a 100644 --- a/core/backup_impl_create_backup.go +++ b/core/backup_impl_create_backup.go @@ -750,18 +750,17 @@ func (b *BackupContext) executeCreateBackup(ctx context.Context, request *backup } func (b *BackupContext) writeBackupInfoMeta(ctx context.Context, id string) error { + log.Info("Start write backup meta", zap.String("backupName", id)) + b.meta.CleanBinlogInfo(id) + log.Info("Clean binlog info done", zap.String("backupName", id)) backupInfo := b.meta.GetFullMeta(id) log.Debug("Final backupInfo", zap.String("backupInfo", backupInfo.String())) output, _ := serialize(backupInfo) - log.Debug("backup meta", zap.String("value", string(output.BackupMetaBytes))) - log.Debug("collection meta", zap.String("value", string(output.CollectionMetaBytes))) - log.Debug("partition meta", zap.String("value", string(output.PartitionMetaBytes))) - log.Debug("segment meta", zap.String("value", string(output.SegmentMetaBytes))) collectionBackups := backupInfo.GetCollectionBackups() - collectionPositions := make(map[string][]*backuppb.ChannelPosition, 0) + collectionPositions := make(map[string][]*backuppb.ChannelPosition, len(collectionBackups)) for _, collectionBackup := range collectionBackups { - collectionCPs := make([]*backuppb.ChannelPosition, 0) + collectionCPs := make([]*backuppb.ChannelPosition, 0, len(collectionBackup.GetChannelCheckpoints())) for vCh, position := range collectionBackup.GetChannelCheckpoints() { pCh := strings.Split(vCh, "_")[0] + "_" + strings.Split(vCh, "_")[1] collectionCPs = append(collectionCPs, &backuppb.ChannelPosition{ diff --git a/core/backup_meta.go b/core/backup_meta.go index 9d38ca9f..82136a84 100644 --- a/core/backup_meta.go +++ b/core/backup_meta.go @@ -111,6 +111,21 @@ func treeToLevel(backup *backuppb.BackupInfo) (LeveledBackupInfo, error) { }, nil } +func (meta *MetaManager) CleanBinlogInfo(id string) { + meta.mu.Lock() + defer meta.mu.Unlock() + collections := meta.collections[id] + for _, collection := range collections { + for _, partition := range meta.partitions[collection.GetCollectionId()] { + for _, segment := range meta.segments[partition.GetPartitionId()] { + segment.Binlogs = nil + segment.Statslogs = nil + segment.Deltalogs = nil + } + } + } +} + func serialize(backup *backuppb.BackupInfo) (*BackupMetaBytes, error) { level, err := treeToLevel(backup) if err != nil {