Merge branch 'labring:main' into config

CHANGELOG/CHANGELOG-5.0.0-beta5.md

@@ -0,0 +1,157 @@
+Welcome to the v5.0.0-beta5 release of Sealos!🎉🎉!
+
+
+
+## Changelog
+### New Features
+* 8f135d6d708fb6e097f979ff98149b100cc779ef: feat(desktop): add appToken for providers & fix ssr error (#4566) (@xudaotutou)
+* b766fb5abeb3c2837140088dfac90778c076af70: feat(desktop): multi region (#4558) (@xudaotutou)
+* 559154f5370c41495ea94a9590b5b91e407e3fca: feat(frontend): static host (#4634) (@xudaotutou)
+* 0180fad6c713439c96655a275f033cb079436344: feat: @sealos/ui useMessage component (#4523) (@zjy365)
+* 3c3563b12a3f34558dccff7cd555694832e49acc: feat: Name modification and deployment count display (#4453) (@zjy365)
+* dc60d199e4b2c35b033124a24495c83ab6f97ec2: feat: launchpad api createApp with extra parameters (#4598) (@zjy365)
+* 649c535284cd61ad0241411e9381cadf33175cc4: feat: object storage cluster init. (#4510) (@lingdie)
+* f3de924d870cef4b81d6248e4a68aba4fcb2a4fc: feat: support multi domains for ingress webhooks. (#4491) (@lingdie)
+* 3bdcc735af3d2c5106550443a57058ca82d49e11: feat:Desktop add standard OAuth2 login support  (#4671) (@zjy365)
+* 2719247f315e212817300dc6f3253fd7f27b0307: feat:Improve App Store rendering speed (#4609) (@zjy365)
+* dfb748f9e8aff53109d3ce76d5689b1895d57cc7: feat:app logo colorful & blue bg (#4488) (@zjy365)
+* 9125932cd491cdebbb579371bfe9a1439ec55602: feat:db adapt mongo instance count  (#4490) (@zjy365)
+* aec7020f77a2ecaf677a48a30db9fe13f1ea2575: feat:db provider adapted to 0.8.1 (#4508) (@zjy365)
+* 046bb27270afeaa5536141516757f4876e6ef26d: feat:desktop support system configuration (#4538) (@zjy365)
+* 84e033d06941d1dac64570fecb6620209e3dbc03: feat:desktop support wechat public account login  (#4519) (@zjy365)
+* 7155582f1d8d277cad9962b9c3fd52c831e6bfd0: feat:desktop supports invitations  (#4613) (@zjy365)
+* 63a693af6aa7296f72770d7a926f9429ade10118: feat:docs sale banner (#4564) (@zjy365)
+* 08a9f55ddbf6774cbc9571e57f3eec9c57c66510: feat:docs self-hosting entrance (#4521) (@zjy365)
+* 929faae785d9e5ceb6be843c313106e4b15f4117: feat:invite app add deployment files (#4614) (@zjy365)
+* ae7f63f9b2f848c55bb815ad5b4fb6bf24776230: feat:launchpad monitor api & template ssr (#4482) (@zjy365)
+* b6b8108d6e445537d2e9ba68f12a0dab62e2d01f: feat:optimize app store loading (#4607) (@zjy365)
+* cd56ca8768452608e6f560c0ff4fcc9bfb73f01d: feat:template carousel image (#4485) (@zjy365)
+* 16b92b814a4c9628414e67ddd5ec27ad96e2ef11: feat:template deploy add template-static (#4457) (@zjy365)
+* 0495264c9c54d01330a9bd879d66645f57ff2799: feat:template language detection & instance api (#4497) (@zjy365)
+* 61e69210ff8b261dc44741faf0b019fbb97b9a73: feat:template obtains service resources (#4502) (@zjy365)
+* e0cc9d611982009fee2a5a19ec941c39df2a7c5c: feat:template session store (#4526) (@zjy365)
+* f193f89a1129790e53b4eb1d42eb6dcf82a2bad3: feat:template support application marketplace category feature  (#4583) (@zjy365)
+* b2beb0a00b5b4987cd33a73f1a40ea2201ac0c0b: feat:template support base64  (#4627) (@zjy365)
+* 53ecc9e6901a6e4f18b2b1b5b35a662e1d7a3ee5: feat:template support categories & fix launchpad table (#4541) (@zjy365)
+* ff26b697c71ee6c663d7597300e624a54e951682: feat:template support parsing option lists (#4533) (@zjy365)
+* 5492617d6f56ae6a2f325cbdd424d5dc2ed2ab34: small feat: add database tips one user select 1 replicas (#4507) (@o0vO)
+* 26e06adefc8ff7486bf813dcae45b0fa43b8f482: style & feat: new UI components & automatically re-watch (#4433) (@Wishrem)
+### Bug fixes
+* 8fd13f63288df3bf70fd03b1fe7e0b9bf630fefe: fix(costcenter):fix rerender when select date (#4574) (@xudaotutou)
+* 1eafe70326daf678bbf9fa16c32705b72419fea5: fix(desktop):add cloudflare turnstile (#4595) (@xudaotutou)
+* 873894f6d2e5f3962295da2f0eb3d1cbaa36dc33: fix(desktop):fix verification code restriction (#4449) (@xudaotutou)
+* a748715a85d35ecf6b3ec6419f6db731ff246d2c: fix(objectstorage):fix create bucket error (#4480) (@xudaotutou)
+* 375b561a0b4ad574f787be58f8cd17b9e9239aa4: fix(objectstorage):fix predictCard (#4501) (@xudaotutou)
+* a6aade62ca56b09f18d671fc2e14ffea45c9d2cb: fix: Add "Deploy" translation key to English locale (#4569) (@tianshanghong)
+* 317ee5c2610f2f71738dd367f30696c116bce80a: fix: db role and select default backup repository (#4676) (@zjy365)
+* 1e54b69df7f872bac945977f3f64fb3fb8d71911: fix: dbprovider resource already exists problem (#4560) (@zjy365)
+* ad83b8433bb8fdb76d1483781997135d5edf7ce8: fix: desktop applaunchpad costcenter styles (#4667) (@zjy365)
+* d334b54a3325e70c92035bb46c02c4b8010772ed: fix: issue #4536 (#4540) (@fengxsong)
+* b93ca1eca586fc3f445723eda44958db46502c3a: fix: local page auto fall into online site. (#4513) (@o0vO)
+* 20a027d9e68a140aea18e8a77a22e40d9906c15c: fix: revised version (#4604) (@rtpacks)
+* 6eaec67f7fa481553cf11f17b529648c887fcf32: fix:db auth & template develop (#4445) (@zjy365)
+* c35ee00d79226322a566d945fc074779e876129c: fix:db external network connection (#4615) (@zjy365)
+* 6727ced5a59d4beb3baf7ba4b266f05994ab410e: fix:dbprovider auto backup (#4515) (@zjy365)
+* 73b12c5cf3e6251c13b1623a472d90c4b80a681f: fix:dbprovider maximum storage (#4459) (@zjy365)
+* fe50d0986028ac1e78b174c8ac0133ea5874b8a7: fix:dbprovider number of database instances (#4580) (@zjy365)
+* 900d3fc8d2701ddb5560446c8442dbc3d241d482: fix:desktop keep last workspace state (#4646) (@zjy365)
+* ac3aeefe6221ac6d9c40dc402054e20c340aeed7: fix:desktop modal header style & costcenter breakpoints (#4675) (@zjy365)
+* 851cd69542f2bd68dc12ddb78f2f41fc19a0ccc8: fix:desktop password login and static host (#4639) (@zjy365)
+* ee6b12979f6d3f65e6b3fc0f3d9a6a4400617fdd: fix:desktop region info (#4631) (@zjy365)
+* 2b1ab315ecc4e2266fa0164c10c7216e1fad669a: fix:docs link address (#4666) (@zjy365)
+* 8d394239d36fa3d8d6f6b6daa1d7d528dac4c20a: fix:docs reference scripts path problem (#4535) (@zjy365)
+* d5c2fecfa757be8cf55a1c06bbfe63c493ef45a3: fix:docs wow.min.js (#4489) (@zjy365)
+* 6c596ecf555349027c763a578bdbed12e57bba32: fix:license add CUSTOM_BASE_PATH env (#4654) (@zjy365)
+* d73becbf408d9165934f6c41a14105a03c92265c: fix:privoder logo svg (#4446) (@zjy365)
+* d7269a9cf9f70e9ee19479368a9104b249462b45: fix:template redirection problem (#4472) (@zjy365)
+* 4ff45a89d7426b328d65489048c2a80bc67f7a5f: fix:template service nodeport (#4503) (@zjy365)
+* 047428cb2dd2b69e294478a9dcd731859c311cba: fix:terminal already exists (#4651) (@zjy365)
+### Other work
+* c3fa80d518fd59685eaa2eed59ab2cb4dc35e9c2: :bug: fix net interface sort (#4672) (@cuisongliu)
+* 7e97c72bb4460ab31c059d4673451835369bce73: Add Object Storage guide docs (#4579) (@nowinkeyy)
+* 6d73bbfc3c036d84cac0ca24b52acd680c643377: Add category to template apps. (#4539) (@zzjin)
+* 16c10cf3ace8b3fb37eff3940442c71773baff0a: Build kubepanel (#4551) (@lingdie)
+* dabac703d3cea9eac536fa40fdd18ccbd17e6a35: Disable desktop app draggable. (#4571) (@zzjin)
+* 883b7df5d9f5c1c67a0822bf7a4cc5eb6c7a76b5: Feat/e2e test (#4486) (@bxy4543)
+* 4a0a90cbbe71dbe68cbd744bdb9a0e0147963eb2: Fix adminer namespace method. (#4581) (@zzjin)
+* 5c73e629fd35c0b4ea3e2aa47e5ed1d2ae55e30e: Fix controller ingress label override. (#4597) (@zzjin)
+* 1c960df32f4f8e41d59ce87a12413e388407cada: Fix controller memory usage, (#4589) (@zzjin)
+* 8ca706929e4a078b2f7ac9f9658892c352638942: Fix dead link (#4529) (@fanux)
+* 4ab847e9fcbf13813dcf97d38c3bf0103072b65f: Fix init admin account & kbcli addon enable snapshot-controller (#4643) (@bxy4543)
+* 3a7a3cd072ddf93c1e7cba20ccd626183fa1884c: Fix service/launchpad typo. (#4655) (@zzjin)
+* 2085ee63cf2f394b9342a642c96dbcb65dbf71d3: Fix title typo. (#4628) (@zzjin)
+* af438cf5244fecc016579531f11d9728aab418be: Fix waiting for table creation (#4638) (@bxy4543)
+* 14e397c38dbef4819b18cafbaeb12bf17b2a14af: Fix/traffic db name (#4548) (@bxy4543)
+* 48b18e08854f2462d604e058a88c5e0adcafb2a0: Launchpad (#4645) (@wallyxjh)
+* 0755c905e0bab5ebaee3096d5f34243b001c6085: License with accountv2 (#4600) (@bxy4543)
+* 2337ab1150cc9867bcb67c0b71bb5036582f9403: Optimize monitor minio (#4679) (@bxy4543)
+* 9318fd5a979ae8caa68bb89d4b222ca71b16ca76: Optimize the bucket controller logic of object storage (#4504) (@nowinkeyy)
+* b496a20599359d469c750bbfe8822088a006cdb2: Optimize/debt resume speed (#4658) (@bxy4543)
+* e819739117a410d7574746d03261b0cb22b10801: Optimize/monitor (#4481) (@bxy4543)
+* a35524dfd3c7cacbc43ee92ac68cc7b35f235587: Separate database svc and minio svc (#4447) (@nowinkeyy)
+* 0d9d19d4c6f0ceff1696a7c467bdfad7028fad99: The init job waits for the user table to be created before execution. (#4619) (@bxy4543)
+* 5e0ea541ca1a4354f315aa9b64f7842f65f98453: Transfer service (#4594) (@bxy4543)
+* 0350700b558d738e87a75724e09edc19ee429d3e: Unified account (#4576) (@bxy4543)
+* a6e09647921a4e083163a245faee923c233fbd9c: Update README.md (#4492) (@fanux)
+* 6b17b9503b4603bf6ef45674d8120f6b57105d3b: Update README.md (#4647) (@fanux)
+* 9a9533ba5f2a08695c35e6d064f7b554ddc4043f: Update cockroach (#4605) (@wallyxjh)
+* d0b30e6e66e9297f45aa0ed2afea9aaf897426dc: Update doc typo. (#4656) (@zzjin)
+* 295e6a543f598b50a3dddeb935a6a70b6ce74a5f: Update doc typos and show `quick-start` tab as default. (#4577) (@zzjin)
+* 4488fb61fdd4764e5f6a74213f7add08e4dc52fa: Update gen-apply-cluster.md (#4530) (@lamking)
+* e87219f94e2a1135fb11e2ed4ec1aebb197c23b2: adapt ephemeral storage limit range (#4456) (@bxy4543)
+* 055fe0bbc8103907e2e13d77311be4a05e0f10a6: adapt install script for cockroachdb. (#4593) (@lingdie)
+* e45e3b79bd3805d659d5c4aae0370adc0fb32fb2: add cost service swagger  (#4474) (@bxy4543)
+* 8d6e23fd7c4810322575caf7c02f179b781f5abe: add init job (#4629) (@bxy4543)
+* b12e933026792c720111a99c98c6b327e8a870c1: add limits to monitor service (#4652) (@nowinkeyy)
+* 15279bcdae9412c8f57a155d5d65ae3bb3444e62: add minio service module to workflows (#4450) (@nowinkeyy)
+* df38d07149d4d6839bfff414083dcf4077b986f1: add object storage key secret (#4623) (@nowinkeyy)
+* 145ce7832f275a582c63101fb13c62f0b065d6a1: add pull file func (#4563) (@nowinkeyy)
+* 9110edea0aaab5fa5cf8f5a58e9d6fb028e3ad48: change admin to username (#4557) (@nowinkeyy)
+* cda0c5cba91f09780faaf57c79b91eb07c3afab7: create region (#4620) (@bxy4543)
+* e7b1c54e2dc042328261773faaee020a2ed46bf9: docs: Add new blog (#4500) (@yangchuansheng)
+* 972784f0047365de226cdc9ed8fcd879676eafc9: docs: Add templates documentation (#4458) (@yangchuansheng)
+* 15fee3b856ca7f56be8a16cea7bb792ab8bbfc2c: docs: Update docs for installing sealos (#4660) (@yangchuansheng)
+* 301d44f7dd51f2788508edcaa332053217d0162a: docs: add blog for affine (#4661) (@yangchuansheng)
+* fee8721f53a2a6b18d0d3a1add1babb0dc2e4b30: docs: add component Highlight (#4471) (@yangchuansheng)
+* b4daa2df1181598e396f0804b0d0a12fd16079f4: docs: add master services agreement (#4452) (@yangchuansheng)
+* 43632605a34ad3ea3ee7e29ca234d0493679c327: docs: add static host doc (#4640) (@nowinkeyy)
+* 9d606dc1f4c03a29a2fa4062f94d061339e6690b: docs: update lychee config (#4498) (@yangchuansheng)
+* 5318703c0b64bc4e8e57548411593a40c5239aa1: docs:offline recharge promotions (#4575) (@zjy365)
+* 87269b7997005ea27d0bb35a1eb0dfe234d9ab63: feat(object-storage):add sort feature (#4464) (@xudaotutou)
+* 8abdf56a684331ed2fc7c754a9f34290f6c3f9a4: feat(sealos create): Add command preview functionality (#4291) (@LZiHaN)
+* c3d58c03b2013ba6bda1b6853e738c55ad8dd424: fix create region (#4632) (@bxy4543)
+* 2122fb726a2669b340a67be949e46081480f572d: fix env name error in deploy.yaml.tmpl (#4624) (@nowinkeyy)
+* 60e490dbd8dea94881dbcf0f835fb35618342545: fix gen region uid (#4611) (@bxy4543)
+* 60c481977f39e1565ff055e482913eaed15b93d2: fix install cli with proxy (#4465) (@bxy4543)
+* d72a2b8a1d0b2d79fcaa753b2de07921a7acaa66: fix link 404 (#4664) (@nowinkeyy)
+* 2113509f408f47621f62ae9dfe30c0e2e05582e5: fix link 404 err0r (#4669) (@nowinkeyy)
+* 5e5e95c759c9d02c38086ac17b974bfaf44bb71b: fix minio init error (#4555) (@nowinkeyy)
+* ae7d7edaefd7244224221a78385664b1f6e69cf2: fix object storage workflow error (#4542) (@nowinkeyy)
+* e41b40f1b03a8906792ef19a40cc7b3873248eb1: fix scripts (#4612) (@bxy4543)
+* 424fcc7275bf4735a4412fe2a084e309d6b591f0: fix user change ns annotations (#4592) (@bxy4543)
+* b9418ed6b7ece3977ba078cd37a11926cf0c59b0: fix(object-storage):preview & copyURL (#4444) (@xudaotutou)
+* 07fdde5bb10e01befc3283b982975bc5c0b9b7c4: k3s support version doc (#4455) (@bxy4543)
+* c6e55b37dc22887d0e4d41003a40d974d73302ca: k3s support version doc (#4461) (@bxy4543)
+* 2857df524bb018a104785062b9b5722ff9f0731f: optimize default kube runtime config; (#4487) (@bxy4543)
+* e1aa395aaf4c827f8f0b96f7ca852b6c6b8ad3e1: optimize init minio sh (#4537) (@nowinkeyy)
+* 5cb7e44a3cc829e04b0aa67595a8b49c433c7105: query traffic at a specified time (#4657) (@nowinkeyy)
+* e6bf29b3c54b0bcf863532c57772ff7ce888eff9: refactor(costcenter): service api (#4475) (@xudaotutou)
+* 70b1f5c6ecffe8c4301a06eb6ee2099ef0085835: refactor:license optimized deployment command form  (#4532) (@zjy365)
+* 9487f0be1d68edc99b3cb7e6caef7428ae5cb291: refactor:providers invite app (#4599) (@zjy365)
+* 21f2c3a58aa601a55ba9358e699aef2f9e2adb7e: refine cluster image(objectstorage-controller、minio-service) (#4512) (@nowinkeyy)
+* a0b3363d9880ecbfe61f47835e2e764c641c7ead: release: update sealos version in install.sh. (#4678) (@lingdie)
+* 91142fd90b301c5ceb9a298ad6a5396637f543d7: style(kubepanel): adjust style (#4546) (@mlhiter)
+* 2057bca8f047a934ff54e26eff123ee7b2644a8c: style(kubepanel): improve the overall style (#4494) (@mlhiter)
+* 1fb9c119613f355077c2d37a59ba75de1c43349f: style: Update UI styles for AppLaunchpad (#4649) (@zjy365)
+* 53631c6ef01d95e326eba89902626fd026090b3a: style: switch sealos coin (#4509) (@xudaotutou)
+* 307e8682c0f8e7dd8a652bde19bf49c9428c9890: update cpu and memory config requirements for deploying sealos (#4625) (@nowinkeyy)
+* 24417328a1ad238d9ace220eee644a2ed5f3923f: user registration switch (#4473) (@bxy4543)
+* 967c5bf77d3515857074f34619289316a7d23884: 🤖 add release changelog using rebot. (#4442) (@sealos-release-robot)
+
+**Full Changelog**: https://github.com/labring/sealos/compare/v5.0.0-beta4...v5.0.0-beta5
+
+See [the CHANGELOG](https://github.com/labring/sealos/blob/main/CHANGELOG/CHANGELOG.md) for more details.
+
+Your patronage towards Sealos is greatly appreciated 🎉🎉.
+
+If you encounter any problems during its usage, please create an issue in the [GitHub repository](https://github.com/labring/sealos), we're committed to resolving your problem as soon as possible.

CHANGELOG/CHANGELOG.md

@@ -2,6 +2,7 @@
 
 All notable changes to this project will be documented in this file.
 
+- [CHANGELOG-5.0.0-beta5.md](./CHANGELOG-5.0.0-beta5.md)
 - [CHANGELOG-5.0.0-beta4.md](./CHANGELOG-5.0.0-beta4.md)
 - [CHANGELOG-5.0.0-beta3.md](./CHANGELOG-5.0.0-beta3.md)
 - [CHANGELOG-5.0.0-beta2.md](./CHANGELOG-5.0.0-beta2.md)

controllers/pkg/objectstorage/objectstorage.go

@@ -17,13 +17,19 @@ package objectstorage
 import (
 	"context"
 	"fmt"
+
+	"github.com/prometheus/prom2json"
+
 	"regexp"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/labring/sealos/controllers/pkg/utils/env"
+
 	"github.com/prometheus/common/model"
 
+	"github.com/minio/madmin-go/v3"
 	"github.com/minio/minio-go/v7"
 	"github.com/prometheus/client_golang/api"
 	v1 "github.com/prometheus/client_golang/api/prometheus/v1"
@@ -44,6 +50,18 @@ func ListUserObjectStorageBucket(client *minio.Client, username string) ([]strin
 	return expectBuckets, nil
 }
 
+func ListAllObjectStorageBucket(client *minio.Client) ([]string, error) {
+	buckets, err := client.ListBuckets(context.Background())
+	if err != nil {
+		return nil, err
+	}
+	var allBuckets []string
+	for _, bucket := range buckets {
+		allBuckets = append(allBuckets, bucket.Name)
+	}
+	return allBuckets, nil
+}
+
 func GetObjectStorageSize(client *minio.Client, bucket string) (int64, int64) {
 	objects := client.ListObjects(context.Background(), bucket, minio.ListObjectsOptions{
 		Recursive: true,
@@ -99,9 +117,11 @@ func GetUserObjectStorageFlow(client *minio.Client, promURL, username, instance
 	return totalFlow, nil
 }
 
+var timeoutDuration = time.Duration(env.GetInt64EnvWithDefault(EnvPromQueryObsTimeoutSecEnv, 10)) * time.Second
+
 const (
-	timeoutDuration = 5 * time.Second
-	timeFormat      = "2006-01-02 15:04:05"
+	EnvPromQueryObsTimeoutSecEnv = "PROM_QUERY_OBS_TIMEOUT_SEC"
+	timeFormat                   = "2006-01-02 15:04:05"
 )
 
 var (
@@ -164,3 +184,60 @@ func extractValues(result1, result2 model.Value) (int64, int64) {
 	val2, _ := strconv.ParseInt(rcvdStr2, 10, 64)
 	return val1, val2
 }
+
+type MetricData struct {
+	// key: bucket name, value: usage
+	Usage map[string]int64
+}
+
+type Metrics map[string]MetricData
+
+func QueryUserUsage(client *madmin.MetricsClient) (Metrics, error) {
+	obMetrics := make(Metrics)
+	bucketMetrics, err := client.BucketMetrics(context.TODO())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get bucket metrics: %w", err)
+	}
+	for _, bucketMetric := range bucketMetrics {
+		if !isUsageBytesTargetMetric(bucketMetric.Name) {
+			continue
+		}
+		for _, metrics := range bucketMetric.Metrics {
+			promMetrics := metrics.(prom2json.Metric)
+			floatValue, err := strconv.ParseFloat(promMetrics.Value, 64)
+			if err != nil {
+				return nil, fmt.Errorf("failed to parse %s to float value", promMetrics.Value)
+			}
+			intValue := int64(floatValue)
+			if bucket := promMetrics.Labels["bucket"]; bucket != "" {
+				user := getUserWithBucket(bucket)
+				metricData, exists := obMetrics[user]
+				if !exists {
+					metricData = MetricData{
+						Usage: make(map[string]int64),
+					}
+				}
+				metricData.Usage[bucket] += intValue
+				obMetrics[user] = metricData
+			}
+		}
+	}
+
+	return obMetrics, err
+}
+
+func isUsageBytesTargetMetric(name string) bool {
+	targetMetrics := []string{
+		"minio_bucket_usage_total_bytes",
+	}
+	for _, target := range targetMetrics {
+		if name == target {
+			return true
+		}
+	}
+	return false
+}
+
+func getUserWithBucket(bucket string) string {
+	return strings.Split(bucket, "-")[0]
+}

controllers/pkg/utils/env/env.go

@@ -27,6 +27,15 @@ func GetEnvWithDefault(key, defaultValue string) string {
 	return defaultValue
 }
 
+func GetBoolWithDefault(key string, defaultValue bool) bool {
+	if env, ok := os.LookupEnv(key); ok && env != "" {
+		if value, err := strconv.ParseBool(env); err == nil {
+			return value
+		}
+	}
+	return defaultValue
+}
+
 func GetInt64EnvWithDefault(key string, defaultValue int64) int64 {
 	if env, ok := os.LookupEnv(key); ok && env != "" {
 		if value, err := strconv.ParseInt(env, 10, 64); err == nil {