Check tarball checksum on tiup-server (#1163)

* Handle unknown component correctlly

* Check tarball

* Fix lint

* Update pkg/repository/model/error.go

Co-authored-by: Allen Zhong <[email protected]>

Co-authored-by: Allen Zhong <[email protected]>
Co-authored-by: Ti Chi Robot <[email protected]>

cmd/mirror.go

@@ -489,6 +489,41 @@ func newMirrorPublishCmd() *cobra.Command {
 	return cmd
 }
 
+func doPublish(
+	component, version, entry, desc string,
+	publishInfo *model.PublishInfo,
+	hashes map[string]string, length int64,
+	standalone, hidden bool,
+	privPath, goos, goarch string,
+	flagSet set.StringSet,
+) error {
+	env := environment.GlobalEnv()
+	m, err := env.V1Repository().FetchComponentManifest(component, true)
+	if err != nil {
+		if errors.Cause(err) == repository.ErrUnknownComponent {
+			fmt.Printf("Creating component %s\n", component)
+			publishInfo.Stand = &standalone
+			publishInfo.Hide = &hidden
+		} else {
+			return err
+		}
+	} else if flagSet.Exist("standalone") || flagSet.Exist("hide") {
+		fmt.Println("This is not a new component, --standalone and --hide flag will be omitted")
+	}
+
+	m = repository.UpdateManifestForPublish(m, component, version, entry, goos, goarch, desc, v1manifest.FileHash{
+		Hashes: hashes,
+		Length: uint(length),
+	})
+
+	manifest, err := sign(privPath, m)
+	if err != nil {
+		return err
+	}
+
+	return env.V1Repository().Mirror().Publish(manifest, publishInfo)
+}
+
 func validatePlatform(goos, goarch string) error {
 	// Only support any/any, don't support linux/any, any/amd64 .etc.
 	if goos == "any" && goarch == "any" {

pkg/repository/model/error.go

@@ -25,7 +25,11 @@ var (
 	// ErrorMissingOwner indicates that the owner is not found
 	ErrorMissingOwner = errors.New("owner not found")
 	// ErrorWrongSignature indicates that the signature is not correct
-	ErrorWrongSignature = errors.New("the signature is not correct")
+	ErrorWrongSignature = errors.New("invalid signature")
+	// ErrorWrongChecksum indicates that the checksum of tar file is not correct
+	ErrorWrongChecksum = errors.New("checksum mismatch")
+	// ErrorWrongFileName indicates that the name of tar file is not correct
+	ErrorWrongFileName = errors.New("incorrect file name")
 	// ErrorWrongManifestType indicates that the manifest type is not expected
 	ErrorWrongManifestType = errors.New("the manifest type is not expected")
 	// ErrorWrongManifestVersion indicates that the manifest version is not expected

pkg/repository/model/model.go

@@ -264,7 +264,7 @@ func (m *model) Publish(manifest *v1manifest.Manifest, info ComponentInfo) error
 		}
 
 		if info.Filename() != "" {
-			if err := m.txn.Write(info.Filename(), info); err != nil {
+			if err := m.checkAndWrite(signed, info); err != nil {
 				return err
 			}
 		}
@@ -274,6 +274,29 @@ func (m *model) Publish(manifest *v1manifest.Manifest, info ComponentInfo) error
 	})
 }
 
+func (m *model) checkAndWrite(manifest *v1manifest.Component, info ComponentData) error {
+	fname := info.Filename()
+	for _, plat := range manifest.Platforms {
+		for _, vi := range plat {
+			if vi.URL[1:] == fname {
+				if err := m.txn.Write(fname, info); err != nil {
+					return err
+				}
+				reader, err := m.txn.Read(fname)
+				if err != nil {
+					return err
+				}
+				defer reader.Close()
+				if err := utils.CheckSHA256(reader, vi.Hashes["sha256"]); err == nil {
+					return nil
+				}
+				return ErrorWrongChecksum
+			}
+		}
+	}
+	return ErrorWrongFileName
+}
+
 func findKeyOwnerFromIndex(signed *v1manifest.Index, keyID string) (string, *v1manifest.Owner) {
 	for on := range signed.Owners {
 		for k := range signed.Owners[on].Keys {

pkg/repository/v1_repository.go

@@ -40,8 +40,8 @@ import (
 	"golang.org/x/mod/semver"
 )
 
-// errUnknownComponent represents the specific component cannot be found in index.json
-var errUnknownComponent = stderrors.New("unknown component")
+// ErrUnknownComponent represents the specific component cannot be found in index.json
+var ErrUnknownComponent = stderrors.New("unknown component")
 
 // V1Repository represents a remote repository viewed with the v1 manifest design.
 type V1Repository struct {
@@ -110,7 +110,7 @@ func (r *V1Repository) UpdateComponents(specs []ComponentSpec) error {
 	for _, spec := range specs {
 		manifest, err := r.updateComponentManifest(spec.ID, false)
 		if err != nil {
-			if errors.Cause(err) == errUnknownComponent {
+			if errors.Cause(err) == ErrUnknownComponent {
 				fmt.Println(color.YellowString("The component `%s` not found (may be deleted from repository); skipped", spec.ID))
 			} else {
 				errs = append(errs, err.Error())
@@ -420,7 +420,7 @@ func (r *V1Repository) updateComponentManifest(id string, withYanked bool) (*v1m
 
 	item, ok := components[id]
 	if !ok {
-		return nil, errUnknownComponent
+		return nil, ErrUnknownComponent
 	}
 	var snapshot v1manifest.Snapshot
 	_, _, err = r.local.LoadManifest(&snapshot)
@@ -684,7 +684,7 @@ func (r *V1Repository) UpdateComponentManifests() error {
 
 	for name := range index.Components {
 		_, err = r.updateComponentManifest(name, false)
-		if err != nil && errors.Cause(err) != errUnknownComponent {
+		if err != nil && errors.Cause(err) != ErrUnknownComponent {
 			return err
 		}
 	}

pkg/repository/v1_repository_test.go

@@ -301,7 +301,7 @@ func TestYanked(t *testing.T) {
 
 	_, err = repo.updateComponentManifest("bar", false)
 	assert.NotNil(t, err)
-	assert.Equal(t, errors.Cause(err), errUnknownComponent)
+	assert.Equal(t, errors.Cause(err), ErrUnknownComponent)
 }
 
 func TestUpdateComponent(t *testing.T) {

server/handler/component.go

@@ -99,10 +99,15 @@ func (h *componentSigner) sign(r *http.Request, m *v1manifest.RawManifest) (sr *
 	switch err := h.mirror.Publish(manifest, info); err {
 	case model.ErrorConflict:
 		return nil, ErrorManifestConflict
+	case model.ErrorWrongSignature:
+		return nil, ErrorForbiden
+	case model.ErrorWrongChecksum, model.ErrorWrongFileName:
+		return nil, ErrorInvalidTarball
 	case nil:
 		return nil, nil
 	default:
 		h.sm.Delete(sid)
+		log.Errorf("Publish component: %s", err.Error())
 		return nil, ErrorInternalError
 	}
 }