feat: generate robots.txt with tunasync

manager/server.go

@@ -78,6 +78,9 @@ func GetTUNASyncManager(cfg *Config) *Manager {
 	// flush disabled jobs
 	s.engine.DELETE("/jobs/disabled", s.flushDisabledJobs)
 
+	// generate robots.txt
+	s.engine.GET("/robots.txt", s.generateRobotsTxt)
+
 	// list workers
 	s.engine.GET("/workers", s.listWorkers)
 	// worker online
@@ -152,6 +155,25 @@ func (s *Manager) listAllJobs(c *gin.Context) {
 	c.JSON(http.StatusOK, webMirStatusList)
 }
 
+func (s *Manager) generateRobotsTxt(c *gin.Context) {
+	s.rwmu.RLock()
+	mirrorStatusList, err := s.adapter.ListAllMirrorStatus()
+	s.rwmu.RUnlock()
+	if err != nil {
+		err := fmt.Errorf("failed to list all mirror status: %s",
+			err.Error(),
+		)
+		c.Error(err)
+		s.returnErrJSON(c, http.StatusInternalServerError, err)
+		return
+	}
+	content := "# robots.txt generated by tunasync-seu (seu-mirrors/tunasync)\nUser-agent: *\n\nDisallow: /-/\n"
+	for _, m := range mirrorStatusList {
+		content += fmt.Sprintf("Disallow: /%s/\n", m.Name)
+	}
+	c.String(http.StatusOK, content)
+}
+
 // flushDisabledJobs deletes all jobs that marks as deleted
 func (s *Manager) flushDisabledJobs(c *gin.Context) {
 	s.rwmu.Lock()