Adds exploitdb threat intelligence information

internal/core/vulndb/exploitdb_service.go

@@ -0,0 +1,127 @@
+package vulndb
+
+import (
+	"context"
+	"log/slog"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"encoding/csv"
+
+	"github.com/l3montree-dev/flawfix/internal/database"
+	"github.com/l3montree-dev/flawfix/internal/database/models"
+	"github.com/pkg/errors"
+)
+
+type exploitDBService struct {
+	nvdService        NVDService
+	exploitRepository exploitRepository
+	httpClient        *http.Client
+}
+
+type exploitRepository interface {
+	SaveBatch(tx database.DB, exploits []models.Exploit) error
+}
+
+func NewExploitDBService(nvdService NVDService, exploitRepository exploitRepository) exploitDBService {
+	return exploitDBService{
+		nvdService:        nvdService,
+		exploitRepository: exploitRepository,
+		httpClient:        &http.Client{},
+	}
+}
+// Infromation provided under GNU General Public License v2.0 or later, by OffSec Services Limited
+var exploitDBURL = "https://gitlab.com/exploit-database/exploitdb/-/raw/main/files_exploits.csv"
+
+func (s *exploitDBService) fetchCSV(ctx context.Context) error {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, exploitDBURL, nil)
+
+	if err != nil {
+		return err
+	}
+
+	res, err := s.httpClient.Do(req)
+	if err != nil {
+		return err
+	}
+
+	defer res.Body.Close()
+	csvReader := csv.NewReader(res.Body)
+	records, err := csvReader.ReadAll()
+	if err != nil {
+		return errors.Wrap(err, "could not read csv")
+	}
+	exploits := make(map[int]models.Exploit, 0)
+	for _, record := range records[1:] {
+		id, err := strconv.ParseInt(record[0], 10, 64)
+		if err != nil {
+			slog.Warn("could not parse exploit id", "id", id)
+			continue
+		}
+		datePublished, err := time.Parse(time.DateOnly, record[3])
+		if err != nil {
+			slog.Warn("could not parse date published", "date", record[3], "exploitId", id)
+			continue
+		}
+		var dateUpdated *time.Time = nil
+		du, err := time.Parse(time.DateOnly, record[9])
+		if err == nil {
+			dateUpdated = &du
+		}
+
+		description := record[2]
+		author := record[4]
+
+		verified := record[10]
+		codes := record[11]
+		tags := record[12]
+		sourceURL := record[16]
+
+		// parse the codes
+		if codes != "" {
+			for _, code := range strings.Split(codes, ";") {
+				if !strings.HasPrefix(code, "CVE-") {
+					continue
+				}
+				exploitModel := models.Exploit{
+					ID: int(id),
+
+					Description: description,
+					Published:   &datePublished,
+					Updated:     dateUpdated,
+					Author:      author,
+					Verified:    verified == "1",
+					SourceURL:   sourceURL,
+					Tags:        tags,
+					CVEID:       code,
+				}
+
+				if existingExploit, exist := exploits[int(id)]; exist {
+					// check the dateUpdated, maybe it is more recent
+					if dateUpdated != nil && existingExploit.Updated != nil && dateUpdated.After(*existingExploit.Updated) {
+						exploits[int(id)] = exploitModel
+					}
+				} else {
+					exploits[int(id)] = exploitModel
+				}
+			}
+		}
+	}
+	// get all values of the map
+	exploitModels := make([]models.Exploit, len(exploits))
+	i := 0
+	for _, exploitModel := range exploits {
+		exploitModels[i] = exploitModel
+		i++
+	}
+
+	return s.exploitRepository.SaveBatch(nil, exploitModels)
+}
+
+func (s exploitDBService) Mirror() error {
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	return s.fetchCSV(ctx)
+}

internal/core/vulndb/scan/purl_comparer.go

@@ -37,7 +37,6 @@ func NewPurlComparer(db core.DB) *purlComparer {
 }
 
 func (comparer *purlComparer) GetVulnsForAll(purls []string) ([]models.VulnInPackage, error) {
-
 	g := errgroup.Group{}
 	g.SetLimit(10) // magic concurrency number - this was just a quick test against a local installed postgresql
 	results := make([][]models.VulnInPackage, len(purls))

internal/core/vulndb/vulndb_handler.go

@@ -28,14 +28,17 @@ func StartMirror(database core.DB, leaderElector leaderElector, configService co
 	cveRepository := repositories.NewCVERepository(database)
 	cweRepository := repositories.NewCWERepository(database)
 	affectedCmpRepository := repositories.NewAffectedCmpRepository(database)
+	exploitRepository := repositories.NewExploitRepository(database)
 
 	nvdService := NewNVDService(cveRepository)
 	epssService := NewEPSSService(nvdService, cveRepository)
 	mitreService := NewMitreService(cweRepository)
 
+	exploitDBService := NewExploitDBService(nvdService, exploitRepository)
 	osvService := NewOSVService(affectedCmpRepository)
+
 	// start the mirror process.
-	vulnDBService := newVulnDBService(leaderElector, mitreService, epssService, nvdService, configService, osvService)
+	vulnDBService := newVulnDBService(leaderElector, mitreService, epssService, nvdService, configService, osvService, exploitDBService)
 
 	vulnDBService.startMirrorDaemon()
 }

internal/core/vulndb/vulndb_service.go

@@ -11,22 +11,24 @@ import (
 type vulnDBService struct {
 	leaderElector leaderElector
 
-	mitreService mitreService
-	epssService  epssService
-	nvdService   NVDService
-	osvService   osvService
+	mitreService     mitreService
+	epssService      epssService
+	nvdService       NVDService
+	osvService       osvService
+	exploitDBService exploitDBService
 
 	configService configService
 }
 
-func newVulnDBService(leaderElector leaderElector, mitreService mitreService, epssService epssService, nvdService NVDService, configService configService, osvService osvService) *vulnDBService {
+func newVulnDBService(leaderElector leaderElector, mitreService mitreService, epssService epssService, nvdService NVDService, configService configService, osvService osvService, exploitDBService exploitDBService) *vulnDBService {
 	return &vulnDBService{
 		leaderElector: leaderElector,
 
-		osvService:   osvService,
-		mitreService: mitreService,
-		epssService:  epssService,
-		nvdService:   nvdService,
+		osvService:       osvService,
+		mitreService:     mitreService,
+		epssService:      epssService,
+		nvdService:       nvdService,
+		exploitDBService: exploitDBService,
 
 		configService: configService,
 	}
@@ -65,6 +67,11 @@ func (v *vulnDBService) mirror() {
 				} else {
 					slog.Info("successfully mirrored nvd")
 				}
+				if err := v.exploitDBService.Mirror(); err != nil {
+					slog.Error("could not mirror exploitdb", "err", err)
+				} else {
+					slog.Info("successfully mirrored exploitdb")
+				}
 				if err := v.epssService.Mirror(); err != nil {
 					slog.Error("could not mirror epss", "err", err)
 				} else {

internal/database/models/cve_model.go

@@ -59,6 +59,8 @@ type CVE struct {
 
 	EPSS       *float32 `json:"epss" gorm:"type:decimal(6,5);"`
 	Percentile *float32 `json:"percentile" gorm:"type:decimal(6,5);"`
+
+	Exploits []*Exploit `json:"exploits" gorm:"foreignKey:CVEID;"`
 }
 
 type Weakness struct {

internal/database/models/exploit_model.go

@@ -0,0 +1,35 @@
+// Copyright (C) 2024 Tim Bastin, l3montree UG (haftungsbeschränkt)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+package models
+
+import "time"
+
+type Exploit struct {
+	ID          int        `json:"id" gorm:"primaryKey;"`
+	Published   *time.Time `json:"published" gorm:"type:date;"`
+	Updated     *time.Time `json:"updated" gorm:"type:date;"`
+	Author      string     `json:"author" gorm:"type:text;"`
+	Type        string     `json:"type" gorm:"type:text;"`
+	Verified    bool       `json:"verified" gorm:"type:boolean;"`
+	SourceURL   string     `json:"sourceURL" gorm:"type:text;"`
+	Description string     `json:"description" gorm:"type:text;"`
+	CVE         CVE        `gorm:"foreignKey:CVEID;constraint:OnDelete:CASCADE;"`
+	CVEID       string     `json:"cveID" gorm:"type:text;"`
+	Tags        string     `json:"tags" gorm:"type:text;"`
+}
+
+func (m Exploit) TableName() string {
+	return "exploits"
+}

internal/database/repositories/exploit_repository.go

@@ -0,0 +1,126 @@
+// Copyright (C) 2024 Tim Bastin, l3montree UG (haftungsbeschränkt)
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as
+// published by the Free Software Foundation, either version 3 of the
+// License, or (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+
+package repositories
+
+import (
+	"fmt"
+	"log/slog"
+
+	"github.com/google/uuid"
+	"github.com/l3montree-dev/flawfix/internal/core"
+	"github.com/l3montree-dev/flawfix/internal/database/models"
+	"gorm.io/gorm"
+	"gorm.io/gorm/clause"
+	"gorm.io/gorm/logger"
+)
+
+type exploitRepository struct {
+	db core.DB
+	Repository[uuid.UUID, models.Exploit, core.DB]
+}
+
+func NewExploitRepository(db core.DB) *exploitRepository {
+	if err := db.AutoMigrate(&models.Exploit{}); err != nil {
+		panic(err)
+	}
+	return &exploitRepository{
+		db:         db,
+		Repository: newGormRepository[uuid.UUID, models.Exploit](db),
+	}
+}
+
+func (g *exploitRepository) createInBatches(tx core.DB, exploits []models.Exploit, batchSize int) error {
+	// Collect all CVE IDs from the exploits
+	var cveIDs []string
+	for _, exploit := range exploits {
+		cveIDs = append(cveIDs, exploit.CVEID)
+	}
+
+	// Query to find valid CVE IDs
+	var validCveIDs []string
+	if err := g.GetDB(tx).Model(&models.CVE{}).Where("cve IN ?", cveIDs).Pluck("cve", &validCveIDs).Error; err != nil {
+		return fmt.Errorf("error fetching valid CVE IDs: %w", err)
+	}
+
+	// Create a map for quick lookup of valid CVE IDs
+	validCveIDMap := make(map[string]struct{})
+	for _, id := range validCveIDs {
+		validCveIDMap[id] = struct{}{}
+	}
+
+	// Filter out exploits with invalid CVE IDs
+	var validExploits []models.Exploit
+	invalidCodes := make([]string, 0)
+	for _, exploit := range exploits {
+		if _, exists := validCveIDMap[exploit.CVEID]; exists {
+			validExploits = append(validExploits, exploit)
+		} else {
+			invalidCodes = append(invalidCodes, exploit.CVEID)
+		}
+	}
+
+	// save the invalid codes in a json file
+	if len(invalidCodes) > 0 {
+		slog.Warn("could not find CVE IDs", "ids", invalidCodes)
+	}
+
+	if len(validExploits) == 0 {
+		slog.Warn("no valid exploits to insert")
+		return nil
+	}
+
+	err := g.GetDB(tx).Session(
+		&gorm.Session{
+			Logger: logger.Default.LogMode(logger.Silent),
+		}).Clauses(
+		clause.OnConflict{
+			UpdateAll: true,
+		},
+	).CreateInBatches(&validExploits, batchSize).Error
+
+	// Check if we got a protocol error since we are inserting more than 65535 parameters
+	if err != nil && err.Error() == "extended protocol limited to 65535 parameters; extended protocol limited to 65535 parameters" {
+		newBatchSize := batchSize / 2
+		if newBatchSize < 1 {
+			// We can't reduce the batch size anymore
+			// Let's try to save the CVEs one by one
+			// This will be slow but it will work
+			for _, exploit := range validExploits {
+				tmpPkg := exploit
+				if err := g.GetDB(tx).Session(
+					&gorm.Session{
+						// Logger: logger.Default.LogMode(logger.Silent),
+					}).Clauses(
+					clause.OnConflict{
+						UpdateAll: true,
+					},
+				).Create(&tmpPkg).Error; err != nil {
+					// Log that we weren't able to save the CVE
+					slog.Error("unable to save exploit", "exploit", exploit.CVEID, "err", err)
+				}
+			}
+			return nil
+		}
+		slog.Warn("protocol error, trying to reduce batch size", "newBatchSize", newBatchSize, "oldBatchSize", batchSize, "err", err)
+		return g.createInBatches(tx, validExploits, newBatchSize)
+	}
+
+	return err
+}
+
+func (g *exploitRepository) SaveBatch(tx core.DB, exploits []models.Exploit) error {
+	return g.createInBatches(tx, exploits, 1000)
+}