add WAF/CC logs for all applications

backend/init.go

@@ -191,6 +191,14 @@ func InitDatabase() {
 			utils.DebugPrintln("InitDatabase ALTER TABLE destinations add constraint", err)
 		}
 	}
+
+	// v1.4.2fix4
+	if !dal.ExistConstraint("ip_policies", "create_time") {
+		_ = dal.ExecSQL(`ALTER TABLE "ip_policies" ADD COLUMN "create_time" BIGINT, ADD COLUMN "description" VARCHAR(1024) DEFAULT ''`)
+		if err != nil {
+			utils.DebugPrintln("InitDatabase ALTER TABLE ip_policies add COLUMN", err)
+		}
+	}
 }
 
 // LoadAppConfiguration ...

data/firewall_cc_log.go

@@ -16,9 +16,6 @@ const (
 	sqlCreateTableIfNotExistsCCLog = `CREATE TABLE IF NOT EXISTS "cc_logs"("id" bigserial primary key,"request_time" bigint,"client_ip" VARCHAR(256) NOT NULL,"host" VARCHAR(256) NOT NULL,"method" VARCHAR(16) NOT NULL,"url_path" VARCHAR(2048) NOT NULL,"url_query" VARCHAR(2048) NOT NULL DEFAULT '',"content_type" VARCHAR(128) NOT NULL DEFAULT '',"user_agent" VARCHAR(1024) NOT NULL DEFAULT '',"cookies" VARCHAR(1024) NOT NULL DEFAULT '',"raw_request" VARCHAR(16384) NOT NULL,"action" bigint,"app_id" bigint)`
 	sqlInsertCCLog                 = `INSERT INTO "cc_logs"("id","request_time","client_ip","host","method","url_path","url_query","content_type","user_agent","cookies","raw_request","action","app_id") VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13)`
 	sqlSelectCCLogByID             = `SELECT "id","request_time","client_ip","host","method","url_path","url_query","content_type","user_agent","cookies","raw_request","action","app_id" FROM "cc_logs" WHERE "id"=$1`
-	sqlSelectSimpleCCLogs          = `SELECT "id","request_time","client_ip","host","method","url_path","action","app_id" FROM "cc_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3 ORDER BY "request_time" DESC LIMIT $4 OFFSET $5`
-	sqlSelectCCLogsCount           = `SELECT COUNT(1) FROM "cc_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3`
-	sqlSelectAllCCLogsCount        = `SELECT COUNT(1) FROM "cc_logs" WHERE "request_time" BETWEEN $1 AND $2`
 	sqlDeleteCCLogsBeforeTime      = `DELETE FROM "cc_logs" WHERE "request_time"<$1`
 )
 
@@ -53,6 +50,7 @@ func (dal *MyDAL) InsertCCLog(requestTime int64, clientIP string, host string, m
 // SelectCCLogsCount ...
 func (dal *MyDAL) SelectCCLogsCount(appID int64, startTime int64, endTime int64) (int64, error) {
 	var count int64
+	const sqlSelectCCLogsCount = `SELECT COUNT(1) FROM "cc_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3`
 	err := dal.db.QueryRow(sqlSelectCCLogsCount, appID, startTime, endTime).Scan(&count)
 	if err != nil {
 		utils.DebugPrintln("SelectCCLogsCount QueryRow", err)
@@ -62,6 +60,7 @@ func (dal *MyDAL) SelectCCLogsCount(appID int64, startTime int64, endTime int64)
 
 // SelectAllCCLogsCount ...
 func (dal *MyDAL) SelectAllCCLogsCount(startTime int64, endTime int64) (int64, error) {
+	const sqlSelectAllCCLogsCount = `SELECT COUNT(1) FROM "cc_logs" WHERE "request_time" BETWEEN $1 AND $2`
 	stmt, err := dal.db.Prepare(sqlSelectAllCCLogsCount)
 	if err != nil {
 		utils.DebugPrintln("SelectAllCCLogsCount Prepare", err)
@@ -100,9 +99,10 @@ func (dal *MyDAL) SelectCCLogByID(id int64) (*models.CCLog, error) {
 	return ccLog, err
 }
 
-// SelectCCLogs ...
-func (dal *MyDAL) SelectCCLogs(appID int64, startTime int64, endTime int64, requestCount int64, offset int64) []*models.SimpleCCLog {
+// SelectCCLogsByAppID ...
+func (dal *MyDAL) SelectCCLogsByAppID(appID int64, startTime int64, endTime int64, requestCount int64, offset int64) []*models.SimpleCCLog {
 	simpleCCLogs := []*models.SimpleCCLog{}
+	const sqlSelectSimpleCCLogs = `SELECT "id","request_time","client_ip","host","method","url_path","action","app_id" FROM "cc_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3 ORDER BY "request_time" DESC LIMIT $4 OFFSET $5`
 	rows, err := dal.db.Query(sqlSelectSimpleCCLogs, appID, startTime, endTime, requestCount, offset)
 	if err != nil {
 		utils.DebugPrintln("SelectCCLogs Query", err)
@@ -118,3 +118,23 @@ func (dal *MyDAL) SelectCCLogs(appID int64, startTime int64, endTime int64, requ
 	}
 	return simpleCCLogs
 }
+
+// SelectCCLogs of all applications
+func (dal *MyDAL) SelectCCLogs(startTime int64, endTime int64, requestCount int64, offset int64) []*models.SimpleCCLog {
+	simpleCCLogs := []*models.SimpleCCLog{}
+	const sqlSelectSimpleCCLogs = `SELECT "id","request_time","client_ip","host","method","url_path","action","app_id" FROM "cc_logs" WHERE "request_time" BETWEEN $1 AND $2 ORDER BY "request_time" DESC LIMIT $3 OFFSET $4`
+	rows, err := dal.db.Query(sqlSelectSimpleCCLogs, startTime, endTime, requestCount, offset)
+	if err != nil {
+		utils.DebugPrintln("SelectCCLogs Query", err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		simpleCCLog := &models.SimpleCCLog{}
+		err = rows.Scan(&simpleCCLog.ID, &simpleCCLog.RequestTime, &simpleCCLog.ClientIP, &simpleCCLog.Host, &simpleCCLog.Method, &simpleCCLog.UrlPath, &simpleCCLog.Action, &simpleCCLog.AppID)
+		if err != nil {
+			utils.DebugPrintln("SelectCCLogs rows.Scan", err)
+		}
+		simpleCCLogs = append(simpleCCLogs, simpleCCLog)
+	}
+	return simpleCCLogs
+}

data/firewall_group_policy_log.go

@@ -13,11 +13,10 @@ import (
 )
 
 const (
-	sqlCreateTableIfNotExistsGroupHitLog  = `CREATE TABLE IF NOT EXISTS "group_hit_logs"("id" bigserial primary key,"request_time" bigint,"client_ip" VARCHAR(256) NOT NULL,"host" VARCHAR(256) NOT NULL,"method" VARCHAR(16) NOT NULL,"url_path" VARCHAR(2048) NOT NULL,"url_query" VARCHAR(2048) NOT NULL DEFAULT '',"content_type" VARCHAR(128) NOT NULL DEFAULT '',"user_agent" VARCHAR(1024) NOT NULL DEFAULT '',"cookies" VARCHAR(1024) NOT NULL DEFAULT '',"raw_request" VARCHAR(16384) NOT NULL,"action" bigint,"policy_id" bigint,"vuln_id" bigint,"app_id" bigint)`
-	sqlInsertGroupHitLog                  = `INSERT INTO "group_hit_logs"("id","request_time","client_ip","host","method","url_path","url_query","content_type","user_agent","cookies","raw_request","action","policy_id","vuln_id","app_id") VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)`
-	sqlSelectGroupHitLogByID              = `SELECT "id","request_time","client_ip","host","method","url_path","url_query","content_type","user_agent","cookies","raw_request","action","policy_id","vuln_id","app_id" FROM "group_hit_logs" WHERE "id"=$1`
-	sqlSelectSimpleGroupHitLogs           = `SELECT "id","request_time","client_ip","host","method","url_path","action","policy_id","app_id" FROM "group_hit_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3 ORDER BY "request_time" DESC LIMIT $4 OFFSET $5`
-	sqlSelectGroupHitLogsCount            = `SELECT COUNT(1) FROM "group_hit_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3`
+	sqlCreateTableIfNotExistsGroupHitLog = `CREATE TABLE IF NOT EXISTS "group_hit_logs"("id" bigserial primary key,"request_time" bigint,"client_ip" VARCHAR(256) NOT NULL,"host" VARCHAR(256) NOT NULL,"method" VARCHAR(16) NOT NULL,"url_path" VARCHAR(2048) NOT NULL,"url_query" VARCHAR(2048) NOT NULL DEFAULT '',"content_type" VARCHAR(128) NOT NULL DEFAULT '',"user_agent" VARCHAR(1024) NOT NULL DEFAULT '',"cookies" VARCHAR(1024) NOT NULL DEFAULT '',"raw_request" VARCHAR(16384) NOT NULL,"action" bigint,"policy_id" bigint,"vuln_id" bigint,"app_id" bigint)`
+	sqlInsertGroupHitLog                 = `INSERT INTO "group_hit_logs"("id","request_time","client_ip","host","method","url_path","url_query","content_type","user_agent","cookies","raw_request","action","policy_id","vuln_id","app_id") VALUES($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15)`
+	sqlSelectGroupHitLogByID             = `SELECT "id","request_time","client_ip","host","method","url_path","url_query","content_type","user_agent","cookies","raw_request","action","policy_id","vuln_id","app_id" FROM "group_hit_logs" WHERE "id"=$1`
+
 	sqlSelectGroupHitLogsCountByVulnID    = `SELECT COUNT(1) FROM "group_hit_logs" WHERE "app_id"=$1 AND "vuln_id"=$2 AND "request_time" BETWEEN $3 AND $4`
 	sqlSelectAllGroupHitLogsCount         = `SELECT COUNT(1) FROM "group_hit_logs" WHERE "request_time" BETWEEN $1 AND $2`
 	sqlSelectAllGroupHitLogsCountByVulnID = `SELECT COUNT(1) FROM "group_hit_logs" WHERE "vuln_id"=$1 AND "request_time" BETWEEN $2 AND $3`
@@ -54,8 +53,9 @@ func (dal *MyDAL) InsertGroupHitLog(requestTime int64, clientIP string, host str
 	return err
 }
 
-// SelectGroupHitLogsCount ...
-func (dal *MyDAL) SelectGroupHitLogsCount(appID int64, startTime int64, endTime int64) (int64, error) {
+// SelectGroupHitLogsCountByAppID ...
+func (dal *MyDAL) SelectGroupHitLogsCountByAppID(appID int64, startTime int64, endTime int64) (int64, error) {
+	const sqlSelectGroupHitLogsCount = `SELECT COUNT(1) FROM "group_hit_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3`
 	stmt, err := dal.db.Prepare(sqlSelectGroupHitLogsCount)
 	if err != nil {
 		utils.DebugPrintln("SelectGroupHitLogsCount Prepare", err)
@@ -69,6 +69,22 @@ func (dal *MyDAL) SelectGroupHitLogsCount(appID int64, startTime int64, endTime
 	return count, err
 }
 
+// SelectGroupHitLogsCount of all applications
+func (dal *MyDAL) SelectGroupHitLogsCount(startTime int64, endTime int64) (int64, error) {
+	const sqlSelectGroupHitLogsCount = `SELECT COUNT(1) FROM "group_hit_logs" WHERE "request_time" BETWEEN $1 AND $2`
+	stmt, err := dal.db.Prepare(sqlSelectGroupHitLogsCount)
+	if err != nil {
+		utils.DebugPrintln("SelectGroupHitLogsCount Prepare", err)
+	}
+	defer stmt.Close()
+	var count int64
+	err = stmt.QueryRow(startTime, endTime).Scan(&count)
+	if err != nil {
+		utils.DebugPrintln("SelectGroupHitLogsCount QueryRow", err)
+	}
+	return count, err
+}
+
 // SelectGroupHitLogsCountByVulnID ...
 func (dal *MyDAL) SelectGroupHitLogsCountByVulnID(appID int64, vulnID int64, startTime int64, endTime int64) (int64, error) {
 	stmt, err := dal.db.Prepare(sqlSelectGroupHitLogsCountByVulnID)
@@ -143,15 +159,41 @@ func (dal *MyDAL) SelectGroupHitLogByID(id int64) (*models.GroupHitLog, error) {
 	return groupHitLog, err
 }
 
-// SelectGroupHitLogs ...
-func (dal *MyDAL) SelectGroupHitLogs(appID int64, startTime int64, endTime int64, requestCount int64, offset int64) []*models.SimpleGroupHitLog {
+// SelectGroupHitLogsByAppID ...
+func (dal *MyDAL) SelectGroupHitLogsByAppID(appID int64, startTime int64, endTime int64, requestCount int64, offset int64) []*models.SimpleGroupHitLog {
 	simpleGroupHitLogs := []*models.SimpleGroupHitLog{}
+	const sqlSelectSimpleGroupHitLogs = `SELECT "id","request_time","client_ip","host","method","url_path","action","policy_id","app_id" FROM "group_hit_logs" WHERE "app_id"=$1 AND "request_time" BETWEEN $2 AND $3 ORDER BY "request_time" DESC LIMIT $4 OFFSET $5`
 	stmt, err := dal.db.Prepare(sqlSelectSimpleGroupHitLogs)
 	if err != nil {
-		utils.DebugPrintln("SelectGroupHitLogs Prepare", err)
+		utils.DebugPrintln("SelectGroupHitLogsByAppID Prepare", err)
 	}
 	defer stmt.Close()
 	rows, err := stmt.Query(appID, startTime, endTime, requestCount, offset)
+	if err != nil {
+		utils.DebugPrintln("SelectGroupHitLogsByAppID Query", err)
+	}
+	defer rows.Close()
+	for rows.Next() {
+		simpleGroupHitLog := &models.SimpleGroupHitLog{}
+		err = rows.Scan(&simpleGroupHitLog.ID, &simpleGroupHitLog.RequestTime, &simpleGroupHitLog.ClientIP, &simpleGroupHitLog.Host, &simpleGroupHitLog.Method, &simpleGroupHitLog.UrlPath, &simpleGroupHitLog.Action, &simpleGroupHitLog.PolicyID, &simpleGroupHitLog.AppID)
+		if err != nil {
+			utils.DebugPrintln("SelectGroupHitLogsByAppID rows.Scan", err)
+		}
+		simpleGroupHitLogs = append(simpleGroupHitLogs, simpleGroupHitLog)
+	}
+	return simpleGroupHitLogs
+}
+
+// SelectGroupHitLogs of all applications
+func (dal *MyDAL) SelectGroupHitLogs(startTime int64, endTime int64, requestCount int64, offset int64) []*models.SimpleGroupHitLog {
+	simpleGroupHitLogs := []*models.SimpleGroupHitLog{}
+	const sqlSelectAllGroupHitLogs = `SELECT "id","request_time","client_ip","host","method","url_path","action","policy_id","app_id" FROM "group_hit_logs" WHERE "request_time" BETWEEN $1 AND $2 ORDER BY "request_time" DESC LIMIT $3 OFFSET $4`
+	stmt, err := dal.db.Prepare(sqlSelectAllGroupHitLogs)
+	if err != nil {
+		utils.DebugPrintln("SelectGroupHitLogs Prepare", err)
+	}
+	defer stmt.Close()
+	rows, err := stmt.Query(startTime, endTime, requestCount, offset)
 	if err != nil {
 		utils.DebugPrintln("SelectGroupHitLogs Query", err)
 	}

data/firewall_ip_policy.go

@@ -14,26 +14,26 @@ import (
 
 // CreateTableIfNotExistsIPPolicies ...
 func (dal *MyDAL) CreateTableIfNotExistsIPPolicies() error {
-	const sqlCreateTableIfNotExistsIPPolicies = `CREATE TABLE IF NOT EXISTS "ip_policies"("id" bigserial PRIMARY KEY, "ip_addr" VARCHAR(128) NOT NULL, "is_allow" boolean, "apply_to_waf" boolean, "apply_to_cc" boolean)`
+	const sqlCreateTableIfNotExistsIPPolicies = `CREATE TABLE IF NOT EXISTS "ip_policies"("id" bigserial PRIMARY KEY, "ip_addr" VARCHAR(128) NOT NULL, "is_allow" boolean, "apply_to_waf" boolean, "apply_to_cc" boolean, "create_time" BIGINT, "description" VARCHAR(1024) DEFAULT '')`
 	_, err := dal.db.Exec(sqlCreateTableIfNotExistsIPPolicies)
 	return err
 }
 
 // InsertIPPolicy Insert IP Address to "ip_policies"
-func (dal *MyDAL) InsertIPPolicy(ipAddr string, isAllow bool, applyToWAF bool, applyToCC bool) (newID int64) {
-	const sqlInsertIPPolicy = `INSERT INTO "ip_policies"("id","ip_addr","is_allow","apply_to_waf","apply_to_cc") VALUES($1,$2,$3,$4,$5) RETURNING "id"`
+func (dal *MyDAL) InsertIPPolicy(ipAddr string, isAllow bool, applyToWAF bool, applyToCC bool, createTime int64, description string) (newID int64) {
+	const sqlInsertIPPolicy = `INSERT INTO "ip_policies"("id","ip_addr","is_allow","apply_to_waf","apply_to_cc","create_time","description") VALUES($1,$2,$3,$4,$5,$6,$7) RETURNING "id"`
 	snowID := utils.GenSnowflakeID()
-	err := dal.db.QueryRow(sqlInsertIPPolicy, snowID, ipAddr, isAllow, applyToWAF, applyToCC).Scan(&newID)
+	err := dal.db.QueryRow(sqlInsertIPPolicy, snowID, ipAddr, isAllow, applyToWAF, applyToCC, createTime, description).Scan(&newID)
 	if err != nil {
 		utils.DebugPrintln("InsertIPPolicy", err)
 	}
 	return newID
 }
 
 // UpdateIPPolicy update IP address and policy
-func (dal *MyDAL) UpdateIPPolicy(id int64, ipAddr string, isAllow bool, applyToWAF bool, applyToCC bool) error {
-	const sqlUpdateIPPolicy = `UPDATE "ip_policies" SET "ip_addr"=$1,"is_allow"=$2,"apply_to_waf"=$3,"apply_to_cc"=$4 WHERE "id"=$5`
-	_, err := dal.db.Exec(sqlUpdateIPPolicy, ipAddr, isAllow, applyToWAF, applyToCC, id)
+func (dal *MyDAL) UpdateIPPolicy(id int64, ipAddr string, isAllow bool, applyToWAF bool, applyToCC bool, description string) error {
+	const sqlUpdateIPPolicy = `UPDATE "ip_policies" SET "ip_addr"=$1,"is_allow"=$2,"apply_to_waf"=$3,"apply_to_cc"=$4,"description"=$5 WHERE "id"=$6`
+	_, err := dal.db.Exec(sqlUpdateIPPolicy, ipAddr, isAllow, applyToWAF, applyToCC, description, id)
 	return err
 }
 
@@ -46,7 +46,7 @@ func (dal *MyDAL) DeleteIPPolicyByID(id int64) error {
 
 // LoadIPPolicies return the list of IPPolicy
 func (dal *MyDAL) LoadIPPolicies() []*models.IPPolicy {
-	const sqlSelectAllowList = `SELECT "id","ip_addr","is_allow","apply_to_waf","apply_to_cc" FROM "ip_policies"`
+	const sqlSelectAllowList = `SELECT "id","ip_addr","is_allow","apply_to_waf","apply_to_cc","create_time","description" FROM "ip_policies"`
 	rows, err := dal.db.Query(sqlSelectAllowList)
 	if err != nil {
 		utils.DebugPrintln("GetIPPolicies", err)
@@ -60,7 +60,10 @@ func (dal *MyDAL) LoadIPPolicies() []*models.IPPolicy {
 			&ipPolicy.IPAddr,
 			&ipPolicy.IsAllow,
 			&ipPolicy.ApplyToWAF,
-			&ipPolicy.ApplyToCC)
+			&ipPolicy.ApplyToCC,
+			&ipPolicy.CreateTime,
+			&ipPolicy.Description,
+		)
 		if err != nil {
 			utils.DebugPrintln("GetIPPolicies rows.Scan", err)
 		}

firewall/ip_policy.go

@@ -14,6 +14,7 @@ import (
 	"janusec/models"
 	"janusec/utils"
 	"strconv"
+	"time"
 )
 
 var globalIPPolicies []*models.IPPolicy
@@ -45,31 +46,22 @@ func UpdateIPPolicy(body []byte, clientIP string, authUser *models.AuthUser) (*m
 		return nil, err
 	}
 	ipPolicy := rpcIPRequest.Object
-
-	/*
-		ipPolicyI := param["object"].(map[string]interface{})
-		id, _ := strconv.ParseInt(ipPolicyI["id"].(string), 10, 64)
-		ipAddr := ipPolicyI["ip_addr"].(string)
-		ipAddr = strings.Trim(ipAddr, " ")
-		isAllow := ipPolicyI["is_allow"].(bool)
-		applyToWAF := ipPolicyI["apply_to_waf"].(bool)
-		applyToCC := ipPolicyI["apply_to_cc"].(bool)
-	*/
-
 	if ipPolicy.ID == 0 {
 		// New IP
-		ipPolicy.ID = data.DAL.InsertIPPolicy(ipPolicy.IPAddr, ipPolicy.IsAllow, ipPolicy.ApplyToWAF, ipPolicy.ApplyToCC)
+		ipPolicy.CreateTime = time.Now().Unix()
+		ipPolicy.ID = data.DAL.InsertIPPolicy(ipPolicy.IPAddr, ipPolicy.IsAllow, ipPolicy.ApplyToWAF, ipPolicy.ApplyToCC, ipPolicy.CreateTime, ipPolicy.Description)
 		globalIPPolicies = append(globalIPPolicies, ipPolicy)
 		go utils.OperationLog(clientIP, authUser.Username, "Add IP Policy", ipPolicy.IPAddr)
 		data.UpdateFirewallLastModified()
 		return ipPolicy, nil
 	}
 	// Update
-	err := data.DAL.UpdateIPPolicy(ipPolicy.ID, ipPolicy.IPAddr, ipPolicy.IsAllow, ipPolicy.ApplyToWAF, ipPolicy.ApplyToCC)
+	err := data.DAL.UpdateIPPolicy(ipPolicy.ID, ipPolicy.IPAddr, ipPolicy.IsAllow, ipPolicy.ApplyToWAF, ipPolicy.ApplyToCC, ipPolicy.Description)
 	if err != nil {
 		utils.DebugPrintln("UpdateIPPolicy", err)
 		return nil, err
 	}
+	globalIPPolicies = data.DAL.LoadIPPolicies()
 	go utils.OperationLog(clientIP, authUser.Username, "Update IP Policy", ipPolicy.IPAddr)
 	data.UpdateFirewallLastModified()
 	return ipPolicy, nil