Skip to content

Commit

Permalink
Merge pull request #96 from cedricve/develop
Browse files Browse the repository at this point in the history
Persistence: adding dropbox support
  • Loading branch information
cedricve authored Apr 7, 2023
2 parents bbbed49 + 43c1666 commit 026bf93
Show file tree
Hide file tree
Showing 17 changed files with 525 additions and 5 deletions.
1 change: 1 addition & 0 deletions machinery/data/config/config.json
Original file line number Diff line number Diff line change
Expand Up @@ -98,6 +98,7 @@
"region": "eu-west1"
},
"kstorage": {},
"dropbox": {},
"mqtturi": "tcp://mqtt.kerberos.io:1883",
"mqtt_username": "",
"mqtt_password": "",
Expand Down
4 changes: 4 additions & 0 deletions machinery/go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@ require (
github.com/appleboy/gin-jwt/v2 v2.9.1
github.com/cedricve/go-onvif v0.0.0-20200222191200-567e8ce298f6
github.com/deepch/vdk v0.0.19
github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5
github.com/eclipse/paho.mqtt.golang v1.4.2
github.com/elastic/go-sysinfo v1.9.0
github.com/gin-contrib/cors v1.4.0
Expand Down Expand Up @@ -68,6 +69,7 @@ require (
github.com/goccy/go-json v0.10.0 // indirect
github.com/gofrs/uuid v3.2.0+incompatible // indirect
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
github.com/golang/protobuf v1.5.2 // indirect
github.com/google/pprof v0.0.0-20210423192551-a2663126120b // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/joeshaw/multierror v0.0.0-20140124173710-69b34d4ec901 // indirect
Expand Down Expand Up @@ -114,12 +116,14 @@ require (
go4.org/unsafe/assume-no-moving-gc v0.0.0-20220617031537-928513b29760 // indirect
golang.org/x/crypto v0.4.0 // indirect
golang.org/x/net v0.4.0 // indirect
golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5 // indirect
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect
golang.org/x/sys v0.3.0 // indirect
golang.org/x/text v0.5.0 // indirect
golang.org/x/time v0.0.0-20211116232009-f0f3c7e86c11 // indirect
golang.org/x/tools v0.1.12 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/appengine v1.6.6 // indirect
google.golang.org/grpc v1.32.0 // indirect
google.golang.org/protobuf v1.28.1 // indirect
gopkg.in/ini.v1 v1.42.0 // indirect
Expand Down
272 changes: 271 additions & 1 deletion machinery/go.sum

Large diffs are not rendered by default.

23 changes: 19 additions & 4 deletions machinery/src/cloud/Cloud.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,22 @@ func HandleUpload(configuration *models.Configuration, communication *models.Com
uploaded, configured, err = UploadS3(configuration, fileName)
} else if config.Cloud == "kstorage" {
uploaded, configured, err = UploadKerberosVault(configuration, fileName)
} else if config.Cloud == "dropbox" {
uploaded, configured, err = UploadDropbox(configuration, fileName)
} else if config.Cloud == "gdrive" {
// Todo: implement gdrive upload
} else if config.Cloud == "onedrive" {
// Todo: implement onedrive upload
} else if config.Cloud == "minio" {
// Todo: implement minio upload
} else if config.Cloud == "webdav" {
// Todo: implement webdav upload
} else if config.Cloud == "ftp" {
// Todo: implement ftp upload
} else if config.Cloud == "sftp" {
// Todo: implement sftp upload
}
// And so on... (have a look here -> https://github.com/kerberos-io/agent/issues/95)

// Check if the file is uploaded, if so, remove it.
if uploaded {
Expand Down Expand Up @@ -564,7 +579,9 @@ func VerifyPersistence(c *gin.Context) {
err := c.BindJSON(&config)
if err != nil || config.Cloud != "" {

if config.Cloud == "s3" {
if config.Cloud == "dropbox" {
VerifyDropbox(config, c)
} else if config.Cloud == "s3" {

// timestamp_microseconds_instanceName_regionCoordinates_numberOfChanges_token
// 1564859471_6-474162_oprit_577-283-727-375_1153_27.mp4
Expand Down Expand Up @@ -645,9 +662,7 @@ func VerifyPersistence(c *gin.Context) {
})
}
}
}

if config.Cloud == "kstorage" {
} else if config.Cloud == "kstorage" {

uri := config.KStorage.URI
accessKey := config.KStorage.AccessKey
Expand Down
135 changes: 135 additions & 0 deletions machinery/src/cloud/Dropbox.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,135 @@
// Package cloud contains the Dropbox implementation of the Cloud interface.
// It uses the Dropbox SDK to upload files to Dropbox.
package cloud

import (
"bytes"
"errors"
"os"

"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/users"
"github.com/gin-gonic/gin"
"github.com/kerberos-io/agent/machinery/src/log"
"github.com/kerberos-io/agent/machinery/src/models"
)

// UploadDropbox uploads the file to your Dropbox account using the access token and directory.
func UploadDropbox(configuration *models.Configuration, fileName string) (bool, bool, error) {

config := configuration.Config
token := config.Dropbox.AccessToken
directory := config.Dropbox.Directory
if directory != "" {
// Check if trailing slash if not we'll add one.
if directory[len(directory)-1:] != "/" {
directory = directory + "/"
}
}

if token == "" {
err := "UploadDropbox: Dropbox not properly configured"
log.Log.Info(err)
return false, true, errors.New(err)
}

// Upload to Dropbox
log.Log.Info("UploadDropbox: Uploading to Dropbox")
log.Log.Info("UploadDropbox: Upload started for " + fileName)
fullname := "data/recordings/" + fileName

dConfig := dropbox.Config{
Token: token,
LogLevel: dropbox.LogInfo, // if needed, set the desired logging level. Default is off
}

file, err := os.OpenFile(fullname, os.O_RDWR, 0755)
if file != nil {
defer file.Close()
}

if err == nil {
// Upload the file
dbf := files.New(dConfig)
res, err := dbf.Upload(&files.UploadArg{
CommitInfo: files.CommitInfo{
Path: "/" + directory + fileName,
Mode: &files.WriteMode{
Tagged: dropbox.Tagged{
Tag: "overwrite",
},
},
},
}, file)

if err != nil {
log.Log.Error("UploadDropbox: Error uploading file: " + err.Error())
return false, false, err
}

log.Log.Info("UploadDropbox: File uploaded successfully, " + res.Name)
return true, true, nil
}

log.Log.Error("UploadDropbox: Error opening file: " + err.Error())
return false, true, err
}

// VerifyDropbox verifies if the Dropbox token is valid and it is able to upload a file.
func VerifyDropbox(config models.Config, c *gin.Context) {

token := config.Dropbox.AccessToken
directory := config.Dropbox.Directory
if directory != "" {
// Check if trailing slash if not we'll add one.
if directory[len(directory)-1:] != "/" {
directory = directory + "/"
}
}

if token != "" {
dConfig := dropbox.Config{
Token: token,
LogLevel: dropbox.LogInfo, // if needed, set the desired logging level. Default is off
}
dbx := users.New(dConfig)
_, err := dbx.GetCurrentAccount()
if err != nil {
c.JSON(400, models.APIResponse{
Data: "Something went wrong while reaching the Dropbox API: " + err.Error(),
})
} else {

// Upload the file
content := TestFile
file := bytes.NewReader(content)

dbf := files.New(dConfig)
_, err := dbf.Upload(&files.UploadArg{
CommitInfo: files.CommitInfo{
Path: "/" + directory + "kerbers-agent-test.mp4",
Mode: &files.WriteMode{
Tagged: dropbox.Tagged{
Tag: "overwrite",
},
},
},
}, file)

if err != nil {
c.JSON(400, models.APIResponse{
Data: "Something went wrong while reaching the Dropbox API: " + err.Error(),
})
} else {
c.JSON(200, models.APIResponse{
Data: "Dropbox is working fine.",
})
}
}
} else {
c.JSON(400, models.APIResponse{
Data: "Dropbox token is not set.",
})
}
}
4 changes: 4 additions & 0 deletions machinery/src/cloud/TestFile.go

Large diffs are not rendered by default.

7 changes: 7 additions & 0 deletions machinery/src/components/Config.go
Original file line number Diff line number Diff line change
Expand Up @@ -414,6 +414,13 @@ func OverrideWithEnvironmentVariables(configuration *models.Configuration) {
configuration.Config.KStorage.Directory = value
break

/* When storing in dropbox */
case "AGENT_DROPBOX_ACCESS_TOKEN":
configuration.Config.Dropbox.AccessToken = value
break
case "AGENT_DROPBOX_DIRECTORY":
configuration.Config.Dropbox.Directory = value
break
}
}
}
Expand Down
7 changes: 7 additions & 0 deletions machinery/src/models/Config.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ type Config struct {
Cloud string `json:"cloud" bson:"cloud"`
S3 *S3 `json:"s3,omitempty" bson:"s3,omitempty"`
KStorage *KStorage `json:"kstorage,omitempty" bson:"kstorage,omitempty"`
Dropbox *Dropbox `json:"dropbox,omitempty" bson:"dropbox,omitempty"`
MQTTURI string `json:"mqtturi" bson:"mqtturi,omitempty"`
MQTTUsername string `json:"mqtt_username" bson:"mqtt_username"`
MQTTPassword string `json:"mqtt_password" bson:"mqtt_password"`
Expand Down Expand Up @@ -148,3 +149,9 @@ type KStorage struct {
Provider string `json:"provider,omitempty" bson:"provider,omitempty"`
Directory string `json:"directory,omitempty" bson:"directory,omitempty"`
}

// Dropbox integration
type Dropbox struct {
AccessToken string `json:"access_token,omitempty" bson:"access_token,omitempty"`
Directory string `json:"directory,omitempty" bson:"directory,omitempty"`
}
4 changes: 4 additions & 0 deletions ui/public/locales/de/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,10 @@
"kerberosvault_secretkey": "Geheimer Schlüssel",
"kerberosvault_description_secretkey": "Der geheime Schlüssel des Kerberos Vault Accounts.",
"verify_connection": "Verbindung prüfen",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"remove_after_upload": "Once recordings are uploaded to some persistence, you might want to remove them from the local Kerberos Agent.",
"remove_after_upload_description": "Remove recordings after they are uploaded successfully.",
"remove_after_upload_enabled": "Enabled delete on upload"
Expand Down
4 changes: 4 additions & 0 deletions ui/public/locales/en/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@
"kerberosvault_description_accesskey": "The access key of your Kerberos Vault account.",
"kerberosvault_secretkey": "Secret key",
"kerberosvault_description_secretkey": "The secret key of your Kerberos Vault account.",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"verify_connection": "Verify Connection",
"remove_after_upload": "Once recordings are uploaded to some persistence, you might want to remove them from the local Kerberos Agent.",
"remove_after_upload_description": "Remove recordings after they are uploaded successfully.",
Expand Down
4 changes: 4 additions & 0 deletions ui/public/locales/es/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@
"kerberosvault_description_accesskey": "The access key of your Kerberos Vault account.",
"kerberosvault_secretkey": "Secret key",
"kerberosvault_description_secretkey": "The secret key of your Kerberos Vault account.",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"verify_connection": "Verify Connection",
"remove_after_upload": "Once recordings are uploaded to some persistence, you might want to remove them from the local Kerberos Agent.",
"remove_after_upload_description": "Remove recordings after they are uploaded successfully.",
Expand Down
4 changes: 4 additions & 0 deletions ui/public/locales/fr/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@
"kerberosvault_description_accesskey": "La clé d'accès de votre compte Kerberos Vault.",
"kerberosvault_secretkey": "Clé secrète",
"kerberosvault_description_secretkey": "La clé secrète de votre compte Kerberos Vault.",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"verify_connection": "Vérifier la connexion",
"remove_after_upload": "Once recordings are uploaded to some persistence, you might want to remove them from the local Kerberos Agent.",
"remove_after_upload_description": "Remove recordings after they are uploaded successfully.",
Expand Down
4 changes: 4 additions & 0 deletions ui/public/locales/ja/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@
"kerberosvault_description_accesskey": "Kerberos Vault アカウントのアクセス キー。",
"kerberosvault_secretkey": "秘密鍵",
"kerberosvault_description_secretkey": "Kerberos Vault アカウントの秘密鍵。",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"verify_connection": "接続の確認",
"remove_after_upload": "Once recordings are uploaded to some persistence, you might want to remove them from the local Kerberos Agent.",
"remove_after_upload_description": "Remove recordings after they are uploaded successfully.",
Expand Down
4 changes: 4 additions & 0 deletions ui/public/locales/nl/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,10 @@
"kerberosvault_description_accesskey": "De access key van jouw Kerberos Vault account.",
"kerberosvault_secretkey": "Secret key",
"kerberosvault_description_secretkey": "De secret key van jouw Kerberos Vault account.",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"verify_connection": "Verifieer verbinding",
"remove_after_upload": "Wanneer opnames opgeslagen zijn, kan je ze verwijderen in de lokale Kerberos Agent.",
"remove_after_upload_description": "Verwijder opnames wanneer ze zijn geupload naar een opslag locatie.",
Expand Down
4 changes: 4 additions & 0 deletions ui/public/locales/pl/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@
"kerberosvault_description_accesskey": "The access key of your Kerberos Vault account.",
"kerberosvault_secretkey": "Secret key",
"kerberosvault_description_secretkey": "The secret key of your Kerberos Vault account.",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"verify_connection": "Verify Connection",
"remove_after_upload": "Once recordings are uploaded to some persistence, you might want to remove them from the local Kerberos Agent.",
"remove_after_upload_description": "Remove recordings after they are uploaded successfully.",
Expand Down
4 changes: 4 additions & 0 deletions ui/public/locales/pt/translation.json
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@
"kerberosvault_description_accesskey": "A chave de acesso da sua conta do Kerberos Vault.",
"kerberosvault_secretkey": "Chave secreta(Secret key)",
"kerberosvault_description_secretkey": "A chave secreta da sua conta do Kerberos Vault.",
"dropbox_directory": "Directory",
"dropbox_description_directory": "The sub directory where the recordings will be stored in your Dropbox account.",
"dropbox_accesstoken": "Access token",
"dropbox_description_accesstoken": "The access token of your Dropbox account/app.",
"verify_connection": "Verificar conexão",
"remove_after_upload": "Once recordings are uploaded to some persistence, you might want to remove them from the local Kerberos Agent.",
"remove_after_upload_description": "Remove recordings after they are uploaded successfully.",
Expand Down
Loading

0 comments on commit 026bf93

Please sign in to comment.