Merge pull request #1062 from wakatime/develop

Release v1.94.1

USAGE.md

@@ -102,7 +102,7 @@ some/submodule/name = new project name
 | status_bar_enabled             | Turns on wakatime status bar for certain editors. | _bool_ | `true` |
 | status_bar_coding_activity     | Enables displaying Today's code stats in the status bar of some editors. When false, only the WakaTime icon is displayed in the status bar. | _bool_ | `true` |
 | status_bar_hide_categories     | When `true`, --today only displays the total code stats, never displaying Categories in the output. | _bool_ | `false` |
-| offline                        | Enables saving code stats locally to ~/.wakatime.bdb when offline, and syncing to the dashboard later when back online. | _bool_ | `true` |
+| offline                        | Enables saving code stats locally to ~/.wakatime/offline_heartbeats.bdb when offline, and syncing to the dashboard later when back online. | _bool_ | `true` |
 | proxy                          | Optional proxy configuration. Supports HTTPS, SOCKS and NTLM proxies. For ex: `https://user:pass@host:port`, `socks5://user:pass@host:port`, `domain\\user:pass` | _string_ | |
 | no_ssl_verify                  | Disables SSL certificate verification for HTTPS requests. By default, SSL certificates are verified. | _bool_ | `false` |
 | ssl_certs_file                 | Path to a CA certs file. By default, uses bundled Letsencrypt CA cert along with system ca certs. | _filepath_ | |

cmd/heartbeat/heartbeat_test.go

@@ -372,7 +372,10 @@ func TestSendHeartbeats_ExtraHeartbeats_Sanitize(t *testing.T) {
 	db, err := bolt.Open(offlineQueueFile.Name(), 0600, nil)
 	require.NoError(t, err)
 
-	defer db.Close()
+	defer func() {
+		err = db.Close()
+		require.NoError(t, err)
+	}()
 
 	tx, err := db.Begin(true)
 	require.NoError(t, err)

cmd/logfile/logfile.go

@@ -11,10 +11,7 @@ import (
 	"github.com/spf13/viper"
 )
 
-const (
-	defaultFile   = "wakatime.log"
-	defaultFolder = ".wakatime"
-)
+const defaultFile = "wakatime.log"
 
 // Params contains log file parameters.
 type Params struct {
@@ -60,7 +57,7 @@ func LoadParams(v *viper.Viper) (Params, error) {
 
 	folder, err := ini.WakaResourcesDir()
 	if err != nil {
-		return Params{}, fmt.Errorf("failed getting user's home directory: %s", err)
+		return Params{}, fmt.Errorf("failed getting resource directory: %s", err)
 	}
 
 	params.File = filepath.Join(folder, defaultFile)

cmd/offlinecount/offlinecount_test.go

@@ -27,7 +27,9 @@ func TestOfflineCount_Empty(t *testing.T) {
 	require.NoError(t, err)
 
 	insertHeartbeatRecords(t, db, "heartbeats", []heartbeatRecord{})
-	db.Close()
+
+	err = db.Close()
+	require.NoError(t, err)
 
 	v := viper.New()
 	v.Set("verbose", true)
@@ -89,7 +91,8 @@ func TestOfflineCount(t *testing.T) {
 		},
 	})
 
-	db.Close()
+	err = db.Close()
+	require.NoError(t, err)
 
 	v := viper.New()
 	v.Set("offline-count", true)

cmd/offlineprint/offlineprint_test.go

@@ -43,7 +43,8 @@ func TestPrintOfflineHeartbeats(t *testing.T) {
 		},
 	})
 
-	db.Close()
+	err = db.Close()
+	require.NoError(t, err)
 
 	v := viper.New()
 	v.Set("print-offline-heartbeats", 10)

cmd/offlinesync/offlinesync.go

@@ -2,6 +2,7 @@ package offlinesync
 
 import (
 	"fmt"
+	"os"
 
 	cmdapi "github.com/wakatime/wakatime-cli/cmd/api"
 	"github.com/wakatime/wakatime-cli/cmd/params"
@@ -25,8 +26,16 @@ func Run(v *viper.Viper) (int, error) {
 		)
 	}
 
-	err = SyncOfflineActivity(v, queueFilepath)
+	queueFilepathLegacy, err := offline.QueueFilepathLegacy()
 	if err != nil {
+		log.Warnf("legacy offline sync failed: failed to load offline queue filepath: %s", err)
+	}
+
+	if err = syncOfflineActivityLegacy(v, queueFilepathLegacy); err != nil {
+		log.Warnf("legacy offline sync failed: %s", err)
+	}
+
+	if err = SyncOfflineActivity(v, queueFilepath); err != nil {
 		if errwaka, ok := err.(wakaerror.Error); ok {
 			return errwaka.ExitCode(), fmt.Errorf("offline sync failed: %s", errwaka.Message())
 		}
@@ -42,6 +51,49 @@ func Run(v *viper.Viper) (int, error) {
 	return exitcode.Success, nil
 }
 
+// syncOfflineActivityLegacy syncs the old offline activity by sending heartbeats
+// from the legacy offline queue to the WakaTime API.
+func syncOfflineActivityLegacy(v *viper.Viper, queueFilepath string) error {
+	if queueFilepath == "" {
+		return nil
+	}
+
+	paramOffline := params.LoadOfflineParams(v)
+
+	paramAPI, err := params.LoadAPIParams(v)
+	if err != nil {
+		return fmt.Errorf("failed to load API parameters: %w", err)
+	}
+
+	apiClient, err := cmdapi.NewClientWithoutAuth(paramAPI)
+	if err != nil {
+		return fmt.Errorf("failed to initialize api client: %w", err)
+	}
+
+	if paramOffline.QueueFileLegacy != "" {
+		queueFilepath = paramOffline.QueueFileLegacy
+	}
+
+	handle := heartbeat.NewHandle(apiClient,
+		offline.WithSync(queueFilepath, paramOffline.SyncMax),
+		apikey.WithReplacing(apikey.Config{
+			DefaultAPIKey: paramAPI.Key,
+			MapPatterns:   paramAPI.KeyPatterns,
+		}),
+	)
+
+	_, err = handle(nil)
+	if err != nil {
+		return err
+	}
+
+	if err := os.Remove(queueFilepath); err != nil {
+		log.Warnf("failed to delete legacy offline file: %s", err)
+	}
+
+	return nil
+}
+
 // SyncOfflineActivity syncs offline activity by sending heartbeats
 // from the offline queue to the WakaTime API.
 func SyncOfflineActivity(v *viper.Viper, queueFilepath string) error {

cmd/offlinesync/offlinesync_internal_test.go

@@ -0,0 +1,147 @@
+package offlinesync
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/spf13/viper"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	bolt "go.etcd.io/bbolt"
+)
+
+func TestSyncOfflineActivityLegacy(t *testing.T) {
+	testServerURL, router, tearDown := setupTestServer()
+	defer tearDown()
+
+	var (
+		plugin   = "plugin/0.0.1"
+		numCalls int
+	)
+
+	router.HandleFunc("/users/current/heartbeats.bulk", func(w http.ResponseWriter, req *http.Request) {
+		numCalls++
+
+		// check request
+		assert.Equal(t, http.MethodPost, req.Method)
+		assert.Equal(t, []string{"application/json"}, req.Header["Accept"])
+		assert.Equal(t, []string{"application/json"}, req.Header["Content-Type"])
+		assert.Equal(t, []string{"Basic MDAwMDAwMDAtMDAwMC00MDAwLTgwMDAtMDAwMDAwMDAwMDAw"}, req.Header["Authorization"])
+		assert.True(t, strings.HasSuffix(req.Header["User-Agent"][0], plugin), fmt.Sprintf(
+			"%q should have suffix %q",
+			req.Header["User-Agent"][0],
+			plugin,
+		))
+
+		expectedBody, err := os.ReadFile("testdata/api_heartbeats_request_template.json")
+		require.NoError(t, err)
+
+		body, err := io.ReadAll(req.Body)
+		require.NoError(t, err)
+
+		assert.JSONEq(t, string(expectedBody), string(body))
+
+		// send response
+		w.WriteHeader(http.StatusCreated)
+
+		f, err := os.Open("testdata/api_heartbeats_response.json")
+		require.NoError(t, err)
+		defer f.Close()
+
+		_, err = io.Copy(w, f)
+		require.NoError(t, err)
+	})
+
+	// setup offline queue
+	f, err := os.CreateTemp(t.TempDir(), "")
+	require.NoError(t, err)
+
+	// early close to avoid file locking in Windows
+	f.Close()
+
+	db, err := bolt.Open(f.Name(), 0600, nil)
+	require.NoError(t, err)
+
+	dataGo, err := os.ReadFile("testdata/heartbeat_go.json")
+	require.NoError(t, err)
+
+	dataPy, err := os.ReadFile("testdata/heartbeat_py.json")
+	require.NoError(t, err)
+
+	dataJs, err := os.ReadFile("testdata/heartbeat_js.json")
+	require.NoError(t, err)
+
+	insertHeartbeatRecords(t, db, "heartbeats", []heartbeatRecord{
+		{
+			ID:        "1592868367.219124-file-coding-wakatime-cli-heartbeat-/tmp/main.go-true",
+			Heartbeat: string(dataGo),
+		},
+		{
+			ID:        "1592868386.079084-file-debugging-wakatime-summary-/tmp/main.py-false",
+			Heartbeat: string(dataPy),
+		},
+		{
+			ID:        "1592868394.084354-file-building-wakatime-todaygoal-/tmp/main.js-false",
+			Heartbeat: string(dataJs),
+		},
+	})
+
+	err = db.Close()
+	require.NoError(t, err)
+
+	v := viper.New()
+	v.Set("api-url", testServerURL)
+	v.Set("key", "00000000-0000-4000-8000-000000000000")
+	v.Set("sync-offline-activity", 100)
+	v.Set("plugin", plugin)
+
+	err = syncOfflineActivityLegacy(v, f.Name())
+	require.NoError(t, err)
+
+	assert.NoFileExists(t, f.Name())
+
+	assert.Eventually(t, func() bool { return numCalls == 1 }, time.Second, 50*time.Millisecond)
+}
+
+func setupTestServer() (string, *http.ServeMux, func()) {
+	router := http.NewServeMux()
+	srv := httptest.NewServer(router)
+
+	return srv.URL, router, func() { srv.Close() }
+}
+
+type heartbeatRecord struct {
+	ID        string
+	Heartbeat string
+}
+
+func insertHeartbeatRecords(t *testing.T, db *bolt.DB, bucket string, hh []heartbeatRecord) {
+	for _, h := range hh {
+		insertHeartbeatRecord(t, db, bucket, h)
+	}
+}
+
+func insertHeartbeatRecord(t *testing.T, db *bolt.DB, bucket string, h heartbeatRecord) {
+	t.Helper()
+
+	err := db.Update(func(tx *bolt.Tx) error {
+		b, err := tx.CreateBucketIfNotExists([]byte(bucket))
+		if err != nil {
+			return fmt.Errorf("failed to create bucket: %s", err)
+		}
+
+		err = b.Put([]byte(h.ID), []byte(h.Heartbeat))
+		if err != nil {
+			return fmt.Errorf("failed put heartbeat: %s", err)
+		}
+
+		return nil
+	})
+	require.NoError(t, err)
+}

cmd/offlinesync/offlinesync_test.go

@@ -93,7 +93,8 @@ func TestSyncOfflineActivity(t *testing.T) {
 		},
 	})
 
-	db.Close()
+	err = db.Close()
+	require.NoError(t, err)
 
 	v := viper.New()
 	v.Set("api-url", testServerURL)
@@ -181,7 +182,8 @@ func TestSyncOfflineActivity_MultipleApiKey(t *testing.T) {
 		},
 	})
 
-	db.Close()
+	err = db.Close()
+	require.NoError(t, err)
 
 	v := viper.New()
 	v.Set("api-url", testServerURL)

cmd/params/params.go

@@ -6,6 +6,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"io"
 	"net/url"
 	"os"
 	"os/exec"
@@ -129,10 +130,11 @@ type (
 
 	// Offline contains offline related parameters.
 	Offline struct {
-		Disabled  bool
-		QueueFile string
-		PrintMax  int
-		SyncMax   int
+		Disabled        bool
+		QueueFile       string
+		QueueFileLegacy string
+		PrintMax        int
+		SyncMax         int
 	}
 
 	// ProjectParams params for project name sanitization.
@@ -653,10 +655,11 @@ func LoadOfflineParams(v *viper.Viper) Offline {
 	}
 
 	return Offline{
-		Disabled:  disabled,
-		QueueFile: vipertools.GetString(v, "offline-queue-file"),
-		PrintMax:  v.GetInt("print-offline-heartbeats"),
-		SyncMax:   syncMax,
+		Disabled:        disabled,
+		QueueFile:       vipertools.GetString(v, "offline-queue-file"),
+		QueueFileLegacy: vipertools.GetString(v, "offline-queue-file-legacy"),
+		PrintMax:        v.GetInt("print-offline-heartbeats"),
+		SyncMax:         syncMax,
 	}
 }
 
@@ -730,7 +733,7 @@ func readExtraHeartbeats() ([]heartbeat.Heartbeat, error) {
 	in := bufio.NewReader(os.Stdin)
 
 	input, err := in.ReadString('\n')
-	if err != nil {
+	if err != nil && err != io.EOF {
 		log.Debugf("failed to read data from stdin: %s", err)
 	}
 
@@ -1036,10 +1039,11 @@ func (p Heartbeat) String() string {
 // String implements fmt.Stringer interface.
 func (p Offline) String() string {
 	return fmt.Sprintf(
-		"disabled: %t, print max: %d, queue file: '%s', num sync max: %d",
+		"disabled: %t, print max: %d, queue file: '%s', queue file legacy: '%s', num sync max: %d",
 		p.Disabled,
 		p.PrintMax,
 		p.QueueFile,
+		p.QueueFileLegacy,
 		p.SyncMax,
 	)
 }