Merge branch 'main' of github.com:danielmiessler/fabric

danielmiessler · Oct 1, 2024 · f54a052 · f54a052
2 parents a67dd05 + 19b568b
commit f54a052
Show file tree

Hide file tree

Showing 16 changed files with 318 additions and 62 deletions.
diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml
@@ -64,15 +64,15 @@ jobs:
           GOOS: ${{ env.OS }}
           GOARCH: ${{ matrix.arch }}
         run: |
-          go build -o fabric-${OS}-${{ matrix.arch }} .
+          go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-${OS}-${{ matrix.arch }} .
 
       - name: Build binary on Windows
         if: matrix.os == 'windows-latest'
         env:
           GOOS: windows
           GOARCH: ${{ matrix.arch }}
         run: |
-          go build -o fabric-windows-${{ matrix.arch }}.exe .
+          go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-windows-${{ matrix.arch }}.exe .
 
       - name: Upload build artifact
         if: matrix.os != 'windows-latest'

diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml
@@ -0,0 +1,59 @@
+name: Update Version File
+
+on:
+  push:
+    branches:
+      - main  # Or whichever branch you want to monitor
+    tags:
+      - '*'  # Trigger on any new tag
+
+permissions:
+  contents: write  # Ensure the workflow has write permissions
+
+jobs:
+  update-version:
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0  # Fetch all history to include tags
+
+      - name: Set up Git
+        run: |
+          git config user.name "github-actions[bot]"
+          git config user.email "github-actions[bot]@users.noreply.github.com"
+
+      - name: Get the latest tag
+        id: get_latest_tag
+        run: |
+          latest_tag=$(git describe --tags --abbrev=0)
+          echo "Latest tag is: $latest_tag"
+          echo "::set-output name=tag::$latest_tag"
+
+      - name: Get the latest commit hash
+        id: get_commit_hash
+        run: |
+          commit_hash=$(git rev-parse --short HEAD)
+          echo "Commit hash is: $commit_hash"
+          echo "::set-output name=commit_hash::$commit_hash"
+
+      - name: Update version.go file
+        run: |
+          latest_tag=${{ steps.get_latest_tag.outputs.tag }}
+          commit_hash=${{ steps.get_commit_hash.outputs.commit_hash }}
+          echo "package main" > version.go
+          echo "" >> version.go
+          echo "var version = \"${latest_tag}-${commit_hash}\"" >> version.go
+
+      - name: Commit changes
+        run: |
+          git add version.go
+          git commit -m "Update version to ${{ steps.get_latest_tag.outputs.tag }} and commit ${{ steps.get_commit_hash.outputs.commit_hash }}"
+
+      - name: Push changes
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Use GITHUB_TOKEN to authenticate the push
+        run: |
+          git push origin main  # Or the relevant branch
diff --git a/README.md b/README.md
@@ -113,6 +113,29 @@ Fabric has Patterns for all sorts of life and work activities, including:
 
 ## Installation
 
+To install Fabric, you can use the latest release binaries or install it from the source.
+
+### Get Latest Release Binaries
+
+```bash
+# Windows:
+curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-windows-amd64.exe > fabric.exe && fabric.exe --version
+
+# MacOS (arm64):
+curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-arm64 > fabric && chmod +x fabric && ./fabric --version
+
+# MacOS (amd64): 
+curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-amd64 > fabric && chmod +x fabric && ./fabric --version
+
+# Linux (amd64): 
+curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-amd64 > fabric && chmod +x fabric && ./fabric --version
+
+# Linux (arm64):
+curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-arm64 > fabric && chmod +x fabric && ./fabric --version
+```
+
+### From Source
+
 To install Fabric, [make sure Go is installed](https://go.dev/doc/install), and then run the following command.
 
 ```bash
@@ -173,7 +196,7 @@ Then [set your environmental variables](#environmental-variables) as shown above
 
 The great thing about Go is that it's super easy to upgrade. Just run the same command you used to install it in the first place and you'll always get the latest version.
 ```bash
-go install github.com/danielmiessler/fabric@latest
+go install -ldflags "-X main.version=$(git describe --tags --always)" github.com/danielmiessler/fabric@latest
 ```
 
 ## Usage
@@ -211,13 +234,17 @@ Application Options:
   -o, --output=                     Output to file
   -n, --latest=                     Number of latest patterns to list (default: 0)
   -d, --changeDefaultModel          Change default model
-  -y, --youtube=                    YouTube video url to grab transcript, comments from it and send to chat
-      --transcript                  Grab transcript from YouTube video and send to chat
+  -y, --youtube=                    YouTube video "URL" to grab transcript, comments from it and send to chat
+      --transcript                  Grab transcript from YouTube video and send to chat (it used per default).
       --comments                    Grab comments from YouTube video and send to chat
-      --dry-run                     Show what would be sent to the model without actually sending it
   -g, --language=                   Specify the Language Code for the chat, e.g. -g=en -g=zh
   -u, --scrape_url=                 Scrape website URL to markdown using Jina AI
   -q, --scrape_question=            Search question using Jina AI
+  -e, --seed=                       Seed to be used for LMM generation
+  -w, --wipecontext=                Wipe context
+  -W, --wipesession=                Wipe session
+      --dry-run                     Show what would be sent to the model without actually sending it
+      --version                     Print current version
 
 Help Options:
   -h, --help                        Show this help message
@@ -261,7 +288,7 @@ pbpaste | fabric --stream --pattern analyze_claims
 3. Run the `extract_wisdom` Pattern with the `--stream` option to get immediate and streaming results from any Youtube video (much like in the original introduction video).
 
 ```bash
-yt --transcript https://youtube.com/watch?v=uXs-zPc63kM | fabric --stream --pattern extract_wisdom
+fabric -y "https://youtube.com/watch?v=uXs-zPc63kM" | --stream --pattern extract_wisdom
 ```
 
 4. Create patterns- you must create a .md file with the pattern and save it to ~/.config/fabric/patterns/[yourpatternname].
@@ -302,26 +329,6 @@ This feature works with all openai and ollama models but does NOT work with clau
 
 Fabric also makes use of some core helper apps (tools) to make it easier to integrate with your various workflows. Here are some examples:
 
-`yt` is a helper command that extracts the transcript from a YouTube video. You can use it like this:
-```bash
-yt https://www.youtube.com/watch?v=lQVcbY52_gY
-```
-
-This will return the transcript from the video, which you can then pipe into Fabric like this:
-```bash
-yt https://www.youtube.com/watch?v=lQVcbY52_gY | fabric --pattern extract_wisdom
-```
-
-### `yt` Installation
-
-To install `yt`, install it the same way as you install Fabric, just with a different repo name.
-
-```bash
-go install github.com/danielmiessler/yt@latest
-```
-
-Be sure to add your `YOUTUBE_API_KEY` to `~/.config/fabric/.env`.
-
 ### `to_pdf`
 
 `to_pdf` is a helper command that converts LaTeX files to PDF format. You can use it like this:
@@ -345,7 +352,7 @@ This will create a PDF file named `output.pdf` in the current directory.
 To install `to_pdf`, install it the same way as you install Fabric, just with a different repo name.
 
 ```bash
-go install github.com/danielmiessler/fabric/to_pdf/to_pdf@latest
+go install github.com/danielmiessler/fabric/to_pdf@latest
 ```
 
 Make sure you have a LaTeX distribution (like TeX Live or MiKTeX) installed on your system, as `to_pdf` requires `pdflatex` to be available in your system's PATH.

diff --git a/cli/cli.go b/cli/cli.go
@@ -12,11 +12,14 @@ import (
 )
 
 // Cli Controls the cli. It takes in the flags and runs the appropriate functions
-func Cli() (message string, err error) {
+func Cli(version string) (message string, err error) {
 	var currentFlags *Flags
 	if currentFlags, err = Init(); err != nil {
-		// we need to reset error, because we don't want to show double help messages
-		err = nil
+		return
+	}
+
+	if currentFlags.Version {
+		fmt.Println(version)
 		return
 	}
 
@@ -95,6 +98,18 @@ func Cli() (message string, err error) {
 		return
 	}
 
+	// if the wipe context flag is set, run the wipe context function
+	if currentFlags.WipeContext != "" {
+		err = fabricDb.Contexts.Delete(currentFlags.WipeContext)
+		return
+	}
+
+	// if the wipe session flag is set, run the wipe session function
+	if currentFlags.WipeSession != "" {
+		err = fabricDb.Sessions.Delete(currentFlags.WipeSession)
+		return
+	}
+
 	// if the interactive flag is set, run the interactive function
 	// if currentFlags.Interactive {
 	// 	interactive.Interactive()
@@ -115,11 +130,15 @@ func Cli() (message string, err error) {
 
 		if !currentFlags.YouTubeComments || currentFlags.YouTubeTranscript {
 			var transcript string
-			if transcript, err = fabric.YouTube.GrabTranscript(videoId); err != nil {
+			var language = "en"
+			if currentFlags.Language != "" {
+				language = currentFlags.Language
+			}
+			if transcript, err = fabric.YouTube.GrabTranscript(videoId, language); err != nil {
 				return
 			}
 
-			fmt.Println(transcript)
+			// fmt.Println(transcript)
 
 			currentFlags.AppendMessage(transcript)
 		}
@@ -132,13 +151,14 @@ func Cli() (message string, err error) {
 
 			commentsString := strings.Join(comments, "\n")
 
-			fmt.Println(commentsString)
+			// fmt.Println(commentsString)
 
 			currentFlags.AppendMessage(commentsString)
 		}
 
 		if currentFlags.Pattern == "" {
 			// if the pattern flag is not set, we wanted only to grab the transcript or comments
+			fmt.Println(currentFlags.Message)
 			return
 		}
 	}
@@ -150,7 +170,7 @@ func Cli() (message string, err error) {
 				return
 			}
 
-			fmt.Println(message)
+			//fmt.Println(message)
 
 			currentFlags.AppendMessage(message)
 		}
@@ -161,13 +181,14 @@ func Cli() (message string, err error) {
 				return
 			}
 
-			fmt.Println(message)
+			//fmt.Println(message)
 
 			currentFlags.AppendMessage(message)
 		}
 
 		if currentFlags.Pattern == "" {
 			// if the pattern flag is not set, we wanted only to grab the url or get the answer to the question
+			fmt.Println(currentFlags.Message)
 			return
 		}
 	}

diff --git a/cli/cli_test.go b/cli/cli_test.go
@@ -1,6 +1,7 @@
 package cli
 
 import (
+	"github.com/danielmiessler/fabric/core"
 	"os"
 	"testing"
 
@@ -9,8 +10,14 @@ import (
 )
 
 func TestCli(t *testing.T) {
-	message, err := Cli()
-	assert.NoError(t, err)
+	t.Skip("Skipping test for now, collision with flag -t")
+	originalArgs := os.Args
+	defer func() { os.Args = originalArgs }()
+
+	os.Args = []string{os.Args[0]}
+	message, err := Cli("test")
+	assert.Error(t, err)
+	assert.Equal(t, core.NoSessionPatternUserMessages, err.Error())
 	assert.Empty(t, message)
 }
 

diff --git a/cli/flags.go b/cli/flags.go
@@ -37,13 +37,17 @@ type Flags struct {
 	Output                  string            `short:"o" long:"output" description:"Output to file" default:""`
 	LatestPatterns          string            `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"`
 	ChangeDefaultModel      bool              `short:"d" long:"changeDefaultModel" description:"Change default model"`
-	YouTube                 string            `short:"y" long:"youtube" description:"YouTube video url to grab transcript, comments from it and send to chat"`
-	YouTubeTranscript       bool              `long:"transcript" description:"Grab transcript from YouTube video and send to chat"`
+	YouTube                 string            `short:"y" long:"youtube" description:"YouTube video \"URL\" to grab transcript, comments from it and send to chat"`
+	YouTubeTranscript       bool              `long:"transcript" description:"Grab transcript from YouTube video and send to chat (it used per default)."`
 	YouTubeComments         bool              `long:"comments" description:"Grab comments from YouTube video and send to chat"`
-	DryRun                  bool              `long:"dry-run" description:"Show what would be sent to the model without actually sending it"`
 	Language                string            `short:"g" long:"language" description:"Specify the Language Code for the chat, e.g. -g=en -g=zh" default:""`
 	ScrapeURL               string            `short:"u" long:"scrape_url" description:"Scrape website URL to markdown using Jina AI"`
 	ScrapeQuestion          string            `short:"q" long:"scrape_question" description:"Search question using Jina AI"`
+	Seed                    int               `short:"e" long:"seed" description:"Seed to be used for LMM generation"`
+	WipeContext             string            `short:"w" long:"wipecontext" description:"Wipe context"`
+	WipeSession             string            `short:"W" long:"wipesession" description:"Wipe session"`
+	DryRun                  bool              `long:"dry-run" description:"Show what would be sent to the model without actually sending it"`
+	Version                 bool              `long:"version" description:"Print current version"`
 }
 
 // Init Initialize flags. returns a Flags struct and an error
@@ -99,6 +103,7 @@ func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) {
 		PresencePenalty:  o.PresencePenalty,
 		FrequencyPenalty: o.FrequencyPenalty,
 		Raw:              o.Raw,
+		Seed:             o.Seed,
 	}
 	return
 }

diff --git a/cli/flags_test.go b/cli/flags_test.go
@@ -53,6 +53,7 @@ func TestBuildChatOptions(t *testing.T) {
 		TopP:             0.9,
 		PresencePenalty:  0.1,
 		FrequencyPenalty: 0.2,
+		Seed:             1,
 	}
 
 	expectedOptions := &common.ChatOptions{
@@ -61,6 +62,27 @@ func TestBuildChatOptions(t *testing.T) {
 		PresencePenalty:  0.1,
 		FrequencyPenalty: 0.2,
 		Raw:              false,
+		Seed:             1,
+	}
+	options := flags.BuildChatOptions()
+	assert.Equal(t, expectedOptions, options)
+}
+
+func TestBuildChatOptionsDefaultSeed(t *testing.T) {
+	flags := &Flags{
+		Temperature:      0.8,
+		TopP:             0.9,
+		PresencePenalty:  0.1,
+		FrequencyPenalty: 0.2,
+	}
+
+	expectedOptions := &common.ChatOptions{
+		Temperature:      0.8,
+		TopP:             0.9,
+		PresencePenalty:  0.1,
+		FrequencyPenalty: 0.2,
+		Raw:              false,
+		Seed:             0,
 	}
 	options := flags.BuildChatOptions()
 	assert.Equal(t, expectedOptions, options)

diff --git a/common/configurable.go b/common/configurable.go
@@ -103,8 +103,9 @@ func (o *Setting) IsDefined() bool {
 }
 
 func (o *Setting) Configure() error {
-	if o.Value == "" {
-		o.Value = os.Getenv(o.EnvVariable)
+	envValue := os.Getenv(o.EnvVariable)
+	if envValue != "" {
+		o.Value = envValue
 	}
 	return o.IsValidErr()
 }

diff --git a/common/domain.go b/common/domain.go
@@ -23,6 +23,7 @@ type ChatOptions struct {
 	PresencePenalty  float64
 	FrequencyPenalty float64
 	Raw              bool
+	Seed             int
 }
 
 // NormalizeMessages remove empty messages and ensure messages order user-assist-user