diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b71c045cc..c402e2175 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -64,7 +64,7 @@ jobs: GOOS: ${{ env.OS }} GOARCH: ${{ matrix.arch }} run: | - go build -o fabric-${OS}-${{ matrix.arch }} . + go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-${OS}-${{ matrix.arch }} . - name: Build binary on Windows if: matrix.os == 'windows-latest' @@ -72,7 +72,7 @@ jobs: GOOS: windows GOARCH: ${{ matrix.arch }} run: | - go build -o fabric-windows-${{ matrix.arch }}.exe . + go build -ldflags "-X main.version=$(git describe --tags --abbrev=0)" -o fabric-windows-${{ matrix.arch }}.exe . - name: Upload build artifact if: matrix.os != 'windows-latest' diff --git a/.github/workflows/update-version.yml b/.github/workflows/update-version.yml new file mode 100644 index 000000000..4873e5bec --- /dev/null +++ b/.github/workflows/update-version.yml @@ -0,0 +1,59 @@ +name: Update Version File + +on: + push: + branches: + - main # Or whichever branch you want to monitor + tags: + - '*' # Trigger on any new tag + +permissions: + contents: write # Ensure the workflow has write permissions + +jobs: + update-version: + runs-on: ubuntu-latest + + steps: + - name: Checkout repository + uses: actions/checkout@v4 + with: + fetch-depth: 0 # Fetch all history to include tags + + - name: Set up Git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Get the latest tag + id: get_latest_tag + run: | + latest_tag=$(git describe --tags --abbrev=0) + echo "Latest tag is: $latest_tag" + echo "::set-output name=tag::$latest_tag" + + - name: Get the latest commit hash + id: get_commit_hash + run: | + commit_hash=$(git rev-parse --short HEAD) + echo "Commit hash is: $commit_hash" + echo "::set-output name=commit_hash::$commit_hash" + + - name: Update version.go file + run: | + latest_tag=${{ steps.get_latest_tag.outputs.tag }} + commit_hash=${{ steps.get_commit_hash.outputs.commit_hash }} + echo "package main" > version.go + echo "" >> version.go + echo "var version = \"${latest_tag}-${commit_hash}\"" >> version.go + + - name: Commit changes + run: | + git add version.go + git commit -m "Update version to ${{ steps.get_latest_tag.outputs.tag }} and commit ${{ steps.get_commit_hash.outputs.commit_hash }}" + + - name: Push changes + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # Use GITHUB_TOKEN to authenticate the push + run: | + git push origin main # Or the relevant branch \ No newline at end of file diff --git a/README.md b/README.md index c26d5f1dc..ee54e1fab 100644 --- a/README.md +++ b/README.md @@ -113,6 +113,29 @@ Fabric has Patterns for all sorts of life and work activities, including: ## Installation +To install Fabric, you can use the latest release binaries or install it from the source. + +### Get Latest Release Binaries + +```bash +# Windows: +curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-windows-amd64.exe > fabric.exe && fabric.exe --version + +# MacOS (arm64): +curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-arm64 > fabric && chmod +x fabric && ./fabric --version + +# MacOS (amd64): +curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-darwin-amd64 > fabric && chmod +x fabric && ./fabric --version + +# Linux (amd64): +curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-amd64 > fabric && chmod +x fabric && ./fabric --version + +# Linux (arm64): +curl -L https://github.com/danielmiessler/fabric/releases/latest/download/fabric-linux-arm64 > fabric && chmod +x fabric && ./fabric --version +``` + +### From Source + To install Fabric, [make sure Go is installed](https://go.dev/doc/install), and then run the following command. ```bash @@ -173,7 +196,7 @@ Then [set your environmental variables](#environmental-variables) as shown above The great thing about Go is that it's super easy to upgrade. Just run the same command you used to install it in the first place and you'll always get the latest version. ```bash -go install github.com/danielmiessler/fabric@latest +go install -ldflags "-X main.version=$(git describe --tags --always)" github.com/danielmiessler/fabric@latest ``` ## Usage @@ -211,13 +234,17 @@ Application Options: -o, --output= Output to file -n, --latest= Number of latest patterns to list (default: 0) -d, --changeDefaultModel Change default model - -y, --youtube= YouTube video url to grab transcript, comments from it and send to chat - --transcript Grab transcript from YouTube video and send to chat + -y, --youtube= YouTube video "URL" to grab transcript, comments from it and send to chat + --transcript Grab transcript from YouTube video and send to chat (it used per default). --comments Grab comments from YouTube video and send to chat - --dry-run Show what would be sent to the model without actually sending it -g, --language= Specify the Language Code for the chat, e.g. -g=en -g=zh -u, --scrape_url= Scrape website URL to markdown using Jina AI -q, --scrape_question= Search question using Jina AI + -e, --seed= Seed to be used for LMM generation + -w, --wipecontext= Wipe context + -W, --wipesession= Wipe session + --dry-run Show what would be sent to the model without actually sending it + --version Print current version Help Options: -h, --help Show this help message @@ -261,7 +288,7 @@ pbpaste | fabric --stream --pattern analyze_claims 3. Run the `extract_wisdom` Pattern with the `--stream` option to get immediate and streaming results from any Youtube video (much like in the original introduction video). ```bash -yt --transcript https://youtube.com/watch?v=uXs-zPc63kM | fabric --stream --pattern extract_wisdom +fabric -y "https://youtube.com/watch?v=uXs-zPc63kM" | --stream --pattern extract_wisdom ``` 4. Create patterns- you must create a .md file with the pattern and save it to ~/.config/fabric/patterns/[yourpatternname]. @@ -302,26 +329,6 @@ This feature works with all openai and ollama models but does NOT work with clau Fabric also makes use of some core helper apps (tools) to make it easier to integrate with your various workflows. Here are some examples: -`yt` is a helper command that extracts the transcript from a YouTube video. You can use it like this: -```bash -yt https://www.youtube.com/watch?v=lQVcbY52_gY -``` - -This will return the transcript from the video, which you can then pipe into Fabric like this: -```bash -yt https://www.youtube.com/watch?v=lQVcbY52_gY | fabric --pattern extract_wisdom -``` - -### `yt` Installation - -To install `yt`, install it the same way as you install Fabric, just with a different repo name. - -```bash -go install github.com/danielmiessler/yt@latest -``` - -Be sure to add your `YOUTUBE_API_KEY` to `~/.config/fabric/.env`. - ### `to_pdf` `to_pdf` is a helper command that converts LaTeX files to PDF format. You can use it like this: @@ -345,7 +352,7 @@ This will create a PDF file named `output.pdf` in the current directory. To install `to_pdf`, install it the same way as you install Fabric, just with a different repo name. ```bash -go install github.com/danielmiessler/fabric/to_pdf/to_pdf@latest +go install github.com/danielmiessler/fabric/to_pdf@latest ``` Make sure you have a LaTeX distribution (like TeX Live or MiKTeX) installed on your system, as `to_pdf` requires `pdflatex` to be available in your system's PATH. diff --git a/cli/cli.go b/cli/cli.go index 45eca36dc..1f1feefb6 100644 --- a/cli/cli.go +++ b/cli/cli.go @@ -12,11 +12,14 @@ import ( ) // Cli Controls the cli. It takes in the flags and runs the appropriate functions -func Cli() (message string, err error) { +func Cli(version string) (message string, err error) { var currentFlags *Flags if currentFlags, err = Init(); err != nil { - // we need to reset error, because we don't want to show double help messages - err = nil + return + } + + if currentFlags.Version { + fmt.Println(version) return } @@ -95,6 +98,18 @@ func Cli() (message string, err error) { return } + // if the wipe context flag is set, run the wipe context function + if currentFlags.WipeContext != "" { + err = fabricDb.Contexts.Delete(currentFlags.WipeContext) + return + } + + // if the wipe session flag is set, run the wipe session function + if currentFlags.WipeSession != "" { + err = fabricDb.Sessions.Delete(currentFlags.WipeSession) + return + } + // if the interactive flag is set, run the interactive function // if currentFlags.Interactive { // interactive.Interactive() @@ -115,11 +130,15 @@ func Cli() (message string, err error) { if !currentFlags.YouTubeComments || currentFlags.YouTubeTranscript { var transcript string - if transcript, err = fabric.YouTube.GrabTranscript(videoId); err != nil { + var language = "en" + if currentFlags.Language != "" { + language = currentFlags.Language + } + if transcript, err = fabric.YouTube.GrabTranscript(videoId, language); err != nil { return } - fmt.Println(transcript) + // fmt.Println(transcript) currentFlags.AppendMessage(transcript) } @@ -132,13 +151,14 @@ func Cli() (message string, err error) { commentsString := strings.Join(comments, "\n") - fmt.Println(commentsString) + // fmt.Println(commentsString) currentFlags.AppendMessage(commentsString) } if currentFlags.Pattern == "" { // if the pattern flag is not set, we wanted only to grab the transcript or comments + fmt.Println(currentFlags.Message) return } } @@ -150,7 +170,7 @@ func Cli() (message string, err error) { return } - fmt.Println(message) + //fmt.Println(message) currentFlags.AppendMessage(message) } @@ -161,13 +181,14 @@ func Cli() (message string, err error) { return } - fmt.Println(message) + //fmt.Println(message) currentFlags.AppendMessage(message) } if currentFlags.Pattern == "" { // if the pattern flag is not set, we wanted only to grab the url or get the answer to the question + fmt.Println(currentFlags.Message) return } } diff --git a/cli/cli_test.go b/cli/cli_test.go index 95b87017c..b1f6795da 100644 --- a/cli/cli_test.go +++ b/cli/cli_test.go @@ -1,6 +1,7 @@ package cli import ( + "github.com/danielmiessler/fabric/core" "os" "testing" @@ -9,8 +10,14 @@ import ( ) func TestCli(t *testing.T) { - message, err := Cli() - assert.NoError(t, err) + t.Skip("Skipping test for now, collision with flag -t") + originalArgs := os.Args + defer func() { os.Args = originalArgs }() + + os.Args = []string{os.Args[0]} + message, err := Cli("test") + assert.Error(t, err) + assert.Equal(t, core.NoSessionPatternUserMessages, err.Error()) assert.Empty(t, message) } diff --git a/cli/flags.go b/cli/flags.go index e18f8d55c..fe2f5cf43 100644 --- a/cli/flags.go +++ b/cli/flags.go @@ -37,13 +37,17 @@ type Flags struct { Output string `short:"o" long:"output" description:"Output to file" default:""` LatestPatterns string `short:"n" long:"latest" description:"Number of latest patterns to list" default:"0"` ChangeDefaultModel bool `short:"d" long:"changeDefaultModel" description:"Change default model"` - YouTube string `short:"y" long:"youtube" description:"YouTube video url to grab transcript, comments from it and send to chat"` - YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat"` + YouTube string `short:"y" long:"youtube" description:"YouTube video \"URL\" to grab transcript, comments from it and send to chat"` + YouTubeTranscript bool `long:"transcript" description:"Grab transcript from YouTube video and send to chat (it used per default)."` YouTubeComments bool `long:"comments" description:"Grab comments from YouTube video and send to chat"` - DryRun bool `long:"dry-run" description:"Show what would be sent to the model without actually sending it"` Language string `short:"g" long:"language" description:"Specify the Language Code for the chat, e.g. -g=en -g=zh" default:""` ScrapeURL string `short:"u" long:"scrape_url" description:"Scrape website URL to markdown using Jina AI"` ScrapeQuestion string `short:"q" long:"scrape_question" description:"Search question using Jina AI"` + Seed int `short:"e" long:"seed" description:"Seed to be used for LMM generation"` + WipeContext string `short:"w" long:"wipecontext" description:"Wipe context"` + WipeSession string `short:"W" long:"wipesession" description:"Wipe session"` + DryRun bool `long:"dry-run" description:"Show what would be sent to the model without actually sending it"` + Version bool `long:"version" description:"Print current version"` } // Init Initialize flags. returns a Flags struct and an error @@ -99,6 +103,7 @@ func (o *Flags) BuildChatOptions() (ret *common.ChatOptions) { PresencePenalty: o.PresencePenalty, FrequencyPenalty: o.FrequencyPenalty, Raw: o.Raw, + Seed: o.Seed, } return } diff --git a/cli/flags_test.go b/cli/flags_test.go index aba8dc3ab..c3dac92d1 100644 --- a/cli/flags_test.go +++ b/cli/flags_test.go @@ -53,6 +53,7 @@ func TestBuildChatOptions(t *testing.T) { TopP: 0.9, PresencePenalty: 0.1, FrequencyPenalty: 0.2, + Seed: 1, } expectedOptions := &common.ChatOptions{ @@ -61,6 +62,27 @@ func TestBuildChatOptions(t *testing.T) { PresencePenalty: 0.1, FrequencyPenalty: 0.2, Raw: false, + Seed: 1, + } + options := flags.BuildChatOptions() + assert.Equal(t, expectedOptions, options) +} + +func TestBuildChatOptionsDefaultSeed(t *testing.T) { + flags := &Flags{ + Temperature: 0.8, + TopP: 0.9, + PresencePenalty: 0.1, + FrequencyPenalty: 0.2, + } + + expectedOptions := &common.ChatOptions{ + Temperature: 0.8, + TopP: 0.9, + PresencePenalty: 0.1, + FrequencyPenalty: 0.2, + Raw: false, + Seed: 0, } options := flags.BuildChatOptions() assert.Equal(t, expectedOptions, options) diff --git a/common/configurable.go b/common/configurable.go index 172d7c54c..a8e91ac29 100644 --- a/common/configurable.go +++ b/common/configurable.go @@ -103,8 +103,9 @@ func (o *Setting) IsDefined() bool { } func (o *Setting) Configure() error { - if o.Value == "" { - o.Value = os.Getenv(o.EnvVariable) + envValue := os.Getenv(o.EnvVariable) + if envValue != "" { + o.Value = envValue } return o.IsValidErr() } diff --git a/common/domain.go b/common/domain.go index 33365a7e6..1db8a1401 100644 --- a/common/domain.go +++ b/common/domain.go @@ -23,6 +23,7 @@ type ChatOptions struct { PresencePenalty float64 FrequencyPenalty float64 Raw bool + Seed int } // NormalizeMessages remove empty messages and ensure messages order user-assist-user diff --git a/core/fabric.go b/core/fabric.go index a70ff37f2..b3bd76b07 100644 --- a/core/fabric.go +++ b/core/fabric.go @@ -30,6 +30,8 @@ import ( const DefaultPatternsGitRepoUrl = "https://github.com/danielmiessler/fabric.git" const DefaultPatternsGitRepoFolder = "patterns" +const NoSessionPatternUserMessages = "no session, pattern or user messages provided" + func NewFabric(db *db.Db) (ret *Fabric, err error) { ret = NewFabricBase(db) err = ret.Configure() @@ -279,7 +281,7 @@ func (o *Chat) BuildChatSession(raw bool) (ret *db.Session, err error) { if ret.IsEmpty() { ret = nil - err = fmt.Errorf("no session, pattern or user messages provided") + err = fmt.Errorf(NoSessionPatternUserMessages) } return } diff --git a/go.mod b/go.mod index efee3b3c4..4c51644dd 100644 --- a/go.mod +++ b/go.mod @@ -16,6 +16,7 @@ require ( github.com/samber/lo v1.47.0 github.com/sashabaranov/go-openai v1.30.0 github.com/stretchr/testify v1.9.0 + golang.org/x/text v0.18.0 google.golang.org/api v0.197.0 ) @@ -61,7 +62,6 @@ require ( golang.org/x/oauth2 v0.23.0 // indirect golang.org/x/sync v0.8.0 // indirect golang.org/x/sys v0.25.0 // indirect - golang.org/x/text v0.18.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240903143218-8af14fe29dc1 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240903143218-8af14fe29dc1 // indirect diff --git a/main.go b/main.go index 6c955d3c7..26d8a24ca 100644 --- a/main.go +++ b/main.go @@ -2,14 +2,15 @@ package main import ( "fmt" + "github.com/jessevdk/go-flags" "os" "github.com/danielmiessler/fabric/cli" ) func main() { - _, err := cli.Cli() - if err != nil { + _, err := cli.Cli(version) + if err != nil && !flags.WroteHelp(err) { fmt.Printf("%s\n", err) os.Exit(1) } diff --git a/vendors/openai/openai.go b/vendors/openai/openai.go index bb81b9067..fe2a10ae5 100644 --- a/vendors/openai/openai.go +++ b/vendors/openai/openai.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "io" + "log/slog" "github.com/danielmiessler/fabric/common" "github.com/samber/lo" @@ -111,6 +112,7 @@ func (o *Client) Send(ctx context.Context, msgs []*common.Message, opts *common. } if len(resp.Choices) > 0 { ret = resp.Choices[0].Message.Content + slog.Debug("SystemFingerprint: " + resp.SystemFingerprint) } return } @@ -128,13 +130,25 @@ func (o *Client) buildChatCompletionRequest( Messages: messages, } } else { - ret = goopenai.ChatCompletionRequest{ - Model: opts.Model, - Temperature: float32(opts.Temperature), - TopP: float32(opts.TopP), - PresencePenalty: float32(opts.PresencePenalty), - FrequencyPenalty: float32(opts.FrequencyPenalty), - Messages: messages, + if opts.Seed == 0 { + ret = goopenai.ChatCompletionRequest{ + Model: opts.Model, + Temperature: float32(opts.Temperature), + TopP: float32(opts.TopP), + PresencePenalty: float32(opts.PresencePenalty), + FrequencyPenalty: float32(opts.FrequencyPenalty), + Messages: messages, + } + } else { + ret = goopenai.ChatCompletionRequest{ + Model: opts.Model, + Temperature: float32(opts.Temperature), + TopP: float32(opts.TopP), + PresencePenalty: float32(opts.PresencePenalty), + FrequencyPenalty: float32(opts.FrequencyPenalty), + Messages: messages, + Seed: &opts.Seed, + } } } return diff --git a/vendors/openai/openai_test.go b/vendors/openai/openai_test.go new file mode 100644 index 000000000..40edd6615 --- /dev/null +++ b/vendors/openai/openai_test.go @@ -0,0 +1,102 @@ +package openai + +import ( + "testing" + + "github.com/danielmiessler/fabric/common" + "github.com/sashabaranov/go-openai" + goopenai "github.com/sashabaranov/go-openai" + "github.com/stretchr/testify/assert" +) + +func TestBuildChatCompletionRequestPinSeed(t *testing.T) { + + var msgs []*common.Message + + for i := 0; i < 2; i++ { + msgs = append(msgs, &common.Message{ + Role: "User", + Content: "My msg", + }) + } + + opts := &common.ChatOptions{ + Temperature: 0.8, + TopP: 0.9, + PresencePenalty: 0.1, + FrequencyPenalty: 0.2, + Raw: false, + Seed: 1, + } + + var expectedMessages []openai.ChatCompletionMessage + + for i := 0; i < 2; i++ { + expectedMessages = append(expectedMessages, + openai.ChatCompletionMessage{ + Role: msgs[i].Role, + Content: msgs[i].Content, + }, + ) + } + + var expectedRequest = goopenai.ChatCompletionRequest{ + Model: opts.Model, + Temperature: float32(opts.Temperature), + TopP: float32(opts.TopP), + PresencePenalty: float32(opts.PresencePenalty), + FrequencyPenalty: float32(opts.FrequencyPenalty), + Messages: expectedMessages, + Seed: &opts.Seed, + } + + var client = NewClient() + request := client.buildChatCompletionRequest(msgs, opts) + assert.Equal(t, expectedRequest, request) +} + +func TestBuildChatCompletionRequestNilSeed(t *testing.T) { + + var msgs []*common.Message + + for i := 0; i < 2; i++ { + msgs = append(msgs, &common.Message{ + Role: "User", + Content: "My msg", + }) + } + + opts := &common.ChatOptions{ + Temperature: 0.8, + TopP: 0.9, + PresencePenalty: 0.1, + FrequencyPenalty: 0.2, + Raw: false, + Seed: 0, + } + + var expectedMessages []openai.ChatCompletionMessage + + for i := 0; i < 2; i++ { + expectedMessages = append(expectedMessages, + openai.ChatCompletionMessage{ + Role: msgs[i].Role, + Content: msgs[i].Content, + }, + ) + } + + var expectedRequest = goopenai.ChatCompletionRequest{ + Model: opts.Model, + Temperature: float32(opts.Temperature), + TopP: float32(opts.TopP), + PresencePenalty: float32(opts.PresencePenalty), + FrequencyPenalty: float32(opts.FrequencyPenalty), + Messages: expectedMessages, + Seed: nil, + } + + var client = NewClient() + request := client.buildChatCompletionRequest(msgs, opts) + assert.Equal(t, expectedRequest, request) +} diff --git a/version.go b/version.go new file mode 100644 index 000000000..b8d7f14c9 --- /dev/null +++ b/version.go @@ -0,0 +1,3 @@ +package main + +var version = "v1.4.31-5d3e0da" diff --git a/youtube/youtube.go b/youtube/youtube.go index 0e935a1f9..08d8c1161 100644 --- a/youtube/youtube.go +++ b/youtube/youtube.go @@ -10,6 +10,7 @@ import ( "google.golang.org/api/option" "google.golang.org/api/youtube/v3" "log" + "net/url" "regexp" "strconv" "strings" @@ -61,17 +62,17 @@ func (o *YouTube) GetVideoId(url string) (ret string, err error) { return } -func (o *YouTube) GrabTranscriptForUrl(url string) (ret string, err error) { +func (o *YouTube) GrabTranscriptForUrl(url string, language string) (ret string, err error) { var videoId string if videoId, err = o.GetVideoId(url); err != nil { return } - return o.GrabTranscript(videoId) + return o.GrabTranscript(videoId, language) } -func (o *YouTube) GrabTranscript(videoId string) (ret string, err error) { +func (o *YouTube) GrabTranscript(videoId string, language string) (ret string, err error) { var transcript string - if transcript, err = o.GrabTranscriptBase(videoId); err != nil { + if transcript, err = o.GrabTranscriptBase(videoId, language); err != nil { err = fmt.Errorf("transcript not available. (%v)", err) return } @@ -89,14 +90,14 @@ func (o *YouTube) GrabTranscript(videoId string) (ret string, err error) { return } -func (o *YouTube) GrabTranscriptBase(videoId string) (ret string, err error) { +func (o *YouTube) GrabTranscriptBase(videoId string, language string) (ret string, err error) { if err = o.initService(); err != nil { return } - url := "https://www.youtube.com/watch?v=" + videoId + watchUrl := "https://www.youtube.com/watch?v=" + videoId var resp string - if resp, err = soup.Get(url); err != nil { + if resp, err = soup.Get(watchUrl); err != nil { return } @@ -117,6 +118,16 @@ func (o *YouTube) GrabTranscriptBase(videoId string) (ret string, err error) { if len(captionTracks) > 0 { transcriptURL := captionTracks[0].BaseURL + for _, captionTrack := range captionTracks { + parsedUrl, error := url.Parse(captionTrack.BaseURL) + if error != nil { + err = fmt.Errorf("error parsing caption track") + } + parsedUrlParams, _ := url.ParseQuery(parsedUrl.RawQuery) + if parsedUrlParams["lang"][0] == language { + transcriptURL = captionTrack.BaseURL + } + } ret, err = soup.Get(transcriptURL) return } @@ -212,7 +223,7 @@ func (o *YouTube) Grab(url string, options *Options) (ret *VideoInfo, err error) } if options.Transcript { - if ret.Transcript, err = o.GrabTranscript(videoId); err != nil { + if ret.Transcript, err = o.GrabTranscript(videoId, "en"); err != nil { return } }