From 3233692b66609e3b56cdfe882707643a4aaeff8c Mon Sep 17 00:00:00 2001 From: lehel Date: Thu, 25 Sep 2025 14:07:42 +0200 Subject: [PATCH] ollama server --- config.go | 22 +++++++++++++++ config.yaml | 2 +- llm.go | 77 ++++++++++++++++++++++++++++++---------------------- main.go | 48 ++++++++++---------------------- run.sh | 2 +- test_copy.sh | 21 ++++++++++++++ 6 files changed, 104 insertions(+), 68 deletions(-) create mode 100644 config.go create mode 100755 test_copy.sh diff --git a/config.go b/config.go new file mode 100644 index 0000000..df22298 --- /dev/null +++ b/config.go @@ -0,0 +1,22 @@ +package main + +import "io/ioutil" +import "gopkg.in/yaml.v3" + +// Config holds all prompts and settings +type Config struct { + LLM struct { + ExtractKeywordsPrompt string `yaml:"extract_keywords_prompt"` + DisambiguatePrompt string `yaml:"disambiguate_prompt"` + } `yaml:"llm"` +} + +var appConfig Config + +func loadConfig(path string) error { + data, err := ioutil.ReadFile(path) + if err != nil { + return err + } + return yaml.Unmarshal(data, &appConfig) +} diff --git a/config.yaml b/config.yaml index e64deae..a4ffdc0 100644 --- a/config.yaml +++ b/config.yaml @@ -1,4 +1,4 @@ llm: - extract_keywords_prompt: "Extract 3–5 key veterinary-related terms from this user message: {{.Message}}" + extract_keywords_prompt: "Translate [{{.Message}}] to English, then output only 3–5 comma-separated veterinary-related keywords derived strictly from [{{.Message}}]. example output [\"keyword1\",\"keyword2\"] No other text, no extra punctuation, no explanations, no quotes, no formatting." disambiguate_prompt: "Given these possible vet visit reasons: {{.Entries}}, choose the single best match for this user message: {{.Message}}. Reply with the id or none." diff --git a/llm.go b/llm.go index 5f39127..702b430 100644 --- a/llm.go +++ b/llm.go @@ -6,6 +6,7 @@ import ( "encoding/json" "net/http" "strings" + "text/template" "github.com/sirupsen/logrus" ) @@ -16,31 +17,46 @@ type LLMClient struct { BaseURL string } +// renderPrompt renders a Go template with the given data +func renderPrompt(tmplStr string, data any) (string, error) { + tmpl, err := template.New("").Parse(tmplStr) + if err != nil { + return "", err + } + var buf bytes.Buffer + if err := tmpl.Execute(&buf, data); err != nil { + return "", err + } + return buf.String(), nil +} + // ExtractKeywords calls LLM to extract keywords from user message -func (llm *LLMClient) ExtractKeywords(ctx context.Context, message string) ([]string, error) { +func (llm *LLMClient) ExtractKeywords(ctx context.Context, message string) (map[string]interface{}, error) { prompt, err := renderPrompt(appConfig.LLM.ExtractKeywordsPrompt, map[string]string{"Message": message}) if err != nil { logrus.WithError(err).Error("[CONFIG] Failed to render ExtractKeywords prompt") return nil, err } logrus.WithField("prompt", prompt).Info("[LLM] ExtractKeywords prompt") - resp, err := llm.openAICompletion(ctx, prompt) + format := map[string]interface{}{ + "type": "object", + "properties": map[string]interface{}{ + "translate": map[string]interface{}{"type": "string"}, + "keyword": map[string]interface{}{"type": "array", "items": map[string]interface{}{"type": "string"}}, + "animal": map[string]interface{}{"type": "string"}, + }, + "required": []string{"translate", "keyword", "animal"}, + } + resp, err := llm.openAICompletion(ctx, prompt, format) logrus.WithFields(logrus.Fields{"response": resp, "err": err}).Info("[LLM] ExtractKeywords response") if err != nil { return nil, err } - var keywords []string - if err := json.Unmarshal([]byte(resp), &keywords); err == nil { - return keywords, nil + var result map[string]interface{} + if err := json.Unmarshal([]byte(resp), &result); err != nil { + return nil, err } - // fallback: try splitting by comma - for _, k := range bytes.Split([]byte(resp), []byte{','}) { - kw := strings.TrimSpace(string(k)) - if kw != "" { - keywords = append(keywords, kw) - } - } - return keywords, nil + return result, nil } // DisambiguateBestMatch calls LLM to pick best match from candidates @@ -52,7 +68,7 @@ func (llm *LLMClient) DisambiguateBestMatch(ctx context.Context, message string, return "", err } logrus.WithField("prompt", prompt).Info("[LLM] DisambiguateBestMatch prompt") - resp, err := llm.openAICompletion(ctx, prompt) + resp, err := llm.openAICompletion(ctx, prompt, nil) logrus.WithFields(logrus.Fields{"response": resp, "err": err}).Info("[LLM] DisambiguateBestMatch response") if err != nil { return "", err @@ -64,24 +80,21 @@ func (llm *LLMClient) DisambiguateBestMatch(ctx context.Context, message string, return id, nil } -// openAICompletion is a minimal OpenAI API call (text-davinci-003 or gpt-3.5-turbo-instruct) -func (llm *LLMClient) openAICompletion(ctx context.Context, prompt string) (string, error) { +// openAICompletion calls Ollama API with prompt and structure, returns structured result +func (llm *LLMClient) openAICompletion(ctx context.Context, prompt string, format map[string]interface{}) (string, error) { apiURL := llm.BaseURL if apiURL == "" { - apiURL = "https://api.openai.com/v1/completions" + apiURL = "http://localhost:11434/api/chat" } - logrus.WithFields(logrus.Fields{"api_url": apiURL, "prompt": prompt}).Info("[LLM] openAICompletion POST") + logrus.WithFields(logrus.Fields{"api_url": apiURL, "prompt": prompt, "format": format}).Info("[LLM] openAICompletion POST") body := map[string]interface{}{ - "model": "text-davinci-003", - "prompt": prompt, - "max_tokens": 64, - "temperature": 0, + "model": "qwen3:latest", + "messages": []map[string]string{{"role": "user", "content": prompt}}, + "stream": false, + "format": format, } jsonBody, _ := json.Marshal(body) req, _ := http.NewRequestWithContext(ctx, "POST", apiURL, bytes.NewBuffer(jsonBody)) - if llm.APIKey != "" { - req.Header.Set("Authorization", "Bearer "+llm.APIKey) - } req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) @@ -91,18 +104,18 @@ func (llm *LLMClient) openAICompletion(ctx context.Context, prompt string) (stri } defer resp.Body.Close() var result struct { - Choices []struct { - Text string `json:"text"` - } `json:"choices"` + Message struct { + Content string `json:"content"` + } `json:"message"` } if err := json.NewDecoder(resp.Body).Decode(&result); err != nil { logrus.WithError(err).Error("[LLM] openAICompletion decode error") return "", err } - if len(result.Choices) == 0 { - logrus.Warn("[LLM] openAICompletion: no choices returned") + if result.Message.Content == "" { + logrus.Warn("[LLM] openAICompletion: no content returned") return "", nil } - logrus.WithField("text", result.Choices[0].Text).Info("[LLM] openAICompletion: got text") - return result.Choices[0].Text, nil + logrus.WithField("content", result.Message.Content).Info("[LLM] openAICompletion: got content") + return result.Message.Content, nil } diff --git a/main.go b/main.go index c20acee..8a6d0df 100644 --- a/main.go +++ b/main.go @@ -1,7 +1,6 @@ package main import ( - "bytes" "context" "fmt" "html/template" @@ -55,24 +54,6 @@ type ChatResponse struct { Notes string `json:"notes,omitempty"` } -// Config holds all prompts and settings -type Config struct { - LLM struct { - ExtractKeywordsPrompt string `yaml:"extract_keywords_prompt"` - DisambiguatePrompt string `yaml:"disambiguate_prompt"` - } `yaml:"llm"` -} - -var appConfig Config - -func loadConfig(path string) error { - data, err := ioutil.ReadFile(path) - if err != nil { - return err - } - return yaml.Unmarshal(data, &appConfig) -} - // naiveKeywordExtract splits message into lowercase words (placeholder for LLM) func naiveKeywordExtract(msg string) []string { // TODO: Replace with LLM call @@ -118,19 +99,6 @@ func sumProcedures(procs []Procedure) (int, int) { return totalPrice, totalDuration } -// renderPrompt renders a Go template with the given data -func renderPrompt(tmplStr string, data any) (string, error) { - tmpl, err := template.New("").Parse(tmplStr) - if err != nil { - return "", err - } - var buf bytes.Buffer - if err := tmpl.Execute(&buf, data); err != nil { - return "", err - } - return buf.String(), nil -} - var uiTemplate *template.Template func loadUITemplate(path string) error { @@ -175,7 +143,19 @@ func main() { } ctx := context.Background() keywords, err := llm.ExtractKeywords(ctx, req.Message) - candidates := findCandidates(keywords) + kwIface := keywords["keyword"] + var kwArr []string + switch v := kwIface.(type) { + case []interface{}: + for _, item := range v { + if s, ok := item.(string); ok { + kwArr = append(kwArr, s) + } + } + case []string: + kwArr = v + } + candidates := findCandidates(kwArr) bestID := "" if len(candidates) > 0 && err == nil { bestID, err = llm.DisambiguateBestMatch(ctx, req.Message, candidates) @@ -216,7 +196,7 @@ func main() { } // logRequest logs incoming chat requests and extracted info -func logRequest(req ChatRequest, keywords []string, candidates []Reason, bestID string, err error) { +func logRequest(req ChatRequest, keywords map[string]interface{}, candidates []Reason, bestID string, err error) { logrus.WithFields(logrus.Fields{ "message": req.Message, "keywords": keywords, diff --git a/run.sh b/run.sh index a736063..f0bc61b 100755 --- a/run.sh +++ b/run.sh @@ -1,3 +1,3 @@ -export OPENAI_BASE_URL=http://localhost:1234/v1/completions +export OPENAI_BASE_URL=http://localhost:11434/api/chat export OPENAI_API_KEY=sk-no-key-needed # (if LM Studio doesn't require a real key) go run . diff --git a/test_copy.sh b/test_copy.sh new file mode 100755 index 0000000..f740e90 --- /dev/null +++ b/test_copy.sh @@ -0,0 +1,21 @@ +curl -X POST http://localhost:11434/api/chat \ + -H 'Content-Type: application/json' \ + -d '{ + "model": "qwen3:latest", + "messages": [ + { + "role": "user", + "content": "Translate [\"megy a kutyam hasa\"] to English, then output only 3–5 comma-separated veterinary-related keywords derived strictly from [\"megy a kutyam hasa\"]. example output [\"keyword1\",\"keyword2\"] No other text, no extra punctuation, no explanations, no quotes, no formatting." + } + ], + "stream": false, + "format": { + "type": "object", + "properties": { + "translate": { "type": "string" }, + "keyword": { "type": "array", "items": { "type": "string" } }, + "animal": { "type": "string" } + }, + "required": ["translate", "keyword", "animal"] + } + }' \ No newline at end of file