Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: copilot-extensions/rag-extension
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: main
Choose a base ref
...
head repository: Cirou/rag-extension-poc
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: main
Choose a head ref
Able to merge. These branches can be automatically merged.
  • 5 commits
  • 7 files changed
  • 2 contributors

Commits on Jan 28, 2025

  1. Copy the full SHA
    07a58aa View commit details
  2. RITM1186197 specification

    Cirou committed Jan 28, 2025
    Copy the full SHA
    00bc4a9 View commit details
  3. docx conversion

    Aiello authored and Cirou committed Jan 28, 2025
    Copy the full SHA
    78299f6 View commit details
  4. Switched to ModelGPT4

    Cirou committed Jan 28, 2025
    Copy the full SHA
    104af91 View commit details

Commits on Jan 30, 2025

  1. Upload sample docs

    Cirou committed Jan 30, 2025
    Copy the full SHA
    f73e0b4 View commit details
2 changes: 1 addition & 1 deletion agent/service.go
Original file line number Diff line number Diff line change
@@ -156,7 +156,7 @@ func (s *Service) generateCompletion(ctx context.Context, integrationID, apiToke
messages = append(messages, req.Messages...)

chatReq := &copilot.ChatCompletionsRequest{
Model: copilot.ModelGPT35,
Model: copilot.ModelGPT4,
Messages: messages,
Stream: true,
}
4 changes: 4 additions & 0 deletions config/info.go
Original file line number Diff line number Diff line change
@@ -32,21 +32,25 @@ func New() (*Info, error) {
if port == "" {
return nil, fmt.Errorf("%s environment variable required", portEnv)
}
fmt.Println("PORT:", port)

fqdn := os.Getenv(fqdnEnv)
if fqdn == "" {
return nil, fmt.Errorf("%s environment variable required", fqdnEnv)
}
fmt.Println("FQDN:", fqdn)

clientID := os.Getenv(clientIdEnv)
if clientID == "" {
return nil, fmt.Errorf("%s environment variable required", clientIdEnv)
}
fmt.Println("CLIENT_ID:", clientID)

clientSecret := os.Getenv(clientSecretEnv)
if clientSecret == "" {
return nil, fmt.Errorf("%s environment variable required", clientSecretEnv)
}
fmt.Println("CLIENT_SECRET:", clientSecret)

return &Info{
Port: port,
Binary file not shown.
Binary file not shown.
11 changes: 1 addition & 10 deletions go.mod
Original file line number Diff line number Diff line change
@@ -3,17 +3,8 @@ module github.com/copilot-extensions/rag-extension
go 1.21.6

require (
github.com/google/go-github/v57 v57.0.0
github.com/google/uuid v1.6.0
github.com/invopop/jsonschema v0.12.0
github.com/wk8/go-ordered-map/v2 v2.1.8
golang.org/x/oauth2 v0.22.0
)

require (
github.com/bahlo/generic-list-go v0.2.0 // indirect
github.com/buger/jsonparser v1.1.1 // indirect
github.com/google/go-querystring v1.1.0 // indirect
github.com/mailru/easyjson v0.7.7 // indirect
gopkg.in/yaml.v3 v3.0.1 // indirect
)
require github.com/google/go-cmp v0.6.0 // indirect
27 changes: 0 additions & 27 deletions go.sum
Original file line number Diff line number Diff line change
@@ -1,33 +1,6 @@
github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk=
github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg=
github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs=
github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/go-github/v57 v57.0.0 h1:L+Y3UPTY8ALM8x+TV0lg+IEBI+upibemtBD8Q9u7zHs=
github.com/google/go-github/v57 v57.0.0/go.mod h1:s0omdnye0hvK/ecLvpsGfJMiRt85PimQh4oygmLIxHw=
github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8=
github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI=
github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0=
github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y=
github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0=
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc=
github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw=
golang.org/x/oauth2 v0.22.0 h1:BzDx2FehcG7jJwgWLELCdmLuxk2i+x9UDpSiss2u0ZA=
golang.org/x/oauth2 v0.22.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
154 changes: 153 additions & 1 deletion main.go
Original file line number Diff line number Diff line change
@@ -9,14 +9,25 @@ import (
"net/http"
"net/url"
"os"
"io"
"strings"
"log"
"path/filepath"
"archive/zip"
"bytes"

"github.com/copilot-extensions/rag-extension/agent"
"github.com/copilot-extensions/rag-extension/config"
"github.com/copilot-extensions/rag-extension/oauth"

)

func main() {

if err := convertDocx(); err != nil {
log.Fatalf("Errore durante la conversione: %v", err)
}

if err := run(); err != nil {
fmt.Println(err)
os.Exit(1)
@@ -57,7 +68,27 @@ func run() error {
// the signature with one of these keys verifies that the request to the
// completions API comes from GitHub and not elsewhere on the internet.
func fetchPublicKey() (*ecdsa.PublicKey, error) {
resp, err := http.Get("https://api.github.com/meta/public_keys/copilot_api")

// URL dell'API di GitHub
url := "https://api.github.com/meta/public_keys/copilot_api"

// Crea la richiesta
req, err := http.NewRequest("GET", url, nil)
if err != nil {
return nil, fmt.Errorf("failed to create request: %w", err)
}

// Aggiungi l'intestazione User-Agent
req.Header.Set("User-Agent", "rag-extension-poc/1.0")

token := os.Getenv("GITHUB_TOKEN") // Assicurati di configurare questa variabile
if token != "" {
req.Header.Set("Authorization", "Bearer "+token)
}

// Invia la richiesta
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("failed to fetch public key: %w", err)
}
@@ -109,3 +140,124 @@ func fetchPublicKey() (*ecdsa.PublicKey, error) {

return ecdsaKey, nil
}

// convertDocx legge i file .docx, estrae il contenuto testuale e lo salva in .md
func convertDocx() error {
inputDir := "./documents"
outputDir := "./data"
processedDir := filepath.Join(inputDir, "processed")

// Crea cartelle necessarie
if err := os.MkdirAll(outputDir, os.ModePerm); err != nil {
return fmt.Errorf("impossibile creare la cartella di output: %w", err)
}
if err := os.MkdirAll(processedDir, os.ModePerm); err != nil {
return fmt.Errorf("impossibile creare la cartella processed: %w", err)
}

// Leggi i file nella cartella input
files, err := os.ReadDir(inputDir)
if err != nil {
return fmt.Errorf("impossibile leggere la cartella %s: %w", inputDir, err)
}

for _, file := range files {
// Processa solo file con estensione .docx
if filepath.Ext(file.Name()) != ".docx" {
continue
}

inputFilePath := filepath.Join(inputDir, file.Name())
fmt.Printf("Elaborazione del file: %s\n", inputFilePath)

// Estrai il contenuto testuale dal file .docx
text, err := extractTextFromDocx(inputFilePath)
if err != nil {
log.Printf("Errore nell'elaborazione del file %s: %v", inputFilePath, err)
continue
}

// Scrivi il testo estratto in un file .md
outputFileName := strings.TrimSuffix(file.Name(), ".docx") + ".md"
outputFilePath := filepath.Join(outputDir, outputFileName)
err = os.WriteFile(outputFilePath, []byte(text), 0644)
if err != nil {
log.Printf("Errore nella scrittura del file %s: %v", outputFilePath, err)
continue
}

// Sposta il file originale nella cartella processed
processedFilePath := filepath.Join(processedDir, file.Name())
err = os.Rename(inputFilePath, processedFilePath)
if err != nil {
log.Printf("Errore nello spostamento del file %s nella cartella processed: %v", inputFilePath, err)
continue
}

fmt.Printf("File convertito e salvato: %s\n", outputFilePath)
}

return nil
}

// extractTextFromDocx estrae il contenuto testuale da un file .docx
func extractTextFromDocx(filePath string) (string, error) {
// Apri il file .docx come archivio ZIP
reader, err := zip.OpenReader(filePath)
if err != nil {
return "", fmt.Errorf("impossibile aprire il file .docx: %w", err)
}
defer reader.Close()

var documentXML *zip.File
for _, file := range reader.File {
if file.Name == "word/document.xml" {
documentXML = file
break
}
}

if documentXML == nil {
return "", fmt.Errorf("document.xml non trovato nel file .docx")
}

// Leggi il contenuto di document.xml
rc, err := documentXML.Open()
if err != nil {
return "", fmt.Errorf("impossibile aprire document.xml: %w", err)
}
defer rc.Close()

// Estrai il testo eliminando i tag XML
var buffer bytes.Buffer
_, err = io.Copy(&buffer, rc)
if err != nil {
return "", fmt.Errorf("impossibile leggere document.xml: %w", err)
}

// Rimuovi i tag XML
text := stripXMLTags(buffer.String())
return text, nil
}

// stripXMLTags rimuove i tag XML da una stringa
func stripXMLTags(input string) string {
var output strings.Builder
inTag := false

for _, char := range input {
if char == '<' {
inTag = true
continue
}
if char == '>' {
inTag = false
continue
}
if !inTag {
output.WriteRune(char)
}
}

return strings.TrimSpace(output.String())
}