Skip to content

Commit

Permalink
update upgrade script
Browse files Browse the repository at this point in the history
  • Loading branch information
@itizir
itizir committed Dec 13, 2023
1 parent 0e7deb9 commit defa76a
Show file tree
Hide file tree
Showing 3 changed files with 114 additions and 45 deletions.
5 changes: 4 additions & 1 deletion upgrade/go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,4 +2,7 @@ module github.com/mattn/go-sqlite3/upgrade

go 1.16

require github.com/PuerkitoBio/goquery v1.7.1 // indirect
require (
golang.org/x/crypto v0.16.0
golang.org/x/net v0.19.0
)
48 changes: 39 additions & 9 deletions upgrade/go.sum
View file
Original file line number Diff line number Diff line change
@@ -1,12 +1,42 @@
github.com/PuerkitoBio/goquery v1.7.1 h1:oE+T06D+1T7LNrn91B4aERsRIeCLJ/oPSa6xB9FPnz4=
github.com/PuerkitoBio/goquery v1.7.1/go.mod h1:XY0pP4kfraEmmV1O7Uf6XyjoslwsneBbgeDjLYuN8xY=
github.com/andybalholm/cascadia v1.2.0 h1:vuRCkM5Ozh/BfmsaTm26kbjm0mIOM3yS5Ek/F5h18aE=
github.com/andybalholm/cascadia v1.2.0/go.mod h1:YCyR8vOZT9aZ1CHEd8ap0gMVm2aFgxBp0T0eFw1RUQY=
golang.org/x/net v0.0.0-20180218175443-cbe0f9307d01/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q=
golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo=
golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=
golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU=
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
106 changes: 71 additions & 35 deletions upgrade/upgrade.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"archive/zip"
"bufio"
"bytes"
"encoding/csv"
"encoding/hex"
"errors"
"flag"
Expand All @@ -15,19 +16,30 @@ import (
"io/ioutil"
"log"
"net/http"
"net/url"
"os"
"os/exec"
"path/filepath"
"strings"
"time"

"github.com/PuerkitoBio/goquery"
"golang.org/x/crypto/sha3"
"golang.org/x/net/html"
)

var (
cFlags = "-DSQLITE_ENABLE_UPDATE_DELETE_LIMIT=1"
cleanup = true

errNotFound = errors.New("records not found")
)

const (
sqliteAddress = "https://www.sqlite.org"
downloadPage = "download.html"

sqliteDownloadDataTag = "Download product data for scripts to read"
urlColumnName = "RELATIVE-URL"
hashColumnName = "SHA3-HASH"
)

func main() {
Expand Down Expand Up @@ -88,65 +100,89 @@ func main() {
}
}

func findAddress(n *html.Node, prefix string) (string, string, error) {
if n.Type == html.CommentNode && strings.Contains(n.Data, sqliteDownloadDataTag) {
data := strings.TrimSpace(n.Data[strings.Index(n.Data, "\n")+1:])
r := csv.NewReader(strings.NewReader(data))
records, err := r.ReadAll()
if err != nil {
return "", "", err
}

for i, r := range records {
if len(r) != 5 {
return "", "", fmt.Errorf("expected record of length 5, got: %v", r)
}
if i == 0 && (r[2] != urlColumnName || r[4] != hashColumnName) {
return "", "", fmt.Errorf("unexpected columns: %v", r)
}
if strings.Contains(r[2], prefix) {
return r[2], r[4], nil
}
}
}

mainErr := errNotFound
for c := n.FirstChild; c != nil; c = c.NextSibling {
path, hash, err := findAddress(c, prefix)
if err == nil {
return path, hash, nil
}
if err != errNotFound {
mainErr = err
}
}
return "", "", mainErr
}

func download(prefix string) (content, hash []byte, err error) {
year := time.Now().Year()
u, err := url.Parse(sqliteAddress)
if err != nil {
return nil, nil, err
}

site := "https://www.sqlite.org/download.html"
//fmt.Printf("scraping %v\n", site)
doc, err := goquery.NewDocument(site)
dwnldPage, err := http.Get(u.JoinPath(downloadPage).String())
if err != nil {
return nil, nil, err
}
defer dwnldPage.Body.Close()
node, err := html.Parse(dwnldPage.Body)
if err != nil {
return nil, nil, err
}

url, hashString := "", ""
doc.Find("tr").EachWithBreak(func(_ int, s *goquery.Selection) bool {
found := false
s.Find("a").Each(func(_ int, s *goquery.Selection) {
if strings.HasPrefix(s.Text(), prefix) {
found = true
url = fmt.Sprintf("https://www.sqlite.org/%d/", year) + s.Text()
}
})
if found {
s.Find("td").Each(func(_ int, s *goquery.Selection) {
text := s.Text()
split := strings.Split(text, "(sha3: ")
if len(split) < 2 {
return
}
text = split[1]
hashString = strings.Split(text, ")")[0]
})
}
return !found
})
relPath, hashString, err := findAddress(node, prefix)
if err != nil {
return nil, nil, fmt.Errorf("failed to find download info in the document: %w", err)
}

targetHash, err := hex.DecodeString(hashString)
if err != nil || len(targetHash) != 32 {
return nil, nil, fmt.Errorf("unable to find valid sha3-256 hash on sqlite.org: %q", hashString)
return nil, nil, fmt.Errorf("unable to find valid sha3-256 hash on download page: %q", hashString)
}

if url == "" {
return nil, nil, fmt.Errorf("unable to find prefix '%s' on sqlite.org", prefix)
if relPath == "" {
return nil, nil, fmt.Errorf("unable to find prefix '%s' on download page", prefix)
}

fmt.Printf("Downloading %v\n", url)
resp, err := http.Get(url)
src := u.JoinPath(relPath).String()
fmt.Printf("Downloading %v\n", src)
resp, err := http.Get(src)
if err != nil {
return nil, nil, err
}
defer resp.Body.Close()

// Ready Body Content
shasum := sha3.New256()
content, err = ioutil.ReadAll(io.TeeReader(resp.Body, shasum))
defer resp.Body.Close()
if err != nil {
return nil, nil, err
}

computedHash := shasum.Sum(nil)
if !bytes.Equal(targetHash, computedHash) {
return nil, nil, fmt.Errorf("invalid hash of file downloaded from %q: got %x instead of %x", url, computedHash, targetHash)
return nil, nil, fmt.Errorf("invalid hash of file downloaded from %q: got %x instead of %x", src, computedHash, targetHash)
}

return content, computedHash, nil
Expand Down

0 comments on commit defa76a

Please sign in to comment.