Skip to content

Commit c34b2dd

Browse files
xtucxtuc
committed
port to Cloudflare Worker
1 parent cfd93dc commit c34b2dd

File tree

14 files changed

+763
-179
lines changed

14 files changed

+763
-179
lines changed

‎README.md‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,13 @@ In `tools/` run `npm install`.
4141
bash ./scripts/test-process-version.sh package-name package-version
4242
```
4343

44+
### Compile for Cloudflare Workers
45+
46+
```
47+
cd ./worker
48+
GOOS=js GOARCH=wasm tinygo build -o wasm.wasm ./main.go
49+
```
50+
4451
## License
4552

4653
Each library hosted on cdnjs is released under its own license. This cdnjs repository is published under [MIT license](LICENSE).

‎algolia/index.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -55,7 +55,8 @@ func getAlternativeNames(name string) []string {
5555
return names
5656
}
5757

58-
var githubURL = regexp.MustCompile(`github\.com[/|:]([\w\.-]+)\/([\w\.-]+)\/?`)
58+
// var githubURL = regexp.MustCompile(`github\.com[/|:]([\w\.-]+)\/([\w\.-]+)\/?`)
59+
var githubURL = regexp.MustCompile(`github\.com/.*`)
5960

6061
func getGitHubMeta(repo *packages.Repository) (*GitHubMeta, error) {
6162
if repo == nil {

‎functions/algolia-pump/main.go‎ renamed to ‎functions/algolia-pump/algolia_pump.go‎

Lines changed: 7 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -17,10 +17,10 @@ import (
1717

1818
"github.com/cdnjs/tools/algolia"
1919
"github.com/cdnjs/tools/audit"
20-
"github.com/cdnjs/tools/gcp"
2120
"github.com/cdnjs/tools/kv"
2221
"github.com/cdnjs/tools/metrics"
2322
"github.com/cdnjs/tools/packages"
23+
"github.com/cdnjs/tools/r2"
2424
"github.com/cdnjs/tools/sentry"
2525
)
2626

@@ -45,17 +45,14 @@ func getExistingVersionsFromAggregatedMetadata(p *packages.Package) ([]string, e
4545
return versions, nil
4646
}
4747

48-
func Invoke(ctx context.Context, e gcp.GCSEvent) error {
48+
func Run(ctx context.Context, bucket string, file string, pkgName string, currVersion string, config string) error {
4949
sentry.Init()
5050
defer sentry.PanicHandler()
5151

52-
log.Printf("File: %v\n", e.Name)
53-
log.Printf("Metadata: %v\n", e.Metadata)
52+
log.Printf("File: %v\n", file)
53+
log.Printf("Config: %v\n", config)
5454

55-
pkgName := e.Metadata["package"].(string)
56-
currVersion := e.Metadata["version"].(string)
57-
58-
configStr, err := b64.StdEncoding.DecodeString(e.Metadata["config"].(string))
55+
configStr, err := b64.StdEncoding.DecodeString(config)
5956
if err != nil {
6057
return fmt.Errorf("could not decode config: %v", err)
6158
}
@@ -76,7 +73,7 @@ func Invoke(ctx context.Context, e gcp.GCSEvent) error {
7673
if err != nil {
7774
return fmt.Errorf("failed to retrieve existing versions: %s", err)
7875
}
79-
archive, err := gcp.ReadObject(ctx, e.Bucket, e.Name)
76+
archive, err := r2.ReadObject(ctx, bucket, file)
8077
if err != nil {
8178
return fmt.Errorf("could not read object: %v", err)
8279
}
@@ -102,7 +99,7 @@ func Invoke(ctx context.Context, e gcp.GCSEvent) error {
10299
}
103100
return nil
104101
}
105-
if err := gcp.Inflate(bytes.NewReader(archive), onFile); err != nil {
102+
if err := r2.Inflate(bytes.NewReader(archive), onFile); err != nil {
106103
return fmt.Errorf("could not inflate archive: %s", err)
107104
}
108105

‎gcp/gcp.go‎

Lines changed: 17 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,25 @@
11
package gcp
22

3-
import (
4-
"os"
5-
"path"
3+
// import (
4+
// "os"
5+
// "path"
66

7-
"github.com/cdnjs/tools/util"
7+
// "github.com/cdnjs/tools/util"
88

9-
"golang.org/x/net/context"
10-
"google.golang.org/api/option"
9+
// "golang.org/x/net/context"
10+
// "google.golang.org/api/option"
1111

12-
"cloud.google.com/go/storage"
13-
)
12+
// "cloud.google.com/go/storage"
13+
// )
1414

15-
func getCredentialsFile() string {
16-
home, err := os.UserHomeDir()
17-
util.Check(err)
15+
// func getCredentialsFile() string {
16+
// home, err := os.UserHomeDir()
17+
// util.Check(err)
1818

19-
return path.Join(home, "google_storage_cdnjs_assets.json")
20-
}
19+
// return path.Join(home, "google_storage_cdnjs_assets.json")
20+
// }
2121

22-
// GetStorageClient gets the GCP Storage Client.
23-
func GetStorageClient(ctx context.Context) (*storage.Client, error) {
24-
return storage.NewClient(ctx, option.WithCredentialsFile(getCredentialsFile()))
25-
}
22+
// // GetStorageClient gets the GCP Storage Client.
23+
// func GetStorageClient(ctx context.Context) (*storage.Client, error) {
24+
// return storage.NewClient(ctx, option.WithCredentialsFile(getCredentialsFile()))
25+
// }

‎gcp/gcs.go‎

Lines changed: 18 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -2,41 +2,38 @@ package gcp
22

33
import (
44
"archive/tar"
5-
"bufio"
6-
"bytes"
75
"compress/gzip"
86
"context"
97
"io"
108
"time"
119

1210
"github.com/pkg/errors"
13-
14-
"cloud.google.com/go/storage"
1511
)
1612

1713
func ReadObject(ctx context.Context, bucket string, name string) ([]byte, error) {
18-
client, err := storage.NewClient(ctx)
19-
if err != nil {
20-
return nil, errors.Wrap(err, "could not create client")
21-
}
14+
return nil, nil
15+
// client, err := storage.NewClient(ctx)
16+
// if err != nil {
17+
// return nil, errors.Wrap(err, "could not create client")
18+
// }
2219

23-
bkt := client.Bucket(bucket)
24-
obj := bkt.Object(name)
20+
// bkt := client.Bucket(bucket)
21+
// obj := bkt.Object(name)
2522

26-
r, err := obj.NewReader(ctx)
27-
if err != nil {
28-
return nil, errors.Wrap(err, "could not create client")
29-
}
30-
defer r.Close()
23+
// r, err := obj.NewReader(ctx)
24+
// if err != nil {
25+
// return nil, errors.Wrap(err, "could not create client")
26+
// }
27+
// defer r.Close()
3128

32-
var buff bytes.Buffer
33-
w := bufio.NewWriter(&buff)
29+
// var buff bytes.Buffer
30+
// w := bufio.NewWriter(&buff)
3431

35-
if _, err := io.Copy(w, r); err != nil {
36-
return nil, errors.Wrap(err, "could not read object")
37-
}
32+
// if _, err := io.Copy(w, r); err != nil {
33+
// return nil, errors.Wrap(err, "could not read object")
34+
// }
3835

39-
return buff.Bytes(), nil
36+
// return buff.Bytes(), nil
4037
}
4138

4239
// GCSEvent is the payload of a GCS event.

‎gcp/incoming.go‎

Lines changed: 38 additions & 44 deletions
Original file line numberDiff line numberDiff line change
@@ -2,61 +2,55 @@ package gcp
22

33
import (
44
"bytes"
5-
"context"
6-
"encoding/json"
7-
"fmt"
8-
"io"
95
"os"
106

117
"github.com/cdnjs/tools/packages"
128
"github.com/cdnjs/tools/version"
13-
14-
"cloud.google.com/go/storage"
15-
"github.com/pkg/errors"
9+
// "cloud.google.com/go/storage"
1610
)
1711

1812
var (
1913
GCS_BUCKET = os.Getenv("GCS_BUCKET")
2014
)
2115

2216
func AddIncomingFile(fileName string, buff bytes.Buffer, pckg *packages.Package, v version.Version) error {
23-
// Create GCS connection
24-
ctx := context.Background()
25-
client, err := storage.NewClient(ctx)
26-
if err != nil {
27-
return fmt.Errorf("HTTP response error: %v", err)
28-
}
29-
30-
bucket := client.Bucket(GCS_BUCKET)
31-
obj := bucket.Object(fileName)
32-
w := obj.NewWriter(ctx)
33-
w.ACL = []storage.ACLRule{
34-
{Entity: storage.AllUsers, Role: storage.RoleReader},
35-
}
36-
37-
if _, err := io.Copy(w, bytes.NewReader(buff.Bytes())); err != nil {
38-
return fmt.Errorf("Failed to copy to bucket: %v", err)
39-
}
40-
if err := w.Close(); err != nil {
41-
return fmt.Errorf("Failed to close: %v", err)
42-
}
43-
44-
configBytes, err := json.Marshal(pckg)
45-
if err != nil {
46-
return fmt.Errorf("failed to marshal filemap: %v", err)
47-
}
48-
49-
// update the metadata once the object is written
50-
_, err = obj.Update(ctx, storage.ObjectAttrsToUpdate{
51-
Metadata: map[string]string{
52-
"version": v.Version,
53-
"package": *pckg.Name,
54-
"config": string(configBytes),
55-
},
56-
})
57-
if err != nil {
58-
return errors.Wrap(err, "could not update metadata")
59-
}
17+
// // Create GCS connection
18+
// ctx := context.Background()
19+
// client, err := storage.NewClient(ctx)
20+
// if err != nil {
21+
// return fmt.Errorf("HTTP response error: %v", err)
22+
// }
23+
24+
// bucket := client.Bucket(GCS_BUCKET)
25+
// obj := bucket.Object(fileName)
26+
// w := obj.NewWriter(ctx)
27+
// w.ACL = []storage.ACLRule{
28+
// {Entity: storage.AllUsers, Role: storage.RoleReader},
29+
// }
30+
31+
// if _, err := io.Copy(w, bytes.NewReader(buff.Bytes())); err != nil {
32+
// return fmt.Errorf("Failed to copy to bucket: %v", err)
33+
// }
34+
// if err := w.Close(); err != nil {
35+
// return fmt.Errorf("Failed to close: %v", err)
36+
// }
37+
38+
// configBytes, err := json.Marshal(pckg)
39+
// if err != nil {
40+
// return fmt.Errorf("failed to marshal filemap: %v", err)
41+
// }
42+
43+
// // update the metadata once the object is written
44+
// _, err = obj.Update(ctx, storage.ObjectAttrsToUpdate{
45+
// Metadata: map[string]string{
46+
// "version": v.Version,
47+
// "package": *pckg.Name,
48+
// "config": string(configBytes),
49+
// },
50+
// })
51+
// if err != nil {
52+
// return errors.Wrap(err, "could not update metadata")
53+
// }
6054

6155
return nil
6256
}

‎git/github.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ func getRepo(gitURL string) string {
3737
// "[email protected]:chris-pearce/backpack.css.git"
3838
// "git+https://github.com/18F/web-design-standards.git"
3939
// "https://github.com/epeli/underscore.string"
40-
re := regexp.MustCompile(`.*github\.com[:|/](.*?)(?:\.git)?$`)
40+
// re := regexp.MustCompile(`.*github\.com[:|/](.*?)(?:\.git)?$`)
41+
re := regexp.MustCompile(`.*github\.com/.*`)
4142
return re.ReplaceAllString(gitURL, "$1")
4243
}
4344

‎r2/r2.go‎

Lines changed: 50 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,50 @@
1+
package r2
2+
3+
import (
4+
"archive/tar"
5+
"compress/gzip"
6+
"context"
7+
"io"
8+
9+
"github.com/pkg/errors"
10+
)
11+
12+
func ReadObject(ctx context.Context, bucket string, name string) ([]byte, error) {
13+
panic("TODO")
14+
}
15+
16+
func Inflate(gzipStream io.Reader, onFile func(string, io.Reader) error) error {
17+
uncompressedStream, err := gzip.NewReader(gzipStream)
18+
if err != nil {
19+
return errors.Wrap(err, "ExtractTarGz: NewReader failed")
20+
}
21+
22+
tarReader := tar.NewReader(uncompressedStream)
23+
24+
for {
25+
header, err := tarReader.Next()
26+
27+
if err == io.EOF {
28+
break
29+
}
30+
31+
if err != nil {
32+
return errors.Wrap(err, "ExtractTarGz: Next() failed")
33+
}
34+
35+
switch header.Typeflag {
36+
case tar.TypeDir:
37+
// do nothing
38+
case tar.TypeReg:
39+
if err := onFile(header.Name, tarReader); err != nil {
40+
return errors.Wrap(err, "failed to handle file")
41+
}
42+
default:
43+
return errors.Errorf(
44+
"ExtractTarGz: uknown type: %x in %s",
45+
header.Typeflag,
46+
header.Name)
47+
}
48+
}
49+
return nil
50+
}

‎sandbox/sandbox.go‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -16,11 +16,6 @@ import (
1616
"github.com/pkg/errors"
1717
)
1818

19-
var (
20-
DOCKER_IMAGE = os.Getenv("DOCKER_IMAGE")
21-
CONTAINER_NAME_RE = regexp.MustCompile(`[^a-zA-Z0-9-_]+`)
22-
)
23-
2419
func Setup() (string, string, error) {
2520
tmpDir := os.TempDir()
2621
inDir, err := ioutil.TempDir(tmpDir, "in")
@@ -36,6 +31,7 @@ func Setup() (string, string, error) {
3631
}
3732

3833
func Init(ctx context.Context) error {
34+
DOCKER_IMAGE = os.Getenv("DOCKER_IMAGE")
3935
if DOCKER_IMAGE == "" {
4036
return errors.New("DOCKER_IMAGE needs to be present")
4137
}
@@ -72,8 +68,10 @@ func Run(ctx context.Context, containerName, in, out string) (string, error) {
7268

7369
// Sanitize the container's name because some package use special character
7470
// in their versions and Docker doesn't accept that.
71+
CONTAINER_NAME_RE = regexp.MustCompile(`[^a-zA-Z0-9-_]+`)
7572
containerName = CONTAINER_NAME_RE.ReplaceAllString(containerName, "-")
7673

74+
DOCKER_IMAGE = os.Getenv("DOCKER_IMAGE")
7775
resp, err := cli.ContainerCreate(ctx,
7876
&container.Config{
7977
Image: DOCKER_IMAGE,

0 commit comments

Comments
 (0)