Fix a bug in cron API data exporting (#2112)

Co-authored-by: Azeem Shaikh <[email protected]>

Author: azeemshaikh38

cron/internal/config/config.go

@@ -79,7 +79,7 @@ type config struct {
 	// Raw results.
 	RawResultDataBucketURL string `yaml:"raw-result-data-bucket-url"`
 	RawBigQueryTable       string `yaml:"raw-bigquery-table"`
-	ExportResultsBucketURL string `yaml:"export-results-bucket-url"`
+	APIResultsBucketURL    string `yaml:"api-results-bucket-url"`
 }
 
 func getParsedConfigFromFile(byteValue []byte) (config, error) {
@@ -234,8 +234,8 @@ func GetMetricExporter() (string, error) {
 	return getStringConfigValue(metricExporter, configYAML, "MetricExporter", "metric-exporter")
 }
 
-// GetBQExportResultsBucketURL returns the bucket URL for storing cron job results.
-func GetBQExportResultsBucketURL() (string, error) {
+// GetAPIResultsBucketURL returns the bucket URL for storing cron job results.
+func GetAPIResultsBucketURL() (string, error) {
 	return getStringConfigValue(apiResultsBucketURL, configYAML,
-		"ExportResultsBucketURL", "export-results-bucket-url")
+		"APIResultsBucketURL", "api-results-bucket-url")
 }

cron/internal/config/config.yaml

@@ -29,5 +29,5 @@ result-data-bucket-url: gs://ossf-scorecard-data2
 # Raw results.
 raw-result-data-bucket-url: gs://ossf-scorecard-rawdata
 raw-bigquery-table: scorecard-rawdata
-# export-bucket
-export-results-bucket-url: gs://ossf-scorecard-cron-releasetest-results
+# API results bucket
+api-results-bucket-url: gs://ossf-scorecard-cron-results

cron/internal/config/config_test.go

@@ -37,9 +37,9 @@ const (
 	prodShardSize           int    = 10
 	prodMetricExporter      string = "stackdriver"
 	// Raw results.
-	prodRawBucket                = "gs://ossf-scorecard-rawdata"
-	prodRawBigQueryTable         = "scorecard-rawdata"
-	prodBigQueryExportsBucketURL = "gs://ossf-scorecard-cron-releasetest-results"
+	prodRawBucket        = "gs://ossf-scorecard-rawdata"
+	prodRawBigQueryTable = "scorecard-rawdata"
+	prodAPIBucketURL     = "gs://ossf-scorecard-cron-results"
 )
 
 func getByteValueFromFile(filename string) ([]byte, error) {
@@ -75,7 +75,7 @@ func TestYAMLParsing(t *testing.T) {
 				MetricExporter:         prodMetricExporter,
 				RawResultDataBucketURL: prodRawBucket,
 				RawBigQueryTable:       prodRawBigQueryTable,
-				ExportResultsBucketURL: prodBigQueryExportsBucketURL,
+				APIResultsBucketURL:    prodAPIBucketURL,
 			},
 		},
 
@@ -348,16 +348,16 @@ func TestGetMetricExporter(t *testing.T) {
 }
 
 //nolint:paralleltest // Since os.Setenv is used.
-func TestGetBigQueryExportsBucketURL(t *testing.T) {
+func TestGetAPIResultsBucketURL(t *testing.T) {
 	t.Run("GetBigQueryExportsBucketURL", func(t *testing.T) {
 		bigqueryExportsBucketURL := apiResultsBucketURL
 		os.Unsetenv(bigqueryExportsBucketURL)
-		bucket, err := GetBQExportResultsBucketURL()
+		bucket, err := GetAPIResultsBucketURL()
 		if err != nil {
 			t.Errorf("failed to get production bucket URL from config: %v", err)
 		}
-		if bucket != prodBigQueryExportsBucketURL {
-			t.Errorf("test failed: expected - %s, got = %s", prodBigQueryExportsBucketURL, bucket)
+		if bucket != prodAPIBucketURL {
+			t.Errorf("test failed: expected - %s, got = %s", prodAPIBucketURL, bucket)
 		}
 	})
 }

cron/internal/worker/main.go

@@ -43,12 +43,17 @@ import (
 	"github.com/ossf/scorecard/v4/stats"
 )
 
+const (
+	resultsFile    = "results.json"
+	rawResultsFile = "raw.json"
+)
+
 var ignoreRuntimeErrors = flag.Bool("ignoreRuntimeErrors", false, "if set to true any runtime errors will be ignored")
 
 // nolint: gocognit
 func processRequest(ctx context.Context,
 	batchRequest *data.ScorecardBatchRequest,
-	blacklistedChecks []string, bucketURL, rawBucketURL, exportBucketURL string,
+	blacklistedChecks []string, bucketURL, rawBucketURL, apiBucketURL string,
 	checkDocs docs.Doc,
 	repoClient clients.RepoClient, ossFuzzRepoClient clients.RepoClient,
 	ciiClient clients.CIIBestPracticesClient,
@@ -139,29 +144,29 @@ func processRequest(ctx context.Context,
 		if err := format.AsRawJSON(&result, &exportRawBuffer); err != nil {
 			return fmt.Errorf("error during result.AsRawJSON for export: %w", err)
 		}
-		exportPath := fmt.Sprintf("%s/result.json", repo.URI())
-		exportCommitSHAPath := fmt.Sprintf("%s/%s/result.json", repo.URI(), result.Repo.CommitSHA)
-		exportRawPath := fmt.Sprintf("%s/raw.json", repo.URI())
-		exportRawCommitSHAPath := fmt.Sprintf("%s/%s/raw.json", repo.URI(), result.Repo.CommitSHA)
+		exportPath := fmt.Sprintf("%s/%s", repo.URI(), resultsFile)
+		exportCommitSHAPath := fmt.Sprintf("%s/%s/%s", repo.URI(), result.Repo.CommitSHA, resultsFile)
+		exportRawPath := fmt.Sprintf("%s/%s", repo.URI(), rawResultsFile)
+		exportRawCommitSHAPath := fmt.Sprintf("%s/%s/%s", repo.URI(), result.Repo.CommitSHA, rawResultsFile)
 
 		// Raw result.
 		if err := format.AsRawJSON(&result, &rawBuffer); err != nil {
 			return fmt.Errorf("error during result.AsRawJSON: %w", err)
 		}
 
 		// These are results without the commit SHA which represents the latest commit.
-		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportPath, exportBuffer.Bytes()); err != nil {
+		if err := data.WriteToBlobStore(ctx, apiBucketURL, exportPath, exportBuffer.Bytes()); err != nil {
 			return fmt.Errorf("error during writing to exportBucketURL: %w", err)
 		}
 		// Export result based on commitSHA.
-		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportCommitSHAPath, exportBuffer.Bytes()); err != nil {
+		if err := data.WriteToBlobStore(ctx, apiBucketURL, exportCommitSHAPath, exportBuffer.Bytes()); err != nil {
 			return fmt.Errorf("error during exportBucketURL with commit SHA: %w", err)
 		}
 		// Export raw result.
-		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawPath, exportRawBuffer.Bytes()); err != nil {
+		if err := data.WriteToBlobStore(ctx, apiBucketURL, exportRawPath, exportRawBuffer.Bytes()); err != nil {
 			return fmt.Errorf("error during writing to exportBucketURL for raw results: %w", err)
 		}
-		if err := data.WriteToBlobStore(ctx, exportBucketURL, exportRawCommitSHAPath, exportRawBuffer.Bytes()); err != nil {
+		if err := data.WriteToBlobStore(ctx, apiBucketURL, exportRawCommitSHAPath, exportRawBuffer.Bytes()); err != nil {
 			return fmt.Errorf("error during exportBucketURL for raw results with commit SHA: %w", err)
 		}
 	}
@@ -238,7 +243,7 @@ func main() {
 		panic(err)
 	}
 
-	exportBucketURL, err := config.GetBQExportResultsBucketURL()
+	apiBucketURL, err := config.GetAPIResultsBucketURL()
 	if err != nil {
 		panic(err)
 	}
@@ -278,7 +283,7 @@ func main() {
 			break
 		}
 		if err := processRequest(ctx, req, blacklistedChecks,
-			bucketURL, rawBucketURL, exportBucketURL, checkDocs,
+			bucketURL, rawBucketURL, apiBucketURL, checkDocs,
 			repoClient, ossFuzzRepoClient, ciiClient, vulnsClient, logger); err != nil {
 			// TODO(log): Previously Warn. Consider logging an error here.
 			logger.Info(fmt.Sprintf("error processing request: %v", err))