Skip to content

Commit 2435533

Browse files
hongalexhongalex
authored
chore(internal/godocfx): support individual package .repo-metadata.json (#13859)
The single internal/.repo-metadata-full.json file was removed in favor of individual .repo-metadata.json files located within package directories. `godocfx` was dependent on fetching this. This PR updates the metadata resolution to: 1. Search upwards from the package directory to the module root for a `.repo-metadata.json` file. 2. Load the API description from the local metadata file. 3. Provide a fallback mechanism for tests and older module versions. Also updated test goldens to reflect metadata resolution changes and improved symbol linking. Fixes: b/462186715
1 parent 8a2a3b8 commit 2435533

File tree

4 files changed

+182
-128
lines changed

4 files changed

+182
-128
lines changed

‎internal/godocfx/godocfx_test.go‎

Lines changed: 57 additions & 37 deletions
Original file line numberDiff line numberDiff line change
@@ -20,8 +20,6 @@ import (
2020
"flag"
2121
"fmt"
2222
"io/fs"
23-
"net/http"
24-
"net/http/httptest"
2523
"os"
2624
"path/filepath"
2725
"testing"
@@ -40,28 +38,13 @@ func TestMain(m *testing.M) {
4038
os.Exit(m.Run())
4139
}
4240

43-
func fakeMetaServer() *httptest.Server {
44-
meta := repoMetadata{
45-
"cloud.google.com/go/storage": repoMetadataItem{
46-
Description: "Storage API",
47-
},
48-
"cloud.google.com/iam/apiv1beta1": repoMetadataItem{
49-
Description: "IAM",
50-
},
51-
"cloud.google.com/go/cloudbuild/apiv1/v2": repoMetadataItem{
52-
Description: "Cloud Build API",
53-
},
54-
}
55-
return httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
56-
json.NewEncoder(w).Encode(meta)
57-
}))
58-
}
59-
6041
func TestParse(t *testing.T) {
6142
mod := "cloud.google.com/go/bigquery"
62-
metaServer := fakeMetaServer()
63-
defer metaServer.Close()
64-
r, err := parse(mod+"/...", ".", []string{"README.md"}, nil, &friendlyAPINamer{metaURL: metaServer.URL})
43+
r, err := parse(mod+"/...", ".", []string{"README.md"}, nil, &friendlyAPINamer{
44+
Fallbacks: map[string]string{
45+
"cloud.google.com/go/bigquery": "BigQuery API",
46+
},
47+
})
6548
if err != nil {
6649
t.Fatalf("Parse: %v", err)
6750
}
@@ -132,9 +115,11 @@ func TestGoldens(t *testing.T) {
132115
extraFiles := []string{"README.md"}
133116

134117
testMod := indexEntry{Path: "cloud.google.com/go/storage", Version: "v1.33.0"}
135-
metaServer := fakeMetaServer()
136-
defer metaServer.Close()
137-
namer := &friendlyAPINamer{metaURL: metaServer.URL}
118+
namer := &friendlyAPINamer{
119+
Fallbacks: map[string]string{
120+
"cloud.google.com/go/storage": "Storage API",
121+
},
122+
}
138123
ok := processMods([]indexEntry{testMod}, gotDir, namer, extraFiles, false)
139124
if !ok {
140125
t.Fatalf("failed to process modules")
@@ -304,36 +289,71 @@ Deprecated: use Reader.Attrs.Size.`,
304289
}
305290

306291
func TestFriendlyAPIName(t *testing.T) {
307-
metaServer := fakeMetaServer()
308-
defer metaServer.Close()
309-
namer := &friendlyAPINamer{metaURL: metaServer.URL}
292+
tmpDir := t.TempDir()
293+
meta := repoMetadata{
294+
Description: "Storage API",
295+
}
296+
b, _ := json.Marshal(meta)
297+
if err := os.WriteFile(filepath.Join(tmpDir, ".repo-metadata.json"), b, 0644); err != nil {
298+
t.Fatal(err)
299+
}
300+
301+
badDir := t.TempDir()
302+
if err := os.WriteFile(filepath.Join(badDir, ".repo-metadata.json"), []byte("invalid json"), 0644); err != nil {
303+
t.Fatal(err)
304+
}
305+
306+
namer := &friendlyAPINamer{}
310307

311308
tests := []struct {
312309
importPath string
310+
module *packages.Module
313311
want string
312+
wantErr bool
314313
}{
315314
{
316315
importPath: "cloud.google.com/go/storage",
317-
want: "Storage API",
316+
module: &packages.Module{
317+
Path: "cloud.google.com/go/storage",
318+
Dir: tmpDir,
319+
},
320+
want: "Storage API",
318321
},
319322
{
320-
importPath: "cloud.google.com/iam/apiv1beta1",
321-
want: "IAM v1beta1",
323+
importPath: "cloud.google.com/go/storage/apiv1",
324+
module: &packages.Module{
325+
Path: "cloud.google.com/go/storage",
326+
Dir: tmpDir,
327+
},
328+
want: "Storage API v1",
322329
},
323330
{
324-
importPath: "cloud.google.com/go/cloudbuild/apiv1/v2",
325-
want: "Cloud Build API v1",
331+
importPath: "cloud.google.com/go/storage/apiv1beta1",
332+
module: &packages.Module{
333+
Path: "cloud.google.com/go/storage",
334+
Dir: tmpDir,
335+
},
336+
want: "Storage API v1beta1",
337+
},
338+
{
339+
importPath: "cloud.google.com/go/bad",
340+
module: &packages.Module{
341+
Path: "cloud.google.com/go/bad",
342+
Dir: badDir,
343+
},
344+
wantErr: true,
326345
},
327346
{
328347
importPath: "not found",
329-
want: "",
348+
module: nil,
349+
wantErr: true,
330350
},
331351
}
332352

333353
for _, test := range tests {
334-
got, err := namer.friendlyAPIName(test.importPath)
335-
if err != nil {
336-
t.Errorf("friendlyAPIName(%q) got err: %v", test.importPath, err)
354+
got, err := namer.friendlyAPIName(test.importPath, test.module)
355+
if (err != nil) != test.wantErr {
356+
t.Errorf("friendlyAPIName(%q) got err: %v, wantErr %v", test.importPath, err, test.wantErr)
337357
continue
338358
}
339359
if got != test.want {

‎internal/godocfx/main.go‎

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -113,9 +113,7 @@ func main() {
113113
log.Println("No modules to process")
114114
return
115115
}
116-
namer := &friendlyAPINamer{
117-
metaURL: "https://raw.githubusercontent.com/googleapis/google-cloud-go/main/internal/.repo-metadata-full.json",
118-
}
116+
namer := &friendlyAPINamer{}
119117
if ok := processMods(mods, *outDir, namer, nil, *print); !ok {
120118
os.Exit(1)
121119
}

‎internal/godocfx/parse.go‎

Lines changed: 77 additions & 41 deletions
Original file line numberDiff line numberDiff line change
@@ -29,7 +29,6 @@ import (
2929
"go/printer"
3030
"go/token"
3131
"log"
32-
"net/http"
3332
"os"
3433
"path/filepath"
3534
"regexp"
@@ -160,7 +159,7 @@ func parse(glob string, workingDir string, optionalExtraFiles []string, filter [
160159
for _, pi := range pkgInfos {
161160
link := newLinker(pi)
162161
topLevelDecls := pkgsite.TopLevelDecls(pi.Doc)
163-
friendly, err := namer.friendlyAPIName(pi.Doc.ImportPath)
162+
friendly, err := namer.friendlyAPIName(pi.Doc.ImportPath, pi.Pkg.Module)
164163
if err != nil {
165164
return nil, err
166165
}
@@ -654,67 +653,104 @@ func hasPrefix(s string, prefixes []string) bool {
654653
return false
655654
}
656655

657-
// repoMetadata is the JSON format of the .repo-metadata-full.json file.
658-
// See https://raw.githubusercontent.com/googleapis/google-cloud-go/main/internal/.repo-metadata-full.json.
659-
type repoMetadata map[string]repoMetadataItem
660-
661-
type repoMetadataItem struct {
662-
Description string `json:"description"`
656+
// repoMetadata is the JSON format of the .repo-metadata.json file.
657+
type repoMetadata struct {
658+
Description string `json:"description"`
659+
DistributionName string `json:"distribution_name"`
663660
}
664661

665662
type friendlyAPINamer struct {
666-
// metaURL is the URL to .repo-metadata-full.json, which contains metadata
667-
// about packages in this repo. See
668-
// https://raw.githubusercontent.com/googleapis/google-cloud-go/main/internal/.repo-metadata-full.json.
669-
metaURL string
670-
671-
// metadata caches the repo metadata results.
672-
metadata repoMetadata
673-
// getOnce ensures we only fetch the metadata JSON once.
674-
getOnce sync.Once
663+
// metadata caches the repo metadata results by directory.
664+
metadata map[string]repoMetadata
665+
mu sync.Mutex
666+
667+
// Fallbacks maps importPath -> description. Used when decentralized
668+
// metadata is not found.
669+
Fallbacks map[string]string
675670
}
676671

677672
// vNumberRE is a heuristic for API versions.
678673
var vNumberRE = regexp.MustCompile(`apiv[0-9][^/]*`)
679674

680675
// friendlyAPIName returns the friendlyAPIName for the given import path.
681-
// We rely on the .repo-metadata-full.json file to get the description of the
682-
// API for the given import path. We use the importPath to parse out the
683-
// API version, since that isn't included in the metadata.
676+
// We rely on the .repo-metadata.json file in the package directory (or its
677+
// parents up to the module root) to get the description of the API for the
678+
// given import path. We use the importPath to parse out the API version, since
679+
// that isn't included in the metadata.
680+
//
681+
// If no API description is found, it falls back to the Fallbacks map.
684682
//
685683
// If no API description is found, friendlyAPIName returns "".
686684
// If no API version is found, friendlyAPIName only returns the description.
687685
//
688686
// See https://github.com/googleapis/google-cloud-go/issues/5949.
689-
func (d *friendlyAPINamer) friendlyAPIName(importPath string) (string, error) {
690-
var err error
691-
d.getOnce.Do(func() {
692-
resp, getErr := http.Get(d.metaURL)
693-
if err != nil {
694-
err = fmt.Errorf("error getting repo metadata: %v", getErr)
695-
return
687+
func (d *friendlyAPINamer) friendlyAPIName(importPath string, module *packages.Module) (string, error) {
688+
var description string
689+
690+
if module != nil {
691+
// Determine the directory for the package.
692+
relPath := strings.TrimPrefix(importPath, module.Path)
693+
relPath = strings.TrimPrefix(relPath, "/")
694+
pkgDir := filepath.Join(module.Dir, relPath)
695+
696+
// Search upwards from pkgDir to module.Dir for .repo-metadata.json.
697+
var meta repoMetadata
698+
found := false
699+
700+
currDir := pkgDir
701+
for {
702+
d.mu.Lock()
703+
if d.metadata == nil {
704+
d.metadata = make(map[string]repoMetadata)
705+
}
706+
cached, ok := d.metadata[currDir]
707+
d.mu.Unlock()
708+
709+
if ok {
710+
meta = cached
711+
found = true
712+
break
713+
}
714+
715+
metaFile := filepath.Join(currDir, ".repo-metadata.json")
716+
b, err := os.ReadFile(metaFile)
717+
if err == nil {
718+
if err := json.Unmarshal(b, &meta); err != nil {
719+
return "", fmt.Errorf("failed to decode repo metadata in %s: %v", metaFile, err)
720+
}
721+
d.mu.Lock()
722+
d.metadata[currDir] = meta
723+
d.mu.Unlock()
724+
found = true
725+
break
726+
}
727+
728+
if currDir == module.Dir || currDir == "." || currDir == "/" {
729+
break
730+
}
731+
parent := filepath.Dir(currDir)
732+
if parent == currDir {
733+
break
734+
}
735+
currDir = parent
696736
}
697-
defer resp.Body.Close()
698-
if decodeErr := json.NewDecoder(resp.Body).Decode(&d.metadata); err != nil {
699-
err = fmt.Errorf("failed to decode repo metadata: %v", decodeErr)
700-
return
737+
if found {
738+
description = meta.Description
701739
}
702-
})
703-
if err != nil {
704-
return "", err
705740
}
706-
if d.metadata == nil {
707-
return "", fmt.Errorf("no metadata found: earlier error fetching?")
741+
742+
if description == "" && d.Fallbacks != nil {
743+
description = d.Fallbacks[importPath]
708744
}
709-
pkg, ok := d.metadata[importPath]
710-
if !ok {
711-
return "", nil
745+
746+
if description == "" {
747+
return "", fmt.Errorf("no description found for %q and no fallback provided", importPath)
712748
}
713749

714750
if apiV := vNumberRE.FindString(importPath); apiV != "" {
715751
version := strings.TrimPrefix(apiV, "api")
716-
return fmt.Sprintf("%s %s", pkg.Description, version), nil
752+
return fmt.Sprintf("%s %s", description, version), nil
717753
}
718754

719-
return pkg.Description, nil
755+
return description, nil
720756
}

0 commit comments

Comments
 (0)