Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

more work on backstage publish #190

Merged
merged 4 commits into from
Apr 11, 2023
Merged

more work on backstage publish #190

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
1098609
more work on backstage publish
theganyo Apr 6, 2023
619b497
Merge branch 'main' into theganyo/issue189
theganyo Apr 7, 2023
ceae231
naming improvements, add owner
theganyo Apr 10, 2023
f438682
fix test
theganyo Apr 10, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion cmd/registry-connect/discover/apigee/edge/edge_client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,6 +33,15 @@ func oauthTestServer(t *testing.T) *httptest.Server {
AccessToken: "token",
}

m.HandleFunc("/noauth", (func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
w.WriteHeader(http.StatusUnauthorized)
default:
w.WriteHeader(http.StatusMethodNotAllowed)
}
}))

m.HandleFunc("/oauth", (func(w http.ResponseWriter, r *http.Request) {
switch r.Method {
case http.MethodPost:
Expand Down Expand Up @@ -64,10 +73,12 @@ func TestNewEdgeClient(t *testing.T) {

var err error

SetOAuthURL(ts.URL + "/noauth")

_, err = NewEdgeClient(opts)
errorContains(t, err, "401")

SetOAuthURL(ts.URL + "/oauth") // intercept OAuth url
SetOAuthURL(ts.URL + "/oauth")

_, err = NewEdgeClient(opts)
if err != nil {
Expand Down
295 changes: 5 additions & 290 deletions cmd/registry-connect/publish/backstage/backstage.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,30 +15,12 @@
package backstage

import (
"context"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/apigee/registry-experimental/cmd/registry-connect/publish/backstage/encoding"
"github.com/apigee/registry/pkg/application/apihub"
"github.com/apigee/registry/pkg/connection"
"github.com/apigee/registry/pkg/log"
"github.com/apigee/registry/pkg/mime"
"github.com/apigee/registry/pkg/names"
"github.com/apigee/registry/pkg/visitor"
"github.com/apigee/registry/rpc"
"github.com/spf13/cobra"
"google.golang.org/protobuf/proto"
"gopkg.in/yaml.v3"
)

const (
apiHubTag = "apihub"
apiLinkFormat = "https://pantheon.corp.google.com/apigee/hub/apis/%s/overview?project=%s"
taxonomiesLinkFormat = "https://pantheon.corp.google.com/apigee/hub/settings/taxonomies?project=%s"
)
var ownerName, ownerDesc string

func Command() *cobra.Command {
var filter string
Expand Down Expand Up @@ -67,276 +49,9 @@ func Command() *cobra.Command {
},
}
cmd.Flags().StringVar(&filter, "filter", "", "filter selected apis")
cmd.Flags().StringVar(&ownerName, "owner-name", "", "Apigee contact name")
cmd.Flags().StringVar(&ownerDesc, "owner-desc", "", "Apigee contact description")
_ = cmd.MarkFlagRequired("owner-name")
_ = cmd.MarkFlagRequired("owner-desc")
return cmd
}

func recommendedOrLatestVersion(ctx context.Context, client connection.RegistryClient, a *rpc.Api) (*rpc.ApiVersion, error) {
n, _ := names.ParseApi(a.Name)
versionName := n.Version("-")

if a.RecommendedVersion != "" {
rv, err := names.ParseVersion(a.RecommendedVersion)
if err != nil {
return nil, err
}
versionName = rv
}

var version *rpc.ApiVersion
err := visitor.ListVersions(ctx, client, versionName, "", func(ctx context.Context, av *rpc.ApiVersion) error {
version = av
return nil
})
return version, err
}

func primaryOrLatestSpec(ctx context.Context, client connection.RegistryClient, av *rpc.ApiVersion) (*rpc.ApiSpec, error) {
n, _ := names.ParseVersion(av.Name)
specName := n.Spec("-")

if av.PrimarySpec != "" {
as, err := names.ParseSpec(av.PrimarySpec)
if err != nil {
return nil, err
}
specName = as
}

var spec *rpc.ApiSpec
err := visitor.ListSpecs(ctx, client, specName, "", true, func(ctx context.Context, as *rpc.ApiSpec) error {
spec = as
return nil
})
return spec, err
}

func name(str string) string {
if len(str) > 63 {
str = str[0:63]
}
return str
}

func required(value string) string {
if value == "" {
return "unknown"
}
return value
}

type catalog struct {
client connection.RegistryClient
config connection.Config
filter string
root string
filesByKind map[string][]string
}

func (c *catalog) Run(ctx context.Context) error {
c.filesByKind = map[string][]string{}

if err := c.createGroups(ctx); err != nil {
return err
}
if err := c.createAPIs(ctx); err != nil {
return err
}
return c.writeCatalog()
}

func (c *catalog) createGroups(ctx context.Context) error {
taxonomiesName, err := names.ParseArtifact(c.config.FQName("artifacts/apihub-taxonomies"))
if err != nil {
return err
}
return visitor.GetArtifact(ctx, c.client, taxonomiesName, true, func(ctx context.Context, a *rpc.Artifact) error {
message, err := mime.MessageForMimeType(a.GetMimeType())
if err == nil {
err = proto.Unmarshal(a.GetContents(), message)
}
if err != nil {
return err
}
artifactName, _ := names.ParseArtifact(a.Name)
taxonomies := message.(*apihub.TaxonomyList)
for _, t := range taxonomies.GetTaxonomies() {
if t.Id == "apihub-team" {
for _, team := range t.Elements {
metadata := encoding.Metadata{
Name: name(team.Id),
Namespace: artifactName.ProjectID(),
Title: team.DisplayName,
Description: team.Description,
Tags: []string{apiHubTag},
Links: []encoding.Link{
{
URL: fmt.Sprintf(taxonomiesLinkFormat, artifactName.ProjectID()),
Title: "API Hub Taxonomies",
// Icon: "", // requires backstage setup
},
},
}
group := encoding.Group{
Type: "team",
}
if err := c.addEntity(metadata, group); err != nil {
return err
}
}
}
}
return nil
})
}

func (c *catalog) createAPIs(ctx context.Context) error {
project, err := names.ParseProject("projects/" + c.config.Project)
if err != nil {
return err
}
return visitor.ListAPIs(ctx, c.client, project.Api("-"), c.filter, func(ctx context.Context, a *rpc.Api) error {
log.FromContext(ctx).Infof("publishing %s", a.Name)

apiName, _ := names.ParseApi(a.Name)
name := apiName.ApiID
metadata := encoding.Metadata{
Name: name,
Namespace: project.ProjectID,
Title: a.DisplayName,
Description: a.Description,
Labels: a.Labels, // TODO: not viewable in backstage
Annotations: a.Annotations, // TODO: not viewable in backstage
Tags: []string{apiHubTag},
Links: []encoding.Link{ // TODO: not viewable in backstage
{
URL: fmt.Sprintf(apiLinkFormat, apiName.ApiID, project.ProjectID),
Title: "API Hub",
},
},
}
var style, lifecycle, definition, owner string

owner = a.Labels["apihub-team"]
style = strings.TrimPrefix(a.Labels["apihub-style"], "apihub-")
lifecycle = a.Labels["apihub-lifecycle"]
// TODO: add contact as user?
// primaryContact = a.Labels["apihub-primary-contact"]
// primaryContactDescription = a.Labels["apihub-primary-contact-description"]

av, err := recommendedOrLatestVersion(ctx, c.client, a)
if err != nil {
return err
}
if av != nil {
lifecycle = av.State

as, err := primaryOrLatestSpec(ctx, c.client, av)
if err != nil {
return err
}
if as != nil && as.MimeType != "application/x.proto+zip" { // no binary
// overrides API style as Backstage actually just expects spec style
// TODO: full mime list
if strings.Contains(as.MimeType, "openapi") || strings.Contains(as.MimeType, "yaml") {
style = "openapi"
} else if strings.Contains(as.MimeType, "proto") || strings.Contains(as.MimeType, "grpc") {
style = "grpc"
}
definition = string(as.Contents)
}
// the following doesn't work to write the spec content in a separate local file,
// perhaps related to: https://github.com/backstage/backstage/issues/14372
// if as != nil && len(as.Contents) > 0 {
// file := filepath.Join(c.root, metadata.Name+".spec")
// if strings.Contains(as.MimeType, "yaml") || strings.Contains(as.MimeType, "openapi") {
// definition = "$yaml: " + file
// } else if strings.Contains(as.MimeType, "json") {
// definition = "$json: " + file
// } else if strings.Contains(as.MimeType, "text") {
// definition = "$text: " + file
// }
// if err := os.MkdirAll(filepath.Dir(file), os.FileMode(0755)); err != nil { // rwx,rx,rx
// return err
// }
// if err := os.WriteFile(file, as.Contents, os.FileMode(0644)); err != nil {
// return err
// }
// }
}

api := encoding.Api{
Type: required(style), // backstage well-known types: openapi, asyncapi, graphql, grpc
Lifecycle: required(lifecycle), // backstage well-known types: experimental, production, deprecated
Owner: required(owner),
Definition: required(definition),
}
return c.addEntity(metadata, api)
})
}

func (c *catalog) writeYAML(file string, data *encoding.Envelope) error {
file = filepath.Join(c.root, file)
if err := os.MkdirAll(filepath.Dir(file), os.FileMode(0755)); err != nil { // rwx,rx,rx
return err
}

out, err := os.OpenFile(file, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, os.FileMode(0644)) // rw,r,r
if err != nil {
return err
}
defer out.Close()
enc := yaml.NewEncoder(out)
return enc.Encode(data)
}

func (c *catalog) addEntity(metadata encoding.Metadata, entity interface{}) error {
envelope, err := encoding.NewEnvelope(metadata, entity)
if err != nil {
return err
}
file := metadata.Name + ".yaml"
c.filesByKind[envelope.Kind] = append(c.filesByKind[envelope.Kind], file)
return c.writeYAML(filepath.Join(envelope.Kind+"s", file), envelope)
}

func (c *catalog) writeCatalog() error {
subCatalogs := []string{}

for k, fs := range c.filesByKind {
relativeFs := []string{}
for _, f := range fs {
relativeFs = append(relativeFs, "./"+f)
}
location := encoding.Location{
Targets: relativeFs,
}
pluralKinds := k + "s"
catalog := fmt.Sprintf("All-%s.yaml", pluralKinds)
subCatalogs = append(subCatalogs, "./"+filepath.Join(pluralKinds, catalog))
metadata := encoding.Metadata{
Name: "APIHub-" + pluralKinds,
Description: fmt.Sprintf("API Hub %s for Backstage.io", pluralKinds),
}
envelope, err := encoding.NewEnvelope(metadata, location)
if err != nil {
return err
}
file := filepath.Join(pluralKinds, catalog)
if err := c.writeYAML(file, envelope); err != nil {
return err
}
}

location := encoding.Location{
Targets: subCatalogs,
}
metadata := encoding.Metadata{
Name: "APIHub-" + c.config.Project,
Description: fmt.Sprintf("API Hub project %s export for Backstage.io", c.config.Project),
}
envelope, err := encoding.NewEnvelope(metadata, location)
if err != nil {
return err
}
return c.writeYAML("APIHub-Catalog.yaml", envelope)
}
Loading