Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 18 additions & 2 deletions .githooks/pre-commit
Original file line number Diff line number Diff line change
Expand Up @@ -14,10 +14,26 @@ if [ -z "$changed" ]
then
exit 0
else
# check required tools
for tool in swag jq yq api-spec-converter openapi; do
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partially fixed. Tool checks were added, but openapi is now required in preflight and still not pinned in mise.toml. Please add it so mise install matches the hook requirements.

command -v "$tool" >/dev/null 2>&1 || { echo "❌ $tool not found. Run 'mise install' to install all required tools."; exit 1; }
done

# regenerate docs
swag init --generatedTime --parseDependency --parseInternal -g handlers/main.go -d api/ api/*
go run docs/annotate_dtos/main.go
swag init --generatedTime --parseDependency --parseDependencyLevel 3 --parseInternal -g handlers/main.go -d api/ api/*
swag fmt -d ./api
go run v3gen/main.go
bash docs/fix_openapi_spec.sh
api-spec-converter --from=swagger_2 --to=openapi_3 -s yaml ./docs/swagger.yaml > ./docs/v3/openapi3.yaml
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This hook now requires jq, yq, and api-spec-converter. Please add command -v checks with install hints so failures are clear for non-mise setups.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

We should not require anyone to install these on their machines imo. That's why the dependency manager exists. These tools will never run in CI or in prod, it will always run locally.

api-spec-converter --from=swagger_2 --to=openapi_3 ./docs/swagger.json > ./docs/v3/openapi3.json
# add region descriptions and EU server (swag only supports a single host)
yq -i '.servers[0].description = "US Region" | .servers += [{"url": "https://eu.getconvoy.cloud/api", "description": "EU Region"}]' ./docs/v3/openapi3.yaml
jq '.servers[0].description = "US Region" | .servers += [{"url": "https://eu.getconvoy.cloud/api", "description": "EU Region"}]' ./docs/v3/openapi3.json > ./docs/v3/openapi3.json.tmp && mv ./docs/v3/openapi3.json.tmp ./docs/v3/openapi3.json
# validate specs
openapi swagger validate ./docs/swagger.json
openapi swagger validate ./docs/swagger.yaml
openapi spec validate ./docs/v3/openapi3.yaml

git add docs/ # add all files under the generated doc folder to git
fi

Expand Down
28 changes: 27 additions & 1 deletion .mise-tasks/gen/docs.sh
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,33 @@

set -e

# preflight: check required tools
for tool in swag jq yq api-spec-converter openapi; do
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Partially fixed. Preflight is in place, but it now checks openapi and mise.toml still does not include it.

command -v "$tool" >/dev/null 2>&1 || { echo "❌ $tool not found. Run 'mise install' to install required tools."; exit 1; }
done

echo "Generating docs"

#generate custom swag tags
go run docs/annotate_dtos/main.go

#generate v2 openapi specs
swag init --generatedTime --parseDependency --parseDependencyLevel 3 --parseInternal -g handlers/main.go -d api/ api/*
swag fmt -d ./api
go run v3gen/main.go

# fix openapi2 specs (structural fixes, add proper produce/consume tags, replace x-nullable..)
bash docs/fix_openapi_spec.sh

#generate v3 specs
api-spec-converter --from=swagger_2 --to=openapi_3 -s yaml ./docs/swagger.yaml > ./docs/v3/openapi3.yaml
Copy link
Copy Markdown
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Looks good. Please add a small required-tools preflight (jq, yq, api-spec-converter) so it fails fast with a clear message.

Copy link
Copy Markdown
Collaborator Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Same thing here. Forcing people to install a tool they'll only use to generate docs and making sure everyone is on the same version is a problem we struggled with for a long time and it makes no sense to keep supporting it

api-spec-converter --from=swagger_2 --to=openapi_3 ./docs/swagger.json > ./docs/v3/openapi3.json

# add region descriptions and EU server (swag only supports a single host)
yq -i '.servers[0].description = "US Region" | .servers += [{"url": "https://eu.getconvoy.cloud/api", "description": "EU Region"}]' ./docs/v3/openapi3.yaml
jq '.servers[0].description = "US Region" | .servers += [{"url": "https://eu.getconvoy.cloud/api", "description": "EU Region"}]' ./docs/v3/openapi3.json > ./docs/v3/openapi3.json.tmp && mv ./docs/v3/openapi3.json.tmp ./docs/v3/openapi3.json

# validate specs
echo "Validating specs..."
openapi swagger validate ./docs/swagger.json
openapi swagger validate ./docs/swagger.yaml
openapi spec validate ./docs/v3/openapi3.yaml
18 changes: 17 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -103,7 +103,23 @@ migrate_create:
@go run cmd/main.go migrate create

generate_docs:
swag init --generatedTime --parseDependency --parseInternal -d api/ api/*
@echo "Checking required tools (run 'mise install' to install all)..."
@for tool in swag jq yq api-spec-converter openapi; do \
command -v "$$tool" >/dev/null 2>&1 || { echo "❌ $$tool not found. Run 'mise install' to install all required tools."; exit 1; }; \
done
@echo "Generating docs..."
go run docs/annotate_dtos/main.go
swag init --generatedTime --parseDependency --parseDependencyLevel 3 --parseInternal -g handlers/main.go -d api/ api/*
swag fmt -d ./api
bash docs/fix_openapi_spec.sh
api-spec-converter --from=swagger_2 --to=openapi_3 -s yaml ./docs/swagger.yaml > ./docs/v3/openapi3.yaml
api-spec-converter --from=swagger_2 --to=openapi_3 ./docs/swagger.json > ./docs/v3/openapi3.json
yq -i '.servers[0].description = "US Region" | .servers += [{"url": "https://eu.getconvoy.cloud/api", "description": "EU Region"}]' ./docs/v3/openapi3.yaml
jq '.servers[0].description = "US Region" | .servers += [{"url": "https://eu.getconvoy.cloud/api", "description": "EU Region"}]' ./docs/v3/openapi3.json > ./docs/v3/openapi3.json.tmp && mv ./docs/v3/openapi3.json.tmp ./docs/v3/openapi3.json
@echo "Validating specs..."
openapi swagger validate ./docs/swagger.json
openapi swagger validate ./docs/swagger.yaml
openapi spec validate ./docs/v3/openapi3.yaml

run_dependencies:
docker compose -f docker-compose.dep.yml up -d
21 changes: 12 additions & 9 deletions api/handlers/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ package handlers
// This is the main doc file, swag cli needs it to be named main.go

// @title Convoy API Reference
// @version 24.1.4
// @description Convoy is a fast and secure webhooks proxy. This document contains datastore.s API specification.
// @version 26.3.5
// @description Convoy is a fast and secure webhooks proxy. This document contains datastore's API specification.
// @termsOfService https://getconvoy.io/terms

// @contact.name Convoy Support
Expand All @@ -15,16 +15,13 @@ package handlers
// @license.url https://www.mozilla.org/en-US/MPL/2.0/

// @schemes https
// @host dashboard.getconvoy.io
// @host us.getconvoy.cloud
// @BasePath /api

// @securityDefinitions.apikey ApiKeyAuth
// @in header
// @name Authorization

// @tag.name Organisations
// @tag.description Organisation related APIs

// @tag.name Subscriptions
// @tag.description Subscription related APIs

Expand All @@ -43,14 +40,20 @@ package handlers
// @tag.name Delivery Attempts
// @tag.description Delivery Attempt related APIs

// @tag.name Projects
// @tag.description Project related APIs

// @tag.name Portal Links
// @tag.description Portal Links related APIs

// @tag.name Meta Events
// @tag.description Meta Events related APIs

// @tag.name EventTypes
// @tag.description Event Types related APIs

// @tag.name Filters
// @tag.description Filters related APIs

// @tag.name Onboard
// @tag.description Onboard related APIs

// Stub represents empty json or arbitrary json bodies for our doc annotations
type Stub struct{}
8 changes: 4 additions & 4 deletions api/models/models.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import (

type PagedResponse struct {
Content interface{} `json:"content,omitempty"`
Pagination *datastore.PaginationData `json:"pagination,omitempty"`
Pagination *datastore.PaginationData `json:"pagination,omitempty" extensions:"x-nullable"`
}

type Organisation struct {
Expand All @@ -39,14 +39,14 @@ type APIKeyByIDResponse struct {
Name string `json:"name"`
Role auth.Role `json:"role"`
Type datastore.KeyType `json:"key_type"`
ExpiresAt null.Time `json:"expires_at,omitempty"`
ExpiresAt null.Time `json:"expires_at,omitempty" extensions:"x-nullable"`
CreatedAt time.Time `json:"created_at,omitempty"`
UpdatedAt time.Time `json:"updated_at,omitempty"`
}

type UserInviteTokenResponse struct {
Token *datastore.OrganisationInvite `json:"token"`
User *datastore.User `json:"user"`
Token *datastore.OrganisationInvite `json:"token" extensions:"x-nullable"`
User *datastore.User `json:"user" extensions:"x-nullable"`
}

type DeliveryAttempt struct {
Expand Down
4 changes: 2 additions & 2 deletions api/models/project.go
Original file line number Diff line number Diff line change
Expand Up @@ -203,8 +203,8 @@ type ProjectResponse struct {
}

type CreateProjectResponse struct {
APIKey *datastore.APIKeyResponse `json:"api_key"`
Project *ProjectResponse `json:"project"`
APIKey *datastore.APIKeyResponse `json:"api_key" extensions:"x-nullable"`
Project *ProjectResponse `json:"project" extensions:"x-nullable"`
}

func NewListProjectResponse(projects []*datastore.Project) []*ProjectResponse {
Expand Down
4 changes: 2 additions & 2 deletions api/models/source.go
Original file line number Diff line number Diff line change
Expand Up @@ -231,8 +231,8 @@ type CustomResponse struct {
}

type UpdateCustomResponse struct {
Body *string `json:"body"`
ContentType *string `json:"content_type"`
Body *string `json:"body" extensions:"x-nullable"`
ContentType *string `json:"content_type" extensions:"x-nullable"`
}

type VerifierConfig struct {
Expand Down
204 changes: 204 additions & 0 deletions docs/annotate_dtos/main.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,204 @@
// tools/annotate_dtos/main.go
//
//nolint:all
package main

import (
"fmt"
"go/ast"
"go/parser"
"go/token"
"io/ioutil"
"os"
"path/filepath"
"strings"
)

func main() {
dryRun := len(os.Args) > 1 && os.Args[1] == "--dry-run"

dtoDir := "./api/models"
fmt.Printf("Using AST to add nullable extensions in: %s\n", dtoDir)

err := filepath.Walk(dtoDir, func(path string, info os.FileInfo, err error) error {
if err != nil {
return err
}

if !info.IsDir() && strings.HasSuffix(path, ".go") {
processFileWithAST(path, dryRun)
}
return nil
})

if err != nil {
fmt.Printf("Error: %v\n", err)
}
}

func processFileWithAST(filePath string, dryRun bool) {
content, err := ioutil.ReadFile(filePath)
if err != nil {
fmt.Printf("Error reading %s: %v\n", filePath, err)
return
}

// Check if file contains Response structs
if !strings.Contains(string(content), "Response") {
return
}

fset := token.NewFileSet()
node, err := parser.ParseFile(fset, filePath, content, parser.ParseComments)
if err != nil {
fmt.Printf("Error parsing %s: %v\n", filePath, err)
return
}

var fieldsToFix []fieldInfo

ast.Inspect(node, func(n ast.Node) bool {
typeSpec, ok := n.(*ast.TypeSpec)
if !ok {
return true
}

structType, ok := typeSpec.Type.(*ast.StructType)
if !ok {
return true
}

// Only process Response structs
if !strings.Contains(typeSpec.Name.Name, "Response") {
return true
}

for _, field := range structType.Fields.List {
if field.Tag == nil || len(field.Names) == 0 {
continue
}

tagValue := field.Tag.Value
// Remove backticks
tagValue = strings.Trim(tagValue, "`")

// Skip if already has extensions:"x-nullable"
if strings.Contains(tagValue, `extensions:"x-nullable"`) {
continue
}

// Check if field type is nullable
fieldType := getFieldTypeString(field.Type)
if isNullableTypeAST(fieldType) {
pos := fset.Position(field.Pos())
fieldsToFix = append(fieldsToFix, fieldInfo{
name: field.Names[0].Name,
typeStr: fieldType,
tag: tagValue,
line: pos.Line,
filename: filePath,
})
}
}
return true
})

if len(fieldsToFix) == 0 {
return
}

if dryRun {
fmt.Printf("[DRY] %s: %d fields need extensions:\"x-nullable\"\n",
filepath.Base(filePath), len(fieldsToFix))
for _, field := range fieldsToFix {
fmt.Printf(" - %s (%s)\n", field.name, field.typeStr)
}
return
}

// Actually fix the file
lines := strings.Split(string(content), "\n")
modified := false

for _, field := range fieldsToFix {
lineIndex := field.line - 1
if lineIndex >= len(lines) {
continue
}

line := lines[lineIndex]

// Find the tag and add extensions:"x-nullable"
tagStart := strings.Index(line, "`")
tagEnd := strings.LastIndex(line, "`")
if tagStart == -1 || tagEnd == -1 {
continue
}

tagContent := line[tagStart+1 : tagEnd]
newTagContent := tagContent + ` extensions:"x-nullable"`
newLine := line[:tagStart+1] + newTagContent + line[tagEnd:]

lines[lineIndex] = newLine
modified = true

fmt.Printf("✓ %s: %s\n", filepath.Base(filePath), field.name)
}

if modified {
output := strings.Join(lines, "\n")
if err := ioutil.WriteFile(filePath, []byte(output), 0644); err != nil {
fmt.Printf("Error writing %s: %v\n", filePath, err)
}
}
}

type fieldInfo struct {
name string
typeStr string
tag string
line int
filename string
}

func getFieldTypeString(expr ast.Expr) string {
switch t := expr.(type) {
case *ast.Ident:
return t.Name
case *ast.SelectorExpr:
xIdent, ok := t.X.(*ast.Ident)
if !ok {
return ""
}
return fmt.Sprintf("%s.%s", xIdent.Name, t.Sel.Name)
case *ast.StarExpr:
return "*" + getFieldTypeString(t.X)
case *ast.ArrayType:
return "[]" + getFieldTypeString(t.Elt)
default:
return ""
}
}

func isNullableTypeAST(typeStr string) bool {
// Check for null package types
if strings.HasPrefix(typeStr, "null.") {
return true
}

// Check for sql.Null types
if strings.HasPrefix(typeStr, "sql.Null") {
return true
}

// Check for pointer types
if strings.HasPrefix(typeStr, "*") {
// Don't treat pointers to slices/maps as nullable for JSON
if strings.HasPrefix(typeStr, "*[]") || strings.HasPrefix(typeStr, "*map[") {
return false
}
return true
}

return false
}
Loading
Loading