Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ jobs:
- run: go test -cover -coverprofile=coverage.out ./...
- name: Check coverage # TODO: increase minimum after adding more tests
run: |
awk -v minimum=0 '
awk -v minimum=80 '
NR > 1 {
total += $2
if ($3 > 0) covered += $2
Expand Down
16 changes: 10 additions & 6 deletions errdoc-gen/README.md
Original file line number Diff line number Diff line change
@@ -1,6 +1,10 @@
## Usage

```shell script
# eg: ./errdoc-gen --source devel/pingap/tidb --module github.com/pingcap/tidb --output devel/pingap/tidb/errors.toml
./errdoc-gen --source /path/to/source/code --module ${module-name} --output /path/to/errors.toml
```
## Usage

```shell script
# eg: ./errdoc-gen --source devel/pingap/tidb --module github.com/pingcap/tidb --output devel/pingap/tidb/errors.toml
./errdoc-gen --source /path/to/source/code --module ${module-name} --output /path/to/errors.toml
```

Only run errdoc-gen on trusted source trees. It compiles and runs a generated Go program that imports the scanned packages, executing their variable initializers and `init()` functions, including those of their dependencies. Input validation does not sandbox this execution.

`--module` and package import paths must match `[A-Za-z0-9_]+([.-][A-Za-z0-9_]+)*(/[A-Za-z0-9_]+([.-][A-Za-z0-9_]+)*)*`. This accepts domains, hyphenated names, and version numbers while rejecting empty components and traversal. `--ignore` accepts comma-separated paths using the same pattern, with an optional `./` prefix and trailing `/`, relative to `--source`; surrounding whitespace and blank entries are ignored. Parse failures, invalid paths, and scans with no error declarations or no normalized errors fail without overwriting the output.
74 changes: 59 additions & 15 deletions errdoc-gen/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"os"
"os/exec"
"path/filepath"
"regexp"
"slices"
"strings"
"text/template"
Expand Down Expand Up @@ -55,6 +56,13 @@ func fatal(format string, args ...any) {
const autoDirectoryName = "_errdoc-generator"
const entryFileName = "main.go"

const pathElementPattern = `[A-Za-z0-9_]+([.-][A-Za-z0-9_]+)*`

var (
importPathRegexp = regexp.MustCompile(`^` + pathElementPattern + `(/` + pathElementPattern + `)*$`)
ignoreDirectoryRegexp = regexp.MustCompile(`^(\./)?` + pathElementPattern + `(/` + pathElementPattern + `)*/?$`)
)

func main() {
flag.Parse()
if opt.source == "" {
Expand All @@ -68,8 +76,8 @@ func main() {
opt.source = source

_, err = os.Stat(filepath.Join(opt.source, "go.mod"))
if os.IsNotExist(err) {
fatal("The source directory is not the root path of codebase(go.mod not found)")
if err != nil {
fatal("Cannot access go.mod in the source directory: %v", err)
}

if opt.output == "" {
Expand All @@ -79,7 +87,7 @@ func main() {

errNames, err := errdoc(opt.source, opt.module)
if err != nil {
log("Extract the error documentation failed: %+v", err)
fatal("Extract the error documentation failed: %+v", err)
}

targetDir := filepath.Join(opt.source, autoDirectoryName)
Expand Down Expand Up @@ -107,7 +115,7 @@ import (
"github.com/BurntSushi/toml"
"github.com/pingcap/errors"
{{- range $decl := .}}
{{$decl.PackageName}} "{{- $decl.ImportPath}}"
{{$decl.PackageName}} {{printf "%q" $decl.ImportPath}}
{{- end}}
)

Expand Down Expand Up @@ -163,6 +171,10 @@ func main() {
dedup[codeText.String()] = s
}
}
if len(dedup) == 0 {
println("No normalized errors found; refusing to overwrite the output")
os.Exit(1)
Comment thread
kennytm marked this conversation as resolved.
Resolved
Comment thread
kennytm marked this conversation as resolved.
Resolved
}

var sorted []spec
for _, item := range dedup {
Expand Down Expand Up @@ -225,6 +237,10 @@ func main() {
cmd.Dir = opt.source
data, err := cmd.CombinedOutput()
if err != nil {
if !opt.retainCode {
out.Close()
os.RemoveAll(targetDir)
}
fatal("Generate %s failed: %v, output:\n%s", opt.output, err, string(data))
}

Expand All @@ -238,16 +254,26 @@ type errDecl struct {
}

func errdoc(source, module string) ([]*errDecl, error) {
if err := validateImportPath(module); err != nil {
return nil, fmt.Errorf("invalid --module: %w", err)
}
source, err := filepath.Abs(source)
if err != nil {
return nil, err
}

dedup := map[string]*errDecl{}

ignored := strings.Split(opt.ignore, ",")
for i := range ignored {
ignored[i] = filepath.Join(source, ignored[i])
ignored := []string{filepath.Join(source, autoDirectoryName)}
for _, dir := range strings.Split(opt.ignore, ",") {
dir = strings.TrimSpace(dir)
if dir == "" {
continue
}
if !ignoreDirectoryRegexp.MatchString(dir) {
return nil, fmt.Errorf("invalid --ignore directory %q: must be a subdirectory of the source", dir)
}
ignored = append(ignored, filepath.Join(source, dir))
}
err = filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
if err != nil {
Expand All @@ -265,8 +291,7 @@ func errdoc(source, module string) ([]*errDecl, error) {
fset := token.NewFileSet()
file, err := parser.ParseFile(fset, path, nil, parser.ParseComments)
if err != nil {
// Ignore invalid source file
return nil
return fmt.Errorf("parse %s: %w", path, err)
}
errNames := export(file)
if len(errNames) < 1 {
Expand All @@ -277,26 +302,45 @@ func errdoc(source, module string) ([]*errDecl, error) {
if err != nil {
return err
}
packageName := strings.ReplaceAll(subPath, "/", "_")
if decl, found := dedup[packageName]; found {
importPath := module
if subPath != "." {
importPath += "/" + filepath.ToSlash(subPath)
}
if err := validateImportPath(importPath); err != nil {
return fmt.Errorf("invalid package directory %q: %w", dirPath, err)
}
if decl, found := dedup[importPath]; found {
decl.ErrNames = append(decl.ErrNames, errNames...)
} else {
decl := &errDecl{
ImportPath: filepath.Join(module, subPath),
PackageName: packageName,
ImportPath: importPath,
PackageName: fmt.Sprintf("errpkg%d", len(dedup)),
ErrNames: errNames,
}
dedup[packageName] = decl
dedup[importPath] = decl
}
return nil
})
if err != nil {
return nil, err
}
if len(dedup) == 0 {
return nil, fmt.Errorf("no error declarations found; refusing to overwrite the output")
}

var errDecls []*errDecl
for _, decl := range dedup {
errDecls = append(errDecls, decl)
}

return errDecls, err
return errDecls, nil
}

func validateImportPath(importPath string) error {
if !importPathRegexp.MatchString(importPath) {
return fmt.Errorf("invalid import path %q", importPath)
}
return nil
}

func export(f *ast.File) []string {
Expand Down
Loading
Loading