+ golang / go +
+The Go programming language
+ Go + + 123,456 + + + 18,900 + + + 42 stars today + +From fb4348d123ff7f86560e20c4520ba04b63304a0d Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Tue, 16 Jun 2026 00:31:10 +0700
Subject: [PATCH 01/21] replace REST API with HTML/Atom scraper, rename binary
to github
The tool now scrapes github.com HTML pages and Atom feeds instead of
calling api.github.com. This removes the 60-req/hour ceiling and the
need for any API key.
Changes:
- github/: full rewrite with parse.go (HTML+Atom), api.go (15 methods),
ops.go (15 kit Handle registrations), domain.go (kit Domain), ids.go
- cli/: slimmed down to NewApp() that delegates to the kit domain
- cmd/github/: new main using kit.Main; drops old cmd/ghb/
- 17 tests pass with httptest servers, no network needed in CI
- binary renamed from ghb to github; goreleaser and Dockerfile updated
---
.goreleaser.yaml | 37 +--
Dockerfile | 8 +-
README.md | 95 ++++--
cli/cmd_releases.go | 26 --
cli/cmd_repo.go | 36 ---
cli/cmd_search.go | 36 ---
cli/cmd_trending.go | 34 --
cli/cmd_user.go | 37 ---
cli/errors.go | 15 -
cli/output.go | 25 --
cli/root.go | 159 ++-------
cli/version.go | 27 --
cmd/ghb/main.go | 27 --
cmd/github/main.go | 14 +
github/api.go | 232 ++++++++++++++
github/domain.go | 58 ++++
github/github.go | 424 +++++++++---------------
github/github_test.go | 727 +++++++++++++++++++++++++++++-------------
github/ids.go | 37 +++
github/ids_test.go | 40 +++
github/ops.go | 418 ++++++++++++++++++++++++
github/parse.go | 712 +++++++++++++++++++++++++++++++++++++++++
github/types.go | 212 ++++++------
go.mod | 15 +-
go.sum | 54 +++-
25 files changed, 2422 insertions(+), 1083 deletions(-)
delete mode 100644 cli/cmd_releases.go
delete mode 100644 cli/cmd_repo.go
delete mode 100644 cli/cmd_search.go
delete mode 100644 cli/cmd_trending.go
delete mode 100644 cli/cmd_user.go
delete mode 100644 cli/errors.go
delete mode 100644 cli/output.go
delete mode 100644 cli/version.go
delete mode 100644 cmd/ghb/main.go
create mode 100644 cmd/github/main.go
create mode 100644 github/api.go
create mode 100644 github/domain.go
create mode 100644 github/ids.go
create mode 100644 github/ids_test.go
create mode 100644 github/ops.go
create mode 100644 github/parse.go
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index b450097..b7072a2 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -3,22 +3,22 @@
# entries for the package managers (Homebrew, Scoop). `git tag vX.Y.Z && git
# push --tags` fans out to all of them through .github/workflows/release.yml.
#
-# Publish steps that push to a repository you do not own yet (the Homebrew tap,
+# Publish steps that push to a repository we do not own yet (the Homebrew tap,
# the Scoop bucket) self-disable when their token is absent. A release with no
# extra secrets still produces every downloadable artifact and the container
# image; each manager lights up the moment its repository and token exist.
version: 2
-project_name: ghb
+project_name: github-cli
before:
hooks:
- go mod download
builds:
- - id: ghb
- binary: ghb
- main: ./cmd/ghb
+ - id: github
+ binary: github
+ main: ./cmd/github
env:
- CGO_ENABLED=0
flags:
@@ -43,7 +43,7 @@ builds:
archives:
- id: default
- name_template: "ghb_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}"
+ name_template: "github_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}"
format_overrides:
- goos: windows
formats: [zip]
@@ -53,12 +53,12 @@ archives:
nfpms:
- id: linux-packages
- package_name: ghb
+ package_name: github-cli
file_name_template: "{{ .ConventionalFileName }}"
vendor: tamnd
homepage: https://github.com/tamnd/github-cli
maintainer: Duc-Tam Nguyen The Go programming language Production-Grade Container Scheduling Linux kernel source tree My personal dive log application The Go programming language Empowering everyone to build reliable and efficient software. ]*class="[^"]*col-9[^"]*"[^>]*>(.*?)
+ golang / go
+
+
+ kubernetes / kubernetes
+
+
+
+
+`
- start := time.Now()
- body, err := c.get(context.Background(), c.baseURL)
- if err != nil {
- t.Fatal(err)
+func TestParseRepos(t *testing.T) {
+ repos := ParseRepos(reposFixture, "torvalds")
+ if len(repos) != 2 {
+ t.Fatalf("want 2 repos, got %d", len(repos))
+ }
+ r := repos[0]
+ if r.FullName != "torvalds/linux" {
+ t.Errorf("full_name: want torvalds/linux, got %q", r.FullName)
}
- if string(body) != `"recovered"` {
- t.Errorf("body = %q after retries", body)
+ if r.Description != "Linux kernel source tree" {
+ t.Errorf("description: got %q", r.Description)
}
- if hits != 3 {
- t.Errorf("server saw %d hits, want 3", hits)
+ if r.Language != "C" {
+ t.Errorf("language: want C, got %q", r.Language)
}
- if time.Since(start) < 500*time.Millisecond {
- t.Error("retries did not back off")
+ if r.Stars != 182000 {
+ t.Errorf("stars: want 182000, got %d", r.Stars)
}
}
-func TestSearchRepos(t *testing.T) {
- desc := "A great project"
- lang := "Go"
- license := "MIT"
- resp := searchReposResp{
- TotalCount: 1,
- Items: []wireRepo{
- {
- ID: 1,
- FullName: "owner/repo",
- Description: &desc,
- HTMLURL: "https://github.com/owner/repo",
- Stars: 5000,
- Forks: 1200,
- Language: &lang,
- License: &struct {
- SPDXID string `json:"spdx_id"`
- }{SPDXID: license},
- PushedAt: "2024-06-01T12:00:00Z",
- },
- },
- }
-
- c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) {
- if !strings.HasPrefix(r.URL.Path, "/search/repositories") {
- t.Errorf("unexpected path: %s", r.URL.Path)
- }
- _ = json.NewEncoder(w).Encode(resp)
- })
+const atomCommitsFixture = `
+
]*class="[^"]*mb-1[^"]*"[^>]*>\s*(.*?)\s*
`) + reSearchStars = regexp.MustCompile(`([0-9,]+)\s+stars?`) + reSearchLang = regexp.MustCompile(`(?s)]*class="[^"]*search-match[^"]*"[^>]*>([^<]+)<`) + reSearchDate = regexp.MustCompile(`]*class="[^"]*col-9[^"]*"[^>]*>(.*?)
`) + reStarLang = regexp.MustCompile(`itemprop="programmingLanguage"[^>]*>([^<]+)<`) + reStarStars = regexp.MustCompile(`href="[^"]+/stargazers"[^>]*>\s*([0-9,]+)`) +) + +// ── helpers ────────────────────────────────────────────────────────────────── + +// cleanInt parses a comma-formatted or k-suffixed integer string. +// "12,345" → 12345; "3.2k" → 3200; returns 0 on failure. +func cleanInt(s string) int { + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + // handle k suffix + if strings.HasSuffix(s, "k") || strings.HasSuffix(s, "K") { + f, err := strconv.ParseFloat(strings.ReplaceAll(s[:len(s)-1], ",", ""), 64) + if err != nil { + return 0 + } + return int(f * 1000) + } + s = strings.ReplaceAll(s, ",", "") + n, _ := strconv.Atoi(s) + return n +} + +// cleanStr strips HTML tags, decodes HTML entities, and trims whitespace. +func cleanStr(s string) string { + // strip tags + reTag := regexp.MustCompile(`<[^>]+>`) + s = reTag.ReplaceAllString(s, " ") + // decode entities + s = html.UnescapeString(s) + // collapse whitespace + reWS := regexp.MustCompile(`\s+`) + s = reWS.ReplaceAllString(s, " ") + return strings.TrimSpace(s) +} + +// extractSHA extracts the commit SHA from a GitHub Atom entry ID. +// IDs look like: tag:github.com,2008:Grit::Commit/abc1234567890 +func extractSHA(id string) string { + if idx := strings.LastIndex(id, "/"); idx >= 0 { + return id[idx+1:] + } + return id +} + +// first returns the first capture group match, or "". +func first(re *regexp.Regexp, s string) string { + m := re.FindStringSubmatch(s) + if m == nil || len(m) < 2 { + return "" + } + return cleanStr(m[1]) +} + +// ── ParseTrending ──────────────────────────────────────────────────────────── + +// ParseTrending parses the github.com/trending HTML page. +func ParseTrending(body string) []TrendingRepo { + articles := reTrendingArticle.FindAllStringSubmatch(body, -1) + out := make([]TrendingRepo, 0, len(articles)) + for i, m := range articles { + block := m[1] + + fullName := first(reTrendingLink, block) + if fullName == "" { + continue + } + + desc := "" + if dm := reTrendingDesc.FindStringSubmatch(block); dm != nil { + desc = cleanStr(dm[1]) + } + + lang := first(reTrendingLang, block) + stars := 0 + if sm := reTrendingStars.FindStringSubmatch(block); sm != nil { + stars = cleanInt(sm[1]) + } + forks := 0 + if fm := reTrendingForks.FindStringSubmatch(block); fm != nil { + forks = cleanInt(fm[1]) + } + period := 0 + if pm := reTrendingPeriod.FindStringSubmatch(block); pm != nil { + period = cleanInt(pm[1]) + } + + out = append(out, TrendingRepo{ + Rank: i + 1, + FullName: fullName, + Description: desc, + Language: lang, + Stars: stars, + Forks: forks, + PeriodStars: period, + URL: "https://github.com/" + fullName, + }) + } + return out +} + +// ── ParseUser ──────────────────────────────────────────────────────────────── + +// ParseUser parses a github.com/{username} profile page. +func ParseUser(body, username string) (User, error) { + name := first(reUserName, body) + + bio := "" + if bm := reUserBio.FindStringSubmatch(body); bm != nil { + bio = cleanStr(bm[1]) + } else if bm2 := reUserBioAlt.FindStringSubmatch(body); bm2 != nil { + bio = cleanStr(bm2[1]) + } + + company := first(reUserCompany, body) + location := first(reUserLocation, body) + email := first(reUserEmail, body) + blog := first(reUserBlog, body) + + followers := 0 + if fm := reUserFollowers.FindStringSubmatch(body); fm != nil { + followers = cleanInt(fm[1]) + } + following := 0 + if fm := reUserFollowing.FindStringSubmatch(body); fm != nil { + following = cleanInt(fm[1]) + } + repos := 0 + if rm := reUserRepos.FindStringSubmatch(body); rm != nil { + repos = cleanInt(rm[1]) + } + + return User{ + Login: username, + Name: name, + Bio: bio, + Company: company, + Location: location, + Email: email, + Blog: blog, + Followers: followers, + Following: following, + Repos: repos, + URL: userURL(username), + }, nil +} + +// ── ParseRepos ─────────────────────────────────────────────────────────────── + +// ParseRepos parses the github.com/{username}?tab=repositories HTML page. +func ParseRepos(body, username string) []Repo { + items := reRepoItem.FindAllStringSubmatch(body, -1) + out := make([]Repo, 0, len(items)) + for _, m := range items { + block := m[1] + + fullName := first(reRepoItemLink, block) + if fullName == "" { + continue + } + + desc := "" + if dm := reRepoItemDesc.FindStringSubmatch(block); dm != nil { + desc = cleanStr(dm[1]) + } + + lang := first(reRepoItemLang, block) + stars := 0 + if sm := reRepoItemStars.FindStringSubmatch(block); sm != nil { + stars = cleanInt(sm[1]) + } + forks := 0 + if fm := reRepoItemForks.FindStringSubmatch(block); fm != nil { + forks = cleanInt(fm[1]) + } + pushedAt := first(reRepoItemDate, block) + + out = append(out, Repo{ + FullName: fullName, + Description: desc, + Language: lang, + Stars: stars, + Forks: forks, + PushedAt: pushedAt, + URL: "https://github.com/" + fullName, + }) + } + return out +} + +// ── ParseRepo ──────────────────────────────────────────────────────────────── + +// ParseRepo parses the github.com/{owner}/{repo} page. +func ParseRepo(body, owner, repo string) (Repo, error) { + fullName := owner + "/" + repo + + desc := "" + if dm := reRepoDesc.FindStringSubmatch(body); dm != nil { + desc = cleanStr(dm[1]) + } else if dm2 := reRepoDescAlt.FindStringSubmatch(body); dm2 != nil { + desc = cleanStr(dm2[1]) + } + + lang := "" + if lm := reRepoLang.FindStringSubmatch(body); lm != nil { + lang = cleanStr(lm[1]) + } else if lm2 := reRepoLangAlt.FindStringSubmatch(body); lm2 != nil { + lang = cleanStr(lm2[1]) + } + + stars := 0 + if sm := reRepoStars.FindStringSubmatch(body); sm != nil { + stars = cleanInt(sm[1]) + } + forks := 0 + if fm := reRepoForks.FindStringSubmatch(body); fm != nil { + forks = cleanInt(fm[1]) + } + openIssues := 0 + if im := reRepoIssues.FindStringSubmatch(body); im != nil { + openIssues = cleanInt(im[1]) + } + watchers := 0 + if wm := reRepoWatchers.FindStringSubmatch(body); wm != nil { + watchers = cleanInt(wm[1]) + } + + // topics + topicMatches := reRepoTopics.FindAllStringSubmatch(body, -1) + topics := make([]string, 0, len(topicMatches)) + for _, tm := range topicMatches { + t := cleanStr(tm[1]) + if t != "" { + topics = append(topics, t) + } + } + + license := first(reRepoLicense, body) + branch := first(reRepoBranch, body) + if branch == "" { + branch = "main" + } + + isFork := reRepoFork.MatchString(body) + isArchived := reRepoArchive.MatchString(body) + + return Repo{ + FullName: fullName, + Description: desc, + Language: lang, + Stars: stars, + Forks: forks, + Watchers: watchers, + OpenIssues: openIssues, + DefaultBranch: branch, + License: license, + Topics: topics, + Fork: isFork, + Archived: isArchived, + URL: repoURL(owner, repo), + }, nil +} + +// ── Atom feeds ─────────────────────────────────────────────────────────────── + +// ParseAtomCommits parses the /commits/{branch}.atom feed. +func ParseAtomCommits(body string) ([]Commit, error) { + var feed atomFeed + if err := xml.Unmarshal([]byte(body), &feed); err != nil { + return nil, err + } + out := make([]Commit, 0, len(feed.Entries)) + for _, e := range feed.Entries { + sha := extractSHA(e.ID) + if len(sha) > 7 { + sha = sha[:7] + } + out = append(out, Commit{ + SHA: sha, + Message: strings.TrimSpace(e.Title), + Author: strings.TrimSpace(e.Author.Name), + Date: e.Published, + URL: e.Link.Href, + }) + } + return out, nil +} + +// ParseAtomReleases parses the /releases.atom feed. +func ParseAtomReleases(body string) ([]Release, error) { + var feed atomFeed + if err := xml.Unmarshal([]byte(body), &feed); err != nil { + return nil, err + } + out := make([]Release, 0, len(feed.Entries)) + for _, e := range feed.Entries { + tag := lastPathSegment(e.Link.Href) + out = append(out, Release{ + Tag: tag, + Name: strings.TrimSpace(e.Title), + Author: strings.TrimSpace(e.Author.Name), + Published: e.Published, + URL: e.Link.Href, + }) + } + return out, nil +} + +// ParseAtomTags parses the /tags.atom feed. +func ParseAtomTags(body string) ([]Tag, error) { + var feed atomFeed + if err := xml.Unmarshal([]byte(body), &feed); err != nil { + return nil, err + } + out := make([]Tag, 0, len(feed.Entries)) + for _, e := range feed.Entries { + out = append(out, Tag{ + Name: strings.TrimSpace(e.Title), + Updated: e.Updated, + URL: e.Link.Href, + }) + } + return out, nil +} + +// ── ParseIssues ────────────────────────────────────────────────────────────── + +// ParseIssues parses the /{owner}/{repo}/issues HTML page. +func ParseIssues(body, owner, repo, state string) []Issue { + // find all issue-N id blocks + matches := reIssueBlock.FindAllStringSubmatch(body, -1) + out := make([]Issue, 0, len(matches)) + for _, m := range matches { + numStr := m[1] + block := m[2] + num, _ := strconv.Atoi(numStr) + if num == 0 { + continue + } + + title := "" + issueURL := "" + if tm := reIssueTitle.FindStringSubmatch(block); tm != nil { + issueURL = "https://github.com" + tm[1] + title = cleanStr(tm[2]) + } + + createdAt := first(reIssueDate, block) + author := first(reIssueAuthor, block) + + labelMatches := reIssueLabel.FindAllStringSubmatch(block, -1) + labels := make([]string, 0, len(labelMatches)) + for _, lm := range labelMatches { + labels = append(labels, cleanStr(lm[1])) + } + + comments := 0 + if cm := reIssueComments.FindStringSubmatch(block); cm != nil { + comments, _ = strconv.Atoi(cm[1]) + } + + if issueURL == "" { + issueURL = "https://github.com/" + owner + "/" + repo + "/issues/" + numStr + } + + out = append(out, Issue{ + Number: num, + Title: title, + State: state, + Author: author, + Comments: comments, + Labels: strings.Join(labels, ", "), + CreatedAt: createdAt, + URL: issueURL, + }) + } + return out +} + +// ── ParsePulls ─────────────────────────────────────────────────────────────── + +// ParsePulls parses the /{owner}/{repo}/pulls HTML page. +// The PR list uses the same HTML structure as issues with a different URL path. +func ParsePulls(body, owner, repo, state string) []PullRequest { + // use the same issue_N id structure + matches := reIssueBlock.FindAllStringSubmatch(body, -1) + out := make([]PullRequest, 0, len(matches)) + for _, m := range matches { + numStr := m[1] + block := m[2] + num, _ := strconv.Atoi(numStr) + if num == 0 { + continue + } + + title := "" + prURL := "" + prNum := num + if tm := rePRTitle.FindStringSubmatch(block); tm != nil { + prURL = "https://github.com" + tm[1] + n, _ := strconv.Atoi(tm[2]) + if n > 0 { + prNum = n + } + title = cleanStr(tm[3]) + } else if tm2 := reIssueTitle.FindStringSubmatch(block); tm2 != nil { + prURL = "https://github.com" + tm2[1] + title = cleanStr(tm2[2]) + } + + createdAt := first(reIssueDate, block) + author := first(reIssueAuthor, block) + + comments := 0 + if cm := reIssueComments.FindStringSubmatch(block); cm != nil { + comments, _ = strconv.Atoi(cm[1]) + } + + if prURL == "" { + prURL = "https://github.com/" + owner + "/" + repo + "/pull/" + numStr + } + + out = append(out, PullRequest{ + Number: prNum, + Title: title, + State: state, + Author: author, + Comments: comments, + CreatedAt: createdAt, + URL: prURL, + }) + } + return out +} + +// ── ParseSearch ────────────────────────────────────────────────────────────── + +// ParseSearch parses the github.com/search?type=repositories results page. +func ParseSearch(body string) []SearchRepo { + // find all result items by looking for full_name links + nameMatches := reSearchName.FindAllStringSubmatch(body, -1) + if len(nameMatches) == 0 { + nameMatches = reSearchNameB.FindAllStringSubmatch(body, -1) + } + + out := make([]SearchRepo, 0, len(nameMatches)) + // split body on each result card anchor to get per-card blocks + // Use a simpler approach: find all v-align-middle hrefs + reCard := regexp.MustCompile(`(?s)class="v-align-middle[^"]*"[^>]*href="/([^/"]+/[^/"]+)"[^>]*>.*?(?:class="v-align-middle|$)`) + _ = reCard + + for i, nm := range nameMatches { + fullName := nm[1] + // carve out a block around this match to extract nearby metadata + idx := strings.Index(body, nm[0]) + block := "" + if idx >= 0 { + end := idx + 2000 + if end > len(body) { + end = len(body) + } + block = body[idx:end] + } + + desc := "" + if dm := reSearchDesc.FindStringSubmatch(block); dm != nil { + desc = cleanStr(dm[1]) + } + stars := 0 + if sm := reSearchStars.FindStringSubmatch(block); sm != nil { + stars = cleanInt(sm[1]) + } + lang := first(reSearchLang, block) + updatedAt := first(reSearchDate, block) + + out = append(out, SearchRepo{ + Rank: i + 1, + FullName: fullName, + Description: desc, + Language: lang, + Stars: stars, + UpdatedAt: updatedAt, + URL: "https://github.com/" + fullName, + }) + } + return out +} + +// ── ParseFollowers / ParseFollowing ───────────────────────────────────────── + +// ParseFollowers parses the ?tab=followers HTML page. +func ParseFollowers(body string) []User { + return parseUserGrid(body) +} + +// ParseFollowing parses the ?tab=following HTML page. +func ParseFollowing(body string) []User { + return parseUserGrid(body) +} + +func parseUserGrid(body string) []User { + loginMatches := reFollowerLogin.FindAllStringSubmatch(body, -1) + out := make([]User, 0, len(loginMatches)) + seen := map[string]bool{} + for _, lm := range loginMatches { + login := cleanStr(lm[1]) + if login == "" || seen[login] { + continue + } + // skip orgs and special pages + if strings.Contains(login, "/") || strings.HasPrefix(login, "?") { + continue + } + seen[login] = true + + // carve out a block near this login to look for display name + idx := strings.Index(body, lm[0]) + name := "" + if idx >= 0 { + end := idx + 500 + if end > len(body) { + end = len(body) + } + block := body[idx:end] + name = first(reFollowerName, block) + } + + out = append(out, User{ + Login: login, + Name: name, + URL: userURL(login), + }) + } + return out +} + +// ── ParseStars ─────────────────────────────────────────────────────────────── + +// ParseStars parses the ?tab=stars HTML page. +func ParseStars(body string) []StarredRepo { + nameMatches := reStarName.FindAllStringSubmatch(body, -1) + if len(nameMatches) == 0 { + nameMatches = reStarNameB.FindAllStringSubmatch(body, -1) + } + out := make([]StarredRepo, 0, len(nameMatches)) + seen := map[string]bool{} + for _, nm := range nameMatches { + fullName := nm[1] + if fullName == "" || seen[fullName] { + continue + } + if !strings.Contains(fullName, "/") { + continue + } + seen[fullName] = true + + idx := strings.Index(body, nm[0]) + block := "" + if idx >= 0 { + end := idx + 1000 + if end > len(body) { + end = len(body) + } + block = body[idx:end] + } + + desc := "" + if dm := reStarDesc.FindStringSubmatch(block); dm != nil { + desc = cleanStr(dm[1]) + } + lang := first(reStarLang, block) + stars := 0 + if sm := reStarStars.FindStringSubmatch(block); sm != nil { + stars = cleanInt(sm[1]) + } + + out = append(out, StarredRepo{ + FullName: fullName, + Description: desc, + Language: lang, + Stars: stars, + URL: "https://github.com/" + fullName, + }) + } + return out +} diff --git a/github/types.go b/github/types.go index f45ce98..a8579f1 100644 --- a/github/types.go +++ b/github/types.go @@ -1,130 +1,130 @@ +// Package github is the scraper library behind the github CLI. +// It reads public GitHub data from HTML pages, Atom feeds, and +// raw.githubusercontent.com. No API key or authentication is required. +// +// github is an independent tool and is not affiliated with GitHub or Microsoft. package github -import "fmt" - -// Repo is the record emitted for repository commands. -type Repo struct { - Rank int `json:"rank"` - FullName string `json:"full_name"` - Description string `json:"description"` - Language string `json:"language"` - Stars int `json:"stars"` - Forks int `json:"forks"` - License string `json:"license"` - PushedAt string `json:"pushed_at"` - URL string `json:"url"` +// TrendingRepo is one entry from the GitHub trending page. +type TrendingRepo struct { + Rank int `json:"rank" table:"Rank,right"` + FullName string `json:"full_name" table:"Repo"` + Description string `json:"description" table:"Description"` + Language string `json:"language" table:"Lang"` + Stars int `json:"stars" table:"Stars,right"` + Forks int `json:"forks" table:"Forks,right"` + PeriodStars int `json:"period_stars" table:"New Stars,right"` + URL string `json:"url" table:"-" kit:"url"` } -// User is the record emitted for user commands. +// User is a GitHub user profile record. +// It is also used for the followers and following listings; +// counts are 0 on listing pages where they are not shown. type User struct { - Login string `json:"login"` - Name string `json:"name"` - Company string `json:"company"` - Location string `json:"location"` - Followers int `json:"followers"` - Repos int `json:"repos"` - Bio string `json:"bio"` - URL string `json:"url"` + Login string `json:"login" table:"Login"` + Name string `json:"name" table:"Name"` + Bio string `json:"bio" table:"-"` + Company string `json:"company" table:"Company"` + Location string `json:"location" table:"Location"` + Email string `json:"email" table:"-"` + Blog string `json:"blog" table:"-"` + Followers int `json:"followers" table:"Followers,right"` + Following int `json:"following" table:"Following,right"` + Repos int `json:"repos" table:"Repos,right"` + URL string `json:"url" table:"-" kit:"url"` } -// Release is the record emitted for the releases command. -type Release struct { - Rank int `json:"rank"` - TagName string `json:"tag_name"` - Name string `json:"name"` - Prerelease bool `json:"prerelease"` - CreatedAt string `json:"created_at"` - URL string `json:"url"` +// Repo is a repository record used by both repo (single) and repos (list). +type Repo struct { + FullName string `json:"full_name" table:"Repo"` + Description string `json:"description" table:"Description"` + Language string `json:"language" table:"Lang"` + Stars int `json:"stars" table:"Stars,right"` + Forks int `json:"forks" table:"Forks,right"` + Watchers int `json:"watchers" table:"-"` + OpenIssues int `json:"open_issues" table:"Issues,right"` + DefaultBranch string `json:"default_branch" table:"-"` + License string `json:"license" table:"License"` + Topics []string `json:"topics" table:"-"` + Fork bool `json:"fork" table:"-"` + Archived bool `json:"archived" table:"-"` + PushedAt string `json:"pushed_at" table:"Pushed"` + CreatedAt string `json:"created_at" table:"-"` + UpdatedAt string `json:"updated_at" table:"-"` + URL string `json:"url" table:"-" kit:"url"` } -// ─── wire types from GitHub REST API ───────────────────────────────────────── - -type wireRepo struct { - ID int `json:"id"` - FullName string `json:"full_name"` - Description *string `json:"description"` - HTMLURL string `json:"html_url"` - Stars int `json:"stargazers_count"` - Forks int `json:"forks_count"` - Language *string `json:"language"` - License *struct { - SPDXID string `json:"spdx_id"` - } `json:"license"` - PushedAt string `json:"pushed_at"` +// Commit is one entry from the commits Atom feed. +type Commit struct { + SHA string `json:"sha" table:"SHA"` + Message string `json:"message" table:"Message"` + Author string `json:"author" table:"Author"` + Date string `json:"date" table:"Date"` + URL string `json:"url" table:"-" kit:"url"` } -type wireUser struct { - Login string `json:"login"` - Name *string `json:"name"` - Company *string `json:"company"` - Location *string `json:"location"` - Bio *string `json:"bio"` - PublicRepos int `json:"public_repos"` - Followers int `json:"followers"` - HTMLURL string `json:"html_url"` +// Release is one entry from the releases Atom feed. +type Release struct { + Tag string `json:"tag" table:"Tag"` + Name string `json:"name" table:"Name"` + Author string `json:"author" table:"Author"` + Published string `json:"published" table:"Published"` + URL string `json:"url" table:"-" kit:"url"` } -type wireRelease struct { - TagName string `json:"tag_name"` - Name string `json:"name"` - Prerelease bool `json:"prerelease"` - Draft bool `json:"draft"` - CreatedAt string `json:"created_at"` - HTMLURL string `json:"html_url"` +// Tag is one entry from the tags Atom feed. +type Tag struct { + Name string `json:"name" table:"Tag"` + Updated string `json:"updated" table:"Updated"` + URL string `json:"url" table:"-" kit:"url"` } -type searchReposResp struct { - TotalCount int `json:"total_count"` - Items []wireRepo `json:"items"` +// Issue is one issue row scraped from the issues HTML page. +type Issue struct { + Number int `json:"number" table:"#,right"` + Title string `json:"title" table:"Title"` + State string `json:"state" table:"State"` + Author string `json:"author" table:"Author"` + Comments int `json:"comments" table:"Comments,right"` + Labels string `json:"labels" table:"Labels"` + CreatedAt string `json:"created_at" table:"Created"` + URL string `json:"url" table:"-" kit:"url"` } -// ─── converters ────────────────────────────────────────────────────────────── - -func deref(s *string) string { - if s == nil { - return "" - } - return *s +// PullRequest is one PR row scraped from the pulls HTML page. +type PullRequest struct { + Number int `json:"number" table:"#,right"` + Title string `json:"title" table:"Title"` + State string `json:"state" table:"State"` + Author string `json:"author" table:"Author"` + Comments int `json:"comments" table:"Comments,right"` + CreatedAt string `json:"created_at" table:"Created"` + URL string `json:"url" table:"-" kit:"url"` } -func wireRepoToRepo(w wireRepo, rank int) Repo { - lic := "" - if w.License != nil { - lic = w.License.SPDXID - } - return Repo{ - Rank: rank, - FullName: w.FullName, - Description: deref(w.Description), - Language: deref(w.Language), - Stars: w.Stars, - Forks: w.Forks, - License: lic, - PushedAt: w.PushedAt, - URL: w.HTMLURL, - } +// SearchRepo is one repository card from the search results page. +type SearchRepo struct { + Rank int `json:"rank" table:"Rank,right"` + FullName string `json:"full_name" table:"Repo"` + Description string `json:"description" table:"Description"` + Language string `json:"language" table:"Lang"` + Stars int `json:"stars" table:"Stars,right"` + UpdatedAt string `json:"updated_at" table:"Updated"` + URL string `json:"url" table:"-" kit:"url"` } -func wireUserToUser(w wireUser) User { - return User{ - Login: w.Login, - Name: deref(w.Name), - Company: deref(w.Company), - Location: deref(w.Location), - Followers: w.Followers, - Repos: w.PublicRepos, - Bio: deref(w.Bio), - URL: fmt.Sprintf("https://github.com/%s", w.Login), - } +// StarredRepo is one repository card from the stars tab. +type StarredRepo struct { + FullName string `json:"full_name" table:"Repo"` + Description string `json:"description" table:"Description"` + Language string `json:"language" table:"Lang"` + Stars int `json:"stars" table:"Stars,right"` + URL string `json:"url" table:"-" kit:"url"` } -func wireReleaseToRelease(w wireRelease, rank int) Release { - return Release{ - Rank: rank, - TagName: w.TagName, - Name: w.Name, - Prerelease: w.Prerelease, - CreatedAt: w.CreatedAt, - URL: w.HTMLURL, - } +// FileContent is the result of the readme and file commands. +type FileContent struct { + Path string `json:"path" table:"Path"` + Content string `json:"content" table:"-"` + URL string `json:"url" table:"-" kit:"url"` } diff --git a/go.mod b/go.mod index 667b111..fdb0055 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26 require ( github.com/charmbracelet/fang v1.0.0 - github.com/mattn/go-isatty v0.0.22 github.com/spf13/cobra v1.10.2 + github.com/tamnd/any-cli v0.4.0 ) require ( @@ -20,18 +20,27 @@ require ( github.com/clipperhouse/displaywidth v0.4.1 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/mango v0.1.0 // indirect github.com/muesli/mango-cobra v1.2.0 // indirect github.com/muesli/mango-pflag v0.1.0 // indirect github.com/muesli/roff v0.1.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect + golang.org/x/sync v0.20.0 // indirect + golang.org/x/sys v0.42.0 // indirect golang.org/x/text v0.24.0 // indirect + modernc.org/libc v1.72.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.52.0 // indirect ) diff --git a/go.sum b/go.sum index e7d4564..280027e 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,14 @@ github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsV github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= @@ -47,8 +55,12 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -58,17 +70,51 @@ github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tamnd/any-cli v0.4.0 h1:ngyRJBvjZ2X1iBlwlmDLvY2S9aQWlDjVE7CiOwxtt5Y= +github.com/tamnd/any-cli v0.4.0/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= +modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= +modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.52.0 h1:p4dhYh2tXZCiyaqHwRVJDjIGKWyXayiQpThxgDzJaxo= +modernc.org/sqlite v1.52.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= From b37197ef9257cf3194c09b75634515353451ed35 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Fri, 19 Jun 2026 22:48:16 +0700 Subject: [PATCH 02/21] chore: upgrade GitHub Actions to latest versions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Node.js 20 is being deprecated in the Actions runtime. actions/checkout → v7.0.0 browser-actions/setup-chrome → v2.1.2 golangci/golangci-lint-action → v9.2.1 goreleaser/goreleaser-action → v7.2.2 docker/setup-qemu-action → v4.1.0 docker/setup-buildx-action → v4.1.0 docker/login-action → v4.2.0 sigstore/cosign-installer → v4.1.2 anchore/sbom-action → v0.24.0 --- .github/workflows/docs.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a9d2589..8530157 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,14 +26,14 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: submodules: true # Sitemap lastmod comes from the latest content commit. fetch-depth: 0 - name: Checkout tago - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v7.0.0 with: repository: tamnd/tago path: .tago-src @@ -107,7 +107,7 @@ jobs: group: cloudflare-pages-github-cli cancel-in-progress: true steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 1 sparse-checkout: scripts/ From 901392069bdb94ad3b1cfe82593e0c6ea57f91e8 Mon Sep 17 00:00:00 2001 From: tamnd <1218621+tamnd@users.noreply.github.com> Date: Sat, 25 Jul 2026 17:42:00 +0700 Subject: [PATCH 03/21] gh: keyless core, repository reader, and the nine search types The foundation of the rewrite: URI parsing, the surface table, the per-field merge, the repository page decoder, and every search type that answers without a session. Search is the widest surface here. Nine of the ten types return real results to an anonymous client. The tenth is code search, which answers 200 with an empty result set, so it exits 7 with a message saying why rather than reporting no matches. Three things the live probes settled: The sidebarAbout sections block is mostly booleans meaning "this box is on the page", with only releases and usedBy arriving as objects. A typed struct there loses the whole block the first time a member is true instead of {}, so each member is decoded on its own. There is no keyless language histogram. /graphs/languages 301s back to the repository page, show_partial and the other guesses 404, and the sidebar language bar is a loading skeleton on a cold fetch. The primary language comes from a repository search instead, one hop, recorded in _via so nobody mistakes it for a byte count. Tree entry ids use owner/name@ref/path with a slash. A colon there put the filename inside the ref and produced tree URLs nobody could follow. --- gh/base.go | 254 +++++++++++++ gh/client.go | 506 ++++++++++++++++++++++++++ gh/errors.go | 121 +++++++ gh/gh.go | 55 +++ gh/live_test.go | 263 ++++++++++++++ gh/merge.go | 262 ++++++++++++++ gh/repo.go | 519 +++++++++++++++++++++++++++ gh/search.go | 812 ++++++++++++++++++++++++++++++++++++++++++ gh/surface.go | 198 ++++++++++ gh/types.go | 761 +++++++++++++++++++++++++++++++++++++++ gh/uri.go | 580 ++++++++++++++++++++++++++++++ go.mod | 9 +- go.sum | 12 + pkg/page/compact.go | 68 ++++ pkg/page/dom.go | 285 +++++++++++++++ pkg/page/page.go | 252 +++++++++++++ pkg/page/scan.go | 157 ++++++++ pkg/page/selectors.go | 172 +++++++++ 18 files changed, 5282 insertions(+), 4 deletions(-) create mode 100644 gh/base.go create mode 100644 gh/client.go create mode 100644 gh/errors.go create mode 100644 gh/gh.go create mode 100644 gh/live_test.go create mode 100644 gh/merge.go create mode 100644 gh/repo.go create mode 100644 gh/search.go create mode 100644 gh/surface.go create mode 100644 gh/types.go create mode 100644 gh/uri.go create mode 100644 pkg/page/compact.go create mode 100644 pkg/page/dom.go create mode 100644 pkg/page/page.go create mode 100644 pkg/page/scan.go create mode 100644 pkg/page/selectors.go diff --git a/gh/base.go b/gh/base.go new file mode 100644 index 0000000..6ac0744 --- /dev/null +++ b/gh/base.go @@ -0,0 +1,254 @@ +package gh + +import ( + "encoding/json" + "reflect" + "sort" + "strings" + "sync" + "time" +) + +// base.go holds what every record has in common and the guard that keeps the +// records honest. +// +// The guard is decodeExtra. Every decoder runs it, and it puts anything the +// struct did not claim into Extra. The scenario suite then asserts Extra is +// empty. The effect is that the day GitHub adds a field, a test fails and +// names it, instead of the field being silently dropped for a year. + +// Base is embedded in every record. Kind and ID are the identity, URI and URL +// are the two addresses, Sources records where the fields came from, and Extra +// is the data-loss guard. +type Base struct { + Kind string `json:"kind" table:"kind"` + ID string `json:"id" table:"id" kit:"id"` + URI string `json:"uri,omitempty" table:"-"` + URL string `json:"url,omitempty" table:"-"` + Sources []string `json:"sources,omitempty" table:"-"` + Extra json.RawMessage `json:"extra,omitempty" table:"-"` +} + +// setIdentity fills Kind, ID, URI, and URL from a kind and an id. Every +// constructor calls it, so no record can exist with a URI that disagrees with +// its id. +func (b *Base) setIdentity(kind, id string) { + b.Kind = kind + b.ID = id + b.URI = URI(kind, id) + if u, err := Locate(kind, id); err == nil { + b.URL = u + } +} + +// addSource records a URL a field came from. Duplicates are dropped and the +// order is the order they were read, which makes the field useful for +// debugging a merge as well as for provenance. +func (b *Base) addSource(urls ...string) { + for _, u := range urls { + if u == "" { + continue + } + if !contains(b.Sources, u) { + b.Sources = append(b.Sources, u) + } + } +} + +// addExtra files a block of unmodelled keys under the name of the payload block +// they came from. Namespacing matters: "twelve unknown keys" is not actionable, +// "twelve unknown keys in sidebarAbout" is. +func (b *Base) addExtra(name string, raw json.RawMessage) { + if len(raw) == 0 { + return + } + m := map[string]json.RawMessage{} + if len(b.Extra) > 0 { + if err := json.Unmarshal(b.Extra, &m); err != nil { + return + } + } + m[name] = raw + out, err := json.Marshal(m) + if err != nil { + return + } + b.Extra = out +} + +func contains(ss []string, s string) bool { + for _, x := range ss { + if x == s { + return true + } + } + return false +} + +// Actor is a person or an organization as it appears inside another record: on +// a commit, an issue, a release. It is deliberately small. The full account is +// a separate read, and inlining it would turn one request into hundreds. +// Both id forms are kept. The numeric one is what avatar URLs and the older +// links use, the base64 global one is what Relay results carry, and a joiner +// downstream will have one or the other and not both. +type Actor struct { + Login string `json:"login" table:"login"` + Name string `json:"name,omitempty" table:"name"` + Type string `json:"type,omitempty" table:"-"` + NodeID string `json:"node_id,omitempty" table:"-"` + DatabaseID *int `json:"database_id,omitempty" table:"-"` + AvatarURL string `json:"avatar_url,omitempty" table:"-"` + URL string `json:"url,omitempty" table:"-"` + URI string `json:"uri,omitempty" table:"-"` +} + +// actor builds an Actor from a login, filling the derived fields. An empty +// login gives an empty Actor rather than one with a URL to nowhere. +func actor(login string) Actor { + if login == "" { + return Actor{} + } + return Actor{ + Login: login, + URL: BaseURL + "/" + login, + URI: URI(KindUser, login), + } +} + +// --- the data-loss guard --- + +// decodeExtra returns the keys of raw that v did not claim, minus the keys in +// skip. It is what stands between this tool and silently dropping a field +// GitHub added last Tuesday. +// +// skip lists are explicit, short, and commented one entry at a time. A key +// dropped without a reason is a bug waiting to be found by someone six months +// from now, so the convention is that every skip list entry says why. +func decodeExtra(raw json.RawMessage, v any, skip ...string) json.RawMessage { + if len(raw) == 0 { + return nil + } + var m map[string]json.RawMessage + if err := json.Unmarshal(raw, &m); err != nil { + return nil + } + for _, k := range claimedKeys(reflect.TypeOf(v)) { + delete(m, k) + } + for _, k := range skip { + if strings.HasSuffix(k, "*") { + prefix := strings.TrimSuffix(k, "*") + for key := range m { + if strings.HasPrefix(key, prefix) { + delete(m, key) + } + } + continue + } + delete(m, k) + } + // Drop the keys whose value is null or an empty container. A key GitHub + // sends as null carries no information, and reporting it as unmodelled + // data would make Extra noisy enough that nobody would read it. + for k, val := range m { + if isEmptyJSON(val) { + delete(m, k) + } + } + if len(m) == 0 { + return nil + } + out, err := json.Marshal(m) + if err != nil { + return nil + } + return out +} + +func isEmptyJSON(v json.RawMessage) bool { + s := strings.TrimSpace(string(v)) + return s == "" || s == "null" || s == "{}" || s == "[]" || s == `""` +} + +// claimedKeys walks a struct's json tags, following embedded structs, and +// returns every key the type would decode. +var claimedCache sync.Map // reflect.Type -> []string + +func claimedKeys(t reflect.Type) []string { + for t != nil && t.Kind() == reflect.Pointer { + t = t.Elem() + } + if t == nil || t.Kind() != reflect.Struct { + return nil + } + if v, ok := claimedCache.Load(t); ok { + return v.([]string) + } + seen := map[string]bool{} + var walk func(reflect.Type) + walk = func(t reflect.Type) { + for i := range t.NumField() { + f := t.Field(i) + if !f.IsExported() { + continue + } + tag := f.Tag.Get("json") + name, _, _ := strings.Cut(tag, ",") + if name == "-" { + continue + } + if f.Anonymous && name == "" { + ft := f.Type + for ft.Kind() == reflect.Pointer { + ft = ft.Elem() + } + if ft.Kind() == reflect.Struct { + walk(ft) + continue + } + } + if name == "" { + name = f.Name + } + seen[name] = true + } + } + walk(t) + keys := make([]string, 0, len(seen)) + for k := range seen { + keys = append(keys, k) + } + sort.Strings(keys) + claimedCache.Store(t, keys) + return keys +} + +// --- small shared helpers --- + +// parseTime accepts the three time formats GitHub uses across its surfaces: +// RFC 3339 with a zone, RFC 3339 in UTC with a Z, and the datetime attribute +// on aThe Go programming language
- Go - - 123,456 - - - 18,900 - - - 42 stars today - -Production-Grade Container Scheduling
- Go - - 112,000 - - - 40,200 - - - 37 stars today - -Linux kernel source tree
- C - 182,000 - 52,000 -My personal dive log application
- C++ - 1,200 - 400 -The Go programming language
-Go -112,000 stars -Empowering everyone to build reliable and efficient software.
-Rust -90,000 stars -]*class="[^"]*col-9[^"]*"[^>]*>(.*?)
`) - reTrendingLang = regexp.MustCompile(`itemprop="programmingLanguage"[^>]*>\s*([^<]+?)\s*<`) - reTrendingStars = regexp.MustCompile(`href="[^"]+/stargazers"[^>]*>\s*(?:<[^>]+>)*\s*([0-9,]+)`) - reTrendingForks = regexp.MustCompile(`href="[^"]+/network/members"[^>]*>\s*(?:<[^>]+>)*\s*([0-9,]+)`) - reTrendingPeriod = regexp.MustCompile(`([0-9,]+)\s+stars?\s+(?:today|this week|this month)`) -) - -// User profile -var ( - reUserName = regexp.MustCompile(`(?:itemprop="name"|class="[^"]*p-name[^"]*")[^>]*>\s*([^<\n]+?)\s*<`) - reUserBio = regexp.MustCompile(`class="[^"]*p-note[^"]*"[^>]*>\s*]*class="[^"]*mb-1[^"]*"[^>]*>\s*(.*?)\s*
`) - reSearchStars = regexp.MustCompile(`([0-9,]+)\s+stars?`) - reSearchLang = regexp.MustCompile(`(?s)]*class="[^"]*search-match[^"]*"[^>]*>([^<]+)<`) - reSearchDate = regexp.MustCompile(`]*class="[^"]*col-9[^"]*"[^>]*>(.*?)
`) - reStarLang = regexp.MustCompile(`itemprop="programmingLanguage"[^>]*>([^<]+)<`) - reStarStars = regexp.MustCompile(`href="[^"]+/stargazers"[^>]*>\s*([0-9,]+)`) -) - -// ── helpers ────────────────────────────────────────────────────────────────── - -// cleanInt parses a comma-formatted or k-suffixed integer string. -// "12,345" → 12345; "3.2k" → 3200; returns 0 on failure. -func cleanInt(s string) int { - s = strings.TrimSpace(s) - if s == "" { - return 0 - } - // handle k suffix - if strings.HasSuffix(s, "k") || strings.HasSuffix(s, "K") { - f, err := strconv.ParseFloat(strings.ReplaceAll(s[:len(s)-1], ",", ""), 64) - if err != nil { - return 0 - } - return int(f * 1000) - } - s = strings.ReplaceAll(s, ",", "") - n, _ := strconv.Atoi(s) - return n -} - -// cleanStr strips HTML tags, decodes HTML entities, and trims whitespace. -func cleanStr(s string) string { - // strip tags - reTag := regexp.MustCompile(`<[^>]+>`) - s = reTag.ReplaceAllString(s, " ") - // decode entities - s = html.UnescapeString(s) - // collapse whitespace - reWS := regexp.MustCompile(`\s+`) - s = reWS.ReplaceAllString(s, " ") - return strings.TrimSpace(s) -} - -// extractSHA extracts the commit SHA from a GitHub Atom entry ID. -// IDs look like: tag:github.com,2008:Grit::Commit/abc1234567890 -func extractSHA(id string) string { - if idx := strings.LastIndex(id, "/"); idx >= 0 { - return id[idx+1:] - } - return id -} - -// first returns the first capture group match, or "". -func first(re *regexp.Regexp, s string) string { - m := re.FindStringSubmatch(s) - if m == nil || len(m) < 2 { - return "" - } - return cleanStr(m[1]) -} - -// ── ParseTrending ──────────────────────────────────────────────────────────── - -// ParseTrending parses the github.com/trending HTML page. -func ParseTrending(body string) []TrendingRepo { - articles := reTrendingArticle.FindAllStringSubmatch(body, -1) - out := make([]TrendingRepo, 0, len(articles)) - for i, m := range articles { - block := m[1] - - fullName := first(reTrendingLink, block) - if fullName == "" { - continue - } - - desc := "" - if dm := reTrendingDesc.FindStringSubmatch(block); dm != nil { - desc = cleanStr(dm[1]) - } - - lang := first(reTrendingLang, block) - stars := 0 - if sm := reTrendingStars.FindStringSubmatch(block); sm != nil { - stars = cleanInt(sm[1]) - } - forks := 0 - if fm := reTrendingForks.FindStringSubmatch(block); fm != nil { - forks = cleanInt(fm[1]) - } - period := 0 - if pm := reTrendingPeriod.FindStringSubmatch(block); pm != nil { - period = cleanInt(pm[1]) - } - - out = append(out, TrendingRepo{ - Rank: i + 1, - FullName: fullName, - Description: desc, - Language: lang, - Stars: stars, - Forks: forks, - PeriodStars: period, - URL: "https://github.com/" + fullName, - }) - } - return out -} - -// ── ParseUser ──────────────────────────────────────────────────────────────── - -// ParseUser parses a github.com/{username} profile page. -func ParseUser(body, username string) (User, error) { - name := first(reUserName, body) - - bio := "" - if bm := reUserBio.FindStringSubmatch(body); bm != nil { - bio = cleanStr(bm[1]) - } else if bm2 := reUserBioAlt.FindStringSubmatch(body); bm2 != nil { - bio = cleanStr(bm2[1]) - } - - company := first(reUserCompany, body) - location := first(reUserLocation, body) - email := first(reUserEmail, body) - blog := first(reUserBlog, body) - - followers := 0 - if fm := reUserFollowers.FindStringSubmatch(body); fm != nil { - followers = cleanInt(fm[1]) - } - following := 0 - if fm := reUserFollowing.FindStringSubmatch(body); fm != nil { - following = cleanInt(fm[1]) - } - repos := 0 - if rm := reUserRepos.FindStringSubmatch(body); rm != nil { - repos = cleanInt(rm[1]) - } - - return User{ - Login: username, - Name: name, - Bio: bio, - Company: company, - Location: location, - Email: email, - Blog: blog, - Followers: followers, - Following: following, - Repos: repos, - URL: userURL(username), - }, nil -} - -// ── ParseRepos ─────────────────────────────────────────────────────────────── - -// ParseRepos parses the github.com/{username}?tab=repositories HTML page. -func ParseRepos(body, username string) []Repo { - items := reRepoItem.FindAllStringSubmatch(body, -1) - out := make([]Repo, 0, len(items)) - for _, m := range items { - block := m[1] - - fullName := first(reRepoItemLink, block) - if fullName == "" { - continue - } - - desc := "" - if dm := reRepoItemDesc.FindStringSubmatch(block); dm != nil { - desc = cleanStr(dm[1]) - } - - lang := first(reRepoItemLang, block) - stars := 0 - if sm := reRepoItemStars.FindStringSubmatch(block); sm != nil { - stars = cleanInt(sm[1]) - } - forks := 0 - if fm := reRepoItemForks.FindStringSubmatch(block); fm != nil { - forks = cleanInt(fm[1]) - } - pushedAt := first(reRepoItemDate, block) - - out = append(out, Repo{ - FullName: fullName, - Description: desc, - Language: lang, - Stars: stars, - Forks: forks, - PushedAt: pushedAt, - URL: "https://github.com/" + fullName, - }) - } - return out -} - -// ── ParseRepo ──────────────────────────────────────────────────────────────── - -// ParseRepo parses the github.com/{owner}/{repo} page. -func ParseRepo(body, owner, repo string) (Repo, error) { - fullName := owner + "/" + repo - - desc := "" - if dm := reRepoDesc.FindStringSubmatch(body); dm != nil { - desc = cleanStr(dm[1]) - } else if dm2 := reRepoDescAlt.FindStringSubmatch(body); dm2 != nil { - desc = cleanStr(dm2[1]) - } - - lang := "" - if lm := reRepoLang.FindStringSubmatch(body); lm != nil { - lang = cleanStr(lm[1]) - } else if lm2 := reRepoLangAlt.FindStringSubmatch(body); lm2 != nil { - lang = cleanStr(lm2[1]) - } - - stars := 0 - if sm := reRepoStars.FindStringSubmatch(body); sm != nil { - stars = cleanInt(sm[1]) - } - forks := 0 - if fm := reRepoForks.FindStringSubmatch(body); fm != nil { - forks = cleanInt(fm[1]) - } - openIssues := 0 - if im := reRepoIssues.FindStringSubmatch(body); im != nil { - openIssues = cleanInt(im[1]) - } - watchers := 0 - if wm := reRepoWatchers.FindStringSubmatch(body); wm != nil { - watchers = cleanInt(wm[1]) - } - - // topics - topicMatches := reRepoTopics.FindAllStringSubmatch(body, -1) - topics := make([]string, 0, len(topicMatches)) - for _, tm := range topicMatches { - t := cleanStr(tm[1]) - if t != "" { - topics = append(topics, t) - } - } - - license := first(reRepoLicense, body) - branch := first(reRepoBranch, body) - if branch == "" { - branch = "main" - } - - isFork := reRepoFork.MatchString(body) - isArchived := reRepoArchive.MatchString(body) - - return Repo{ - FullName: fullName, - Description: desc, - Language: lang, - Stars: stars, - Forks: forks, - Watchers: watchers, - OpenIssues: openIssues, - DefaultBranch: branch, - License: license, - Topics: topics, - Fork: isFork, - Archived: isArchived, - URL: repoURL(owner, repo), - }, nil -} - -// ── Atom feeds ─────────────────────────────────────────────────────────────── - -// ParseAtomCommits parses the /commits/{branch}.atom feed. -func ParseAtomCommits(body string) ([]Commit, error) { - var feed atomFeed - if err := xml.Unmarshal([]byte(body), &feed); err != nil { - return nil, err - } - out := make([]Commit, 0, len(feed.Entries)) - for _, e := range feed.Entries { - sha := extractSHA(e.ID) - if len(sha) > 7 { - sha = sha[:7] - } - out = append(out, Commit{ - SHA: sha, - Message: strings.TrimSpace(e.Title), - Author: strings.TrimSpace(e.Author.Name), - Date: e.Published, - URL: e.Link.Href, - }) - } - return out, nil -} - -// ParseAtomReleases parses the /releases.atom feed. -func ParseAtomReleases(body string) ([]Release, error) { - var feed atomFeed - if err := xml.Unmarshal([]byte(body), &feed); err != nil { - return nil, err - } - out := make([]Release, 0, len(feed.Entries)) - for _, e := range feed.Entries { - tag := lastPathSegment(e.Link.Href) - out = append(out, Release{ - Tag: tag, - Name: strings.TrimSpace(e.Title), - Author: strings.TrimSpace(e.Author.Name), - Published: e.Published, - URL: e.Link.Href, - }) - } - return out, nil -} - -// ParseAtomTags parses the /tags.atom feed. -func ParseAtomTags(body string) ([]Tag, error) { - var feed atomFeed - if err := xml.Unmarshal([]byte(body), &feed); err != nil { - return nil, err - } - out := make([]Tag, 0, len(feed.Entries)) - for _, e := range feed.Entries { - out = append(out, Tag{ - Name: strings.TrimSpace(e.Title), - Updated: e.Updated, - URL: e.Link.Href, - }) - } - return out, nil -} - -// ── ParseIssues ────────────────────────────────────────────────────────────── - -// ParseIssues parses the /{owner}/{repo}/issues HTML page. -func ParseIssues(body, owner, repo, state string) []Issue { - // find all issue-N id blocks - matches := reIssueBlock.FindAllStringSubmatch(body, -1) - out := make([]Issue, 0, len(matches)) - for _, m := range matches { - numStr := m[1] - block := m[2] - num, _ := strconv.Atoi(numStr) - if num == 0 { - continue - } - - title := "" - issueURL := "" - if tm := reIssueTitle.FindStringSubmatch(block); tm != nil { - issueURL = "https://github.com" + tm[1] - title = cleanStr(tm[2]) - } - - createdAt := first(reIssueDate, block) - author := first(reIssueAuthor, block) - - labelMatches := reIssueLabel.FindAllStringSubmatch(block, -1) - labels := make([]string, 0, len(labelMatches)) - for _, lm := range labelMatches { - labels = append(labels, cleanStr(lm[1])) - } - - comments := 0 - if cm := reIssueComments.FindStringSubmatch(block); cm != nil { - comments, _ = strconv.Atoi(cm[1]) - } - - if issueURL == "" { - issueURL = "https://github.com/" + owner + "/" + repo + "/issues/" + numStr - } - - out = append(out, Issue{ - Number: num, - Title: title, - State: state, - Author: author, - Comments: comments, - Labels: strings.Join(labels, ", "), - CreatedAt: createdAt, - URL: issueURL, - }) - } - return out -} - -// ── ParsePulls ─────────────────────────────────────────────────────────────── - -// ParsePulls parses the /{owner}/{repo}/pulls HTML page. -// The PR list uses the same HTML structure as issues with a different URL path. -func ParsePulls(body, owner, repo, state string) []PullRequest { - // use the same issue_N id structure - matches := reIssueBlock.FindAllStringSubmatch(body, -1) - out := make([]PullRequest, 0, len(matches)) - for _, m := range matches { - numStr := m[1] - block := m[2] - num, _ := strconv.Atoi(numStr) - if num == 0 { - continue - } - - title := "" - prURL := "" - prNum := num - if tm := rePRTitle.FindStringSubmatch(block); tm != nil { - prURL = "https://github.com" + tm[1] - n, _ := strconv.Atoi(tm[2]) - if n > 0 { - prNum = n - } - title = cleanStr(tm[3]) - } else if tm2 := reIssueTitle.FindStringSubmatch(block); tm2 != nil { - prURL = "https://github.com" + tm2[1] - title = cleanStr(tm2[2]) - } - - createdAt := first(reIssueDate, block) - author := first(reIssueAuthor, block) - - comments := 0 - if cm := reIssueComments.FindStringSubmatch(block); cm != nil { - comments, _ = strconv.Atoi(cm[1]) - } - - if prURL == "" { - prURL = "https://github.com/" + owner + "/" + repo + "/pull/" + numStr - } - - out = append(out, PullRequest{ - Number: prNum, - Title: title, - State: state, - Author: author, - Comments: comments, - CreatedAt: createdAt, - URL: prURL, - }) - } - return out -} - -// ── ParseSearch ────────────────────────────────────────────────────────────── - -// ParseSearch parses the github.com/search?type=repositories results page. -func ParseSearch(body string) []SearchRepo { - // find all result items by looking for full_name links - nameMatches := reSearchName.FindAllStringSubmatch(body, -1) - if len(nameMatches) == 0 { - nameMatches = reSearchNameB.FindAllStringSubmatch(body, -1) - } - - out := make([]SearchRepo, 0, len(nameMatches)) - // split body on each result card anchor to get per-card blocks - // Use a simpler approach: find all v-align-middle hrefs - reCard := regexp.MustCompile(`(?s)class="v-align-middle[^"]*"[^>]*href="/([^/"]+/[^/"]+)"[^>]*>.*?(?:class="v-align-middle|$)`) - _ = reCard - - for i, nm := range nameMatches { - fullName := nm[1] - // carve out a block around this match to extract nearby metadata - idx := strings.Index(body, nm[0]) - block := "" - if idx >= 0 { - end := idx + 2000 - if end > len(body) { - end = len(body) - } - block = body[idx:end] - } - - desc := "" - if dm := reSearchDesc.FindStringSubmatch(block); dm != nil { - desc = cleanStr(dm[1]) - } - stars := 0 - if sm := reSearchStars.FindStringSubmatch(block); sm != nil { - stars = cleanInt(sm[1]) - } - lang := first(reSearchLang, block) - updatedAt := first(reSearchDate, block) - - out = append(out, SearchRepo{ - Rank: i + 1, - FullName: fullName, - Description: desc, - Language: lang, - Stars: stars, - UpdatedAt: updatedAt, - URL: "https://github.com/" + fullName, - }) - } - return out -} - -// ── ParseFollowers / ParseFollowing ───────────────────────────────────────── - -// ParseFollowers parses the ?tab=followers HTML page. -func ParseFollowers(body string) []User { - return parseUserGrid(body) -} - -// ParseFollowing parses the ?tab=following HTML page. -func ParseFollowing(body string) []User { - return parseUserGrid(body) -} - -func parseUserGrid(body string) []User { - loginMatches := reFollowerLogin.FindAllStringSubmatch(body, -1) - out := make([]User, 0, len(loginMatches)) - seen := map[string]bool{} - for _, lm := range loginMatches { - login := cleanStr(lm[1]) - if login == "" || seen[login] { - continue - } - // skip orgs and special pages - if strings.Contains(login, "/") || strings.HasPrefix(login, "?") { - continue - } - seen[login] = true - - // carve out a block near this login to look for display name - idx := strings.Index(body, lm[0]) - name := "" - if idx >= 0 { - end := idx + 500 - if end > len(body) { - end = len(body) - } - block := body[idx:end] - name = first(reFollowerName, block) - } - - out = append(out, User{ - Login: login, - Name: name, - URL: userURL(login), - }) - } - return out -} - -// ── ParseStars ─────────────────────────────────────────────────────────────── - -// ParseStars parses the ?tab=stars HTML page. -func ParseStars(body string) []StarredRepo { - nameMatches := reStarName.FindAllStringSubmatch(body, -1) - if len(nameMatches) == 0 { - nameMatches = reStarNameB.FindAllStringSubmatch(body, -1) - } - out := make([]StarredRepo, 0, len(nameMatches)) - seen := map[string]bool{} - for _, nm := range nameMatches { - fullName := nm[1] - if fullName == "" || seen[fullName] { - continue - } - if !strings.Contains(fullName, "/") { - continue - } - seen[fullName] = true - - idx := strings.Index(body, nm[0]) - block := "" - if idx >= 0 { - end := idx + 1000 - if end > len(body) { - end = len(body) - } - block = body[idx:end] - } - - desc := "" - if dm := reStarDesc.FindStringSubmatch(block); dm != nil { - desc = cleanStr(dm[1]) - } - lang := first(reStarLang, block) - stars := 0 - if sm := reStarStars.FindStringSubmatch(block); sm != nil { - stars = cleanInt(sm[1]) - } - - out = append(out, StarredRepo{ - FullName: fullName, - Description: desc, - Language: lang, - Stars: stars, - URL: "https://github.com/" + fullName, - }) - } - return out -} diff --git a/github/types.go b/github/types.go deleted file mode 100644 index a8579f1..0000000 --- a/github/types.go +++ /dev/null @@ -1,130 +0,0 @@ -// Package github is the scraper library behind the github CLI. -// It reads public GitHub data from HTML pages, Atom feeds, and -// raw.githubusercontent.com. No API key or authentication is required. -// -// github is an independent tool and is not affiliated with GitHub or Microsoft. -package github - -// TrendingRepo is one entry from the GitHub trending page. -type TrendingRepo struct { - Rank int `json:"rank" table:"Rank,right"` - FullName string `json:"full_name" table:"Repo"` - Description string `json:"description" table:"Description"` - Language string `json:"language" table:"Lang"` - Stars int `json:"stars" table:"Stars,right"` - Forks int `json:"forks" table:"Forks,right"` - PeriodStars int `json:"period_stars" table:"New Stars,right"` - URL string `json:"url" table:"-" kit:"url"` -} - -// User is a GitHub user profile record. -// It is also used for the followers and following listings; -// counts are 0 on listing pages where they are not shown. -type User struct { - Login string `json:"login" table:"Login"` - Name string `json:"name" table:"Name"` - Bio string `json:"bio" table:"-"` - Company string `json:"company" table:"Company"` - Location string `json:"location" table:"Location"` - Email string `json:"email" table:"-"` - Blog string `json:"blog" table:"-"` - Followers int `json:"followers" table:"Followers,right"` - Following int `json:"following" table:"Following,right"` - Repos int `json:"repos" table:"Repos,right"` - URL string `json:"url" table:"-" kit:"url"` -} - -// Repo is a repository record used by both repo (single) and repos (list). -type Repo struct { - FullName string `json:"full_name" table:"Repo"` - Description string `json:"description" table:"Description"` - Language string `json:"language" table:"Lang"` - Stars int `json:"stars" table:"Stars,right"` - Forks int `json:"forks" table:"Forks,right"` - Watchers int `json:"watchers" table:"-"` - OpenIssues int `json:"open_issues" table:"Issues,right"` - DefaultBranch string `json:"default_branch" table:"-"` - License string `json:"license" table:"License"` - Topics []string `json:"topics" table:"-"` - Fork bool `json:"fork" table:"-"` - Archived bool `json:"archived" table:"-"` - PushedAt string `json:"pushed_at" table:"Pushed"` - CreatedAt string `json:"created_at" table:"-"` - UpdatedAt string `json:"updated_at" table:"-"` - URL string `json:"url" table:"-" kit:"url"` -} - -// Commit is one entry from the commits Atom feed. -type Commit struct { - SHA string `json:"sha" table:"SHA"` - Message string `json:"message" table:"Message"` - Author string `json:"author" table:"Author"` - Date string `json:"date" table:"Date"` - URL string `json:"url" table:"-" kit:"url"` -} - -// Release is one entry from the releases Atom feed. -type Release struct { - Tag string `json:"tag" table:"Tag"` - Name string `json:"name" table:"Name"` - Author string `json:"author" table:"Author"` - Published string `json:"published" table:"Published"` - URL string `json:"url" table:"-" kit:"url"` -} - -// Tag is one entry from the tags Atom feed. -type Tag struct { - Name string `json:"name" table:"Tag"` - Updated string `json:"updated" table:"Updated"` - URL string `json:"url" table:"-" kit:"url"` -} - -// Issue is one issue row scraped from the issues HTML page. -type Issue struct { - Number int `json:"number" table:"#,right"` - Title string `json:"title" table:"Title"` - State string `json:"state" table:"State"` - Author string `json:"author" table:"Author"` - Comments int `json:"comments" table:"Comments,right"` - Labels string `json:"labels" table:"Labels"` - CreatedAt string `json:"created_at" table:"Created"` - URL string `json:"url" table:"-" kit:"url"` -} - -// PullRequest is one PR row scraped from the pulls HTML page. -type PullRequest struct { - Number int `json:"number" table:"#,right"` - Title string `json:"title" table:"Title"` - State string `json:"state" table:"State"` - Author string `json:"author" table:"Author"` - Comments int `json:"comments" table:"Comments,right"` - CreatedAt string `json:"created_at" table:"Created"` - URL string `json:"url" table:"-" kit:"url"` -} - -// SearchRepo is one repository card from the search results page. -type SearchRepo struct { - Rank int `json:"rank" table:"Rank,right"` - FullName string `json:"full_name" table:"Repo"` - Description string `json:"description" table:"Description"` - Language string `json:"language" table:"Lang"` - Stars int `json:"stars" table:"Stars,right"` - UpdatedAt string `json:"updated_at" table:"Updated"` - URL string `json:"url" table:"-" kit:"url"` -} - -// StarredRepo is one repository card from the stars tab. -type StarredRepo struct { - FullName string `json:"full_name" table:"Repo"` - Description string `json:"description" table:"Description"` - Language string `json:"language" table:"Lang"` - Stars int `json:"stars" table:"Stars,right"` - URL string `json:"url" table:"-" kit:"url"` -} - -// FileContent is the result of the readme and file commands. -type FileContent struct { - Path string `json:"path" table:"Path"` - Content string `json:"content" table:"-"` - URL string `json:"url" table:"-" kit:"url"` -} diff --git a/pkg/page/dom.go b/pkg/page/dom.go index 06b298f..051c26e 100644 --- a/pkg/page/dom.go +++ b/pkg/page/dom.go @@ -176,6 +176,139 @@ func Text(n *html.Node) string { func collapse(s string) string { return strings.Join(strings.Fields(s), " ") } +// blockTag is the set of elements that end a line of prose. It does not need to +// be the full HTML block list, only the tags GitHub's renderer actually emits +// into a README, a release note, or a comment body. +var blockTag = map[string]bool{ + "address": true, "article": true, "aside": true, "blockquote": true, + "br": true, "dd": true, "details": true, "div": true, "dl": true, + "dt": true, "figcaption": true, "figure": true, "footer": true, + "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, + "header": true, "hr": true, "li": true, "main": true, "nav": true, + "ol": true, "p": true, "pre": true, "section": true, "summary": true, + "table": true, "tbody": true, "td": true, "th": true, "thead": true, + "tr": true, "ul": true, +} + +// BlockText returns the prose of a subtree with the line structure the markup +// implies, which is what Text deliberately throws away. +// +// Text collapses a whole subtree onto one line, which is right for a label and +// wrong for a document: a twenty-kilobyte README as a single line is not a +// readable rendering of anything. This keeps one line per block element, one +// blank line between paragraphs, and the interior whitespace of a exactly
+// as it was, since indentation is the meaning of a code block rather than
+// decoration on it.
+func BlockText(n *html.Node) string {
+ if n == nil {
+ return ""
+ }
+ var t textLines
+ t.walk(n)
+ return t.done()
+}
+
+// FragmentText is BlockText over an HTML fragment that arrived as a string.
+// Several of GitHub's payloads carry rendered markup as a JSON value rather
+// than as part of the document, so there is no node to walk until this parses
+// one.
+func FragmentText(s string) string {
+ if strings.TrimSpace(s) == "" {
+ return ""
+ }
+ doc, err := html.Parse(strings.NewReader(s))
+ if err != nil {
+ return ""
+ }
+ return BlockText(doc)
+}
+
+// textLines accumulates prose one line at a time. It exists because whether a
+// line keeps its whitespace depends on where the line started, which a single
+// pass over a string builder cannot know after the fact.
+type textLines struct {
+ out []string
+ cur strings.Builder
+ pre int // depth inside
+ raw bool // the line being built started inside a
+}
+
+func (t *textLines) walk(n *html.Node) {
+ switch n.Type {
+ case html.TextNode:
+ t.text(n.Data)
+ return
+ case html.ElementNode:
+ switch n.Data {
+ case "script", "style", "template":
+ return
+ case "pre":
+ t.pre++
+ defer func() { t.pre-- }()
+ }
+ if blockTag[n.Data] {
+ t.brk()
+ }
+ }
+ for c := n.FirstChild; c != nil; c = c.NextSibling {
+ t.walk(c)
+ }
+ if n.Type == html.ElementNode && blockTag[n.Data] {
+ t.brk()
+ }
+}
+
+func (t *textLines) text(s string) {
+ if t.pre == 0 {
+ t.cur.WriteString(s)
+ return
+ }
+ t.raw = true
+ for i, part := range strings.Split(s, "\n") {
+ if i > 0 {
+ t.brk()
+ t.raw = true
+ }
+ t.cur.WriteString(part)
+ }
+}
+
+func (t *textLines) brk() {
+ line := t.cur.String()
+ t.cur.Reset()
+ if t.raw {
+ line = strings.TrimRight(line, " \t\r")
+ } else {
+ line = collapse(line)
+ }
+ t.raw = false
+ t.out = append(t.out, line)
+}
+
+// done joins the lines, dropping runs of blank ones. A rendered document is
+// full of wrapper divs, and one blank line between paragraphs is the intent
+// while six is an artifact of the markup.
+func (t *textLines) done() string {
+ t.brk()
+ var b strings.Builder
+ blank := false
+ for _, line := range t.out {
+ if line == "" {
+ blank = true
+ continue
+ }
+ if blank && b.Len() > 0 {
+ b.WriteString("\n")
+ }
+ blank = false
+ if b.Len() > 0 {
+ b.WriteString("\n")
+ }
+ b.WriteString(line)
+ }
+ return b.String()
+}
+
// RelTime returns the datetime attribute of the first
// descendant. The element's own text is never read: it is localised and
// relative, and parsing it would be a whole class of bug for no gain.
diff --git a/pkg/page/dom_test.go b/pkg/page/dom_test.go
index cf1399e..64f868b 100644
--- a/pkg/page/dom_test.go
+++ b/pkg/page/dom_test.go
@@ -102,3 +102,73 @@ func TestSelMatch(t *testing.T) {
}
}
}
+
+// The markup here is the shape GitHub's markdown renderer emits into a README:
+// a heading, a paragraph broken across source lines, a list, and a fenced code
+// block that came through as .
+func TestBlockText(t *testing.T) {
+ doc := parse(t, `
+gh
+GitHub on
+the command line.
+It brings pull requests to the terminal.
+- one
- two
+func main() {
+ println("hi")
+}
+
+Done.
+ `)
+
+ want := strings.Join([]string{
+ "gh",
+ "",
+ "GitHub on the command line.",
+ "",
+ "It brings pull requests to the terminal.",
+ "",
+ "one",
+ "",
+ "two",
+ "",
+ "func main() {",
+ "\tprintln(\"hi\")",
+ "}",
+ "",
+ "Done.",
+ }, "\n")
+
+ got := BlockText(Find(doc, Sel{Class: "markdown-body"}))
+ if got != want {
+ t.Errorf("BlockText:\n%q\nwant:\n%q", got, want)
+ }
+}
+
+func TestBlockTextKeepsCodeIndentation(t *testing.T) {
+ // A code block's leading whitespace is its meaning, so it survives even
+ // though every other line gets collapsed.
+ got := FragmentText(" indented\n more\n
")
+ if got != " indented\n more" {
+ t.Errorf("FragmentText(pre) = %q", got)
+ }
+}
+
+func TestBlockTextDropsChrome(t *testing.T) {
+ // Wrapper divs are the bulk of GitHub's markup and none of its prose, so a
+ // stack of them must not turn into a stack of blank lines.
+ got := FragmentText(`a
+
+b
`)
+ if got != "a\n\nb" {
+ t.Errorf("FragmentText = %q, want %q", got, "a\n\nb")
+ }
+}
+
+func TestBlockTextEmpty(t *testing.T) {
+ if got := BlockText(nil); got != "" {
+ t.Errorf("BlockText(nil) = %q", got)
+ }
+ if got := FragmentText(" "); got != "" {
+ t.Errorf("FragmentText(blank) = %q", got)
+ }
+}
diff --git a/pkg/page/selectors.go b/pkg/page/selectors.go
index 577e082..4e84cb3 100644
--- a/pkg/page/selectors.go
+++ b/pkg/page/selectors.go
@@ -135,7 +135,11 @@ var (
ProfilePinnedList = Sel{Tag: "ol", Class: "js-pinned-items-reorder-list"}
ProfileOrgAvatar = Sel{Tag: "a", Attr: "data-hovercard-type", AttrValue: "organization"}
ProfileUserLink = Sel{Tag: "a", Attr: "data-hovercard-type", AttrValue: "user"}
- ProfileReadme = Sel{Class: "js-profile-readme"}
+ // ProfileReadme is the box a user profile puts its readme in. An
+ // organization's readme has no class of its own, so the caller falls back
+ // to the markdown article, which is the same on both.
+ // Verified 2026-07-25 against sindresorhus and github.
+ ProfileReadme = Sel{Class: "profile-readme"}
ProfileVCardList = Sel{Class: "vcard-details"}
ProfileAchieve = Sel{Class: "js-profile-achievements"}
)
diff --git a/pkg/render/render.go b/pkg/render/render.go
deleted file mode 100644
index 34393b6..0000000
--- a/pkg/render/render.go
+++ /dev/null
@@ -1,350 +0,0 @@
-// Package render turns slices of record structs into one of the output formats
-// hackernews-cli supports: table, json, jsonl, csv, tsv, url, and raw. It works
-// off struct reflection and json tags, so any record type renders without
-// per-type code.
-package render
-
-import (
- "encoding/csv"
- "encoding/json"
- "fmt"
- "io"
- "reflect"
- "strconv"
- "strings"
- "text/tabwriter"
- "text/template"
- "time"
-)
-
-// Format is an output rendering format.
-type Format string
-
-const (
- FormatTable Format = "table"
- FormatJSON Format = "json"
- FormatJSONL Format = "jsonl"
- FormatCSV Format = "csv"
- FormatTSV Format = "tsv"
- FormatURL Format = "url"
- FormatRaw Format = "raw"
-)
-
-// Valid reports whether f is one of the supported formats.
-func (f Format) Valid() bool {
- switch f {
- case FormatTable, FormatJSON, FormatJSONL, FormatCSV, FormatTSV, FormatURL, FormatRaw:
- return true
- }
- return false
-}
-
-// Renderer writes records in a chosen format.
-type Renderer struct {
- Format Format
- Fields []string
- NoHeader bool
- Template string
- w io.Writer
-}
-
-// New builds a Renderer writing to w.
-func New(w io.Writer, format Format, fields []string, noHeader bool, tmpl string) *Renderer {
- return &Renderer{Format: format, Fields: fields, NoHeader: noHeader, Template: tmpl, w: w}
-}
-
-// Render writes records (a slice of structs, or a single struct) in the configured format.
-func (r *Renderer) Render(records any) error {
- rv := reflect.ValueOf(records)
- if rv.Kind() == reflect.Pointer {
- rv = rv.Elem()
- }
- if rv.Kind() != reflect.Slice {
- s := reflect.MakeSlice(reflect.SliceOf(rv.Type()), 1, 1)
- s.Index(0).Set(rv)
- rv = s
- }
- n := rv.Len()
- items := make([]any, n)
- for i := 0; i < n; i++ {
- items[i] = rv.Index(i).Interface()
- }
-
- if r.Template != "" {
- return r.renderTemplate(items)
- }
- switch r.Format {
- case FormatJSON:
- return r.renderJSON(items)
- case FormatJSONL:
- return r.renderJSONL(items)
- case FormatCSV:
- return r.renderDelimited(items, ',')
- case FormatTSV:
- return r.renderDelimited(items, '\t')
- case FormatURL:
- return r.renderURL(items)
- case FormatRaw:
- return r.renderRaw(items)
- default:
- return r.renderTable(items)
- }
-}
-
-func (r *Renderer) renderJSON(items []any) error {
- enc := json.NewEncoder(r.w)
- enc.SetIndent("", " ")
- if len(items) == 1 {
- return enc.Encode(items[0])
- }
- return enc.Encode(items)
-}
-
-func (r *Renderer) renderJSONL(items []any) error {
- enc := json.NewEncoder(r.w)
- for _, it := range items {
- if err := enc.Encode(it); err != nil {
- return err
- }
- }
- return nil
-}
-
-func (r *Renderer) renderTemplate(items []any) error {
- t, err := template.New("row").Funcs(template.FuncMap{
- "join": func(sep string, v any) string { return joinAny(sep, v) },
- }).Parse(r.Template)
- if err != nil {
- return fmt.Errorf("parse --template: %w", err)
- }
- for _, it := range items {
- if err := t.Execute(r.w, toAnyMap(it)); err != nil {
- return err
- }
- _, _ = fmt.Fprintln(r.w)
- }
- return nil
-}
-
-func (r *Renderer) renderURL(items []any) error {
- for _, it := range items {
- m := toMap(it)
- if u := firstNonEmpty(m["url"], m["hn_url"], m["permalink"]); u != "" {
- _, _ = fmt.Fprintln(r.w, u)
- }
- }
- return nil
-}
-
-func (r *Renderer) renderRaw(items []any) error {
- cols := r.columns(items)
- for _, it := range items {
- m := toMap(it)
- vals := make([]string, 0, len(cols))
- for _, c := range cols {
- vals = append(vals, m[c])
- }
- _, _ = fmt.Fprintln(r.w, strings.Join(vals, " "))
- }
- return nil
-}
-
-func (r *Renderer) renderTable(items []any) error {
- if len(items) == 0 {
- return nil
- }
- cols := r.columns(items)
- tw := tabwriter.NewWriter(r.w, 0, 4, 2, ' ', 0)
- if !r.NoHeader {
- _, _ = fmt.Fprintln(tw, strings.Join(upperAll(cols), "\t"))
- }
- for _, it := range items {
- m := toMap(it)
- cells := make([]string, len(cols))
- for i, c := range cols {
- cells[i] = truncate(m[c], 60)
- }
- _, _ = fmt.Fprintln(tw, strings.Join(cells, "\t"))
- }
- return tw.Flush()
-}
-
-func (r *Renderer) renderDelimited(items []any, comma rune) error {
- if len(items) == 0 {
- return nil
- }
- cols := r.columns(items)
- cw := csv.NewWriter(r.w)
- cw.Comma = comma
- if !r.NoHeader {
- if err := cw.Write(cols); err != nil {
- return err
- }
- }
- for _, it := range items {
- m := toMap(it)
- row := make([]string, len(cols))
- for i, c := range cols {
- row[i] = m[c]
- }
- if err := cw.Write(row); err != nil {
- return err
- }
- }
- cw.Flush()
- return cw.Error()
-}
-
-func (r *Renderer) columns(items []any) []string {
- if len(r.Fields) > 0 {
- return r.Fields
- }
- if len(items) == 0 {
- return nil
- }
- return structJSONKeys(items[0])
-}
-
-func toAnyMap(v any) any {
- data, err := json.Marshal(v)
- if err != nil {
- return v
- }
- var m map[string]any
- if err := json.Unmarshal(data, &m); err != nil {
- return v
- }
- return m
-}
-
-func joinAny(sep string, v any) string {
- switch vv := v.(type) {
- case nil:
- return ""
- case []string:
- return strings.Join(vv, sep)
- case []any:
- parts := make([]string, len(vv))
- for i, e := range vv {
- parts[i] = fmt.Sprintf("%v", e)
- }
- return strings.Join(parts, sep)
- default:
- return fmt.Sprintf("%v", v)
- }
-}
-
-func toMap(v any) map[string]string {
- out := map[string]string{}
- rv := reflect.ValueOf(v)
- if rv.Kind() == reflect.Pointer {
- rv = rv.Elem()
- }
- if rv.Kind() != reflect.Struct {
- return out
- }
- rt := rv.Type()
- for i := 0; i < rt.NumField(); i++ {
- f := rt.Field(i)
- if f.PkgPath != "" {
- continue
- }
- key := jsonKey(f)
- if key == "-" {
- continue
- }
- out[key] = formatValue(rv.Field(i))
- }
- return out
-}
-
-func structJSONKeys(v any) []string {
- rv := reflect.ValueOf(v)
- if rv.Kind() == reflect.Pointer {
- rv = rv.Elem()
- }
- if rv.Kind() != reflect.Struct {
- return nil
- }
- rt := rv.Type()
- var keys []string
- for i := 0; i < rt.NumField(); i++ {
- f := rt.Field(i)
- if f.PkgPath != "" {
- continue
- }
- key := jsonKey(f)
- if key == "-" {
- continue
- }
- keys = append(keys, key)
- }
- return keys
-}
-
-func jsonKey(f reflect.StructField) string {
- tag := f.Tag.Get("json")
- if tag == "" {
- return f.Name
- }
- name := strings.Split(tag, ",")[0]
- if name == "" {
- return f.Name
- }
- return name
-}
-
-func formatValue(v reflect.Value) string {
- switch v.Kind() {
- case reflect.String:
- return v.String()
- case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
- return strconv.FormatInt(v.Int(), 10)
- case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
- return strconv.FormatUint(v.Uint(), 10)
- case reflect.Float32, reflect.Float64:
- return strconv.FormatFloat(v.Float(), 'g', -1, 64)
- case reflect.Bool:
- return strconv.FormatBool(v.Bool())
- case reflect.Slice:
- parts := make([]string, v.Len())
- for i := 0; i < v.Len(); i++ {
- parts[i] = formatValue(v.Index(i))
- }
- return strings.Join(parts, ";")
- case reflect.Struct:
- if t, ok := v.Interface().(time.Time); ok {
- if t.IsZero() {
- return ""
- }
- return t.Format(time.RFC3339)
- }
- }
- return fmt.Sprintf("%v", v.Interface())
-}
-
-func upperAll(ss []string) []string {
- out := make([]string, len(ss))
- for i, s := range ss {
- out[i] = strings.ToUpper(s)
- }
- return out
-}
-
-func firstNonEmpty(ss ...string) string {
- for _, s := range ss {
- if s != "" {
- return s
- }
- }
- return ""
-}
-
-func truncate(s string, n int) string {
- s = strings.ReplaceAll(s, "\n", " ")
- if len([]rune(s)) <= n {
- return s
- }
- rs := []rune(s)
- return string(rs[:n-1]) + "..."
-}
From 17cd10520fdb27886bbb1c444b47cee657460e78 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 19:37:18 +0700
Subject: [PATCH 09/21] gh: people, gists, activity, trending, and repository
statistics
This is the long tail of the read surface: the profile tabs, the
organization roster, gists, the contribution calendar, activity feeds,
trending, topic pages, fork networks, and the contributor graph.
Two findings worth naming. The contributor graph has a data route of its
own, /graphs/contributors-data, which answers 202 with an empty body
while GitHub computes the numbers, so the reader polls rather than
treating that as a failure. And the language histogram does have a
keyless source after all: /{owner}/{repo}/_sidebar is the fragment the
repository page's own front end waits for, it needs no credential, and
it carries the contributor count and the dependent count alongside the
languages. repo.go used to say the histogram had no keyless source. It
does now, and --deep uses it.
The calendar fragment is HTML and answers 406 to a request that asks for
JSON, which the client reports as a response rather than an error, so it
reads on the HTML surface and not the XHR one.
---
gh/base.go | 14 +-
gh/discover.go | 578 ++++++++++++++++++++++++++++++++++++++
gh/gh.go | 1 +
gh/ops.go | 325 ++++++++++++++++++++++
gh/people.go | 738 +++++++++++++++++++++++++++++++++++++++++++++++++
gh/repo.go | 100 ++++++-
gh/types.go | 44 ++-
gh/uri.go | 32 ++-
8 files changed, 1815 insertions(+), 17 deletions(-)
create mode 100644 gh/discover.go
create mode 100644 gh/people.go
diff --git a/gh/base.go b/gh/base.go
index 18c8706..fdd2bee 100644
--- a/gh/base.go
+++ b/gh/base.go
@@ -263,15 +263,21 @@ func claimedKeys(t reflect.Type) []string {
// --- small shared helpers ---
-// parseTime accepts the three time formats GitHub uses across its surfaces:
-// RFC 3339 with a zone, RFC 3339 in UTC with a Z, and the datetime attribute
-// on a element, which is the same thing.
+// parseTime accepts the time formats GitHub uses across its surfaces: RFC 3339
+// with a zone, RFC 3339 in UTC with a Z, the bare date on a commit calendar,
+// and the space-separated form the activity Atom feed puts in its published
+// element, which is the only surface that does not use RFC 3339.
func parseTime(s string) *time.Time {
s = strings.TrimSpace(s)
if s == "" {
return nil
}
- for _, layout := range []string{time.RFC3339, "2006-01-02T15:04:05Z0700", "2006-01-02"} {
+ for _, layout := range []string{
+ time.RFC3339,
+ "2006-01-02T15:04:05Z0700",
+ "2006-01-02 15:04:05 MST",
+ "2006-01-02",
+ } {
if t, err := time.Parse(layout, s); err == nil {
u := t.UTC()
return &u
diff --git a/gh/discover.go b/gh/discover.go
new file mode 100644
index 0000000..87bbb7b
--- /dev/null
+++ b/gh/discover.go
@@ -0,0 +1,578 @@
+package gh
+
+import (
+ "context"
+ "encoding/json"
+ "strconv"
+ "strings"
+ "time"
+
+ "golang.org/x/net/html"
+
+ "github.com/tamnd/github-cli/pkg/page"
+)
+
+// discover.go reads the pages that answer "what is out there": trending, topic
+// pages, fork networks, and repository statistics.
+//
+// Trending is the clearest case for this whole tool. There is no JSON version
+// of it anywhere, with a token or without, so a page decoder is not a fallback
+// here, it is the only implementation that can exist.
+
+// TrendingOptions are the three knobs the trending page has.
+type TrendingOptions struct {
+ // Since is daily, weekly, or monthly. Empty means daily, which is what the
+ // page defaults to.
+ Since string
+ // Language filters by the language slug in the URL, not by a query.
+ Language string
+ // SpokenLanguage is the natural-language filter, a two-letter code.
+ SpokenLanguage string
+ Limit int
+}
+
+// Trending lists the trending repositories. Rank is the position on the page,
+// which is the only ordering the surface has and is worth keeping, since the
+// list has no other stable key.
+func (c *Client) Trending(ctx context.Context, opts TrendingOptions, emit func(Trending) error) error {
+ u := trendingURL("", opts)
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return structureChanged("trending")
+ }
+ period := firstNonEmpty(opts.Since, "daily")
+ rank := 0
+ for _, row := range page.FindAll(doc, page.TrendingRow) {
+ t, ok := trendingRow(row, period, res.FinalURL)
+ if !ok {
+ continue
+ }
+ rank++
+ t.Rank = rank
+ if err := emit(t); err != nil {
+ return err
+ }
+ if opts.Limit > 0 && rank >= opts.Limit {
+ return nil
+ }
+ }
+ if rank == 0 {
+ return structureChanged("trending")
+ }
+ return nil
+}
+
+// TrendingDevelopers lists the trending developers, each with the repository
+// the page picked out for them.
+func (c *Client) TrendingDevelopers(ctx context.Context, opts TrendingOptions, emit func(Account) error) error {
+ u := trendingURL("developers", opts)
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return structureChanged("trending developers")
+ }
+ seen := 0
+ for _, row := range page.FindAll(doc, page.Sel{Tag: "article", Class: "Box-row"}) {
+ a, ok := trendingDev(row, res.FinalURL)
+ if !ok {
+ continue
+ }
+ if err := emit(a); err != nil {
+ return err
+ }
+ seen++
+ if opts.Limit > 0 && seen >= opts.Limit {
+ return nil
+ }
+ }
+ if seen == 0 {
+ return structureChanged("trending developers")
+ }
+ return nil
+}
+
+// trendingURL builds the trending address. The language is a path segment and
+// the period is a query parameter, which is the site's own split and not one
+// worth normalising away.
+func trendingURL(section string, opts TrendingOptions) string {
+ u := BaseURL + "/trending"
+ if section != "" {
+ u += "/" + section
+ } else if opts.Language != "" {
+ u += "/" + strings.ToLower(opts.Language)
+ }
+ var kv []string
+ if opts.Since != "" {
+ kv = append(kv, "since", opts.Since)
+ }
+ if opts.SpokenLanguage != "" {
+ kv = append(kv, "spoken_language_code", opts.SpokenLanguage)
+ }
+ if len(kv) == 0 {
+ return u
+ }
+ return query(u, kv...)
+}
+
+// trendingRow reads one card. The three counts on it are the same shape and
+// only their link tells them apart: stargazers, forks, and the period figure,
+// which has no link at all.
+func trendingRow(row *html.Node, period, source string) (Trending, bool) {
+ h := page.Find(row, page.Sel{Tag: "h2"})
+ if h == nil {
+ return Trending{}, false
+ }
+ a := page.Find(h, page.Sel{Tag: "a", Attr: "href"})
+ if a == nil {
+ return Trending{}, false
+ }
+ id := hrefPath(page.Attr(a, "href"))
+ owner, name, ok := SplitRepo(id)
+ if !ok {
+ return Trending{}, false
+ }
+ t := Trending{Period: period}
+ t.Owner, t.Name = owner, name
+ t.setIdentity(KindRepo, id)
+ t.addSource(source)
+
+ if p := page.Find(row, page.Sel{Tag: "p"}); p != nil {
+ t.Description = page.Text(p)
+ }
+ if l := page.Find(row, page.Sel{Attr: "itemprop", AttrValue: "programmingLanguage"}); l != nil {
+ t.Language = page.Text(l)
+ }
+ if col := page.Find(row, page.Sel{Class: "repo-language-color"}); col != nil {
+ t.LanguageColor = styleColor(page.Attr(col, "style"))
+ }
+ for _, link := range page.FindAll(row, page.Sel{Tag: "a", Attr: "href"}) {
+ n, _, ok := page.ParseCompactCount(page.Text(link))
+ if !ok {
+ continue
+ }
+ switch href := page.Attr(link, "href"); {
+ case strings.HasSuffix(href, "/stargazers"):
+ t.Stars = intp(n)
+ case strings.HasSuffix(href, "/forks"):
+ t.Forks = intp(n)
+ }
+ }
+ if s := page.Find(row, page.Sel{Class: "float-sm-right"}); s != nil {
+ if n, _, ok := page.CountIn(page.Text(s)); ok {
+ t.StarsInPeriod = intp(n)
+ }
+ }
+ for _, img := range page.FindAll(row, page.Sel{Tag: "img", Class: "avatar-user"}) {
+ login := strings.TrimPrefix(page.Attr(img, "alt"), "@")
+ if login == "" {
+ continue
+ }
+ who := actor(login)
+ who.AvatarURL = page.Attr(img, "src")
+ t.BuiltBy = append(t.BuiltBy, who)
+ }
+ return t, true
+}
+
+// trendingDev reads one developer card. The popular repository on it is a
+// pointer, not a record: it has a name and a description and nothing else, so
+// it goes into PinnedRepos where the profile's own picks go.
+func trendingDev(row *html.Node, source string) (Account, bool) {
+ link := page.Find(row, page.Sel{Tag: "h1", Class: "h3"})
+ if link == nil {
+ return Account{}, false
+ }
+ nameLink := page.Find(link, page.Sel{Tag: "a", Attr: "href"})
+ if nameLink == nil {
+ return Account{}, false
+ }
+ login := hrefPath(page.Attr(nameLink, "href"))
+ if login == "" || strings.Contains(login, "/") {
+ return Account{}, false
+ }
+ a := Account{Login: login, Type: "User", Name: page.Text(nameLink)}
+ a.setIdentity(KindUser, login)
+ a.addSource(source)
+ if a.Name == a.Login {
+ a.Name = ""
+ }
+ if img := page.Find(row, page.Sel{Tag: "img", Class: "avatar-user"}); img != nil {
+ a.AvatarURL = page.Attr(img, "src")
+ }
+ if h := page.Find(row, page.Sel{Tag: "h1", Class: "h4"}); h != nil {
+ if repo := page.Find(h, page.Sel{Tag: "a", Attr: "href"}); repo != nil {
+ if id := hrefPath(page.Attr(repo, "href")); strings.Count(id, "/") == 1 {
+ a.PinnedRepos = append(a.PinnedRepos, id)
+ }
+ }
+ }
+ return a, true
+}
+
+// --- topic pages ---
+
+// TopicPage reads one topic. The search result for a topic carries the name and
+// a short blurb; the page carries the long description, the logo, who created
+// the thing, when it was released, the Wikipedia link, and the related topics,
+// which is most of what makes a topic worth having a record for.
+func (c *Client) TopicPage(ctx context.Context, slug string) (*Topic, error) {
+ slug = strings.Trim(slug, "/")
+ if slug == "" || strings.Contains(slug, "/") {
+ return nil, usageBadID("topic", slug, "a topic slug")
+ }
+ res, err := c.GetHTML(ctx, BaseURL+"/topics/"+slug)
+ if err != nil {
+ return nil, err
+ }
+ p := page.Extract(res.FinalURL, res.Body)
+ doc := p.Doc()
+ if doc == nil {
+ return nil, structureChanged(slug)
+ }
+ t := &Topic{Name: slug}
+ t.setIdentity(KindTopic, slug)
+ t.addSource(res.FinalURL)
+ t.GitHubURL = t.URL
+
+ if h := page.Find(doc, page.Sel{Tag: "h1", Class: "h1"}); h != nil {
+ t.DisplayName = page.Text(h)
+ }
+ if trigger := page.Find(doc, page.Sel{Tag: "topic-feeds-toast-trigger"}); trigger != nil {
+ t.DisplayName = firstNonEmpty(page.Attr(trigger, "data-topic-display-name"), t.DisplayName)
+ }
+ if md := page.Find(doc, page.MarkdownBody); md != nil {
+ t.DescriptionHTML = page.OuterHTML(md)
+ t.Description = page.BlockText(md)
+ // The page has one description where the search result has two. The
+ // first paragraph is the same string the short one would be, so it is
+ // filled from here rather than left empty for no reason.
+ t.ShortDescription, _, _ = strings.Cut(t.Description, "\n")
+ }
+ if img := page.Find(doc, page.Sel{Tag: "img", Attr: "alt", AttrSuffix: " logo"}); img != nil {
+ t.LogoURL = page.Attr(img, "src")
+ }
+ if w := page.Find(doc, page.TopicWikipedia); w != nil {
+ t.WikipediaURL = page.Attr(w, "href")
+ }
+ t.CreatedBy = labelledText(doc, "Created by")
+ t.Released = labelledText(doc, "Released")
+ if n := page.Find(doc, page.Sel{Tag: "h2", Class: "h3"}); n != nil {
+ // "Here are 89,195 public repositories matching this topic..."
+ if count, _, ok := page.CountIn(strings.TrimPrefix(page.Text(n), "Here are ")); ok {
+ t.AppliedCount = intp(count)
+ }
+ }
+ for _, dd := range page.FindAll(doc, page.Sel{Tag: "dd"}) {
+ if n, _, ok := page.ParseCompactCount(strings.TrimSuffix(page.Text(dd), " followers")); ok &&
+ strings.HasSuffix(page.Text(dd), "followers") {
+ t.StargazerCount = intp(n)
+ }
+ }
+ for _, rel := range relatedTopics(doc, slug) {
+ t.Related = append(t.Related, rel)
+ }
+ if t.DisplayName == "" && t.Description == "" {
+ return nil, structureChanged(slug)
+ }
+ return t, nil
+}
+
+// relatedTopics reads the sidebar's related topics.
+//
+// They are not in a container. The heading and the links are siblings, and the
+// same link class is on every topic chip of every repository in the result
+// list below, so scoping by class alone pulls in a few hundred unrelated
+// topics. The heading is the only boundary the markup gives, so the walk
+// starts there and stops at the next heading.
+func relatedTopics(doc *html.Node, slug string) []string {
+ var head *html.Node
+ for _, h := range page.FindAll(doc, page.Sel{Tag: "h2"}) {
+ if page.Text(h) == "Related topics" {
+ head = h
+ break
+ }
+ }
+ if head == nil {
+ return nil
+ }
+ var out []string
+ for n := head.NextSibling; n != nil; n = n.NextSibling {
+ if n.Type == html.ElementNode && (n.Data == "h2" || n.Data == "h3") {
+ break
+ }
+ for _, a := range page.FindAll(n, page.Sel{Tag: "a", Class: "topic-tag-link"}) {
+ rel := strings.TrimPrefix(hrefPath(page.Attr(a, "href")), "topics/")
+ if rel != "" && rel != slug && !contains(out, rel) {
+ out = append(out, rel)
+ }
+ }
+ }
+ return out
+}
+
+// labelledText reads the value beside a muted label in the topic sidebar. The
+// label is a span inside the paragraph and the value is the rest of it, which
+// is the only structure the markup offers.
+func labelledText(doc *html.Node, label string) string {
+ for _, p := range page.FindAll(doc, page.Sel{Tag: "p"}) {
+ span := page.Find(p, page.Sel{Tag: "span", Class: "color-fg-muted"})
+ if span == nil || page.Text(span) != label {
+ continue
+ }
+ return strings.TrimSpace(strings.TrimPrefix(page.Text(p), label))
+ }
+ return ""
+}
+
+// --- fork networks ---
+
+// Forks lists the public forks of a repository. The page is the only keyless
+// source: the network graph route needs a session and the search index does not
+// model the parent link.
+func (c *Client) Forks(ctx context.Context, repo string, limit int, emit func(Repo) error) error {
+ if _, _, ok := SplitRepo(repo); !ok {
+ return usageBadID("repository", repo, "owner/name")
+ }
+ base := repoSubURL(repo, "forks")
+ fetch := func(ctx context.Context, token string) ([]Repo, string, error) {
+ u := base
+ if n := pageToken(token); n > 1 {
+ u = query(u, "page", strconv.Itoa(n))
+ }
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return nil, "", err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return nil, "", structureChanged(repo + " forks")
+ }
+ var out []Repo
+ for _, row := range page.FindAll(doc, page.BoxRow) {
+ f, ok := forkRow(row, repo, res.FinalURL)
+ if ok {
+ out = append(out, f)
+ }
+ }
+ return out, railsNext(doc, token), nil
+ }
+ return paginate(ctx, limit, fetch, emit)
+}
+
+// forkRow reads one row of a fork list. The owner and the name are separate
+// links, so the id is assembled rather than read off one href.
+func forkRow(row *html.Node, parent, source string) (Repo, bool) {
+ h := page.Find(row, page.Sel{Tag: "h2"})
+ if h == nil {
+ return Repo{}, false
+ }
+ var owner, name string
+ for _, a := range page.FindAll(h, page.Sel{Tag: "a", Attr: "href"}) {
+ p := hrefPath(page.Attr(a, "href"))
+ switch {
+ case owner == "" && !strings.Contains(p, "/"):
+ owner = p
+ case strings.Count(p, "/") == 1:
+ owner, name, _ = SplitRepo(p)
+ }
+ }
+ if owner == "" || name == "" {
+ return Repo{}, false
+ }
+ id := owner + "/" + name
+ r := Repo{Owner: owner, Name: name, IsFork: true, ForkOf: parent}
+ r.setIdentity(KindRepo, id)
+ r.addSource(source)
+ for _, a := range page.FindAll(row, page.Sel{Tag: "a", Attr: "href"}) {
+ n, _, ok := page.ParseCompactCount(page.Text(a))
+ if !ok {
+ continue
+ }
+ switch href := page.Attr(a, "href"); {
+ case strings.HasSuffix(href, "/stargazers"):
+ r.Stars = intp(n)
+ case strings.HasSuffix(href, "/forks"):
+ r.Forks = intp(n)
+ }
+ }
+ if t := page.Find(row, page.RelTimeEl); t != nil {
+ r.PushedAt = parseTime(page.Attr(t, "datetime"))
+ }
+ return r, true
+}
+
+// --- statistics ---
+
+// Contributors reads the contributor graph's own data route.
+//
+// The route answers 202 with an empty body while GitHub computes the numbers,
+// which is normal rather than an error and is why this polls. A large
+// repository takes a few seconds the first time and is instant afterwards.
+func (c *Client) Contributors(ctx context.Context, repo string, opts ContributorOptions, emit func(Contributor) error) error {
+ if _, _, ok := SplitRepo(repo); !ok {
+ return usageBadID("repository", repo, "owner/name")
+ }
+ u := repoSubURL(repo, "graphs/contributors-data")
+ res, err := c.Poll(ctx, u, SurfaceXHR)
+ if err != nil {
+ return err
+ }
+ var raw []contributorData
+ if err := json.Unmarshal(res.Body, &raw); err != nil {
+ return badPayload(shortURL(u), err)
+ }
+ if len(raw) == 0 {
+ return structureChanged(repo + " contributors")
+ }
+ // The route answers in ascending order of contribution, which is the
+ // reverse of what anyone asking for contributors wants.
+ seen := 0
+ for i := len(raw) - 1; i >= 0; i-- {
+ rec := raw[i].contributor(repo, res.FinalURL, opts.Weeks)
+ if err := emit(rec); err != nil {
+ return err
+ }
+ seen++
+ if opts.Limit > 0 && seen >= opts.Limit {
+ return nil
+ }
+ }
+ return nil
+}
+
+// ContributorOptions is what to do with the week array.
+type ContributorOptions struct {
+ // Weeks keeps the per-week breakdown. It is off by default because the
+ // route sends every week since the repository began for every contributor,
+ // which on an old project is a few hundred entries each and megabytes of
+ // mostly zeroes for an answer whose question was "who wrote this".
+ Weeks bool
+ Limit int
+}
+
+type contributorData struct {
+ Author *struct {
+ ID int `json:"id"`
+ Login string `json:"login"`
+ Avatar string `json:"avatar"`
+ Path string `json:"path"`
+ } `json:"author"`
+ Total int `json:"total"`
+ Weeks []struct {
+ W int64 `json:"w"`
+ A int `json:"a"`
+ D int `json:"d"`
+ C int `json:"c"`
+ } `json:"weeks"`
+}
+
+// contributor folds the week array into the record. The array is six hundred
+// entries for an old repository and nearly all of them are zero, so the
+// summable fields are summed here and the first and last weeks with any
+// activity are kept as dates, which is what a table can show.
+func (d contributorData) contributor(repo, source string, keepWeeks bool) Contributor {
+ rec := Contributor{Repo: repo, Commits: intp(d.Total)}
+ if d.Author != nil {
+ rec.Login = d.Author.Login
+ rec.AvatarURL = d.Author.Avatar
+ if d.Author.ID > 0 {
+ rec.DatabaseID = intp(d.Author.ID)
+ }
+ }
+ rec.setIdentity(KindContributor, repo+"@"+rec.Login)
+ rec.addSource(source)
+
+ adds, dels := 0, 0
+ for _, w := range d.Weeks {
+ adds += w.A
+ dels += w.D
+ if w.A == 0 && w.D == 0 && w.C == 0 {
+ continue
+ }
+ at := time.Unix(w.W, 0).UTC()
+ if rec.FirstWeek == nil {
+ first := at
+ rec.FirstWeek = &first
+ }
+ last := at
+ rec.LastWeek = &last
+ if keepWeeks {
+ rec.Weeks = append(rec.Weeks, ContributorWeek{Week: at, Additions: w.A, Deletions: w.D, Commits: w.C})
+ }
+ }
+ rec.Additions = intp(adds)
+ rec.Deletions = intp(dels)
+ return rec
+}
+
+// Languages reports the language histogram as one record per language. The
+// numbers are on the repository record already; this exists because "what is
+// this written in, in what proportion" is a question worth one command rather
+// than a field selector on another one.
+// Languages reports the language breakdown, largest first.
+//
+// This reads the sidebar fragment rather than a whole repository page, because
+// the fragment is where the numbers are and it is 3 KB where the page is 300.
+// The numbers are percentages: GitHub computes byte counts and publishes only
+// the proportions, so a byte count is not something this can report honestly.
+func (c *Client) Languages(ctx context.Context, repo string, emit func(LanguageShare) error) error {
+ sb, err := c.sidebar(ctx, repo)
+ if err != nil {
+ return err
+ }
+ langs := sb.langs()
+ if len(langs) == 0 {
+ return structureChanged(repo + " languages")
+ }
+ source := repoSubURL(repo, "_sidebar")
+ for _, l := range langs {
+ share := LanguageShare{
+ Repo: repo,
+ Language: l.Name,
+ Percent: l.Percentage,
+ Color: l.Color,
+ }
+ share.setIdentity(KindRepo, repo)
+ share.addSource(source)
+ if err := emit(share); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// Stats is the counts in one record. Everything in it is already on the
+// repository record; the point is a record with nothing else in it, so
+// `github stats x -o json` is a thing you can diff week to week.
+func (c *Client) Stats(ctx context.Context, repo string) (*RepoStats, error) {
+ // Deep, because the contributor and dependent counts are behind their own
+ // fragments and a counts record missing two of the counts is not worth
+ // having.
+ r, err := c.Repo(ctx, repo, RepoOptions{Deep: true})
+ if err != nil {
+ return nil, err
+ }
+ s := &RepoStats{
+ Repo: repo,
+ Stars: r.Stars,
+ Forks: r.Forks,
+ Watchers: r.Watchers,
+ OpenIssues: r.OpenIssues,
+ Commits: r.CommitCount,
+ Releases: r.ReleaseCount,
+ Tags: r.TagCount,
+ Contributors: r.ContributorCount,
+ Dependents: r.DependentCount,
+ PushedAt: r.PushedAt,
+ }
+ s.setIdentity(KindRepo, repo)
+ s.addSource(r.Sources...)
+ return s, nil
+}
diff --git a/gh/gh.go b/gh/gh.go
index 17dc6f6..57e077a 100644
--- a/gh/gh.go
+++ b/gh/gh.go
@@ -19,6 +19,7 @@ const (
BaseURL = "https://github.com"
RawURL = "https://raw.githubusercontent.com"
CodeLoad = "https://codeload.github.com"
+ GistURL = "https://gist.github.com"
GistRaw = "https://gist.githubusercontent.com"
AvatarURL = "https://avatars.githubusercontent.com"
OpenGraph = "https://opengraph.githubassets.com"
diff --git a/gh/ops.go b/gh/ops.go
index 33396bc..8c74e79 100644
--- a/gh/ops.go
+++ b/gh/ops.go
@@ -23,6 +23,8 @@ func registerOps(app *kit.App) {
registerSearchOps(app)
registerContentOps(app)
registerHistoryOps(app)
+ registerPeopleOps(app)
+ registerDiscoverOps(app)
registerMetaOps(app)
}
@@ -1042,6 +1044,329 @@ func listTimeline(ctx context.Context, in timelineIn, emit func(*TimelineItem) e
return in.C.Timeline(ctx, repo, num, in.Limit, byValue(emit))
}
+// --- people ---
+
+type accountListIn struct {
+ C *Client `kit:"inject"`
+ Name string `kit:"arg" help:"a login, a profile URL, or a github:// URI"`
+ Limit int `kit:"flag,inherit"`
+}
+
+type gistIn struct {
+ C *Client `kit:"inject"`
+ Ref string `kit:"arg" help:"a gist id, a gist URL, or a github:// URI"`
+ Content bool `kit:"flag" help:"fetch each file's raw content, one request per file"`
+}
+
+type contributionsIn struct {
+ C *Client `kit:"inject"`
+ Name string `kit:"arg" help:"a login, a profile URL, or a github:// URI"`
+ Year int `kit:"flag" help:"calendar year, defaulting to the rolling last twelve months"`
+}
+
+type activityIn struct {
+ C *Client `kit:"inject"`
+ Ref string `kit:"arg" help:"a login for a person's stream, or owner/name for a repository's"`
+ Limit int `kit:"flag,inherit"`
+}
+
+func registerPeopleOps(app *kit.App) {
+ kit.Handle(app, kit.OpMeta{
+ Name: "followers", Group: "people", URIType: KindUser, List: true,
+ Summary: "List who follows an account",
+ Long: "The tab is 50 people a page and the pager is a plain next link, so an\n" +
+ "account with a hundred thousand followers is two thousand requests. Set\n" +
+ "--limit unless you mean all of them.",
+ Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}},
+ }, listFollowers)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "following", Group: "people", URIType: KindUser, List: true,
+ Summary: "List who an account follows",
+ Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}},
+ }, listFollowing)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "members", Group: "people", URIType: KindUser, List: true,
+ Summary: "List an organization's public members",
+ Long: "Public members only, which is the organization's own choice per person\n" +
+ "and not something a token would widen for someone outside the org.",
+ Args: []kit.Arg{{Name: "name", Help: "organization login, URL, or github:// URI"}},
+ }, listMembers)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "stars", Group: "people", URIType: KindRepo, List: true,
+ Aliases: []string{"starred"},
+ Summary: "List what an account has starred",
+ Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}},
+ }, listStarred)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "owned", Group: "people", URIType: KindRepo, List: true,
+ Summary: "List an account's repositories as the profile shows them",
+ Long: "This reads the profile's repositories tab, which is the only surface that\n" +
+ "lists forks and archived repositories in the account's own order. It is\n" +
+ "not called repos because `github repos --owner name` already exists, goes\n" +
+ "through search, and is the better tool when you want to filter or sort.",
+ Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}},
+ }, listAccountRepos)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "gists", Group: "people", URIType: KindGist, List: true,
+ Summary: "List an account's public gists",
+ Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}},
+ }, listGists)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "gist", Group: "people", URIType: KindGist, Single: true, Resolver: true,
+ Summary: "Read one gist with its files",
+ Long: "The index gives each file's first few lines only, which is what the page\n" +
+ "renders. With --content each file is fetched whole from the raw host.",
+ Args: []kit.Arg{{Name: "ref", Help: "gist id, gist URL, or github:// URI"}},
+ }, getGist)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "contributions", Group: "people", URIType: KindContribution, List: true,
+ Aliases: []string{"calendar"},
+ Summary: "Read an account's contribution calendar, one record per day",
+ Long: "The count is not on the square. Each square points at a tooltip by id and\n" +
+ "the tooltip holds the sentence with the number in it, so this indexes the\n" +
+ "tooltips first and reads the squares against that index.",
+ Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}},
+ }, listContributions)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "activity", Group: "people", URIType: KindEvent, List: true,
+ Aliases: []string{"events", "feed"},
+ Summary: "Read a public activity feed",
+ Long: "One login gives that person's public events; one owner/name gives that\n" +
+ "repository's commit feed. Both are Atom, both are public, and neither has\n" +
+ "a pager, so a feed is however many entries GitHub decided to put in it.",
+ Args: []kit.Arg{{Name: "ref", Help: "a login, or owner/name"}},
+ }, listActivity)
+}
+
+func listFollowers(ctx context.Context, in accountListIn, emit func(*Account) error) error {
+ login, err := ResolveRef(KindUser, in.Name)
+ if err != nil {
+ return err
+ }
+ return in.C.Followers(ctx, login, in.Limit, byValue(emit))
+}
+
+func listFollowing(ctx context.Context, in accountListIn, emit func(*Account) error) error {
+ login, err := ResolveRef(KindUser, in.Name)
+ if err != nil {
+ return err
+ }
+ return in.C.Following(ctx, login, in.Limit, byValue(emit))
+}
+
+func listMembers(ctx context.Context, in accountListIn, emit func(*Account) error) error {
+ login, err := ResolveRef(KindOrg, in.Name)
+ if err != nil {
+ return err
+ }
+ return in.C.Members(ctx, login, in.Limit, byValue(emit))
+}
+
+func listStarred(ctx context.Context, in accountListIn, emit func(*Repo) error) error {
+ login, err := ResolveRef(KindUser, in.Name)
+ if err != nil {
+ return err
+ }
+ return in.C.Starred(ctx, login, in.Limit, byValue(emit))
+}
+
+func listAccountRepos(ctx context.Context, in accountListIn, emit func(*Repo) error) error {
+ login, err := ResolveRef(KindUser, in.Name)
+ if err != nil {
+ return err
+ }
+ return in.C.ReposAsShown(ctx, login, in.Limit, byValue(emit))
+}
+
+func listGists(ctx context.Context, in accountListIn, emit func(*Gist) error) error {
+ login, err := ResolveRef(KindUser, in.Name)
+ if err != nil {
+ return err
+ }
+ return in.C.Gists(ctx, login, in.Limit, byValue(emit))
+}
+
+func getGist(ctx context.Context, in gistIn, emit func(*Gist) error) error {
+ id, err := ResolveRef(KindGist, in.Ref)
+ if err != nil {
+ return err
+ }
+ g, err := in.C.Gist(ctx, id, in.Content)
+ if err != nil {
+ return err
+ }
+ return emit(g)
+}
+
+func listContributions(ctx context.Context, in contributionsIn, emit func(*ContributionDay) error) error {
+ login, err := ResolveRef(KindUser, in.Name)
+ if err != nil {
+ return err
+ }
+ return in.C.Contributions(ctx, login, in.Year, byValue(emit))
+}
+
+// listActivity does not resolve the reference, because the feed takes both a
+// login and an owner/name and the reader tells them apart itself. Sending it
+// through ResolveRef would force a choice that neither kind wins.
+func listActivity(ctx context.Context, in activityIn, emit func(*Event) error) error {
+ return in.C.Activity(ctx, strings.TrimPrefix(in.Ref, BaseURL+"/"), in.Limit, byValue(emit))
+}
+
+// --- discovery and statistics ---
+
+type trendingIn struct {
+ C *Client `kit:"inject"`
+ Since string `kit:"flag" help:"daily, weekly, or monthly"`
+ Language string `kit:"flag" help:"a language slug, as it appears in the trending URL"`
+ SpokenLanguage string `kit:"flag,name=spoken" help:"a two-letter natural language code"`
+ Developers bool `kit:"flag" help:"list trending developers instead of repositories"`
+ Limit int `kit:"flag,inherit"`
+}
+
+type topicIn struct {
+ C *Client `kit:"inject"`
+ Name string `kit:"arg" help:"a topic slug, a topic URL, or a github:// URI"`
+}
+
+type repoListIn struct {
+ C *Client `kit:"inject"`
+ Ref string `kit:"arg" help:"owner/name, or any URL from the repository"`
+ Limit int `kit:"flag,inherit"`
+}
+
+type repoRefIn struct {
+ C *Client `kit:"inject"`
+ Ref string `kit:"arg" help:"owner/name, or any URL from the repository"`
+}
+
+func registerDiscoverOps(app *kit.App) {
+ kit.Handle(app, kit.OpMeta{
+ Name: "trending", Group: "discover", URIType: KindRepo, List: true,
+ Summary: "List what is trending",
+ Long: "Trending is the clearest case for reading pages. There is no JSON version\n" +
+ "of it anywhere, with a token or without, so a page decoder is not a\n" +
+ "fallback here, it is the only implementation that can exist.",
+ }, listTrending)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "topic", Group: "discover", URIType: KindTopic, Single: true, Resolver: true,
+ Summary: "Read one topic page",
+ Long: "The search result for a topic has a name and a blurb. The page has the long\n" +
+ "description, the logo, who created the thing, when it was released, the\n" +
+ "Wikipedia link, and the related topics, which is most of what makes a topic\n" +
+ "worth a record.",
+ Args: []kit.Arg{{Name: "name", Help: "topic slug, URL, or github:// URI"}},
+ }, getTopic)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "forks", Group: "discover", URIType: KindRepo, List: true,
+ Summary: "List a repository's public forks",
+ Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}},
+ }, listForks)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "contributors", Group: "discover", URIType: KindContributor, List: true,
+ Summary: "List contributors with their commit, addition, and deletion counts",
+ Long: "This reads the contributor graph's own data route, which answers 202 with\n" +
+ "an empty body while GitHub computes the numbers. That is normal rather\n" +
+ "than an error, so the first call on a large repository waits a few seconds\n" +
+ "and every call after it is instant.",
+ Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}},
+ }, listContributors)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "languages", Group: "discover", URIType: KindRepo, List: true,
+ Summary: "Report the language histogram, one record per language",
+ Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}},
+ }, listLanguages)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "stats", Group: "discover", URIType: KindRepo, Single: true,
+ Summary: "Report a repository's counts and nothing else",
+ Long: "Every field here is on the repository record too. The point of having it\n" +
+ "separately is that a record with eight numbers in it is something you can\n" +
+ "store once a day and diff, and a record with a readme in it is not.",
+ Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}},
+ }, getStats)
+}
+
+func listTrending(ctx context.Context, in trendingIn, emit func(any) error) error {
+ opts := TrendingOptions{
+ Since: in.Since,
+ Language: in.Language,
+ SpokenLanguage: in.SpokenLanguage,
+ Limit: in.Limit,
+ }
+ if in.Developers {
+ return in.C.TrendingDevelopers(ctx, opts, func(a Account) error { return emit(&a) })
+ }
+ return in.C.Trending(ctx, opts, func(t Trending) error { return emit(&t) })
+}
+
+func getTopic(ctx context.Context, in topicIn, emit func(*Topic) error) error {
+ slug, err := ResolveRef(KindTopic, in.Name)
+ if err != nil {
+ return err
+ }
+ t, err := in.C.TopicPage(ctx, slug)
+ if err != nil {
+ return err
+ }
+ return emit(t)
+}
+
+func listForks(ctx context.Context, in repoListIn, emit func(*Repo) error) error {
+ repo, err := ResolveRepo(in.Ref)
+ if err != nil {
+ return err
+ }
+ return in.C.Forks(ctx, repo, in.Limit, byValue(emit))
+}
+
+type contributorsIn struct {
+ C *Client `kit:"inject"`
+ Ref string `kit:"arg" help:"owner/name, or any URL from the repository"`
+ Weeks bool `kit:"flag" help:"keep the per-week breakdown, which is large"`
+ Limit int `kit:"flag,inherit"`
+}
+
+func listContributors(ctx context.Context, in contributorsIn, emit func(*Contributor) error) error {
+ repo, err := ResolveRepo(in.Ref)
+ if err != nil {
+ return err
+ }
+ return in.C.Contributors(ctx, repo, ContributorOptions{Weeks: in.Weeks, Limit: in.Limit}, byValue(emit))
+}
+
+func listLanguages(ctx context.Context, in repoRefIn, emit func(*LanguageShare) error) error {
+ repo, err := ResolveRepo(in.Ref)
+ if err != nil {
+ return err
+ }
+ return in.C.Languages(ctx, repo, byValue(emit))
+}
+
+func getStats(ctx context.Context, in repoRefIn, emit func(*RepoStats) error) error {
+ repo, err := ResolveRepo(in.Ref)
+ if err != nil {
+ return err
+ }
+ s, err := in.C.Stats(ctx, repo)
+ if err != nil {
+ return err
+ }
+ return emit(s)
+}
+
// --- meta ---
func registerMetaOps(app *kit.App) {
diff --git a/gh/people.go b/gh/people.go
new file mode 100644
index 0000000..23f7c42
--- /dev/null
+++ b/gh/people.go
@@ -0,0 +1,738 @@
+package gh
+
+import (
+ "context"
+ "encoding/xml"
+ "strconv"
+ "strings"
+ "time"
+
+ "golang.org/x/net/html"
+
+ "github.com/tamnd/any-cli/kit/errs"
+ "github.com/tamnd/github-cli/pkg/page"
+)
+
+// people.go is everything that hangs off an account: who follows whom, what
+// they starred, what they published, and what they did.
+//
+// None of it has a JSON payload. The profile tabs, the organization roster, and
+// the gist index are all Rails, and the activity stream is Atom. That is the
+// whole reason this file exists as its own unit: the surfaces here share a
+// pager and a row shape with each other and with nothing else in the tool.
+//
+// The pager is the "rails" one from surface.go: fetch a page, decode the rows,
+// look for the link that says next. Two templates write that link two ways and
+// nextPageHref knows both.
+
+// Followers lists the accounts following a login, newest first, which is the
+// order the page uses and the only order it offers.
+func (c *Client) Followers(ctx context.Context, login string, limit int, emit func(Account) error) error {
+ return c.profileAccounts(ctx, login, "followers", limit, emit)
+}
+
+// Following lists the accounts a login follows.
+func (c *Client) Following(ctx context.Context, login string, limit int, emit func(Account) error) error {
+ return c.profileAccounts(ctx, login, "following", limit, emit)
+}
+
+func (c *Client) profileAccounts(ctx context.Context, login, tab string, limit int, emit func(Account) error) error {
+ if login == "" || strings.Contains(login, "/") {
+ return usageBadID("account", login, "a bare login")
+ }
+ base := query(accountURL(login), "tab", tab)
+ fetch := func(ctx context.Context, token string) ([]Account, string, error) {
+ u := base
+ if n := pageToken(token); n > 1 {
+ u = query(u, "page", strconv.Itoa(n))
+ }
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return nil, "", err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return nil, "", structureChanged(login + " " + tab)
+ }
+ var out []Account
+ for _, row := range page.FindAll(doc, page.Sel{Class: "d-table"}) {
+ a, ok := followRow(row, res.FinalURL)
+ if ok {
+ out = append(out, a)
+ }
+ }
+ return out, railsNext(doc, token), nil
+ }
+ return paginate(ctx, limit, fetch, emit)
+}
+
+// followRow reads one row of a followers or following list. The row has the
+// login twice, once in the avatar link and once in the muted span, and the
+// display name in the primary span when the person set one.
+func followRow(row *html.Node, source string) (Account, bool) {
+ link := page.Find(row, page.Sel{Tag: "a", Attr: "data-hovercard-type", AttrValue: "user"})
+ if link == nil {
+ return Account{}, false
+ }
+ who := actorFromHref(page.Attr(link, "href"))
+ if who.Login == "" {
+ return Account{}, false
+ }
+ a := Account{Login: who.Login, Type: "User"}
+ a.setIdentity(KindUser, who.Login)
+ if n := page.Find(row, page.Sel{Class: "Link--primary"}); n != nil {
+ a.Name = page.Text(n)
+ }
+ if img := page.Find(row, page.Sel{Tag: "img", Class: "avatar-user"}); img != nil {
+ a.AvatarURL = page.Attr(img, "src")
+ }
+ a.addSource(source)
+ return a, true
+}
+
+// Members lists an organization's public members. The roster is at
+// /orgs/{login}/people rather than on the profile, and the profile's avatar
+// strip is a sample of it rather than a short version of it.
+func (c *Client) Members(ctx context.Context, login string, limit int, emit func(Account) error) error {
+ if login == "" || strings.Contains(login, "/") {
+ return usageBadID("organization", login, "a bare login")
+ }
+ base := BaseURL + "/orgs/" + login + "/people"
+ fetch := func(ctx context.Context, token string) ([]Account, string, error) {
+ u := base
+ if n := pageToken(token); n > 1 {
+ u = query(u, "page", strconv.Itoa(n))
+ }
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return nil, "", err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return nil, "", structureChanged(login + " members")
+ }
+ var out []Account
+ for _, li := range page.FindAll(doc, page.Sel{Class: "member-list-item"}) {
+ m, ok := memberRow(li, res.FinalURL)
+ if ok {
+ out = append(out, m)
+ }
+ }
+ return out, railsNext(doc, token), nil
+ }
+ return paginate(ctx, limit, fetch, emit)
+}
+
+// memberRow reads one member. The role is behind a batch-deferred fragment that
+// needs a session, so it is absent here rather than wrong.
+func memberRow(li *html.Node, source string) (Account, bool) {
+ name := page.Find(li, page.Sel{Tag: "a", Attr: "id", AttrPrefix: "member-"})
+ if name == nil {
+ return Account{}, false
+ }
+ who := actorFromHref(page.Attr(name, "href"))
+ if who.Login == "" {
+ return Account{}, false
+ }
+ a := Account{Login: who.Login, Type: "User", Name: page.Text(name)}
+ a.setIdentity(KindUser, who.Login)
+ // The name anchor falls back to the login when the person set no display
+ // name, and a Name that repeats the Login says nothing.
+ if a.Name == a.Login {
+ a.Name = ""
+ }
+ if img := page.Find(li, page.Sel{Tag: "img", Class: "avatar-user"}); img != nil {
+ a.AvatarURL = page.Attr(img, "src")
+ if id := avatarUserID(page.Attr(img, "src")); id > 0 {
+ a.DatabaseID = intp(id)
+ }
+ }
+ a.addSource(source)
+ return a, true
+}
+
+// avatarUserID pulls the numeric account id out of an avatar URL. It is the
+// only place a listing states it, and having it lets a record join to search
+// results, which key on the same number.
+func avatarUserID(src string) int {
+ _, rest, ok := strings.Cut(src, "/u/")
+ if !ok {
+ return 0
+ }
+ digits, _, _ := strings.Cut(rest, "?")
+ n, err := strconv.Atoi(digits)
+ if err != nil {
+ return 0
+ }
+ return n
+}
+
+// Starred lists the repositories an account has starred. It is a different
+// template from the repositories tab, so it gets its own row reader even though
+// the two records are the same shape.
+func (c *Client) Starred(ctx context.Context, login string, limit int, emit func(Repo) error) error {
+ return c.profileRepos(ctx, login, "stars", limit, emit)
+}
+
+// ReposAsShown lists an account's repositories in the order and with the
+// filters the profile tab itself uses. `github repos --user x` runs a search
+// instead, which sorts and pages better; this is what --as-shown selects when
+// the exact page order is the point.
+func (c *Client) ReposAsShown(ctx context.Context, login string, limit int, emit func(Repo) error) error {
+ return c.profileRepos(ctx, login, "repositories", limit, emit)
+}
+
+func (c *Client) profileRepos(ctx context.Context, login, tab string, limit int, emit func(Repo) error) error {
+ if login == "" || strings.Contains(login, "/") {
+ return usageBadID("account", login, "a bare login")
+ }
+ base := query(accountURL(login), "tab", tab)
+ fetch := func(ctx context.Context, token string) ([]Repo, string, error) {
+ u := base
+ if n := pageToken(token); n > 1 {
+ u = query(u, "page", strconv.Itoa(n))
+ }
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return nil, "", err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return nil, "", structureChanged(login + " " + tab)
+ }
+ var out []Repo
+ for _, h := range repoCardHeadings(doc) {
+ if r, ok := repoCard(h, res.FinalURL); ok {
+ out = append(out, r)
+ }
+ }
+ return out, railsNext(doc, token), nil
+ }
+ return paginate(ctx, limit, fetch, emit)
+}
+
+// repoCardHeadings finds the heading of every repository card on a listing
+// page. The repositories tab marks the name with microdata and the stars tab
+// does not, so both hooks are tried and the results are kept in document order
+// rather than merged, since no page uses both.
+func repoCardHeadings(doc *html.Node) []*html.Node {
+ if named := page.FindAll(doc, page.Sel{Attr: "itemprop", AttrValue: "name codeRepository"}); len(named) > 0 {
+ return named
+ }
+ var out []*html.Node
+ for _, h := range page.FindAll(doc, page.Sel{Tag: "h3"}) {
+ if a := page.Find(h, page.Sel{Tag: "a", Attr: "href"}); a != nil {
+ if p := hrefPath(page.Attr(a, "href")); strings.Count(p, "/") == 1 {
+ out = append(out, a)
+ }
+ }
+ }
+ return out
+}
+
+// repoCard reads a repository out of a listing card, walking up from the name
+// link to the row that holds the rest of the fields.
+func repoCard(nameLink *html.Node, source string) (Repo, bool) {
+ id := hrefPath(page.Attr(nameLink, "href"))
+ if strings.Count(id, "/") != 1 {
+ return Repo{}, false
+ }
+ owner, name, ok := SplitRepo(id)
+ if !ok {
+ return Repo{}, false
+ }
+ r := Repo{Owner: owner, Name: name}
+ r.setIdentity(KindRepo, id)
+ r.addSource(source)
+
+ row := cardRow(nameLink)
+ if row == nil {
+ return r, true
+ }
+ if d := page.Find(row, page.Sel{Attr: "itemprop", AttrValue: "description"}); d != nil {
+ r.Description = page.Text(d)
+ }
+ if l := page.Find(row, page.Sel{Attr: "itemprop", AttrValue: "programmingLanguage"}); l != nil {
+ r.Language = page.Text(l)
+ }
+ if c := page.Find(row, page.Sel{Class: "repo-language-color"}); c != nil {
+ r.LanguageColor = styleColor(page.Attr(c, "style"))
+ }
+ for _, a := range page.FindAll(row, page.Sel{Tag: "a", Attr: "href"}) {
+ href := page.Attr(a, "href")
+ n, _, ok := page.ParseCompactCount(page.Text(a))
+ if !ok {
+ continue
+ }
+ switch {
+ case strings.HasSuffix(href, "/stargazers"):
+ r.Stars = intp(n)
+ case strings.HasSuffix(href, "/forks"):
+ r.Forks = intp(n)
+ }
+ }
+ if t := page.Find(row, page.RelTimeEl); t != nil {
+ r.PushedAt = parseTime(page.Attr(t, "datetime"))
+ }
+ for _, tag := range page.FindAll(row, page.Sel{Class: "topic-tag"}) {
+ if s := page.Text(tag); s != "" {
+ r.Topics = append(r.Topics, s)
+ }
+ }
+ return r, true
+}
+
+// cardRow walks up to the element that contains a whole listing card. Four
+// levels is what separates the name link from the row on every template that
+// uses one, and stopping there keeps a malformed page from handing back the
+// document root and with it every field on it.
+func cardRow(n *html.Node) *html.Node {
+ for i := 0; i < 4 && n != nil; i++ {
+ n = n.Parent
+ if n == nil {
+ return nil
+ }
+ if n.Type == html.ElementNode && (n.Data == "li" || page.HasClass(n, "col-12") || page.HasClass(n, "Box-row")) {
+ return n
+ }
+ }
+ return n
+}
+
+// styleColor pulls a colour out of an inline style, which is where GitHub puts
+// the language colour on every listing template.
+func styleColor(style string) string {
+ _, rest, ok := strings.Cut(style, "background-color:")
+ if !ok {
+ return ""
+ }
+ v, _, _ := strings.Cut(rest, ";")
+ return strings.TrimSpace(v)
+}
+
+// --- gists ---
+
+// Gists lists an account's public gists.
+func (c *Client) Gists(ctx context.Context, login string, limit int, emit func(Gist) error) error {
+ if login == "" || strings.Contains(login, "/") {
+ return usageBadID("account", login, "a bare login")
+ }
+ base := GistURL + "/" + login
+ fetch := func(ctx context.Context, token string) ([]Gist, string, error) {
+ u := base
+ if n := pageToken(token); n > 1 {
+ u = query(u, "page", strconv.Itoa(n))
+ }
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return nil, "", err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return nil, "", structureChanged(login + " gists")
+ }
+ var out []Gist
+ for _, snip := range page.FindAll(doc, page.Sel{Class: "gist-snippet"}) {
+ if g, ok := gistSnippet(snip, res.FinalURL); ok {
+ out = append(out, g)
+ }
+ }
+ return out, railsNext(doc, token), nil
+ }
+ return paginate(ctx, limit, fetch, emit)
+}
+
+// gistSnippet reads one entry of a gist index. The counts are in the text of
+// the links beside it: "1 file", "6 forks", "62 stars".
+func gistSnippet(snip *html.Node, source string) (Gist, bool) {
+ var id, owner string
+ for _, a := range page.FindAll(snip, page.Sel{Tag: "a", Attr: "href"}) {
+ p := hrefPath(page.Attr(a, "href"))
+ o, rest, ok := strings.Cut(p, "/")
+ if !ok {
+ continue
+ }
+ hex, _, _ := strings.Cut(rest, "/")
+ if isGistID(hex) {
+ owner, id = o, hex
+ break
+ }
+ }
+ if id == "" {
+ return Gist{}, false
+ }
+ g := Gist{Owner: owner, IsPublic: true}
+ g.setIdentity(KindGist, id)
+ g.addSource(source)
+ for _, a := range page.FindAll(snip, page.Sel{Tag: "a", Attr: "href"}) {
+ text := page.Text(a)
+ n, _, ok := page.CountIn(text)
+ if !ok {
+ continue
+ }
+ switch {
+ case strings.HasSuffix(text, "file"), strings.HasSuffix(text, "files"):
+ g.FileCount = intp(n)
+ case strings.HasSuffix(text, "fork"), strings.HasSuffix(text, "forks"):
+ g.Forks = intp(n)
+ case strings.HasSuffix(text, "star"), strings.HasSuffix(text, "stars"):
+ g.Stars = intp(n)
+ }
+ }
+ if d := page.Find(snip, page.Sel{Class: "gist-snippet-meta"}); d != nil {
+ if p := page.Find(d, page.Sel{Tag: "span", Class: "f6"}); p != nil {
+ g.Description = page.Text(p)
+ }
+ }
+ if t := page.Find(snip, page.RelTimeEl); t != nil {
+ g.UpdatedAt = parseTime(page.Attr(t, "datetime"))
+ }
+ return g, true
+}
+
+// Gist reads one gist and its file list. Contents are a second request per file
+// and are opt-in, because a gist can hold a megabyte of log paste.
+func (c *Client) Gist(ctx context.Context, id string, withContent bool) (*Gist, error) {
+ id = strings.TrimSpace(id)
+ if i := strings.LastIndex(id, "/"); i >= 0 {
+ id = id[i+1:]
+ }
+ if !isGistID(id) {
+ return nil, usageBadID("gist", id, "a hexadecimal gist id")
+ }
+ res, err := c.GetHTML(ctx, GistURL+"/"+id)
+ if err != nil {
+ return nil, err
+ }
+ p := page.Extract(res.FinalURL, res.Body)
+ doc := p.Doc()
+ if doc == nil {
+ return nil, structureChanged(id)
+ }
+ g := &Gist{IsPublic: true}
+ g.setIdentity(KindGist, id)
+ g.addSource(res.FinalURL)
+ g.Owner = hrefOwner(res.FinalURL)
+ if d := page.Find(doc, page.Sel{Attr: "itemprop", AttrValue: "about"}); d != nil {
+ g.Description = page.Text(d)
+ }
+ if t := page.Find(doc, page.RelTimeEl); t != nil {
+ g.UpdatedAt = parseTime(page.Attr(t, "datetime"))
+ }
+ for _, box := range page.FindAll(doc, page.Sel{Class: "file"}) {
+ f, ok := gistFile(box)
+ if !ok {
+ continue
+ }
+ g.Files = append(g.Files, f)
+ }
+ if len(g.Files) == 0 {
+ return nil, structureChanged(id)
+ }
+ g.FileCount = intp(len(g.Files))
+ if withContent {
+ for i := range g.Files {
+ text, err := c.text(ctx, g.Files[i].RawURL)
+ if err != nil {
+ return nil, err
+ }
+ g.Files[i].Content = text
+ }
+ }
+ return g, nil
+}
+
+// gistFile reads one file block. The raw link is the useful half: it is the
+// only address on the page that returns the bytes rather than the rendering.
+func gistFile(box *html.Node) (GistFile, bool) {
+ name := page.Find(box, page.Sel{Class: "gist-blob-name"})
+ if name == nil {
+ return GistFile{}, false
+ }
+ f := GistFile{Name: page.Text(name)}
+ if f.Name == "" {
+ return GistFile{}, false
+ }
+ for _, a := range page.FindAll(box, page.Sel{Tag: "a", Attr: "href", AttrContains: "/raw/"}) {
+ f.RawURL = absoluteGistURL(page.Attr(a, "href"))
+ break
+ }
+ if f.RawURL == "" {
+ return GistFile{}, false
+ }
+ if i := strings.LastIndex(f.Name, "."); i > 0 {
+ f.Language = f.Name[i+1:]
+ }
+ return f, true
+}
+
+// text fetches a URL and returns it as a string. It is the small sibling of
+// Raw, for the addresses that are already absolute.
+func (c *Client) text(ctx context.Context, rawURL string) (string, error) {
+ res, err := c.Get(ctx, rawURL, SurfaceRaw)
+ if err != nil {
+ return "", err
+ }
+ return string(res.Body), nil
+}
+
+func absoluteGistURL(href string) string {
+ if strings.Contains(href, "://") {
+ return href
+ }
+ return GistURL + "/" + strings.TrimPrefix(href, "/")
+}
+
+func hrefOwner(rawURL string) string {
+ p := hrefPath(rawURL)
+ owner, rest, ok := strings.Cut(p, "/")
+ if !ok || !isGistID(rest) {
+ return ""
+ }
+ return owner
+}
+
+// isGistID matches the twenty-or-more hexadecimal characters a gist is named
+// with. Anything shorter is a login or a route word.
+func isGistID(s string) bool {
+ if len(s) < 20 {
+ return false
+ }
+ for _, r := range s {
+ switch {
+ case r >= '0' && r <= '9', r >= 'a' && r <= 'f', r >= 'A' && r <= 'F':
+ default:
+ return false
+ }
+ }
+ return true
+}
+
+// --- the contribution calendar ---
+
+// Contributions reads a year of a profile's contribution graph, one record per
+// day. This is the only representation of the numbers that exists without a
+// token: the GraphQL field that carries them refuses anonymous callers.
+//
+// A year is the largest window the fragment serves. Asking for a wider range
+// gets the last year, so the range is stated rather than inferred.
+func (c *Client) Contributions(ctx context.Context, login string, year int, emit func(ContributionDay) error) error {
+ if login == "" || strings.Contains(login, "/") {
+ return usageBadID("account", login, "a bare login")
+ }
+ if year == 0 {
+ year = time.Now().UTC().Year()
+ }
+ from := strconv.Itoa(year) + "-01-01"
+ to := strconv.Itoa(year) + "-12-31"
+ u := query(BaseURL+"/users/"+login+"/contributions", "from", from, "to", to)
+ // HTML rather than XHR. This is a fragment the front end swaps into the
+ // profile, so it serves markup and answers 406 to a request that asks for
+ // JSON, which the client reports as a response rather than an error and
+ // would show up here as an empty calendar.
+ res, err := c.Get(ctx, u, SurfaceHTML)
+ if err != nil {
+ return err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return structureChanged(login + " contributions")
+ }
+
+ // The count is not on the cell. Each cell points at a tooltip by id and the
+ // tooltip holds the sentence with the number in it, so the tooltips are
+ // indexed first and the cells read against that index.
+ counts := map[string]int{}
+ for _, tip := range page.FindAll(doc, page.Sel{Tag: "tool-tip"}) {
+ counts[page.Attr(tip, "for")] = leadingCount(page.Text(tip))
+ }
+ found := false
+ for _, td := range page.FindAll(doc, page.Sel{Tag: "td", Attr: "data-date"}) {
+ day := ContributionDay{Login: login}
+ at := parseTime(page.Attr(td, "data-date"))
+ if at == nil {
+ continue
+ }
+ found = true
+ day.Date = *at
+ day.Level, _ = strconv.Atoi(page.Attr(td, "data-level"))
+ day.Count = counts[page.Attr(td, "id")]
+ day.Kind = KindContribution
+ day.ID = login + "@" + page.Attr(td, "data-date")
+ day.URI = URI(KindContribution, day.ID)
+ day.URL = accountURL(login)
+ day.addSource(res.FinalURL)
+ if err := emit(day); err != nil {
+ return err
+ }
+ }
+ if !found {
+ return structureChanged(login + " contributions")
+ }
+ return nil
+}
+
+// leadingCount reads the number off the front of "7 contributions on January
+// 4th." and treats "No contributions" as the zero it is.
+func leadingCount(s string) int {
+ field, _, _ := strings.Cut(strings.TrimSpace(s), " ")
+ n, err := strconv.Atoi(strings.ReplaceAll(field, ",", ""))
+ if err != nil {
+ return 0
+ }
+ return n
+}
+
+// --- activity ---
+
+// Activity reads a public event stream. The same feed shape serves an account
+// and a repository, so the argument is either a login or owner/name and the
+// URL is the only thing that differs.
+//
+// This replaces the REST events endpoint outright. The feed is public, cheap,
+// and needs no credential, and the event class is encoded in each entry's id,
+// so the type comes from a field rather than from matching on prose.
+func (c *Client) Activity(ctx context.Context, ref string, limit int, emit func(Event) error) error {
+ ref = strings.Trim(ref, "/")
+ if ref == "" {
+ return usageBadID("account or repository", ref, "a login or owner/name")
+ }
+ u := feedURL(ref + ".atom")
+ if _, _, ok := SplitRepo(ref); ok {
+ u = repoSubURL(ref, "commits.atom")
+ }
+ res, err := c.Get(ctx, u, SurfaceFeed)
+ if err != nil {
+ return err
+ }
+ var feed atomFeed
+ if err := xml.Unmarshal(res.Body, &feed); err != nil {
+ return badPayload(shortURL(u), err)
+ }
+ seen := 0
+ for _, e := range feed.Entries {
+ ev := e.event(res.FinalURL)
+ if err := emit(ev); err != nil {
+ return err
+ }
+ seen++
+ if limit > 0 && seen >= limit {
+ return nil
+ }
+ }
+ if seen == 0 {
+ return errs.NotFound("%s: the feed carried no entries", shortURL(u))
+ }
+ return nil
+}
+
+// atomFeed is the shape all five of GitHub's feeds share. Only the id encoding
+// differs between them, and that is read per entry rather than per feed.
+type atomFeed struct {
+ Title string `xml:"title"`
+ Updated string `xml:"updated"`
+ Entries []atomEntry `xml:"entry"`
+}
+
+type atomEntry struct {
+ ID string `xml:"id"`
+ Title string `xml:"title"`
+ Published string `xml:"published"`
+ Updated string `xml:"updated"`
+ Content string `xml:"content"`
+ Link struct {
+ Href string `xml:"href,attr"`
+ } `xml:"link"`
+ Author struct {
+ Name string `xml:"name"`
+ URI string `xml:"uri"`
+ } `xml:"author"`
+ Thumbnail struct {
+ URL string `xml:"url,attr"`
+ } `xml:"thumbnail"`
+}
+
+// event turns one entry into a record. The id is
+// "tag:github.com,2008:push/15757005823", so the segment after the colon is the
+// event class and the tool never has to read the localised title to find out
+// what happened.
+func (e atomEntry) event(source string) Event {
+ ev := Event{Title: page.FragmentText(e.Title)}
+ _, tail, _ := strings.Cut(e.ID, "2008:")
+ class, rest, _ := strings.Cut(tail, "/")
+ ev.Type = eventType(class)
+ ev.Kind = KindEvent
+ ev.ID = e.ID
+ if ev.Type != "" && rest != "" {
+ ev.ID = ev.Type + "/" + rest
+ }
+ ev.URI = URI(KindEvent, ev.ID)
+ ev.URL = e.Link.Href
+ ev.Target = e.Link.Href
+ ev.At = firstTime(e.Published, e.Updated)
+ ev.BodyHTML = e.Content
+ if e.Author.Name != "" {
+ ev.Actor = actor(e.Author.Name)
+ ev.Actor.AvatarURL = e.Thumbnail.URL
+ }
+ // The alternate link points at whatever the event touched, and the first
+ // two segments of it are the repository whenever there is one.
+ if p := hrefPath(e.Link.Href); strings.Count(p, "/") >= 1 {
+ owner, rest, _ := strings.Cut(p, "/")
+ name, _, _ := strings.Cut(rest, "/")
+ if owner != "" && name != "" && !routeWord[name] {
+ ev.Repo = owner + "/" + name
+ }
+ }
+ ev.addSource(source)
+ return ev
+}
+
+// eventType turns the class out of the entry id into one word.
+//
+// A person's feed names the class in lower case, push or fork or watch. A
+// repository's commit feed names it after the Ruby object that used to render
+// it, Grit::Commit, which is an implementation detail from 2008 and not a thing
+// anyone should have to filter on.
+func eventType(class string) string {
+ if _, tail, ok := strings.Cut(class, "::"); ok {
+ class = tail
+ }
+ return strings.ToLower(class)
+}
+
+func firstTime(ss ...string) *time.Time {
+ for _, s := range ss {
+ if t := parseTime(s); t != nil {
+ return t
+ }
+ }
+ return nil
+}
+
+// --- the rails pager ---
+
+// nextPageToken returns the page number to ask for next, or empty when the page
+// says there is no next.
+//
+// Two templates write the same link two ways: the organization roster marks it
+// rel="next" and the profile tabs use a plain anchor whose text is Next. Both
+// are checked because both are load-bearing.
+func railsNext(doc *html.Node, token string) string {
+ if !hasNextLink(doc) {
+ return ""
+ }
+ return strconv.Itoa(pageToken(token) + 1)
+}
+
+func hasNextLink(doc *html.Node) bool {
+ if page.Find(doc, page.NextPage) != nil {
+ return true
+ }
+ for _, a := range page.FindAll(doc, page.Sel{Tag: "a", Attr: "href"}) {
+ if strings.EqualFold(strings.TrimSpace(page.Text(a)), "next") {
+ return true
+ }
+ }
+ return false
+}
diff --git a/gh/repo.go b/gh/repo.go
index e8577c7..51ae7ac 100644
--- a/gh/repo.go
+++ b/gh/repo.go
@@ -462,10 +462,16 @@ func descriptionFromOG(title, id string) string {
// --- the deep pass ---
-// deepenRepo runs the extra fetch --deep opts into: the dependent count off the
-// dependency graph. The failure is soft. A dependency graph that is disabled is
-// a fact about the repository, not an error in the read.
+// deepenRepo runs the two extra fetches --deep opts into: the sidebar
+// fragment, which is where the language histogram and the contributor count
+// actually live, and the dependency graph for the dependent count. Both
+// failures are soft. A dependency graph that is disabled is a fact about the
+// repository, not an error in the read.
func (c *Client) deepenRepo(ctx context.Context, r *Repo) error {
+ if sb, err := c.sidebar(ctx, r.ID); err == nil && sb != nil {
+ sb.apply(r)
+ r.addSource(repoSubURL(r.ID, "_sidebar"))
+ }
if n, err := c.dependents(ctx, r.ID); err == nil && n != nil {
r.DependentCount = n
}
@@ -473,16 +479,92 @@ func (c *Client) deepenRepo(ctx context.Context, r *Repo) error {
return nil
}
+// sidebarData is the fragment the repository page's own front end fetches to
+// fill the About column in. Everything on it is deferred, which is why a cold
+// page has a skeleton where the language bar goes.
+type sidebarData struct {
+ Languages *struct {
+ Languages []sidebarLanguage `json:"languages"`
+ } `json:"languages"`
+ Contributors *struct {
+ ContributorCount *int `json:"contributorCount"`
+ } `json:"contributors"`
+ UsedBy *struct {
+ DependentsCount *int `json:"dependentsCount"`
+ } `json:"usedBy"`
+}
+
+type sidebarLanguage struct {
+ Name string `json:"name"`
+ Percentage float64 `json:"percentage"`
+ Color string `json:"color"`
+}
+
+// sidebar reads /{owner}/{repo}/_sidebar.
+//
+// This is the answer to a question the rest of this file used to give up on.
+// /{owner}/{repo}/graphs/languages 301s back to the repository page for an
+// anonymous client, and none of show_partial, /languages, or
+// /graphs/languages-data exist, so the conclusion was that the histogram had no
+// keyless source. It has one: the same fragment the page itself waits for, and
+// it needs no credential, only the header that says a script is asking.
+func (c *Client) sidebar(ctx context.Context, id string) (*sidebarData, error) {
+ if _, _, ok := SplitRepo(id); !ok {
+ return nil, usageBadID("repository", id, "owner/name")
+ }
+ res, err := c.Get(ctx, repoSubURL(id, "_sidebar"), SurfaceXHR)
+ if err != nil {
+ return nil, err
+ }
+ var sb sidebarData
+ if err := json.Unmarshal(res.Body, &sb); err != nil {
+ return nil, badPayload(id, err)
+ }
+ return &sb, nil
+}
+
+// apply folds the fragment into the record.
+//
+// The percentages are what the fragment states, so they are stored as
+// percentages times one hundred and marked as such. A byte count and a
+// percentage are not the same number and nothing downstream should be able to
+// mistake one for the other.
+func (sb *sidebarData) apply(r *Repo) {
+ if sb.Contributors != nil && sb.Contributors.ContributorCount != nil {
+ r.ContributorCount = sb.Contributors.ContributorCount
+ }
+ if sb.UsedBy != nil && sb.UsedBy.DependentsCount != nil {
+ r.DependentCount = sb.UsedBy.DependentsCount
+ }
+ langs := sb.langs()
+ if len(langs) == 0 {
+ return
+ }
+ out := map[string]int64{}
+ for _, l := range langs {
+ out[l.Name] = int64(l.Percentage * 100)
+ }
+ r.Languages = out
+ r.Language = topLanguage(out)
+ if r.LanguageColor == "" {
+ r.LanguageColor = langs[0].Color
+ }
+ recordVia(&r.Base, "languages", "sidebar-percent")
+}
+
+func (sb *sidebarData) langs() []sidebarLanguage {
+ if sb == nil || sb.Languages == nil {
+ return nil
+ }
+ return sb.Languages.Languages
+}
+
// searchLanguage asks repository search for the repository by name, because the
// search result carries the primary language and its colour and the repository
// page does not.
//
-// The obvious place to look is /{owner}/{repo}/graphs/languages, and it is a
-// dead end: it 301s back to the repository page for an anonymous client, and
-// none of show_partial, /languages, or /graphs/languages-data exist. The
-// language bar in the sidebar is the other source, and on a cold page it is a
-// skeleton with no /search?l= links in it at all. So the histogram with real
-// byte counts has no keyless source, and the language name does, one search away.
+// This is the shallow path. It is one request and gives the primary language
+// only; the sidebar fragment gives the whole histogram and is what --deep uses.
func (c *Client) searchLanguage(ctx context.Context, id string) (lang, color string, err error) {
owner, name, ok := SplitRepo(id)
if !ok {
diff --git a/gh/types.go b/gh/types.go
index 8c66958..67dbc49 100644
--- a/gh/types.go
+++ b/gh/types.go
@@ -58,6 +58,7 @@ type Repo struct {
TagCount *int `json:"tag_count,omitempty" table:"-"`
FileCount *int `json:"file_count,omitempty" table:"-"`
DependentCount *int `json:"dependent_count,omitempty" table:"-"`
+ ContributorCount *int `json:"contributor_count,omitempty" table:"-"`
// License comes from one sidebar anchor and from nowhere else on any
// keyless surface. See page.LicenseLink.
@@ -763,8 +764,10 @@ type Trending struct {
// --- contributions ---
// Contributor is one person's contribution statistics for a repository. Weeks
-// arrives with the response so it is kept by default, and it is never a table
-// column because a hundred weeks is not a column.
+// arrives with the response but is dropped unless asked for, because the route
+// sends every week since the repository began for every contributor and that is
+// megabytes of mostly zeroes. It is never a table column either way, because a
+// hundred weeks is not a column.
type Contributor struct {
Base
@@ -821,3 +824,40 @@ type Event struct {
Target string `json:"target,omitempty" table:"-"`
At *time.Time `json:"at,omitempty" table:"at,time"`
}
+
+// --- projections ---
+
+// LanguageShare is one language of one repository. The repository record
+// carries the same numbers as a map, which is the right shape to keep and the
+// wrong shape to print, so this is the row form of it.
+type LanguageShare struct {
+ Base
+
+ Repo string `json:"repo" table:"repo"`
+ Language string `json:"language" table:"language"`
+ Percent float64 `json:"percent" table:"percent"`
+ Color string `json:"color,omitempty" table:"-"`
+}
+
+// RepoStats is the counts and nothing else.
+//
+// Every field is already on Repo. The reason to have it separately is that a
+// record with eight numbers in it is something you can store once a day and
+// diff; a record with a readme in it is not.
+type RepoStats struct {
+ Base
+
+ Repo string `json:"repo" table:"repo"`
+
+ Stars *int `json:"stars,omitempty" table:"stars"`
+ Forks *int `json:"forks,omitempty" table:"forks"`
+ Watchers *int `json:"watchers,omitempty" table:"watching"`
+ OpenIssues *int `json:"open_issues,omitempty" table:"issues"`
+ Commits *int `json:"commits,omitempty" table:"commits"`
+ Releases *int `json:"releases,omitempty" table:"releases"`
+ Tags *int `json:"tags,omitempty" table:"tags"`
+ Contributors *int `json:"contributors,omitempty" table:"people"`
+ Dependents *int `json:"dependents,omitempty" table:"used_by"`
+
+ PushedAt *time.Time `json:"pushed_at,omitempty" table:"pushed,time"`
+}
diff --git a/gh/uri.go b/gh/uri.go
index f2221db..02d55b2 100644
--- a/gh/uri.go
+++ b/gh/uri.go
@@ -47,6 +47,14 @@ const (
KindWiki = "wiki"
KindAdvisory = "advisory"
KindCompare = "compare"
+
+ // These three name records GitHub derives rather than serves. There is no
+ // page whose address is one contributor's statistics or one day of a
+ // calendar, so they get a URI and no canonical URL, and Locate points at
+ // the page they were read from instead of inventing one.
+ KindContributor = "contributor"
+ KindContribution = "contribution"
+ KindEvent = "event"
)
// Scheme is the URI scheme this package mints and dereferences.
@@ -163,7 +171,8 @@ func knownKind(k string) bool {
switch k {
case KindRepo, KindUser, KindOrg, KindIssue, KindPR, KindDiscussion, KindCommit,
KindBranch, KindTag, KindRelease, KindFile, KindTree, KindLabel, KindMilestone,
- KindTopic, KindGist, KindPackage, KindAction, KindWiki, KindAdvisory, KindCompare:
+ KindTopic, KindGist, KindPackage, KindAction, KindWiki, KindAdvisory, KindCompare,
+ KindContributor, KindContribution, KindEvent:
return true
}
return false
@@ -477,7 +486,26 @@ func Locate(kind, id string) (string, error) {
case KindAdvisory:
return BaseURL + "/advisories/" + id, nil
case KindGist:
- return "https://gist.github.com/" + id, nil
+ return GistURL + "/" + id, nil
+ case KindContributor:
+ // The id is owner/name@login and the page that states it is the graph,
+ // which is the whole roster rather than the one row. That is the
+ // closest true address, so it is the one given.
+ repo, _, ok := cutRev(id)
+ if !ok {
+ return "", errs.Usage("contributor id %q is not owner/name@login", id)
+ }
+ return BaseURL + "/" + repo + "/graphs/contributors", nil
+ case KindContribution:
+ login, _, ok := cutRev(id)
+ if !ok {
+ return "", errs.Usage("contribution id %q is not login@date", id)
+ }
+ return BaseURL + "/" + login, nil
+ case KindEvent:
+ // An event's address is the thing it happened to, which the feed states
+ // per entry and no rule can reconstruct from the id.
+ return "", errs.Usage("an event has no address of its own; read its url field")
case KindCompare:
repo, rng, ok := cutRev(id)
if !ok {
From 7fcae273a3cc3c8b9c9bf2a3fdc68e4e83058160 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 20:15:44 +0700
Subject: [PATCH 10/21] gh: the graph plane, plus the dependency graph pages
Every reader so far produced records. This turns those records into a graph:
one extractor per type that says what a record points at, five trust levels so a
caller can ask for only the edges that came from an id rather than from prose,
and a walk that follows them.
The crawler is sequential on purpose. Doc 04 gives it a Concurrency knob and it
would be a lie here, because every request already queues through one rate
limiter, so workers would only queue deeper behind the same pacer while making
the output order unpredictable.
deps and dependents are the two dependency graph pages. They are the most
valuable keyless surface on the site and the least reliable one, and they need
two readers rather than one: the rows have different shapes, and so do the two
pagers, ?page=N with a rel="next" anchor on one side and an opaque cursor in a
button on the other.
The empty-page retry in rowPage is not defensive coding, it is the fix for a
real truncation. GitHub answers a cursor page with a 200, the right title, and
no rows often enough to hit on the first repo tried, and the walk cannot tell
that from the end of the list, so it stopped at sixty rows out of two hundred
and then cached the empty page and kept stopping there for fifteen minutes.
rdf and export are byte-plane commands for the same reason cat is: N-Triples is
a serialisation with its own rules, not a record, and pushing it through the
record renderer would produce something that is neither.
---
cli/export.go | 118 ++++++
cli/rdf.go | 121 ++++++
cli/root.go | 2 +
gh/crawl.go | 313 ++++++++++++++
gh/deps.go | 263 ++++++++++++
gh/graph.go | 949 ++++++++++++++++++++++++++++++++++++++++++
gh/ops.go | 194 +++++++++
gh/rdf.go | 617 +++++++++++++++++++++++++++
gh/types.go | 38 ++
pkg/page/selectors.go | 38 +-
10 files changed, 2650 insertions(+), 3 deletions(-)
create mode 100644 cli/export.go
create mode 100644 cli/rdf.go
create mode 100644 gh/crawl.go
create mode 100644 gh/deps.go
create mode 100644 gh/graph.go
create mode 100644 gh/rdf.go
diff --git a/cli/export.go b/cli/export.go
new file mode 100644
index 0000000..8b17e1c
--- /dev/null
+++ b/cli/export.go
@@ -0,0 +1,118 @@
+package cli
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "io"
+ "os"
+
+ "github.com/tamnd/any-cli/kit"
+ "github.com/tamnd/any-cli/kit/errs"
+ "github.com/tamnd/github-cli/gh"
+)
+
+// export.go writes a whole graph to one file. Everything it does can be had by
+// redirecting `github crawl` or `github rdf`, and it exists because what people
+// want at the end of a walk is one file they can load somewhere else, named once
+// rather than assembled out of two commands and a shell operator.
+
+type exportCmd struct {
+ format string
+ out string
+ depth int
+ follow []string
+ minTrust string
+ limit int
+}
+
+func newExportCmd() kit.Command {
+ c := &exportCmd{}
+ return kit.Command{
+ Use: "export ",
+ Short: "Write a whole graph to one file",
+ Long: "export walks from the seed and writes the result in one go. The formats are\n" +
+ "jsonl (one node, edge, or fact per line), json (a single object), and the four\n" +
+ "RDF serialisations nt, ttl, jsonld, and nq.\n\n" +
+ "Without --out it writes to stdout, which makes it a drop-in for a pipeline.",
+ Group: "graph",
+ Args: kit.ExactArgs(1),
+ Flags: c.flags,
+ Run: c.run,
+ }
+}
+
+func (c *exportCmd) flags(f *kit.FlagSet) {
+ f.StringVar(&c.format, "format", "jsonl", "jsonl, json, nt, ttl, jsonld, or nq")
+ f.StringVarP(&c.out, "out", "O", "", "write here instead of stdout")
+ f.IntVar(&c.depth, "depth", 1, "walk this many edges out")
+ f.StringSliceVar(&c.follow, "follow", nil, "predicates to follow (default: the structural ones)")
+ f.StringVar(&c.minTrust, "min-trust", gh.DefaultMinTrust, "drop edges below this rule")
+ f.IntVar(&c.limit, "limit", 0, "stop after this many nodes")
+}
+
+func (c *exportCmd) run(ctx context.Context, args []string) error {
+ cl, err := clientFrom(ctx)
+ if err != nil {
+ return err
+ }
+ kind, id, g, err := buildGraph(ctx, cl, args[0], graphWalk{
+ depth: c.depth,
+ follow: c.follow,
+ minTrust: c.minTrust,
+ limit: c.limit,
+ })
+ if err != nil {
+ return err
+ }
+ gh.SortEdges(g.Edges)
+
+ out := io.Writer(os.Stdout)
+ if c.out != "" {
+ f, err := os.Create(c.out)
+ if err != nil {
+ return err
+ }
+ defer func() { _ = f.Close() }()
+ out = f
+ }
+ w := bufio.NewWriter(out)
+ defer func() { _ = w.Flush() }()
+
+ switch c.format {
+ case "jsonl":
+ return writeJSONL(w, g)
+ case "json":
+ enc := json.NewEncoder(w)
+ enc.SetIndent("", " ")
+ return enc.Encode(g)
+ case gh.FormatNT, gh.FormatTurtle, gh.FormatJSONLD, gh.FormatNQuads:
+ graph, _ := gh.Locate(kind, id)
+ return gh.WriteRDF(w, g, gh.RDFOptions{Format: c.format, Graph: graph})
+ default:
+ return errs.Usage("unknown --format %q", c.format)
+ }
+}
+
+// writeJSONL puts the nodes first and everything that points at them after, so a
+// reader building an index in one pass never sees an edge before both of its
+// ends.
+func writeJSONL(w io.Writer, g *gh.Graph) error {
+ enc := json.NewEncoder(w)
+ for i := range g.Nodes {
+ if err := enc.Encode(&g.Nodes[i]); err != nil {
+ return err
+ }
+ }
+ for i := range g.Edges {
+ if err := enc.Encode(&g.Edges[i]); err != nil {
+ return err
+ }
+ }
+ for i := range g.Facts {
+ if err := enc.Encode(&g.Facts[i]); err != nil {
+ return err
+ }
+ }
+ return nil
+}
diff --git a/cli/rdf.go b/cli/rdf.go
new file mode 100644
index 0000000..acdf944
--- /dev/null
+++ b/cli/rdf.go
@@ -0,0 +1,121 @@
+package cli
+
+import (
+ "bufio"
+ "context"
+ "os"
+
+ "github.com/tamnd/any-cli/kit"
+ "github.com/tamnd/github-cli/gh"
+)
+
+// rdf.go holds the linked-data output. It is a byte-plane command for the same
+// reason cat is: N-Triples and Turtle are not records, they are a serialisation
+// with their own rules, and putting them through the record renderer would
+// produce something that is neither.
+
+type rdfCmd struct {
+ format string
+ graph string
+ depth int
+ follow []string
+ minTrust string
+ limit int
+}
+
+func newRDFCmd() kit.Command {
+ c := &rdfCmd{}
+ return kit.Command{
+ Use: "rdf ",
+ Short: "Write an entity as RDF triples",
+ Long: "rdf serialises one entity, its edges, and its literals. N-Triples is the\n" +
+ "default because it streams line by line, so a deep walk never needs the whole\n" +
+ "graph in memory; Turtle and JSON-LD do need it and are slower on large graphs\n" +
+ "for that reason.\n\n" +
+ "Subjects are the github.com URLs rather than the github:// URIs, so the output\n" +
+ "is dereferenceable by anything on the web. The URI is kept as a gh:uri\n" +
+ "literal, so nothing is lost.\n\n" +
+ "With --depth it walks first and serialises the whole result, which is how you\n" +
+ "get a loadable dataset rather than one subject.",
+ Group: "graph",
+ Args: kit.ExactArgs(1),
+ Flags: c.flags,
+ Run: c.run,
+ }
+}
+
+func (c *rdfCmd) flags(f *kit.FlagSet) {
+ f.StringVar(&c.format, "format", gh.FormatNT, "nt, ttl, jsonld, or nq")
+ f.StringVar(&c.graph, "graph", "", "the named graph for nq output (default: the entity URL)")
+ f.IntVar(&c.depth, "depth", 0, "walk this many edges out before serialising")
+ f.StringSliceVar(&c.follow, "follow", nil, "predicates to follow when walking")
+ f.StringVar(&c.minTrust, "min-trust", gh.DefaultMinTrust, "drop edges below this rule")
+ f.IntVar(&c.limit, "limit", 0, "stop a walk after this many nodes")
+}
+
+func (c *rdfCmd) run(ctx context.Context, args []string) error {
+ cl, err := clientFrom(ctx)
+ if err != nil {
+ return err
+ }
+ kind, id, g, err := buildGraph(ctx, cl, args[0], graphWalk{
+ depth: c.depth,
+ follow: c.follow,
+ minTrust: c.minTrust,
+ limit: c.limit,
+ })
+ if err != nil {
+ return err
+ }
+ gh.SortEdges(g.Edges)
+
+ w := bufio.NewWriter(os.Stdout)
+ defer func() { _ = w.Flush() }()
+
+ graph := c.graph
+ if graph == "" {
+ graph, _ = gh.Locate(kind, id)
+ }
+ return gh.WriteRDF(w, g, gh.RDFOptions{Format: c.format, Graph: graph})
+}
+
+// graphWalk is the set of knobs rdf and export share. They are the same walk
+// with a different writer on the end, so the flags are declared twice and read
+// once.
+type graphWalk struct {
+ depth int
+ follow []string
+ minTrust string
+ limit int
+}
+
+// buildGraph resolves a reference and returns either the one entity or the whole
+// walk, depending on depth. Both come back as a Graph, so the serialisers never
+// need to know which it was.
+//
+// It holds the result in memory, which is the price of the formats that cannot
+// stream. `github crawl` is the streaming answer for a walk too big for this.
+func buildGraph(ctx context.Context, cl *gh.Client, ref string, o graphWalk) (string, string, *gh.Graph, error) {
+ kind, id, g, err := cl.GraphOfRef(ctx, ref)
+ if err != nil {
+ return "", "", nil, err
+ }
+ if o.depth <= 0 {
+ return kind, id, g, nil
+ }
+ g = &gh.Graph{}
+ err = cl.Crawl(ctx, gh.URI(kind, id), gh.CrawlOptions{
+ Depth: o.depth,
+ Follow: o.follow,
+ MinTrust: o.minTrust,
+ Limit: o.limit,
+ }, gh.CrawlSink{
+ Node: func(n *gh.Node) error { g.AddNode(*n); return nil },
+ Edge: func(e *gh.Edge) error { g.Edges = append(g.Edges, *e); return nil },
+ Fact: func(f *gh.Fact) error { g.Facts = append(g.Facts, *f); return nil },
+ })
+ if err != nil {
+ return "", "", nil, err
+ }
+ return kind, id, g, nil
+}
diff --git a/cli/root.go b/cli/root.go
index def5d66..2c78431 100644
--- a/cli/root.go
+++ b/cli/root.go
@@ -38,5 +38,7 @@ func NewApp() *kit.App {
app.AddCommand(newReadmeCmd())
app.AddCommand(newArchiveCmd())
app.AddCommand(newDiffCmd())
+ app.AddCommand(newRDFCmd())
+ app.AddCommand(newExportCmd())
return app
}
diff --git a/gh/crawl.go b/gh/crawl.go
new file mode 100644
index 0000000..8322773
--- /dev/null
+++ b/gh/crawl.go
@@ -0,0 +1,313 @@
+package gh
+
+import (
+ "context"
+ "fmt"
+ "strings"
+
+ "github.com/tamnd/any-cli/kit/errs"
+)
+
+// crawl.go walks the graph. The algorithm is a breadth-first frontier with a
+// visited set and a hard budget, and that is all it should ever be: the
+// interesting decisions here are about what not to follow and when to stop, not
+// about traversal.
+//
+// The walk is sequential. Doc 04 section 5 gives the crawler a Concurrency
+// field, and it would be a lie in this client: every request already queues
+// through one rate limiter, so N workers would only queue N deep behind the same
+// pacer while making the output order unpredictable. If the pacer ever grows a
+// real parallel mode, this is the place to add the workers.
+
+// CrawlOptions bounds a walk. The budgets are the point of the struct. A tool
+// that can accidentally send a million requests at somebody else's servers
+// should be hard to point that way by accident.
+type CrawlOptions struct {
+ Depth int
+ Follow []string
+ Kinds []string
+ MinTrust string
+ Limit int
+
+ NodesOnly bool
+ EdgesOnly bool
+}
+
+// CrawlSink receives what the walk finds. Emission is streaming: a crawl of a
+// large organization must never need the whole graph in memory, and a crawl that
+// is interrupted has already emitted everything it found.
+type CrawlSink struct {
+ Node func(*Node) error
+ Edge func(*Edge) error
+ Fact func(*Fact) error
+}
+
+// defaults fills in the spec's defaults. Depth 1 and the structural follow set
+// are the safe walk: references, stars, follows, and the two dependency
+// predicates fan out without bound, so each has to be asked for by name.
+func (o *CrawlOptions) defaults() {
+ if o.Depth <= 0 {
+ o.Depth = 1
+ }
+ if o.MinTrust == "" {
+ o.MinTrust = DefaultMinTrust
+ }
+ if len(o.Follow) == 0 {
+ o.Follow = DefaultFollow
+ }
+}
+
+// followSet accepts a predicate written either way, so --follow ownedBy and
+// --follow gh:ownedBy mean the same thing. The word "all" turns the filter off.
+func followSet(follow []string) map[string]bool {
+ out := map[string]bool{}
+ for _, f := range follow {
+ for _, part := range strings.Split(f, ",") {
+ part = strings.TrimSpace(part)
+ if part == "" {
+ continue
+ }
+ if part == "all" {
+ return nil
+ }
+ out[strings.TrimPrefix(part, "gh:")] = true
+ }
+ }
+ return out
+}
+
+// kindSet is the same idea for --kinds. An empty set expands every kind.
+func kindSet(kinds []string) map[string]bool {
+ out := map[string]bool{}
+ for _, k := range kinds {
+ for _, part := range strings.Split(k, ",") {
+ part = strings.TrimSpace(part)
+ if part != "" {
+ out[part] = true
+ }
+ }
+ }
+ return out
+}
+
+// splitURI takes a github:// URI apart. The crawler holds URIs rather than
+// records, which is what keeps its memory proportional to the number of nodes
+// seen and not to their size.
+func splitURI(uri string) (kind, id string, ok bool) {
+ k, i, _, err := parseURI(uri)
+ if err != nil {
+ return "", "", false
+ }
+ return k, i, true
+}
+
+// Crawl walks outward from a seed reference. Nodes, edges, and facts come out as
+// they are discovered, and the walk stops cleanly at either bound and reports
+// what it had rather than failing.
+func (c *Client) Crawl(ctx context.Context, seed string, o CrawlOptions, sink CrawlSink) error {
+ o.defaults()
+ kind, id, err := Classify(seed)
+ if err != nil {
+ return err
+ }
+ allow := followSet(o.Follow)
+ expand := kindSet(o.Kinds)
+
+ type item struct {
+ uri string
+ depth int
+ }
+ start := URI(kind, id)
+ visited := map[string]bool{start: true}
+ frontier := []item{{start, 0}}
+ nodes := 0
+
+ for len(frontier) > 0 {
+ cur := frontier[0]
+ frontier = frontier[1:]
+
+ // The limit is checked before the fetch, which is what makes it a
+ // budget rather than a suggestion.
+ if o.Limit > 0 && nodes >= o.Limit {
+ return nil
+ }
+ if err := ctx.Err(); err != nil {
+ return wrapNetwork("", err)
+ }
+ curKind, curID, ok := splitURI(cur.uri)
+ if !ok {
+ continue
+ }
+ rec, err := c.fetchOne(ctx, curKind, curID)
+ if err != nil {
+ // One unreachable node must not end a walk that has already
+ // produced useful output. A deleted repository, a kind this tool
+ // cannot dereference yet, and a page that has moved are all normal
+ // mid-crawl, and the alternative is a two-hour walk that throws
+ // away its results on the last hop.
+ if softSkip(err) {
+ continue
+ }
+ return err
+ }
+ node, edges, facts := Extract(rec)
+ if node.URI == "" {
+ continue
+ }
+ nodes++
+ if !o.EdgesOnly && sink.Node != nil {
+ n := node
+ if err := sink.Node(&n); err != nil {
+ return err
+ }
+ }
+ edges = FilterTrust(edges, o.MinTrust)
+ if !o.NodesOnly {
+ for i := range edges {
+ if sink.Edge != nil {
+ e := edges[i]
+ if err := sink.Edge(&e); err != nil {
+ return err
+ }
+ }
+ }
+ for i := range facts {
+ if sink.Fact != nil {
+ f := facts[i]
+ if err := sink.Fact(&f); err != nil {
+ return err
+ }
+ }
+ }
+ }
+ if cur.depth >= o.Depth {
+ continue
+ }
+ for _, e := range edges {
+ // A language or a licence is a bare string with no page behind it,
+ // so it is an edge but never a target.
+ if !strings.HasPrefix(e.Object, Scheme+"://") {
+ continue
+ }
+ if len(allow) > 0 && !allow[e.Predicate] {
+ continue
+ }
+ objKind, _, ok := splitURI(e.Object)
+ if !ok {
+ continue
+ }
+ if len(expand) > 0 && !expand[objKind] {
+ continue
+ }
+ // Cycles are normal on this graph. A fork points at its parent and
+ // the parent's fork list points back, and the visited set is the
+ // only defence that needs.
+ if visited[e.Object] {
+ continue
+ }
+ visited[e.Object] = true
+ frontier = append(frontier, item{e.Object, cur.depth + 1})
+ }
+ }
+ return nil
+}
+
+// softSkip reports whether an error is one node's problem rather than the
+// walk's. Not found, needs a login, and a kind that has no reader yet are all
+// "skip this one", and anything else stops the crawl.
+func softSkip(err error) bool {
+ switch errs.KindOf(err) {
+ case errs.KindNotFound, errs.KindNeedAuth, errs.KindUnsupported:
+ return true
+ default:
+ return false
+ }
+}
+
+// CrawlPlan is what --dry-run answers with. It is a record rather than a line
+// on stderr so the answer goes through the same renderer, formats, and pipes as
+// every other command, and so a script can size a walk without reading prose.
+type CrawlPlan struct {
+ Base
+
+ Seed string `json:"seed" table:"seed"`
+ Depth int `json:"depth" table:"depth"`
+ Nodes int `json:"nodes" table:"nodes"`
+
+ Note string `json:"note" table:"note"`
+}
+
+// Estimate reads the seed and reports what one more level would cost. It is
+// deliberately a lower bound and the note says so: the first level is countable
+// because the seed's edges are in hand, and everything past it depends on a
+// branching factor that cannot be seen from here without doing the walk.
+func (c *Client) Estimate(ctx context.Context, seed string, o CrawlOptions) (*CrawlPlan, error) {
+ o.defaults()
+ kind, id, err := Classify(seed)
+ if err != nil {
+ return nil, err
+ }
+ rec, err := c.fetchOne(ctx, kind, id)
+ if err != nil {
+ return nil, err
+ }
+ _, edges, _ := Extract(rec)
+ edges = FilterTrust(edges, o.MinTrust)
+ allow := followSet(o.Follow)
+ expand := kindSet(o.Kinds)
+
+ seen := map[string]bool{URI(kind, id): true}
+ next := 0
+ for _, e := range edges {
+ if !strings.HasPrefix(e.Object, Scheme+"://") || seen[e.Object] {
+ continue
+ }
+ if len(allow) > 0 && !allow[e.Predicate] {
+ continue
+ }
+ objKind, _, ok := splitURI(e.Object)
+ if !ok || (len(expand) > 0 && !expand[objKind]) {
+ continue
+ }
+ seen[e.Object] = true
+ next++
+ }
+ nodes := 1 + next
+ if o.Limit > 0 && nodes > o.Limit {
+ nodes = o.Limit
+ }
+ note := fmt.Sprintf("at least %d nodes through depth 1, about one request each", nodes)
+ if o.Depth > 1 {
+ note += fmt.Sprintf(", and more at depth %d depending on how the next level branches", o.Depth)
+ }
+ plan := &CrawlPlan{Seed: seed, Depth: o.Depth, Nodes: nodes, Note: note}
+ plan.setIdentity(kind, id)
+ return plan, nil
+}
+
+// GraphOf builds the node, edges, and facts for one entity. `github graph`,
+// `github edges`, and `github rdf` all call it, so the three never disagree
+// about what an entity's edges are.
+func (c *Client) GraphOf(ctx context.Context, kind, id string) (*Graph, error) {
+ rec, err := c.fetchOne(ctx, kind, id)
+ if err != nil {
+ return nil, err
+ }
+ g := &Graph{}
+ g.Add(rec)
+ return g, nil
+}
+
+// GraphOfRef is GraphOf for a reference that has not been classified yet, and it
+// reports the kind it turned out to be so a caller can say what it read.
+func (c *Client) GraphOfRef(ctx context.Context, ref string) (string, string, *Graph, error) {
+ kind, id, err := Classify(ref)
+ if err != nil {
+ return "", "", nil, err
+ }
+ g, err := c.GraphOf(ctx, kind, id)
+ if err != nil {
+ return "", "", nil, err
+ }
+ return kind, id, g, nil
+}
diff --git a/gh/deps.go b/gh/deps.go
new file mode 100644
index 0000000..4539fba
--- /dev/null
+++ b/gh/deps.go
@@ -0,0 +1,263 @@
+package gh
+
+import (
+ "context"
+ "net/url"
+ "strconv"
+ "strings"
+
+ "golang.org/x/net/html"
+
+ "github.com/tamnd/github-cli/pkg/page"
+)
+
+// deps.go reads the two dependency graph pages. They are the most valuable
+// keyless surface on the site and the least reliable one: the graph is opt-in
+// per repository, the rows are prose, and a package GitHub cannot resolve to a
+// repository is a name and nothing else.
+//
+// Both pages are read rather than one, because they are not two views of the
+// same list. Dependencies come from the manifests in this repository and
+// dependents come from every other repository's manifests, so neither can be
+// derived from the other.
+//
+// The two pagers disagree, which is why there are two of them here. The
+// dependency list is a Rails pager with ?page=N and a rel="next" anchor, and the
+// dependents list is a cursor in a button.
+
+// Dependencies lists what a repository declares in its manifests.
+//
+// A repository with the dependency graph switched off answers with a page and
+// no rows, which is an empty list rather than an error: the difference between
+// "nothing to report" and "not enabled" is not on the page, so claiming to know
+// which one it is would be making it up.
+func (c *Client) Dependencies(ctx context.Context, repo string, limit int, emit func(Dependency) error) error {
+ if _, _, ok := SplitRepo(repo); !ok {
+ return usageBadID("repository", repo, "owner/name")
+ }
+ base := repoSubURL(repo, "network/dependencies")
+ fetch := func(ctx context.Context, token string) ([]Dependency, string, error) {
+ u := base
+ if n := pageToken(token); n > 1 {
+ u = query(u, "page", strconv.Itoa(n))
+ }
+ doc, final, err := c.rowPage(ctx, u, page.DependencyRow)
+ if err != nil {
+ return nil, "", err
+ }
+ if doc == nil {
+ return nil, "", structureChanged(repo + " dependencies")
+ }
+ var out []Dependency
+ for _, row := range page.FindAll(doc, page.BoxRow) {
+ d, ok := dependencyRow(row, repo, final)
+ if ok {
+ out = append(out, d)
+ }
+ }
+ return out, railsNext(doc, token), nil
+ }
+ return paginate(ctx, limit, fetch, emit)
+}
+
+// rowPage reads one page of a dependency graph listing.
+//
+// It exists because GitHub answers a cursor page with a 200, the right title,
+// the right chrome, and no rows at all, often enough to matter. An empty page is
+// indistinguishable from the end of the list, so the walk stops early and
+// reports a third of the dependents as the whole set. Asking a second time gets
+// the rows, so the read is repeated once before an empty page is believed.
+//
+// Dropping the cached copy first is the part that matters. Without it the retry
+// reads the same empty bytes back and the wrong answer sticks for the life of
+// the entry, which is how this was found: --no-cache returned two hundred rows
+// and the cached run returned sixty, over and over.
+//
+// The cost is one extra request for a repository whose dependency graph really
+// is empty, which is the right trade: a repository with the graph switched off
+// is cheap to ask twice, and silently reporting an empty list for one with
+// thousands of dependents is not recoverable by the caller.
+func (c *Client) rowPage(ctx context.Context, u string, rows page.Sel) (*html.Node, string, error) {
+ doc, final, n, err := c.readRows(ctx, u, rows)
+ if err != nil || n > 0 {
+ return doc, final, err
+ }
+ c.cacheDrop(u, SurfaceHTML)
+ doc, final, n, err = c.readRows(ctx, u, rows)
+ if err != nil {
+ return nil, "", err
+ }
+ // An empty page that stays empty is not worth keeping either. The next run
+ // would read it back and stop in the same place without ever asking again.
+ if n == 0 {
+ c.cacheDrop(u, SurfaceHTML)
+ }
+ return doc, final, nil
+}
+
+func (c *Client) readRows(ctx context.Context, u string, rows page.Sel) (*html.Node, string, int, error) {
+ res, err := c.GetHTML(ctx, u)
+ if err != nil {
+ return nil, "", 0, err
+ }
+ doc := page.Extract(res.FinalURL, res.Body).Doc()
+ if doc == nil {
+ return nil, res.FinalURL, 0, nil
+ }
+ return doc, res.FinalURL, len(page.FindAll(doc, rows)), nil
+}
+
+// dependencyRow reads one manifest entry. The interesting half is the line under
+// the package name, which is one span holding the ecosystem, the manifest, who
+// detected it and when, and sometimes the licence, separated by middots.
+func dependencyRow(row *html.Node, repo, source string) (Dependency, bool) {
+ box := page.Find(row, page.DependencyRow)
+ if box == nil {
+ return Dependency{}, false
+ }
+ name := page.Find(box, page.DependencyName)
+ if name == nil {
+ return Dependency{}, false
+ }
+ d := Dependency{Repo: repo, Package: strings.TrimSpace(page.Text(name))}
+ if d.Package == "" {
+ return Dependency{}, false
+ }
+ if a := page.Find(box, page.DependencyLink); a != nil {
+ if p := hrefPath(page.Attr(a, "href")); strings.Count(p, "/") == 1 {
+ d.SourceRepo = p
+ d.setIdentity(KindRepo, p)
+ }
+ }
+ if v := page.Find(box, page.DependencyVersion); v != nil {
+ d.Version = strings.TrimSpace(page.Text(v))
+ }
+ if r := page.Find(box, page.DependencyRelation); r != nil {
+ d.Relationship = strings.ToLower(strings.TrimSpace(page.Text(r)))
+ }
+ if m := page.Find(row, page.DependencyManifest); m != nil {
+ d.Manifest = strings.TrimSpace(page.Text(m))
+ if m.Parent != nil {
+ d.Ecosystem, d.License = manifestLine(page.Text(m.Parent), d.Manifest)
+ }
+ }
+ d.addSource(source)
+ return d, true
+}
+
+// manifestLine takes the middot-separated line apart. The ecosystem is always
+// first and the licence, when there is one, is always last; the middle is the
+// manifest name and the detection note, neither of which needs splitting out
+// here.
+//
+// The note comes in two wordings, "Detected by dependabot on " and
+// "Detected automatically on ", and when there is no licence the note is
+// what sits last, so both prefixes have to be recognised or the date ends up
+// filed as the licence.
+func manifestLine(text, manifest string) (ecosystem, license string) {
+ var parts []string
+ for _, p := range strings.Split(text, "·") {
+ if p = strings.TrimSpace(p); p != "" {
+ parts = append(parts, p)
+ }
+ }
+ if len(parts) == 0 {
+ return "", ""
+ }
+ ecosystem = parts[0]
+ last := parts[len(parts)-1]
+ if last != ecosystem && last != manifest && !strings.HasPrefix(last, "Detected ") {
+ license = last
+ }
+ return ecosystem, license
+}
+
+// Dependents lists the repositories that depend on this one.
+//
+// The list is ordered by stars and it is long: a popular library has tens of
+// thousands of rows at thirty a page, so --limit is the flag that matters here
+// and the walk stops the moment it is reached.
+func (c *Client) Dependents(ctx context.Context, repo string, limit int, emit func(Dependent) error) error {
+ if _, _, ok := SplitRepo(repo); !ok {
+ return usageBadID("repository", repo, "owner/name")
+ }
+ base := repoSubURL(repo, "network/dependents")
+ fetch := func(ctx context.Context, token string) ([]Dependent, string, error) {
+ u := base
+ if token != "" {
+ u = query(u, "dependents_after", token)
+ }
+ doc, final, err := c.rowPage(ctx, u, page.DependentRow)
+ if err != nil {
+ return nil, "", err
+ }
+ if doc == nil {
+ return nil, "", structureChanged(repo + " dependents")
+ }
+ var out []Dependent
+ for _, row := range page.FindAll(doc, page.DependentRow) {
+ d, ok := dependentRow(row, repo, final)
+ if ok {
+ out = append(out, d)
+ }
+ }
+ return out, dependentsCursor(doc), nil
+ }
+ return paginate(ctx, limit, fetch, emit)
+}
+
+// dependentRow reads one dependent. Owner and name are two anchors rather than
+// one, the same shape the fork list uses, so the id is assembled.
+func dependentRow(row *html.Node, repo, source string) (Dependent, bool) {
+ link := page.Find(row, page.DependentRepo)
+ if link == nil {
+ return Dependent{}, false
+ }
+ id := hrefPath(page.Attr(link, "href"))
+ owner, _, ok := SplitRepo(id)
+ if !ok {
+ return Dependent{}, false
+ }
+ d := Dependent{Repo: repo, Dependent: id, Owner: owner}
+ d.setIdentity(KindRepo, id)
+ if u := page.Find(row, page.DependentUser); u != nil {
+ if login := hrefPath(page.Attr(u, "href")); login != "" {
+ d.Owner = login
+ }
+ }
+ if img := page.Find(row, page.Sel{Tag: "img", Class: "avatar"}); img != nil {
+ d.AvatarURL = page.Attr(img, "src")
+ }
+ d.Stars = iconCount(row, page.DependentStars)
+ d.Forks = iconCount(row, page.DependentForks)
+ d.addSource(source)
+ return d, true
+}
+
+// iconCount reads the number beside an icon. The icon is what says which count
+// it is, because the two spans are otherwise identical.
+func iconCount(row *html.Node, sel page.Sel) *int {
+ el := page.Find(row, sel)
+ if el == nil {
+ return nil
+ }
+ if n, _, ok := page.CountIn(page.Text(el)); ok {
+ return intp(n)
+ }
+ return nil
+}
+
+// dependentsCursor pulls the opaque cursor out of the Next button. There is no
+// page number on this listing and no total to count against, so the cursor the
+// page hands back is the only way forward.
+func dependentsCursor(doc *html.Node) string {
+ a := page.Find(doc, page.DependentNext)
+ if a == nil {
+ return ""
+ }
+ u, err := url.Parse(page.Attr(a, "href"))
+ if err != nil {
+ return ""
+ }
+ return u.Query().Get("dependents_after")
+}
diff --git a/gh/graph.go b/gh/graph.go
new file mode 100644
index 0000000..03cb55e
--- /dev/null
+++ b/gh/graph.go
@@ -0,0 +1,949 @@
+package gh
+
+import (
+ "sort"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// graph.go turns records into triples. github.com is already a knowledge graph:
+// a repository names its owner, its topics, its licence, and the repository it
+// was forked from; a pull request names the issue it closes and the branch it
+// targets; a commit names its parents. This file reads those declarations off a
+// record and emits them as typed edges.
+//
+// It is pure. No network, no client, no ordering dependency, which is what makes
+// the whole graph plane testable against a fixture and what makes
+// `github edges golang/go#1 --min-trust id` answer without a request.
+
+// Node is one entity. It is deliberately thin: the label and the two addresses
+// and nothing else, because the full record is one `github get` away by URI and
+// duplicating it here would make a crawl of ten thousand nodes unprintable.
+type Node struct {
+ URI string `json:"uri" table:"uri"`
+ Kind string `json:"kind" table:"kind"`
+ ID string `json:"id" table:"id"`
+ Label string `json:"label,omitempty" table:"label,truncate"`
+ URL string `json:"url,omitempty" table:"url,url"`
+}
+
+// Edge is one directed, typed relation between two entities.
+//
+// There is no inverse flag. Every predicate has exactly one direction, and
+// where the inverse is what you want, the edge is emitted with the other node as
+// its subject rather than with a flag saying to read it backwards.
+type Edge struct {
+ Subject string `json:"subject" table:"subject"`
+ Predicate string `json:"predicate" table:"predicate"`
+ Object string `json:"object" table:"object"`
+ Source string `json:"source" table:"source"`
+ Weight *int `json:"weight,omitempty" table:"weight"`
+ At *time.Time `json:"at,omitempty" table:"-"`
+}
+
+// Fact is a literal statement about a node: a star count, a description, a
+// timestamp.
+//
+// It is a separate type from Edge on purpose. Edge.Object is a URI and every
+// consumer of the graph is entitled to treat it as one, so putting "12000" in
+// that field to carry a star count would break each of them for the sake of
+// saving a struct. RDF emits both; `github edges` emits only edges, which is why
+// its output reads as relations rather than as a flattened record.
+type Fact struct {
+ Subject string `json:"subject" table:"subject"`
+ Predicate string `json:"predicate" table:"predicate"`
+ Value string `json:"value" table:"value,truncate"`
+ Datatype string `json:"datatype,omitempty" table:"-"`
+}
+
+// The five extraction rules, in descending order of trust. Every edge carries
+// the one that produced it, which is the field a consumer uses to decide how
+// much to believe.
+const (
+ // SrcID is derived from the id structure alone. No fetch, always correct.
+ SrcID = "id"
+ // SrcPayload is an explicit reference in a JSON payload or a Relay result.
+ SrcPayload = "payload"
+ // SrcFeed is an explicit reference in an Atom feed.
+ SrcFeed = "feed"
+ // SrcHTML was parsed out of rendered markup with a selector. Good, and it
+ // degrades to a missing edge rather than a wrong one when a template moves.
+ SrcHTML = "html"
+ // SrcText is a pattern matched in free text: #42, a bare SHA. Heuristic, and
+ // dropped by the default --min-trust.
+ SrcText = "text"
+)
+
+// trustRank orders the rules. Higher is more trustworthy.
+var trustRank = map[string]int{
+ SrcID: 4,
+ SrcPayload: 3,
+ SrcFeed: 2,
+ SrcHTML: 1,
+ SrcText: 0,
+}
+
+// DefaultMinTrust keeps everything except free-text guesses.
+const DefaultMinTrust = SrcHTML
+
+// TrustAtLeast reports whether a source meets a floor. An unknown floor lets
+// everything through rather than silently dropping the whole graph, and an
+// unknown source is treated as the weakest thing there is.
+func TrustAtLeast(source, min string) bool {
+ floor, ok := trustRank[min]
+ if !ok {
+ return true
+ }
+ return trustRank[source] >= floor
+}
+
+// TrustLevels is the accepted set, for help text and for validation.
+var TrustLevels = []string{SrcID, SrcPayload, SrcFeed, SrcHTML, SrcText}
+
+// The predicate vocabulary. This is the complete set: an edge this tool emits
+// has its predicate here, and adding a relation means adding a constant first.
+const (
+ // Ownership and membership.
+ PredOwnedBy = "ownedBy"
+ PredMemberOf = "memberOf"
+ PredPartOf = "partOf"
+ PredBelongsToPackage = "belongsToPackage"
+
+ // Derivation. The edges that make a graph worth walking.
+ PredForkOf = "forkOf"
+ PredTemplateOf = "templateOf"
+ PredMirrorOf = "mirrorOf"
+ PredDependsOn = "dependsOn"
+ PredUsedBy = "usedBy"
+
+ // Authorship and activity.
+ PredAuthoredBy = "authoredBy"
+ PredCommittedBy = "committedBy"
+ PredContributedTo = "contributedTo"
+ PredAssignedTo = "assignedTo"
+ PredReviewedBy = "reviewedBy"
+ PredReviewRequestedFrom = "reviewRequestedFrom"
+ PredMergedBy = "mergedBy"
+
+ // Reference.
+ PredReferences = "references"
+ PredCloses = "closes"
+ PredClosedBy = "closedBy"
+ PredDuplicateOf = "duplicateOf"
+ PredSubIssueOf = "subIssueOf"
+ PredLinkedTo = "linkedTo"
+ PredTargetsBranch = "targetsBranch"
+ PredFromBranch = "fromBranch"
+ PredPointsAt = "pointsAt"
+ PredParentOf = "parentOf"
+
+ // Classification.
+ PredHasTopic = "hasTopic"
+ PredHasLabel = "hasLabel"
+ PredInMilestone = "inMilestone"
+ PredWrittenIn = "writtenIn"
+ PredLicensedUnder = "licensedUnder"
+ PredRelatedTopic = "relatedTopic"
+
+ // Social. Opt-in everywhere, because the star list of a popular repository
+ // is thousands of pages and nobody wants that by accident.
+ PredStarredBy = "starredBy"
+ PredFollows = "follows"
+ PredSponsors = "sponsors"
+ PredReactedWith = "reactedWith"
+)
+
+// The literal predicates. These name Fact rows rather than edges.
+const (
+ FactName = "name"
+ FactDescription = "description"
+ FactHomepage = "homepage"
+ FactCreated = "created"
+ FactUpdated = "updated"
+ FactStars = "stars"
+ FactForks = "forks"
+ FactWatchers = "watchers"
+ FactCommits = "commits"
+ FactURI = "uri"
+ FactAvatar = "avatar"
+ FactState = "state"
+ FactCount = "count"
+)
+
+// SocialPredicates are the ones a command has to be asked for by name.
+var SocialPredicates = []string{PredStarredBy, PredFollows, PredSponsors, PredReactedWith}
+
+// DefaultFollow is the crawler's follow set. It deliberately excludes
+// references, starredBy, follows, dependsOn, and usedBy: those five turn a
+// bounded walk into an unbounded one, and each has to be asked for by name.
+var DefaultFollow = []string{PredPartOf, PredOwnedBy, PredForkOf, PredHasTopic, PredAuthoredBy}
+
+// LiteralPredicates are the two whose object is a bare string rather than a
+// URI, because a language and a licence are not github.com entities. RDF gives
+// them synthetic IRIs in the gh: namespace; `github edges` prints them as they
+// are written on the page.
+var LiteralPredicates = map[string]bool{
+ PredWrittenIn: true,
+ PredLicensedUnder: true,
+ PredReactedWith: true,
+}
+
+// --- the builder ---
+
+// builder accumulates one node with its edges and facts while an extractor
+// walks a record.
+type builder struct {
+ node Node
+ edges []Edge
+ facts []Fact
+}
+
+// start sets the node. Every extractor calls it first, and nothing is emitted
+// for a record whose identity did not resolve.
+func (b *builder) start(kind, id, label, url string) {
+ if id == "" {
+ return
+ }
+ if label == "" {
+ label = id
+ }
+ if url == "" {
+ if u, err := Locate(kind, id); err == nil {
+ url = u
+ }
+ }
+ b.node = Node{URI: URI(kind, id), Kind: kind, ID: id, Label: label, URL: url}
+}
+
+// to emits an edge from this node to another entity named by kind and id.
+func (b *builder) to(pred, objKind, objID, source string) {
+ if objID == "" {
+ return
+ }
+ b.toURI(pred, URI(objKind, objID), source)
+}
+
+// toURI is to for an object whose URI is already built.
+func (b *builder) toURI(pred, objURI, source string) {
+ if b.node.URI == "" || objURI == "" {
+ return
+ }
+ b.edges = append(b.edges, Edge{Subject: b.node.URI, Predicate: pred, Object: objURI, Source: source})
+}
+
+// toURL emits an edge to whatever a github.com URL names. Relay results carry
+// links rather than ids for linked pull requests, duplicates, and cross
+// references, and classification is exactly the function that turns one into
+// the other.
+func (b *builder) toURL(pred, rawURL, source string) {
+ if uri := uriOfURL(rawURL); uri != "" {
+ b.toURI(pred, uri, source)
+ }
+}
+
+// raw emits an edge whose object is a bare string rather than a URI: a
+// language, a licence, a reaction.
+func (b *builder) raw(pred, value, source string) {
+ if b.node.URI == "" || value == "" {
+ return
+ }
+ b.edges = append(b.edges, Edge{Subject: b.node.URI, Predicate: pred, Object: value, Source: source})
+}
+
+// from emits an edge whose subject is not this node. A contributor edge points
+// at the repository rather than away from it, and inverting it to make this node
+// the subject would be a lie about which way the relation runs.
+func (b *builder) from(subjURI, pred, objURI, source string) {
+ if subjURI == "" || objURI == "" {
+ return
+ }
+ b.edges = append(b.edges, Edge{Subject: subjURI, Predicate: pred, Object: objURI, Source: source})
+}
+
+// weigh attaches a count to the last edge appended. It is separate from the
+// emitters so the common case stays a one-liner.
+func (b *builder) weigh(n *int) {
+ if n == nil || len(b.edges) == 0 {
+ return
+ }
+ b.edges[len(b.edges)-1].Weight = n
+}
+
+// when attaches a time to the last edge appended.
+func (b *builder) when(t *time.Time) {
+ if t == nil || len(b.edges) == 0 {
+ return
+ }
+ b.edges[len(b.edges)-1].At = t
+}
+
+// fact records a literal. An empty value is skipped, because "this repository
+// has no description" is better said by the absence of a statement than by an
+// empty one.
+func (b *builder) fact(pred, value, datatype string) {
+ if b.node.URI == "" || value == "" {
+ return
+ }
+ b.facts = append(b.facts, Fact{Subject: b.node.URI, Predicate: pred, Value: value, Datatype: datatype})
+}
+
+// num records a count. A nil count is a count the surface did not state, which
+// is not the same as zero and does not become a statement.
+func (b *builder) num(pred string, n *int) {
+ if n == nil {
+ return
+ }
+ b.fact(pred, strconv.Itoa(*n), TypeInteger)
+}
+
+func (b *builder) at(pred string, t *time.Time) {
+ if t == nil || t.IsZero() {
+ return
+ }
+ b.fact(pred, t.UTC().Format(time.RFC3339), TypeDateTime)
+}
+
+// actorEdge emits an edge to a person. The actor's own type is used when it says
+// one, so a bot or an organization does not silently become a user.
+func (b *builder) actorEdge(pred string, a Actor, source string) {
+ if a.Login == "" {
+ return
+ }
+ b.to(pred, actorKind(a), a.Login, source)
+}
+
+func actorKind(a Actor) string {
+ if strings.EqualFold(a.Type, "Organization") {
+ return KindOrg
+ }
+ return KindUser
+}
+
+// uriOfURL classifies a github.com URL into a URI, and answers empty for
+// anything that is not one. Extractors use it rather than Classify directly so
+// a link to an external site drops out instead of producing an error nobody can
+// act on.
+func uriOfURL(raw string) string {
+ if raw == "" {
+ return ""
+ }
+ kind, id, err := Classify(raw)
+ if err != nil {
+ return ""
+ }
+ return URI(kind, id)
+}
+
+// --- extraction ---
+
+// Extract turns one record into its node, its edges, and its facts. A record
+// kind it does not know produces an empty node, which every caller treats as
+// nothing to say rather than as an error.
+func Extract(rec any) (Node, []Edge, []Fact) {
+ b := &builder{}
+ switch r := rec.(type) {
+ case *Repo:
+ b.repo(r)
+ case *Trending:
+ b.trending(r)
+ case *Account:
+ b.account(r)
+ case *Org:
+ b.org(r)
+ case *Issue:
+ b.issue(r)
+ case *PullRequest:
+ b.pull(r)
+ case *Discussion:
+ b.discussion(r)
+ case *Thread:
+ b.thread(r)
+ case *Commit:
+ b.commit(r)
+ case *GitRef:
+ b.gitRef(r)
+ case *Release:
+ b.release(r)
+ case *Topic:
+ b.topic(r)
+ case *Package:
+ b.pkg(r)
+ case *WikiPage:
+ b.wiki(r)
+ case *Gist:
+ b.gist(r)
+ case *File:
+ b.file(r)
+ case *TreeEntry:
+ b.treeEntry(r)
+ case *Contributor:
+ b.contributor(r)
+ case *Dependency:
+ b.dependency(r)
+ case *Dependent:
+ b.dependent(r)
+ case *LanguageShare:
+ b.languageShare(r)
+ case *RepoStats:
+ b.stats(r)
+ default:
+ return Node{}, nil, nil
+ }
+ return b.node, b.edges, b.facts
+}
+
+// repo is the centre of the graph. Everything else hangs off a repository, and
+// most of what a walk finds interesting is stated on this one record.
+func (b *builder) repo(r *Repo) {
+ id := r.ID
+ if id == "" && r.Owner != "" && r.Name != "" {
+ id = r.Owner + "/" + r.Name
+ }
+ b.start(KindRepo, id, id, r.URL)
+
+ // The owner comes from the id, which is why this edge costs nothing. Which
+ // of the two account kinds it is comes from the page, so a repository read
+ // from a surface that did not say defaults to user and is corrected the
+ // moment the owner itself is fetched.
+ if r.Owner != "" {
+ b.to(PredOwnedBy, ownerKind(r), r.Owner, SrcID)
+ }
+ // ForkOf is the "Forked from" line in the header, and it is the only place
+ // any keyless surface names the parent. templateOf and mirrorOf have their
+ // constants in the vocabulary and no producer here, because the page states
+ // that a repository is a template or a mirror without ever naming what it
+ // was generated from or what it mirrors.
+ b.to(PredForkOf, KindRepo, r.ForkOf, SrcHTML)
+
+ for _, t := range r.Topics {
+ b.to(PredHasTopic, KindTopic, t, SrcPayload)
+ }
+ b.raw(PredLicensedUnder, r.License, SrcHTML)
+ b.languages(r)
+
+ for i := range r.Tree {
+ e := &r.Tree[i]
+ if e.URI != "" {
+ b.from(e.URI, PredPartOf, b.node.URI, SrcID)
+ }
+ }
+
+ b.fact(FactName, id, "")
+ b.fact(FactDescription, r.Description, "")
+ b.fact(FactHomepage, r.Homepage, "")
+ b.num(FactStars, r.Stars)
+ b.num(FactForks, r.Forks)
+ b.num(FactWatchers, r.Watchers)
+ b.num(FactCommits, r.CommitCount)
+ b.at(FactCreated, r.CreatedAt)
+ b.at(FactUpdated, firstSetTime(r.PushedAt, r.UpdatedAt))
+ b.fact(FactAvatar, r.OwnerAvatarURL, "")
+ b.fact(FactURI, b.node.URI, "")
+}
+
+// ownerKind decides between a user and an organization. IsOrgOwned is set by
+// the page template, which is the only surface that states it without a token.
+func ownerKind(r *Repo) string {
+ if r.IsOrgOwned {
+ return KindOrg
+ }
+ return KindUser
+}
+
+// languages emits one writtenIn edge per language, weighted by the percentage
+// the histogram gave.
+//
+// The source is honest about where the number came from: Via records
+// sidebar-percent when the histogram was read from the deferred sidebar
+// fragment, which is a payload, and anything else was read off the language bar
+// in the markup.
+func (b *builder) languages(r *Repo) {
+ source := SrcHTML
+ if r.Via["languages"] == "sidebar-percent" {
+ source = SrcPayload
+ }
+ if len(r.Languages) == 0 {
+ b.raw(PredWrittenIn, r.Language, source)
+ return
+ }
+ for _, name := range sortedLanguages(r.Languages) {
+ b.raw(PredWrittenIn, name, source)
+ if n := r.Languages[name]; n > 0 {
+ b.weigh(intp(int(n)))
+ }
+ }
+}
+
+// sortedLanguages orders a histogram by share and then by name, so two runs over
+// the same repository produce the same edge order.
+func sortedLanguages(m map[string]int64) []string {
+ out := make([]string, 0, len(m))
+ for k := range m {
+ out = append(out, k)
+ }
+ sort.SliceStable(out, func(i, j int) bool {
+ if m[out[i]] != m[out[j]] {
+ return m[out[i]] > m[out[j]]
+ }
+ return out[i] < out[j]
+ })
+ return out
+}
+
+// trending is a repository plus the people the trending page credits.
+func (b *builder) trending(t *Trending) {
+ b.repo(&t.Repo)
+ for _, a := range t.BuiltBy {
+ if a.Login != "" {
+ b.from(URI(actorKind(a), a.Login), PredContributedTo, b.node.URI, SrcHTML)
+ }
+ }
+}
+
+func (b *builder) account(a *Account) {
+ kind := KindUser
+ if strings.EqualFold(a.Type, "Organization") {
+ kind = KindOrg
+ }
+ b.start(kind, a.Login, firstNonEmpty(a.Name, a.Login), a.URL)
+
+ for _, org := range a.Organizations {
+ b.to(PredMemberOf, KindOrg, org, SrcHTML)
+ }
+ // A pinned repository the person does not own is pinned work they
+ // contributed to, and this tool cannot tell which from the profile alone.
+ // Only the ones whose id carries this login become ownership edges; the rest
+ // are left out rather than asserted wrongly.
+ for _, repo := range a.PinnedRepos {
+ if owner, _, ok := SplitRepo(repo); ok && strings.EqualFold(owner, a.Login) {
+ b.from(URI(KindRepo, repo), PredOwnedBy, b.node.URI, SrcHTML)
+ }
+ }
+
+ b.fact(FactName, firstNonEmpty(a.Name, a.Login), "")
+ b.fact(FactDescription, a.Bio, "")
+ b.fact(FactHomepage, a.Website, "")
+ b.fact(FactAvatar, a.AvatarURL, "")
+ b.at(FactCreated, a.CreatedAt)
+ b.fact(FactURI, b.node.URI, "")
+ if a.Followers != nil {
+ b.fact("followers", strconv.Itoa(*a.Followers), TypeInteger)
+ }
+}
+
+func (b *builder) org(o *Org) {
+ b.account(&o.Account)
+ // The node kind comes from the template that answered, and this one is the
+ // organization template, so it is an organization whatever the account
+ // record's Type string says.
+ if b.node.URI != "" {
+ b.node.Kind = KindOrg
+ b.node.URI = URI(KindOrg, o.Login)
+ }
+ for _, m := range o.Members {
+ b.from(URI(KindUser, m), PredMemberOf, b.node.URI, SrcHTML)
+ }
+ if o.MemberCount != nil {
+ b.fact("members", strconv.Itoa(*o.MemberCount), TypeInteger)
+ }
+ // TopTopics and TopLanguages are aggregates over the organization's
+ // repositories rather than properties of the organization, so they produce
+ // no hasTopic or writtenIn edge here. The repositories state their own.
+}
+
+// thread covers what issues, pull requests, and discussions share.
+func (b *builder) thread(t *Thread) {
+ kind := t.Kind
+ if kind == "" {
+ kind = KindIssue
+ }
+ b.start(kind, t.ID, t.Title, t.URL)
+
+ if t.Repo != "" {
+ b.to(PredPartOf, KindRepo, t.Repo, SrcID)
+ }
+ b.actorEdge(PredAuthoredBy, t.Author, SrcPayload)
+ for _, l := range t.Labels {
+ if l.Name != "" && t.Repo != "" {
+ b.to(PredHasLabel, KindLabel, t.Repo+"/"+l.Name, SrcPayload)
+ }
+ }
+ if m := t.Milestone; m != nil && m.Number != nil && t.Repo != "" {
+ b.to(PredInMilestone, KindMilestone, t.Repo+"/"+strconv.Itoa(*m.Number), SrcPayload)
+ }
+ for _, a := range t.Assignees {
+ b.actorEdge(PredAssignedTo, a, SrcPayload)
+ }
+ for _, r := range t.Reactions {
+ b.raw(PredReactedWith, strings.ToLower(r.Content), SrcPayload)
+ b.weigh(intp(r.Count))
+ }
+
+ b.fact(FactName, t.Title, "")
+ b.fact(FactState, t.State, "")
+ b.at(FactCreated, t.CreatedAt)
+ b.at(FactUpdated, t.UpdatedAt)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) issue(i *Issue) {
+ b.thread(&i.Thread)
+ b.toURL(PredDuplicateOf, i.DuplicateOf, SrcPayload)
+ for _, u := range i.LinkedPRs {
+ b.toURL(PredLinkedTo, u, SrcPayload)
+ }
+ for _, u := range i.ClosedByPRs {
+ b.toURL(PredClosedBy, u, SrcPayload)
+ }
+ // SubIssueTotal is a count and not a list, so subIssueOf has no producer on
+ // this record. The timeline carries the parent, which is a separate read.
+}
+
+func (b *builder) pull(p *PullRequest) {
+ b.thread(&p.Thread)
+ if p.Repo != "" {
+ b.to(PredTargetsBranch, KindBranch, refID(p.Repo, p.BaseRef), SrcPayload)
+ b.to(PredFromBranch, KindBranch, refID(p.Repo, p.HeadRef), SrcPayload)
+ }
+ if p.MergedBy != nil {
+ b.actorEdge(PredMergedBy, *p.MergedBy, SrcPayload)
+ }
+ for _, a := range p.ReviewRequests {
+ b.actorEdge(PredReviewRequestedFrom, a, SrcPayload)
+ }
+ for _, u := range p.ClosesIssues {
+ b.toURL(PredCloses, u, SrcPayload)
+ }
+}
+
+// refID builds owner/name@ref, and answers empty for an empty ref so that a
+// pull request read from a surface that did not state its base does not point
+// at a branch called nothing.
+func refID(repo, ref string) string {
+ if repo == "" || ref == "" {
+ return ""
+ }
+ return repo + "@" + ref
+}
+
+func (b *builder) discussion(d *Discussion) {
+ b.thread(&d.Thread)
+ // The answer's author is not the discussion's author and there is no
+ // predicate for "answered by" in the vocabulary, so the fact records who it
+ // was rather than inventing one.
+ if d.AnswerAuthor != nil && d.AnswerAuthor.Login != "" {
+ b.fact("answeredBy", d.AnswerAuthor.Login, "")
+ }
+}
+
+func (b *builder) commit(c *Commit) {
+ id := c.ID
+ if id == "" && c.Repo != "" && c.SHA != "" {
+ id = c.Repo + "@" + c.SHA
+ }
+ b.start(KindCommit, id, c.Subject, c.URL)
+
+ if c.Repo != "" {
+ b.to(PredPartOf, KindRepo, c.Repo, SrcID)
+ for _, p := range c.Parents {
+ // The arrow reads "that commit is the parent of this one", which is
+ // why the parent is the subject and not the object.
+ b.from(URI(KindCommit, c.Repo+"@"+p), PredParentOf, b.node.URI, SrcPayload)
+ }
+ }
+ for _, a := range c.Authors {
+ b.actorEdge(PredAuthoredBy, a, SrcPayload)
+ }
+ if c.Committer != nil {
+ b.actorEdge(PredCommittedBy, *c.Committer, SrcPayload)
+ }
+ // GitHub resolved these references itself when it rendered the message, so
+ // they are payload rather than the text rule that would find the same #N in
+ // the raw subject line.
+ for _, ref := range c.IssueRefs {
+ b.toURL(PredReferences, ref.URL, SrcPayload)
+ }
+
+ b.fact(FactName, c.Subject, "")
+ b.at(FactCreated, firstSetTime(c.AuthoredAt, c.CommittedAt))
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) gitRef(r *GitRef) {
+ kind := r.Type
+ if kind != KindTag {
+ kind = KindBranch
+ }
+ id := r.ID
+ if id == "" {
+ id = refID(r.Repo, r.Name)
+ }
+ b.start(kind, id, r.Name, r.URL)
+
+ if r.Repo != "" {
+ b.to(PredPartOf, KindRepo, r.Repo, SrcID)
+ // A ref read from the git protocol carries its object name, which is the
+ // one edge in this whole file that comes from git rather than from
+ // github.com.
+ if r.SHA != "" {
+ b.to(PredPointsAt, KindCommit, r.Repo+"@"+firstNonEmpty(r.PeeledSHA, r.SHA), SrcPayload)
+ }
+ }
+ if r.Author != nil {
+ b.actorEdge(PredAuthoredBy, *r.Author, SrcHTML)
+ }
+ b.fact(FactName, r.Name, "")
+ b.at(FactCreated, r.AuthoredAt)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) release(r *Release) {
+ id := r.ID
+ if id == "" {
+ id = refID(r.Repo, r.Tag)
+ }
+ b.start(KindRelease, id, firstNonEmpty(r.Title, r.Tag), r.URL)
+
+ if r.Repo != "" {
+ b.to(PredPartOf, KindRepo, r.Repo, SrcID)
+ b.to(PredPointsAt, KindCommit, refID(r.Repo, r.CommitSHA), SrcHTML)
+ }
+ if r.Author != nil {
+ // The releases listing comes from the Atom feed, where the author is an
+ // element rather than a selector.
+ b.actorEdge(PredAuthoredBy, *r.Author, SrcFeed)
+ }
+ b.fact(FactName, firstNonEmpty(r.Title, r.Tag), "")
+ b.at(FactCreated, r.PublishedAt)
+ b.at(FactUpdated, r.UpdatedAt)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) topic(t *Topic) {
+ b.start(KindTopic, firstNonEmpty(t.ID, t.Name), firstNonEmpty(t.DisplayName, t.Name), t.URL)
+ for _, rel := range t.Related {
+ b.to(PredRelatedTopic, KindTopic, rel, SrcHTML)
+ }
+ b.fact(FactName, firstNonEmpty(t.DisplayName, t.Name), "")
+ b.fact(FactDescription, firstNonEmpty(t.ShortDescription, t.Description), "")
+ b.num(FactStars, t.StargazerCount)
+ b.num(FactCount, t.AppliedCount)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) pkg(p *Package) {
+ id := p.ID
+ if id == "" && p.Repo != "" {
+ id = p.Repo + "/" + p.Name
+ }
+ b.start(KindPackage, id, p.Name, p.URL)
+ // The direction is the vocabulary's: the package is the subject and the
+ // repository it was published from is the object.
+ b.to(PredBelongsToPackage, KindRepo, p.Repo, SrcPayload)
+ for _, t := range p.Topics {
+ b.to(PredHasTopic, KindTopic, t, SrcPayload)
+ }
+ b.fact(FactName, p.Name, "")
+ b.fact(FactDescription, p.Summary, "")
+ b.at(FactUpdated, p.UpdatedAt)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) wiki(w *WikiPage) {
+ id := w.ID
+ if id == "" && w.Repo != "" {
+ id = w.Repo + "/" + firstNonEmpty(w.Path, w.Title)
+ }
+ b.start(KindWiki, id, w.Title, w.URL)
+ b.to(PredPartOf, KindRepo, w.Repo, SrcID)
+ if w.Author != nil {
+ b.actorEdge(PredAuthoredBy, *w.Author, SrcHTML)
+ }
+ b.fact(FactName, w.Title, "")
+ b.at(FactUpdated, w.UpdatedAt)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) gist(g *Gist) {
+ b.start(KindGist, g.ID, firstNonEmpty(g.Description, g.ID), g.URL)
+ b.to(PredOwnedBy, KindUser, g.Owner, SrcHTML)
+ for _, f := range g.Files {
+ b.raw(PredWrittenIn, f.Language, SrcHTML)
+ }
+ b.fact(FactDescription, g.Description, "")
+ b.num(FactStars, g.Stars)
+ b.num(FactForks, g.Forks)
+ b.at(FactCreated, g.CreatedAt)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) file(f *File) {
+ b.start(KindFile, f.ID, f.Path, f.URL)
+ b.to(PredPartOf, KindRepo, f.Repo, SrcID)
+ b.raw(PredWrittenIn, f.Language, SrcPayload)
+ b.fact(FactName, f.Path, "")
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) treeEntry(t *TreeEntry) {
+ kind := KindFile
+ if strings.Contains(t.Type, "directory") {
+ kind = KindTree
+ }
+ b.start(kind, t.ID, t.Path, t.URL)
+ b.to(PredPartOf, KindRepo, t.Repo, SrcID)
+ b.fact(FactName, t.Path, "")
+ b.fact(FactURI, b.node.URI, "")
+}
+
+// contributor is the one weighted authorship edge, and the weight is what makes
+// `github edges --predicate contributedTo` a ranked list rather than a set.
+func (b *builder) contributor(c *Contributor) {
+ b.start(KindUser, c.Login, c.Login, BaseURL+"/"+c.Login)
+ if c.Repo != "" {
+ b.toURI(PredContributedTo, URI(KindRepo, c.Repo), SrcPayload)
+ b.weigh(c.Commits)
+ b.when(c.LastWeek)
+ }
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func (b *builder) languageShare(l *LanguageShare) {
+ if l.Repo == "" {
+ return
+ }
+ b.start(KindRepo, l.Repo, l.Repo, "")
+ b.raw(PredWrittenIn, l.Language, SrcPayload)
+}
+
+// dependency and dependent are the two halves of the same relation read off two
+// different pages. Both put the page's repository on the subject side, so a
+// dependency row from hugo says hugo dependsOn chroma and a dependent row from
+// hugo says hugo usedBy someone. A package GitHub could not resolve to a
+// repository has nothing to point at and produces no node.
+func (b *builder) dependency(d *Dependency) {
+ if d.SourceRepo == "" || d.Repo == "" {
+ return
+ }
+ b.start(KindRepo, d.SourceRepo, d.SourceRepo, d.URL)
+ b.from(URI(KindRepo, d.Repo), PredDependsOn, b.node.URI, SrcHTML)
+}
+
+func (b *builder) dependent(d *Dependent) {
+ if d.Dependent == "" || d.Repo == "" {
+ return
+ }
+ b.start(KindRepo, d.Dependent, d.Dependent, d.URL)
+ b.from(URI(KindRepo, d.Repo), PredUsedBy, b.node.URI, SrcHTML)
+ if d.Owner != "" {
+ b.to(PredOwnedBy, KindUser, d.Owner, SrcID)
+ }
+ b.num(FactStars, d.Stars)
+ b.num(FactForks, d.Forks)
+}
+
+func (b *builder) stats(s *RepoStats) {
+ if s.Repo == "" {
+ return
+ }
+ b.start(KindRepo, s.Repo, s.Repo, "")
+ b.num(FactStars, s.Stars)
+ b.num(FactForks, s.Forks)
+ b.num(FactWatchers, s.Watchers)
+ b.num(FactCommits, s.Commits)
+ b.at(FactUpdated, s.PushedAt)
+ b.fact(FactURI, b.node.URI, "")
+}
+
+func firstSetTime(ts ...*time.Time) *time.Time {
+ for _, t := range ts {
+ if t != nil && !t.IsZero() {
+ return t
+ }
+ }
+ return nil
+}
+
+// --- the materialised graph ---
+
+// Graph is a set of nodes, edges, and facts held in memory. The streaming
+// commands never build one; `github graph` for a single entity, `github rdf` for
+// the buffered serialisations, and the tests all want the whole thing in hand.
+type Graph struct {
+ Nodes []Node `json:"nodes"`
+ Edges []Edge `json:"edges"`
+ Facts []Fact `json:"facts,omitempty"`
+}
+
+// Add folds a record into the graph, skipping a node already present so that a
+// repeat visit does not duplicate it.
+func (g *Graph) Add(rec any) {
+ node, edges, facts := Extract(rec)
+ if node.URI == "" {
+ return
+ }
+ g.AddNode(node)
+ g.Edges = append(g.Edges, edges...)
+ g.Facts = append(g.Facts, facts...)
+}
+
+// AddNode adds one node if its URI is new.
+func (g *Graph) AddNode(n Node) {
+ if n.URI == "" {
+ return
+ }
+ for _, have := range g.Nodes {
+ if have.URI == n.URI {
+ return
+ }
+ }
+ g.Nodes = append(g.Nodes, n)
+}
+
+// Targets returns the object URIs reachable under an allowed predicate set,
+// which is what the crawler walks. A bare-string object is never a target:
+// there is no page for a language.
+func (g *Graph) Targets(allow map[string]bool) []string {
+ var out []string
+ seen := map[string]bool{}
+ for _, e := range g.Edges {
+ if !strings.HasPrefix(e.Object, Scheme+"://") {
+ continue
+ }
+ if len(allow) > 0 && !allow[e.Predicate] {
+ continue
+ }
+ if !seen[e.Object] {
+ seen[e.Object] = true
+ out = append(out, e.Object)
+ }
+ }
+ return out
+}
+
+// FilterTrust drops the edges below a floor, in place.
+func FilterTrust(edges []Edge, min string) []Edge {
+ out := edges[:0]
+ for _, e := range edges {
+ if TrustAtLeast(e.Source, min) {
+ out = append(out, e)
+ }
+ }
+ return out
+}
+
+// SortEdges gives an export a stable order, which is what makes a diff of two
+// runs readable.
+func SortEdges(edges []Edge) {
+ sort.SliceStable(edges, func(i, j int) bool {
+ a, b := edges[i], edges[j]
+ if a.Subject != b.Subject {
+ return a.Subject < b.Subject
+ }
+ if a.Predicate != b.Predicate {
+ return a.Predicate < b.Predicate
+ }
+ return a.Object < b.Object
+ })
+}
diff --git a/gh/ops.go b/gh/ops.go
index 8c74e79..2160e41 100644
--- a/gh/ops.go
+++ b/gh/ops.go
@@ -25,6 +25,7 @@ func registerOps(app *kit.App) {
registerHistoryOps(app)
registerPeopleOps(app)
registerDiscoverOps(app)
+ registerGraphOps(app)
registerMetaOps(app)
}
@@ -553,6 +554,10 @@ func (c *Client) fetchOne(ctx context.Context, kind, id string) (any, error) {
return nil, errs.Usage("%q is not a file id", id)
}
return c.Blob(ctx, repo, path, BlobOptions{Ref: ref})
+ case KindTopic:
+ return c.TopicPage(ctx, id)
+ case KindGist:
+ return c.Gist(ctx, id, false)
}
return nil, errs.Unsupported("reading a %s is not implemented yet", kind)
}
@@ -1369,6 +1374,195 @@ func getStats(ctx context.Context, in repoRefIn, emit func(*RepoStats) error) er
// --- meta ---
+// --- the graph plane ---
+
+func registerGraphOps(app *kit.App) {
+ kit.Handle(app, kit.OpMeta{
+ Name: "graph", Group: "graph",
+ Summary: "Emit the node, edges, and facts for one entity",
+ Args: []kit.Arg{{Name: "ref", Help: "any github reference"}},
+ }, graph)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "edges", Group: "graph", List: true,
+ Summary: "Emit only the edges for one entity",
+ Long: "Every edge carries the rule that produced it. --min-trust id is the\n" +
+ "interesting case: the edges derived from the id alone need no request at\n" +
+ "all, so `github edges golang/go#1 --min-trust id` answers offline.",
+ Args: []kit.Arg{{Name: "ref", Help: "any github reference"}},
+ }, edges)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "crawl", Group: "graph",
+ Summary: "Walk the graph breadth-first from a seed",
+ Long: "crawl follows only the predicates named by --follow, which defaults to the\n" +
+ "structural ones: references, stars, follows, and the two dependency\n" +
+ "predicates fan out without bound and have to be asked for by name. Nodes\n" +
+ "and edges stream as they are found, so an interrupted walk has still\n" +
+ "emitted everything it reached.",
+ Args: []kit.Arg{{Name: "ref", Help: "seed reference"}},
+ }, crawl)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "deps", Group: "graph", URIType: KindRepo, List: true,
+ Summary: "List what a repository depends on",
+ Long: "The dependency graph is opt-in per repository. A repository with it off\n" +
+ "answers with a page and no rows, which comes back as an empty list rather\n" +
+ "than an error, because the page does not say which of the two it is.",
+ Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}},
+ }, listDeps)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "dependents", Group: "graph", URIType: KindRepo, List: true,
+ Summary: "List the repositories that depend on this one",
+ Long: "The list is ordered by stars and it is long, so --limit is the flag that\n" +
+ "matters: a popular library has tens of thousands of rows at thirty a page.",
+ Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}},
+ }, listDependents)
+}
+
+func graph(ctx context.Context, in bareRefIn, emit func(any) error) error {
+ _, _, g, err := in.C.GraphOfRef(ctx, in.Ref)
+ if err != nil {
+ return err
+ }
+ for i := range g.Nodes {
+ if err := emit(&g.Nodes[i]); err != nil {
+ return err
+ }
+ }
+ for i := range g.Edges {
+ if err := emit(&g.Edges[i]); err != nil {
+ return err
+ }
+ }
+ for i := range g.Facts {
+ if err := emit(&g.Facts[i]); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+type edgesIn struct {
+ C *Client `kit:"inject"`
+ Ref string `kit:"arg" help:"any github reference"`
+ Predicate string `kit:"flag" help:"keep only this predicate"`
+ MinTrust string `kit:"flag,name=min-trust" help:"drop edges below this rule: id, payload, feed, html, text" default:"html"`
+}
+
+func edges(ctx context.Context, in edgesIn, emit func(*Edge) error) error {
+ // The id rules produce their edges without a fetch, so asking for them
+ // alone is answered from the reference and nothing else.
+ if in.MinTrust == SrcID {
+ kind, id, err := Classify(in.Ref)
+ if err != nil {
+ return err
+ }
+ return emitEach(pickEdges(idEdges(kind, id), in), emit)
+ }
+ _, _, g, err := in.C.GraphOfRef(ctx, in.Ref)
+ if err != nil {
+ return err
+ }
+ return emitEach(pickEdges(g.Edges, in), emit)
+}
+
+// idEdges is what the id alone says. It builds the record shell rather than
+// fetching one, which is the whole point: a thread id names its repository and
+// a repository id names its owner, and neither fact needs github.com.
+func idEdges(kind, id string) []Edge {
+ var rec any
+ switch kind {
+ case KindRepo:
+ owner, name, _ := SplitRepo(id)
+ r := &Repo{Owner: owner, Name: name}
+ r.setIdentity(KindRepo, id)
+ rec = r
+ case KindIssue, KindPR, KindDiscussion:
+ repo, num, ok := SplitThreadID(id)
+ if !ok {
+ return nil
+ }
+ n, _ := strconv.Atoi(num)
+ t := &Thread{Repo: repo, Number: n}
+ t.setIdentity(kind, id)
+ rec = &Issue{Thread: *t}
+ default:
+ return nil
+ }
+ _, out, _ := Extract(rec)
+ return FilterTrust(out, SrcID)
+}
+
+func pickEdges(in []Edge, opts edgesIn) []Edge {
+ out := FilterTrust(in, opts.MinTrust)
+ if opts.Predicate == "" {
+ return out
+ }
+ want := strings.TrimPrefix(opts.Predicate, "gh:")
+ kept := out[:0]
+ for _, e := range out {
+ if e.Predicate == want {
+ kept = append(kept, e)
+ }
+ }
+ return kept
+}
+
+type crawlIn struct {
+ C *Client `kit:"inject"`
+ Ref string `kit:"arg" help:"seed reference"`
+ Depth int `kit:"flag" help:"how many edges out to walk" default:"1"`
+ Follow []string `kit:"flag" help:"predicates to follow, or all (default: the structural ones)"`
+ Kinds []string `kit:"flag" help:"expand only these kinds"`
+ MinTrust string `kit:"flag,name=min-trust" help:"drop edges below this rule" default:"html"`
+ DryRun bool `kit:"flag,name=dry-run" help:"print the estimate and stop"`
+ NodesOnly bool `kit:"flag,name=nodes-only" help:"emit nodes only"`
+ EdgesOnly bool `kit:"flag,name=edges-only" help:"emit edges only"`
+ Limit int `kit:"flag,inherit"`
+}
+
+func crawl(ctx context.Context, in crawlIn, emit func(any) error) error {
+ o := CrawlOptions{
+ Depth: in.Depth,
+ Follow: in.Follow,
+ Kinds: in.Kinds,
+ MinTrust: in.MinTrust,
+ Limit: in.Limit,
+ NodesOnly: in.NodesOnly,
+ EdgesOnly: in.EdgesOnly,
+ }
+ if in.DryRun {
+ plan, err := in.C.Estimate(ctx, in.Ref, o)
+ if err != nil {
+ return err
+ }
+ return emit(plan)
+ }
+ return in.C.Crawl(ctx, in.Ref, o, CrawlSink{
+ Node: func(n *Node) error { return emit(n) },
+ Edge: func(e *Edge) error { return emit(e) },
+ Fact: func(f *Fact) error { return emit(f) },
+ })
+}
+
+func listDeps(ctx context.Context, in repoListIn, emit func(*Dependency) error) error {
+ repo, err := ResolveRepo(in.Ref)
+ if err != nil {
+ return err
+ }
+ return in.C.Dependencies(ctx, repo, in.Limit, byValue(emit))
+}
+
+func listDependents(ctx context.Context, in repoListIn, emit func(*Dependent) error) error {
+ repo, err := ResolveRepo(in.Ref)
+ if err != nil {
+ return err
+ }
+ return in.C.Dependents(ctx, repo, in.Limit, byValue(emit))
+}
+
func registerMetaOps(app *kit.App) {
kit.Handle(app, kit.OpMeta{
Name: "url", Group: "meta", Single: true,
diff --git a/gh/rdf.go b/gh/rdf.go
new file mode 100644
index 0000000..cda8f99
--- /dev/null
+++ b/gh/rdf.go
@@ -0,0 +1,617 @@
+package gh
+
+import (
+ "encoding/json"
+ "fmt"
+ "io"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// rdf.go serialises the graph. The graph plane is already triples, so this is a
+// serialisation and not a transformation.
+//
+// The one real decision here is the schema.org alignment. A consumer that has
+// never heard of gh:forkOf still understands schema:author and schema:isPartOf,
+// and using the standard term where one exists is what lets a github export and
+// an hf export join in the same triple store.
+//
+// Subject IRIs are the canonical github.com URLs rather than the github:// URIs.
+// A triple whose subject is https://github.com/golang/go is dereferenceable by
+// anything on the web; one whose subject is github://repo/golang/go is
+// dereferenceable only by this tool. The github:// form survives as a gh:uri
+// literal so nothing is lost.
+
+// The namespaces.
+const (
+ NSSchema = "https://schema.org/"
+ NSGH = "https://github.com/ns#"
+ NSGHR = "https://github.com/"
+ NSRdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
+ NSRdfs = "http://www.w3.org/2000/01/rdf-schema#"
+ NSXsd = "http://www.w3.org/2001/XMLSchema#"
+ NSDoap = "http://usefulinc.com/ns/doap#"
+ NSFoaf = "http://xmlns.com/foaf/0.1/"
+)
+
+var rdfPrefixes = [][2]string{
+ {"schema", NSSchema},
+ {"gh", NSGH},
+ {"ghr", NSGHR},
+ {"rdf", NSRdf},
+ {"rdfs", NSRdfs},
+ {"xsd", NSXsd},
+ {"doap", NSDoap},
+ {"foaf", NSFoaf},
+}
+
+// The datatypes a Fact can carry. They are CURIEs so a Fact reads the same in
+// every serialisation.
+const (
+ TypeInteger = "xsd:integer"
+ TypeDecimal = "xsd:decimal"
+ TypeBoolean = "xsd:boolean"
+ TypeDateTime = "xsd:dateTime"
+)
+
+// rdfTypes is the class mapping. Marking an issue as schema:DiscussionForumPosting
+// is not this tool's invention: it is what GitHub's own structured_data block
+// says, and following the publisher's vocabulary for its own content is the
+// whole point.
+var rdfTypes = map[string][]string{
+ KindRepo: {"schema:SoftwareSourceCode", "doap:Project"},
+ KindUser: {"schema:Person", "foaf:Person"},
+ KindOrg: {"schema:Organization"},
+ KindIssue: {"gh:Issue", "schema:DiscussionForumPosting"},
+ KindPR: {"gh:PullRequest", "schema:DiscussionForumPosting"},
+ KindDiscussion: {"schema:DiscussionForumPosting"},
+ KindCommit: {"gh:Commit"},
+ KindRelease: {"schema:SoftwareApplication"},
+ KindTag: {"gh:Ref"},
+ KindBranch: {"gh:Ref"},
+ KindFile: {"schema:MediaObject"},
+ KindTree: {"schema:MediaObject"},
+ KindTopic: {"schema:DefinedTerm"},
+ KindLabel: {"schema:DefinedTerm"},
+ KindPackage: {"schema:SoftwareApplication"},
+ KindGist: {"schema:SoftwareSourceCode"},
+ KindAction: {"schema:SoftwareApplication"},
+ KindWiki: {"schema:Article"},
+}
+
+// rdfPredicates maps this tool's vocabulary onto RDF terms. A predicate with no
+// entry is emitted in the gh: namespace under its own name, which is what makes
+// adding a predicate to graph.go a one-line change rather than two.
+var rdfPredicates = map[string]string{
+ PredOwnedBy: "schema:author",
+ PredAuthoredBy: "schema:author",
+ PredPartOf: "schema:isPartOf",
+ PredMemberOf: "schema:memberOf",
+ PredHasTopic: "schema:keywords",
+ PredHasLabel: "schema:keywords",
+ PredWrittenIn: "schema:programmingLanguage",
+ PredLicensedUnder: "schema:license",
+ PredReferences: "schema:citation",
+ PredFollows: "schema:follows",
+}
+
+// rdfFacts maps the literal predicates. The counts get gh: terms because
+// schema.org has no stargazer count, and the dates get the standard ones
+// because it does.
+var rdfFacts = map[string]string{
+ FactName: "schema:name",
+ FactDescription: "schema:description",
+ FactHomepage: "schema:url",
+ FactCreated: "schema:dateCreated",
+ FactUpdated: "schema:dateModified",
+ FactStars: "gh:stargazerCount",
+ FactForks: "gh:forkCount",
+ FactWatchers: "gh:watcherCount",
+ FactCommits: "gh:commitCount",
+ FactURI: "gh:uri",
+ FactAvatar: "schema:image",
+}
+
+// The output formats.
+const (
+ FormatNT = "nt"
+ FormatNQuads = "nq"
+ FormatTurtle = "ttl"
+ FormatJSONLD = "jsonld"
+)
+
+// RDFFormats is the accepted set, for help text and validation.
+var RDFFormats = []string{FormatNT, FormatNQuads, FormatTurtle, FormatJSONLD}
+
+// RDFOptions controls a serialisation.
+type RDFOptions struct {
+ Format string
+ // Graph is the fourth position for N-Quads. Putting the source URL there
+ // means the provenance survives into the RDF and a quad store can answer
+ // which page told us this.
+ Graph string
+}
+
+// WriteRDF serialises a whole graph. N-Triples is the default because it
+// streams: nt and nq write a line per triple as it is produced, ttl buffers one
+// subject at a time, and jsonld buffers the lot.
+func WriteRDF(w io.Writer, g *Graph, o RDFOptions) error {
+ switch o.Format {
+ case "", FormatNT:
+ return writeTriples(w, g, "")
+ case FormatNQuads:
+ return writeTriples(w, g, o.Graph)
+ case FormatTurtle:
+ return writeTurtle(w, g)
+ case FormatJSONLD:
+ return writeJSONLD(w, g)
+ default:
+ return fmt.Errorf("unknown rdf format %q, want one of %s", o.Format, strings.Join(RDFFormats, ", "))
+ }
+}
+
+// --- streaming ---
+
+// RDFWriter is the streaming form. A crawl hands it nodes, edges, and facts as
+// it finds them and it writes lines, so `github export --depth 3 --format nt`
+// over a large organization never holds the graph in memory. The buffered
+// formats are handled by collecting into a Graph and calling WriteRDF, and this
+// type reports which is which through Streams.
+type RDFWriter struct {
+ w io.Writer
+ suffix string
+}
+
+// NewRDFWriter returns a streaming writer for nt or nq, and nil for the formats
+// that cannot stream.
+func NewRDFWriter(w io.Writer, o RDFOptions) *RDFWriter {
+ switch o.Format {
+ case "", FormatNT:
+ return &RDFWriter{w: w, suffix: " .\n"}
+ case FormatNQuads:
+ suffix := " .\n"
+ if o.Graph != "" {
+ suffix = " <" + o.Graph + "> .\n"
+ }
+ return &RDFWriter{w: w, suffix: suffix}
+ }
+ return nil
+}
+
+// Streams reports whether a format can be written a triple at a time.
+func Streams(format string) bool {
+ return format == "" || format == FormatNT || format == FormatNQuads
+}
+
+func (r *RDFWriter) Node(n *Node) error { return r.lines(nodeLines(*n)) }
+
+func (r *RDFWriter) Edge(e *Edge) error { return r.lines(edgeLines(*e)) }
+
+func (r *RDFWriter) Fact(f *Fact) error { return r.lines(factLines(*f)) }
+
+func (r *RDFWriter) lines(ls []string) error {
+ for _, l := range ls {
+ if _, err := io.WriteString(r.w, l+r.suffix); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+func writeTriples(w io.Writer, g *Graph, graph string) error {
+ r := NewRDFWriter(w, RDFOptions{Format: FormatNQuads, Graph: graph})
+ for i := range g.Nodes {
+ if err := r.Node(&g.Nodes[i]); err != nil {
+ return err
+ }
+ }
+ for i := range g.Edges {
+ if err := r.Edge(&g.Edges[i]); err != nil {
+ return err
+ }
+ }
+ for i := range g.Facts {
+ if err := r.Fact(&g.Facts[i]); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// nodeLines states a node's classes and its label.
+func nodeLines(n Node) []string {
+ subj := iri(n.URI)
+ var out []string
+ for _, t := range rdfTypes[n.Kind] {
+ out = append(out, subj+" <"+NSRdf+"type> "+expand(t))
+ }
+ if n.Label != "" {
+ out = append(out, subj+" "+expand("rdfs:label")+" "+quote(n.Label))
+ }
+ return out
+}
+
+// edgeLines renders one edge.
+//
+// A weighted edge is reified: a contributor's commit count is a property of the
+// relation and not of either end, and the only honest way to say that in RDF is
+// to give the relation a node of its own.
+func edgeLines(e Edge) []string {
+ subj := iri(e.Subject)
+ obj := objectTerm(e.Predicate, e.Object)
+ out := []string{subj + " " + expand(rdfPredicate(e.Predicate)) + " " + obj}
+ if e.Weight != nil {
+ blank := reifiedNode(e)
+ out = append(out,
+ blank+" <"+NSRdf+"subject> "+subj,
+ blank+" <"+NSRdf+"predicate> "+expand(rdfPredicate(e.Predicate)),
+ blank+" <"+NSRdf+"object> "+obj,
+ blank+" "+expand(weightTerm(e.Predicate))+" "+quote(strconv.Itoa(*e.Weight))+"^^"+expand(TypeInteger),
+ )
+ }
+ if e.At != nil {
+ out = append(out, reifiedNode(e)+" "+expand("schema:dateCreated")+" "+
+ quote(e.At.UTC().Format(time.RFC3339))+"^^"+expand(TypeDateTime))
+ }
+ return out
+}
+
+// weightTerm names what a weight counts. Only contributedTo and reactedWith
+// carry one, and calling both of them "count" would throw away the only thing
+// that makes the number readable.
+func weightTerm(pred string) string {
+ switch pred {
+ case PredContributedTo:
+ return "gh:commitCount"
+ case PredReactedWith:
+ return "gh:reactionCount"
+ default:
+ return "gh:count"
+ }
+}
+
+// reifiedNode names the statement itself. The name is derived from the triple,
+// so two runs produce the same node and a merge of two exports does not
+// duplicate it.
+func reifiedNode(e Edge) string {
+ key := e.Subject + "|" + e.Predicate + "|" + e.Object
+ return "_:stmt-" + strings.NewReplacer("://", "-", "/", "-", "|", "-", "#", "-", "@", "-", " ", "_").Replace(key)
+}
+
+func factLines(f Fact) []string {
+ pred, ok := rdfFacts[f.Predicate]
+ if !ok {
+ pred = "gh:" + f.Predicate
+ }
+ obj := quote(f.Value)
+ if f.Datatype != "" {
+ obj += "^^" + expand(f.Datatype)
+ }
+ return []string{iri(f.Subject) + " " + expand(pred) + " " + obj}
+}
+
+// rdfPredicate maps a predicate onto its RDF term, defaulting to the gh:
+// namespace so a new predicate needs no entry to serialise correctly.
+func rdfPredicate(pred string) string {
+ if p, ok := rdfPredicates[pred]; ok {
+ return p
+ }
+ return "gh:" + pred
+}
+
+// objectTerm renders an edge's object. Most objects are URIs. A language and a
+// licence are bare strings on the record plane, and they get synthetic IRIs
+// here rather than becoming string literals, because gh:language/go is
+// something two exports can join on and "Go" is not.
+func objectTerm(pred, object string) string {
+ if strings.HasPrefix(object, Scheme+"://") {
+ return iri(object)
+ }
+ switch pred {
+ case PredWrittenIn:
+ return "<" + NSGH + "language/" + slug(object) + ">"
+ case PredLicensedUnder:
+ return "<" + NSGH + "license/" + slug(object) + ">"
+ case PredReactedWith:
+ return "<" + NSGH + "reaction/" + slug(object) + ">"
+ }
+ return quote(object)
+}
+
+// slug makes a URI path segment out of a rendered name. Spaces and slashes are
+// the only characters that actually occur here, in names like "Jupyter Notebook"
+// and "BSD 3-Clause", and both have to go.
+func slug(s string) string {
+ s = strings.TrimSpace(s)
+ var b strings.Builder
+ for _, r := range s {
+ switch {
+ case r == ' ' || r == '/' || r == '\\':
+ b.WriteByte('-')
+ case r == '<' || r == '>' || r == '"' || r == '{' || r == '}' || r == '|' || r == '^' || r == '`':
+ // Characters an IRI may not carry. Dropping them beats escaping
+ // them, because nobody wants gh:license/BSD%203-Clause.
+ default:
+ b.WriteRune(r)
+ }
+ }
+ return b.String()
+}
+
+// iri renders a subject or object. A blank node stays a blank node, and a
+// github:// URI becomes the canonical https URL.
+func iri(uri string) string {
+ if strings.HasPrefix(uri, "_:") {
+ return uri
+ }
+ return "<" + IRI(uri) + ">"
+}
+
+// IRI maps a github:// URI to its dereferenceable https form. The github:// form
+// stays on the record plane, where it is a stable key rather than a location.
+//
+// The three derived kinds have no address of their own, so they map into the
+// gh: namespace instead of pretending to be a page.
+func IRI(uri string) string {
+ if !strings.HasPrefix(uri, Scheme+"://") {
+ return uri
+ }
+ rest := strings.TrimPrefix(uri, Scheme+"://")
+ kind, id, ok := strings.Cut(rest, "/")
+ if !ok {
+ return uri
+ }
+ if u, err := Locate(kind, id); err == nil {
+ return u
+ }
+ return NSGH + kind + "/" + slug(id)
+}
+
+func expand(curie string) string {
+ prefix, rest, ok := strings.Cut(curie, ":")
+ if !ok {
+ return "<" + curie + ">"
+ }
+ for _, p := range rdfPrefixes {
+ if p[0] == prefix {
+ return "<" + p[1] + rest + ">"
+ }
+ }
+ return "<" + curie + ">"
+}
+
+func quote(s string) string {
+ var b strings.Builder
+ b.WriteByte('"')
+ for _, r := range s {
+ switch r {
+ case '"':
+ b.WriteString(`\"`)
+ case '\\':
+ b.WriteString(`\\`)
+ case '\n':
+ b.WriteString(`\n`)
+ case '\r':
+ b.WriteString(`\r`)
+ case '\t':
+ b.WriteString(`\t`)
+ default:
+ b.WriteRune(r)
+ }
+ }
+ b.WriteByte('"')
+ return b.String()
+}
+
+// --- turtle ---
+
+// writeTurtle groups by subject, which is the whole reason to prefer Turtle: a
+// node and everything said about it read as one paragraph.
+func writeTurtle(w io.Writer, g *Graph) error {
+ for _, p := range rdfPrefixes {
+ if _, err := fmt.Fprintf(w, "@prefix %s: <%s> .\n", p[0], p[1]); err != nil {
+ return err
+ }
+ }
+ if _, err := io.WriteString(w, "\n"); err != nil {
+ return err
+ }
+
+ bySubject := map[string][][2]string{}
+ var order []string
+ add := func(subj, pred, obj string) {
+ if _, seen := bySubject[subj]; !seen {
+ order = append(order, subj)
+ }
+ bySubject[subj] = append(bySubject[subj], [2]string{pred, obj})
+ }
+ // The line renderers already produce N-Triples, and Turtle is the same
+ // triples with the subject factored out, so this splits each line rather
+ // than growing a second renderer that could disagree with the first.
+ collect := func(lines []string) {
+ for _, l := range lines {
+ subj, pred, obj, ok := splitTriple(l)
+ if !ok {
+ continue
+ }
+ add(subj, shorten(pred), shorten(obj))
+ }
+ }
+ for _, n := range g.Nodes {
+ collect(nodeLines(n))
+ }
+ for _, e := range g.Edges {
+ collect(edgeLines(e))
+ }
+ for _, f := range g.Facts {
+ collect(factLines(f))
+ }
+
+ for _, subj := range order {
+ if _, err := io.WriteString(w, shorten(subj)+"\n"); err != nil {
+ return err
+ }
+ pairs := bySubject[subj]
+ for i, pair := range pairs {
+ end := " ;\n"
+ if i == len(pairs)-1 {
+ end = " .\n\n"
+ }
+ if _, err := io.WriteString(w, " "+pair[0]+" "+pair[1]+end); err != nil {
+ return err
+ }
+ }
+ }
+ return nil
+}
+
+// splitTriple pulls a rendered N-Triples line apart. The grammar is regular
+// enough for this: the subject and the predicate are always angle-bracketed or
+// blank-node terms with no spaces in them, and everything after the second
+// space is the object.
+func splitTriple(line string) (subj, pred, obj string, ok bool) {
+ subj, rest, ok := strings.Cut(line, " ")
+ if !ok {
+ return "", "", "", false
+ }
+ pred, obj, ok = strings.Cut(rest, " ")
+ if !ok {
+ return "", "", "", false
+ }
+ return subj, pred, obj, true
+}
+
+// shorten turns an expanded IRI back into a CURIE where a prefix covers it,
+// which is what makes Turtle readable rather than just grouped.
+func shorten(term string) string {
+ if !strings.HasPrefix(term, "<") {
+ // A literal, possibly with a datatype that is itself an IRI.
+ if i := strings.Index(term, "^^<"); i >= 0 {
+ return term[:i+2] + shorten(term[i+2:])
+ }
+ return term
+ }
+ full := strings.TrimSuffix(strings.TrimPrefix(term, "<"), ">")
+ if full == NSRdf+"type" {
+ return "a"
+ }
+ for _, p := range rdfPrefixes {
+ // ghr is the whole of github.com, so every subject IRI would collapse
+ // into it and read as ghr:golang/go, which is not a legal CURIE local
+ // name once a path has slashes in it. Subjects stay in angle brackets.
+ if p[0] == "ghr" {
+ continue
+ }
+ if rest, found := strings.CutPrefix(full, p[1]); found && rest != "" && !strings.ContainsAny(rest, "/") {
+ return p[0] + ":" + rest
+ }
+ }
+ return term
+}
+
+// --- json-ld ---
+
+// writeJSONLD emits one object per node with its edges and facts folded in, and
+// an inline context so the document stands alone. It buffers everything, which
+// is why the help text points at it for single records rather than crawls.
+func writeJSONLD(w io.Writer, g *Graph) error {
+ ctx := map[string]any{}
+ for _, p := range rdfPrefixes {
+ ctx[p[0]] = p[1]
+ }
+
+ byURI := map[string]map[string]any{}
+ var order []string
+ obj := func(uri string) map[string]any {
+ o, ok := byURI[uri]
+ if !ok {
+ o = map[string]any{"@id": IRI(uri)}
+ byURI[uri] = o
+ order = append(order, uri)
+ }
+ return o
+ }
+ for _, n := range g.Nodes {
+ o := obj(n.URI)
+ if types := rdfTypes[n.Kind]; len(types) > 0 {
+ o["@type"] = types
+ }
+ if n.Label != "" {
+ o["rdfs:label"] = n.Label
+ }
+ if n.URL != "" {
+ o["schema:url"] = n.URL
+ }
+ }
+ for _, e := range g.Edges {
+ o := obj(e.Subject)
+ var value any
+ if strings.HasPrefix(e.Object, Scheme+"://") {
+ value = map[string]any{"@id": IRI(e.Object)}
+ } else if term := objectTerm(e.Predicate, e.Object); strings.HasPrefix(term, "<") {
+ value = map[string]any{"@id": strings.TrimSuffix(strings.TrimPrefix(term, "<"), ">")}
+ } else {
+ value = e.Object
+ }
+ if e.Weight != nil {
+ value = map[string]any{"@id": jsonldID(value), weightTerm(e.Predicate): *e.Weight}
+ }
+ appendValue(o, rdfPredicate(e.Predicate), value)
+ }
+ for _, f := range g.Facts {
+ pred, ok := rdfFacts[f.Predicate]
+ if !ok {
+ pred = "gh:" + f.Predicate
+ }
+ appendValue(obj(f.Subject), pred, jsonldLiteral(f))
+ }
+
+ graph := make([]map[string]any, 0, len(order))
+ for _, uri := range order {
+ graph = append(graph, byURI[uri])
+ }
+ enc := json.NewEncoder(w)
+ enc.SetIndent("", " ")
+ return enc.Encode(map[string]any{"@context": ctx, "@graph": graph})
+}
+
+func jsonldID(v any) any {
+ if m, ok := v.(map[string]any); ok {
+ return m["@id"]
+ }
+ return v
+}
+
+// appendValue keeps repeated predicates as a list rather than letting the last
+// one win, because a repository with twelve topics has twelve of them.
+func appendValue(o map[string]any, pred string, value any) {
+ switch cur := o[pred].(type) {
+ case nil:
+ o[pred] = value
+ case []any:
+ o[pred] = append(cur, value)
+ default:
+ o[pred] = []any{cur, value}
+ }
+}
+
+// jsonldLiteral gives a value its type, so a count arrives as a number and a
+// timestamp as a typed value rather than as prose.
+func jsonldLiteral(f Fact) any {
+ switch f.Datatype {
+ case TypeInteger:
+ if n, err := strconv.ParseInt(f.Value, 10, 64); err == nil {
+ return n
+ }
+ case TypeDecimal:
+ if v, err := strconv.ParseFloat(f.Value, 64); err == nil {
+ return v
+ }
+ case TypeBoolean:
+ return f.Value == "true"
+ case TypeDateTime:
+ return map[string]any{"@value": f.Value, "@type": TypeDateTime}
+ }
+ return f.Value
+}
diff --git a/gh/types.go b/gh/types.go
index 67dbc49..c921b7f 100644
--- a/gh/types.go
+++ b/gh/types.go
@@ -861,3 +861,41 @@ type RepoStats struct {
PushedAt *time.Time `json:"pushed_at,omitempty" table:"pushed,time"`
}
+
+// Dependency is one row of /network/dependencies: a package this repository
+// declares in one of its manifests.
+//
+// The identity is the repository the package resolves to, because that is the
+// only thing on the row with an address on github.com. A package GitHub cannot
+// resolve to a repository has no Kind and no ID, and its name is still on the
+// record, because a dependency list with the unresolvable rows silently dropped
+// is a lie about what the manifest contains.
+type Dependency struct {
+ Base
+
+ Repo string `json:"repo" table:"repo"`
+ Package string `json:"package" table:"package"`
+
+ SourceRepo string `json:"source_repo,omitempty" table:"source"`
+ Version string `json:"version,omitempty" table:"version"`
+ Relationship string `json:"relationship,omitempty" table:"rel"`
+ Ecosystem string `json:"ecosystem,omitempty" table:"ecosystem"`
+ Manifest string `json:"manifest,omitempty" table:"manifest"`
+ License string `json:"license,omitempty" table:"-"`
+}
+
+// Dependent is one row of /network/dependents: a repository that depends on
+// this one. The two counts are on the row, so a caller sorting the dependents
+// of a popular library by stars does not need a fetch per row.
+type Dependent struct {
+ Base
+
+ Repo string `json:"repo" table:"repo"`
+ Dependent string `json:"dependent" table:"dependent"`
+ Owner string `json:"owner" table:"-"`
+
+ Stars *int `json:"stars,omitempty" table:"stars"`
+ Forks *int `json:"forks,omitempty" table:"forks"`
+
+ AvatarURL string `json:"avatar_url,omitempty" table:"-"`
+}
diff --git a/pkg/page/selectors.go b/pkg/page/selectors.go
index 4e84cb3..f0b5193 100644
--- a/pkg/page/selectors.go
+++ b/pkg/page/selectors.go
@@ -139,9 +139,9 @@ var (
// organization's readme has no class of its own, so the caller falls back
// to the markdown article, which is the same on both.
// Verified 2026-07-25 against sindresorhus and github.
- ProfileReadme = Sel{Class: "profile-readme"}
- ProfileVCardList = Sel{Class: "vcard-details"}
- ProfileAchieve = Sel{Class: "js-profile-achievements"}
+ ProfileReadme = Sel{Class: "profile-readme"}
+ ProfileVCardList = Sel{Class: "vcard-details"}
+ ProfileAchieve = Sel{Class: "js-profile-achievements"}
)
// --- discussion pages, /{owner}/{repo}/discussions/{n} ---
@@ -261,6 +261,38 @@ var (
ReleaseAssetLink = Sel{Tag: "a", Attr: "href", AttrPrefix: "/"}
)
+// --- dependency graph, /{owner}/{repo}/network/{dependencies,dependents} ---
+
+// The two dependency pages are the only keyless source for who depends on whom,
+// and they are the most fragile markup this tool reads: the rows are identified
+// by test hooks rather than by classes, and one of those hooks carries GitHub's
+// own typo, "dependendency". It is spelled here the way the page spells it, and
+// a fix on their side will show up as a missing edge rather than a wrong one.
+//
+// The two pages disagree on everything, including which attribute names a row:
+// dependencies uses data-test-selector and dependents uses data-test-id.
+// Verified 2026-07-25 against gohugoio/hugo.
+var (
+ DependencyRow = Sel{Attr: "data-test-selector", AttrValue: "dg-repo-pkg-dependency"}
+ // The name is an anchor when GitHub resolved the package to a repository
+ // and a plain span when it did not, so the class is the only thing both
+ // forms share.
+ DependencyName = Sel{Class: "h4"}
+ DependencyLink = Sel{Tag: "a", Attr: "data-hovercard-type", AttrValue: "dependendency_graph_package"}
+ DependencyVersion = Sel{Tag: "span", Class: "text-mono"}
+ DependencyRelation = Sel{Tag: "a", Attr: "data-test-selector", AttrValue: "relationship-label-link"}
+ DependencyManifest = Sel{Tag: "a", Attr: "data-test-selector", AttrValue: "dg-repo-pkg-manifest"}
+
+ DependentRow = Sel{Attr: "data-test-id", AttrValue: "dg-repo-pkg-dependent"}
+ DependentRepo = Sel{Tag: "a", Attr: "data-hovercard-type", AttrValue: "repository"}
+ DependentUser = Sel{Tag: "a", Attr: "data-hovercard-type", AttrValue: "user"}
+ DependentStars = Sel{Tag: "span", Class: "text-bold", HasDescendantClass: "octicon-star"}
+ DependentForks = Sel{Tag: "span", Class: "text-bold", HasDescendantClass: "octicon-repo-forked"}
+ // The dependents pager is a cursor in a button, not a rel="next" anchor,
+ // so it needs its own selector and its own token.
+ DependentNext = Sel{Tag: "a", Class: "BtnGroup-item", Attr: "href", AttrContains: "dependents_after="}
+)
+
// --- gist, gist.github.com/{id} ---
// Verified 2026-07-25 against gist.github.com.
From a8ab53e2f269467774cbe8c1e6ead07a9304c0b4 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 20:19:14 +0700
Subject: [PATCH 11/21] gh: the page plane and doctor
page prints the whole extraction for one URL and drops nothing. It is the escape
hatch for a field no record models yet, it is the way to tell "the page did not
carry it" apart from "the decoder dropped it", and it is how a recorded fixture
is read back. Every reader in the package now works from the same Client.Page,
so what this prints is what they see rather than a second opinion.
doctor answers the question people actually ask when a command comes back wrong.
The check that matters is the first one: a token in the environment does nothing
here, and the failure that causes is invisible, because the tool keeps working
and stays exactly as rate limited as it was, so the obvious conclusion is that
the token is wrong. Now it says so.
---
cli/page.go | 170 ++++++++++++++++++++++++++++++++++++++++++++++++
cli/root.go | 1 +
gh/client.go | 17 +++++
gh/doctor.go | 178 +++++++++++++++++++++++++++++++++++++++++++++++++++
gh/ops.go | 22 +++++++
5 files changed, 388 insertions(+)
create mode 100644 cli/page.go
create mode 100644 gh/doctor.go
diff --git a/cli/page.go b/cli/page.go
new file mode 100644
index 0000000..1c14fb2
--- /dev/null
+++ b/cli/page.go
@@ -0,0 +1,170 @@
+package cli
+
+import (
+ "bufio"
+ "context"
+ "encoding/json"
+ "os"
+ "sort"
+ "strings"
+
+ "github.com/tamnd/any-cli/kit"
+ "github.com/tamnd/any-cli/kit/errs"
+ "github.com/tamnd/github-cli/gh"
+ "github.com/tamnd/github-cli/pkg/page"
+)
+
+// page.go is the 1:1 view. Everything else in this tool decides what matters on
+// a page and throws the rest away; this command throws nothing away, which makes
+// it three things at once.
+//
+// It is the escape hatch. A consumer who wants a field no record models yet can
+// have it today instead of waiting for a release.
+//
+// It is the debugging tool. When a record comes back thin the first question is
+// always whether the data was missing from the page or dropped by the decoder,
+// and this is the only way to tell the two apart.
+//
+// It is how a recorded fixture is read back, since a fixture is the bytes and
+// nothing else.
+//
+// It is a byte-plane command because its output is one document, not a stream of
+// records, and pretending otherwise would put a table renderer in front of a
+// GraphQL response.
+
+type pageCmd struct {
+ section string
+ query string
+ raw bool
+ compact bool
+}
+
+func newPageCmd() kit.Command {
+ c := &pageCmd{}
+ return kit.Command{
+ Use: "page ",
+ Short: "Print everything a page carries, organised",
+ Long: "page fetches one page and prints the whole extraction as JSON: the React\n" +
+ "route payload, the preloaded Relay queries, GitHub's own schema.org block,\n" +
+ "the ld+json, the og: and twitter: meta, the microdata, and the deferred\n" +
+ "fragments the page names for itself.\n\n" +
+ "--section narrows it to one of payload, queries, structured_data,\n" +
+ "linked_data, partials, meta, microdata, or fragments. --query prints one\n" +
+ "preloaded query result by name, which is where issue and pull request pages\n" +
+ "keep everything. With no argument, --query lists the names.\n\n" +
+ "--raw writes the original markup instead, which is what you want when the\n" +
+ "question is about the HTML rather than about the data in it.",
+ Group: "meta",
+ Args: kit.ExactArgs(1),
+ Flags: c.flags,
+ Run: c.run,
+ }
+}
+
+func (c *pageCmd) flags(f *kit.FlagSet) {
+ f.StringVar(&c.section, "section", "", "print one section only")
+ f.StringVar(&c.query, "query", "", "print one preloaded query by name (empty lists them)")
+ f.BoolVar(&c.raw, "raw", false, "print the original markup instead of the extraction")
+ f.BoolVar(&c.compact, "compact", false, "one line of JSON rather than indented")
+}
+
+func (c *pageCmd) run(ctx context.Context, args []string) error {
+ cl, err := clientFrom(ctx)
+ if err != nil {
+ return err
+ }
+ url, err := pageURL(args[0])
+ if err != nil {
+ return err
+ }
+ p, err := cl.Page(ctx, url)
+ if err != nil {
+ return err
+ }
+
+ w := bufio.NewWriter(os.Stdout)
+ defer func() { _ = w.Flush() }()
+
+ if c.raw {
+ _, err := w.Write(p.HTML)
+ return err
+ }
+
+ enc := json.NewEncoder(w)
+ if !c.compact {
+ enc.SetIndent("", " ")
+ }
+
+ if c.query != "" {
+ q, ok := p.Queries[c.query]
+ if !ok {
+ // The names are the useful half of this failure. Query names are
+ // GitHub's internal Relay identifiers, nobody knows them by heart,
+ // and a bare "not found" would send the reader off to dump the
+ // whole queries section to find out what to ask for.
+ // The message leads with a word rather than the URL because the
+ // error renderer capitalises what it starts with, and a
+ // title-cased URL reads as a typo.
+ return errs.NotFound("no query named %q on %s; it has %s",
+ c.query, url, strings.Join(queryNames(p.Queries), ", "))
+ }
+ return enc.Encode(q)
+ }
+ if c.section != "" {
+ v, err := section(p, c.section)
+ if err != nil {
+ return err
+ }
+ return enc.Encode(v)
+ }
+ return enc.Encode(p)
+}
+
+// pageURL turns anything a person might paste into the page to fetch. A full
+// URL is taken as given, including the parts of the site that name no entity,
+// like /trending and /explore, because the debugging tool is least useful on
+// exactly the pages the model does not cover yet.
+func pageURL(ref string) (string, error) {
+ if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") {
+ return ref, nil
+ }
+ kind, id, err := gh.Classify(ref)
+ if err != nil {
+ return "", err
+ }
+ return gh.Locate(kind, id)
+}
+
+func section(p *page.Page, name string) (any, error) {
+ switch strings.ToLower(strings.TrimSpace(name)) {
+ case "payload":
+ return p.Payload, nil
+ case "queries":
+ return p.Queries, nil
+ case "structured_data", "structured-data", "structured":
+ return p.StructuredData, nil
+ case "linked_data", "linked-data", "ld", "ld+json":
+ return p.LinkedData, nil
+ case "partials":
+ return p.Partials, nil
+ case "meta":
+ return p.Meta, nil
+ case "microdata":
+ return p.Microdata, nil
+ case "fragments":
+ return p.Fragments, nil
+ default:
+ return nil, errs.Usage("unknown --section %q: payload, queries, structured_data, linked_data, partials, meta, microdata, or fragments", name)
+ }
+}
+
+// queryNames is sorted because the map order would otherwise change between two
+// runs against the same bytes, and this output gets diffed.
+func queryNames(q map[string]json.RawMessage) []string {
+ out := make([]string, 0, len(q))
+ for k := range q {
+ out = append(out, k)
+ }
+ sort.Strings(out)
+ return out
+}
diff --git a/cli/root.go b/cli/root.go
index 2c78431..e50961c 100644
--- a/cli/root.go
+++ b/cli/root.go
@@ -38,6 +38,7 @@ func NewApp() *kit.App {
app.AddCommand(newReadmeCmd())
app.AddCommand(newArchiveCmd())
app.AddCommand(newDiffCmd())
+ app.AddCommand(newPageCmd())
app.AddCommand(newRDFCmd())
app.AddCommand(newExportCmd())
return app
diff --git a/gh/client.go b/gh/client.go
index 9f28442..8e2d202 100644
--- a/gh/client.go
+++ b/gh/client.go
@@ -17,6 +17,8 @@ import (
"time"
"github.com/tamnd/any-cli/kit/errs"
+
+ "github.com/tamnd/github-cli/pkg/page"
)
// Client reads github.com. It is safe for concurrent use: the pacer and the
@@ -190,6 +192,21 @@ func (c *Client) GetHTML(ctx context.Context, rawURL string) (*Response, error)
return c.Get(ctx, rawURL, SurfaceHTML)
}
+// Page fetches a URL and hands back the whole extraction, nothing dropped. Every
+// reader in the package works from this, and `github page` prints it, which is
+// what makes the debugging tool show the same view the readers see rather than a
+// second opinion about the page.
+//
+// The URL is taken as given rather than resolved from an entity, because the
+// pages worth inspecting most are the ones the model does not cover yet.
+func (c *Client) Page(ctx context.Context, rawURL string) (*page.Page, error) {
+ res, err := c.GetHTML(ctx, rawURL)
+ if err != nil {
+ return nil, err
+ }
+ return page.Extract(res.FinalURL, res.Body), nil
+}
+
// Stream opens a body without buffering, retrying, or caching. Release assets
// and repository archives go through here: a tarball does not belong in memory
// and does not belong in the cache. The caller closes the reader.
diff --git a/gh/doctor.go b/gh/doctor.go
new file mode 100644
index 0000000..5dcd654
--- /dev/null
+++ b/gh/doctor.go
@@ -0,0 +1,178 @@
+package gh
+
+import (
+ "context"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "time"
+
+ "github.com/tamnd/github-cli/pkg/page"
+)
+
+// doctor.go answers the question people ask when a command comes back wrong: is
+// it me, is it the network, or did GitHub change the page?
+//
+// Every check is a record rather than a paragraph, so the answer can be read by
+// a person and by a script, and so the failing one can be picked out with the
+// same --fields and -o json every other command takes.
+
+// Check is one diagnostic.
+type Check struct {
+ Name string `json:"name" table:"check"`
+ Status string `json:"status" table:"status"`
+ Detail string `json:"detail" table:"detail"`
+}
+
+// The three states a check can be in. Warn exists because most of what goes
+// wrong here is survivable: a token in the environment, a cache that cannot be
+// written, a page that parsed but looks thinner than it should.
+const (
+ StatusOK = "ok"
+ StatusWarn = "warn"
+ StatusFail = "fail"
+)
+
+// tokenVars are the variables people expect to matter and which do not. They are
+// checked by name and never read for their value: this file will not put a
+// credential in a record, and there is nothing here that would use one.
+var tokenVars = []string{"GITHUB_TOKEN", "GH_TOKEN", "GITHUB_API_TOKEN", "GH_ENTERPRISE_TOKEN"}
+
+// Doctor runs the checks in order and emits one record each. It stops for
+// nothing: a failed reachability check makes the page check fail too, and seeing
+// both is more useful than seeing the first one alone.
+func (c *Client) Doctor(ctx context.Context, emit func(*Check) error) error {
+ for _, ck := range []func(context.Context) *Check{
+ c.checkAuthEnv,
+ c.checkReach,
+ c.checkPagePlane,
+ c.checkCache,
+ c.checkPacing,
+ } {
+ if err := emit(ck(ctx)); err != nil {
+ return err
+ }
+ }
+ return nil
+}
+
+// checkAuthEnv is the one people need and do not know to ask for. A token in the
+// environment does nothing here, and the failure it causes is invisible: the
+// tool works, it is just as rate limited as it was before, and the obvious
+// conclusion is that the token is wrong.
+func (c *Client) checkAuthEnv(context.Context) *Check {
+ var set []string
+ for _, v := range tokenVars {
+ if os.Getenv(v) != "" {
+ set = append(set, v)
+ }
+ }
+ if len(set) == 0 {
+ return &Check{Name: "auth", Status: StatusOK,
+ Detail: "no token in the environment, which is what this tool wants"}
+ }
+ return &Check{Name: "auth", Status: StatusWarn,
+ Detail: fmt.Sprintf("%s is set and ignored: github reads public pages and never sends an Authorization header, so a token changes nothing here. Use gh for the authenticated API", strings.Join(set, " and "))}
+}
+
+// checkReach is one small request to the site. robots.txt is the right target:
+// it is a few hundred bytes, it is not behind any of the machinery this tool
+// reads, and it comes back the same for everyone.
+func (c *Client) checkReach(ctx context.Context) *Check {
+ start := time.Now()
+ res, err := c.Get(ctx, BaseURL+"/robots.txt", SurfaceRaw)
+ if err != nil {
+ return &Check{Name: "reach", Status: StatusFail,
+ Detail: fmt.Sprintf("cannot read %s: %v", BaseURL, err)}
+ }
+ return &Check{Name: "reach", Status: StatusOK,
+ Detail: fmt.Sprintf("%s answered %d in %s", BaseURL, res.Status, time.Since(start).Round(time.Millisecond))}
+}
+
+// checkPagePlane reads a repository page and looks for the embedded React
+// payload. This is the check that catches the failure this tool cannot survive:
+// GitHub reorganising the page. Every structureChanged error in the package
+// starts here, so when one fires, this says whether the whole plane moved or
+// only the one selector.
+func (c *Client) checkPagePlane(ctx context.Context) *Check {
+ p, err := c.Page(ctx, BaseURL+"/golang/go")
+ if err != nil {
+ return &Check{Name: "page", Status: StatusFail,
+ Detail: fmt.Sprintf("cannot read a repository page: %v", err)}
+ }
+ switch {
+ case p.Plane == page.PlaneReact && len(p.Payload) > 0:
+ return &Check{Name: "page", Status: StatusOK,
+ Detail: fmt.Sprintf("the react payload is where it should be, %d keys in %d bytes", len(p.Payload), p.Bytes)}
+ case len(p.Microdata) > 0 || len(p.Meta) > 0:
+ return &Check{Name: "page", Status: StatusWarn,
+ Detail: "no react payload, but the meta and microdata are readable: the records will be thinner than they should be. Run github page golang/go to see what came back"}
+ default:
+ return &Check{Name: "page", Status: StatusFail,
+ Detail: "a repository page carried nothing this understands. Either the request was intercepted or the page changed shape. Run github page golang/go --raw to see the bytes"}
+ }
+}
+
+// checkCache reports what is on disk and, more to the point, whether it can be
+// written. A read-only cache directory turns every run into a cold one, which
+// looks like the site being slow rather than like a local problem.
+func (c *Client) checkCache(context.Context) *Check {
+ if c.NoCache {
+ return &Check{Name: "cache", Status: StatusWarn,
+ Detail: "the cache is off for this run, so every request goes to the network"}
+ }
+ if c.CacheDir == "" {
+ return &Check{Name: "cache", Status: StatusWarn, Detail: "no cache directory is configured"}
+ }
+ if err := os.MkdirAll(c.CacheDir, 0o755); err != nil {
+ return &Check{Name: "cache", Status: StatusFail,
+ Detail: fmt.Sprintf("cannot create %s: %v", c.CacheDir, err)}
+ }
+ probe := filepath.Join(c.CacheDir, ".doctor")
+ if err := os.WriteFile(probe, []byte("ok"), 0o644); err != nil {
+ return &Check{Name: "cache", Status: StatusFail,
+ Detail: fmt.Sprintf("%s is not writable: %v", c.CacheDir, err)}
+ }
+ _ = os.Remove(probe)
+
+ n, bytes := cacheSize(c.CacheDir)
+ return &Check{Name: "cache", Status: StatusOK,
+ Detail: fmt.Sprintf("%s holds %d entries, %s, kept for %s", c.CacheDir, n, humanBytes(bytes), c.CacheTTL)}
+}
+
+// checkPacing prints the numbers a run is using. It is not a test of anything;
+// it is here because "why is this slow" and "why did I get rate limited" are
+// both answered by these four values and neither is visible otherwise.
+func (c *Client) checkPacing(context.Context) *Check {
+ return &Check{Name: "pacing", Status: StatusOK,
+ Detail: fmt.Sprintf("%s between requests across %d workers, %s timeout, %d retries, user agent %q",
+ c.Rate, c.Workers, c.HTTP.Timeout, c.Retries, c.UserAgent)}
+}
+
+func cacheSize(dir string) (entries int, bytes int64) {
+ _ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error {
+ if err != nil || d.IsDir() {
+ return nil //nolint:nilerr // a directory that cannot be walked is reported as empty
+ }
+ if info, err := d.Info(); err == nil {
+ entries++
+ bytes += info.Size()
+ }
+ return nil
+ })
+ return entries, bytes
+}
+
+func humanBytes(n int64) string {
+ const unit = 1024
+ if n < unit {
+ return fmt.Sprintf("%d B", n)
+ }
+ div, exp := int64(unit), 0
+ for m := n / unit; m >= unit; m /= unit {
+ div *= unit
+ exp++
+ }
+ return fmt.Sprintf("%.1f %cB", float64(n)/float64(div), "KMGT"[exp])
+}
diff --git a/gh/ops.go b/gh/ops.go
index 2160e41..ebd7542 100644
--- a/gh/ops.go
+++ b/gh/ops.go
@@ -1582,6 +1582,28 @@ func registerMetaOps(app *kit.App) {
"answers it best and what to fall back to when that surface declines. A\n" +
"route that is not in the table is unsupported rather than guessed at.",
}, listRoutes)
+
+ kit.Handle(app, kit.OpMeta{
+ Name: "doctor", Group: "meta", List: true,
+ Summary: "Check the environment, the site, and the cache",
+ Long: "doctor answers the question people ask when a command comes back wrong: is\n" +
+ "it me, is it the network, or did GitHub change the page. It reads a small\n" +
+ "file to check reachability, a repository page to check that the embedded\n" +
+ "payload is still where every reader expects it, and the cache directory to\n" +
+ "check that it can be written.\n\n" +
+ "It also says out loud that GITHUB_TOKEN and GH_TOKEN are ignored, because\n" +
+ "a token in the environment does nothing here and the failure that causes is\n" +
+ "invisible: the tool works, it is just as rate limited as before, and the\n" +
+ "obvious conclusion is that the token is wrong.",
+ }, runDoctor)
+}
+
+type doctorIn struct {
+ C *Client `kit:"inject"`
+}
+
+func runDoctor(ctx context.Context, in doctorIn, emit func(*Check) error) error {
+ return in.C.Doctor(ctx, emit)
}
type parseIn struct {
From 71339d1f3a3fe94e9bd6e3bcd76dd024348a5f6a Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:28:03 +0700
Subject: [PATCH 12/21] Update kit to v0.4.7
Brings the url output format and the table column tags the records rely on.
---
go.mod | 2 +-
go.sum | 6 ++++++
2 files changed, 7 insertions(+), 1 deletion(-)
diff --git a/go.mod b/go.mod
index eba50f1..762fbfe 100644
--- a/go.mod
+++ b/go.mod
@@ -5,7 +5,7 @@ go 1.26
require (
github.com/charmbracelet/fang v1.0.0
github.com/spf13/cobra v1.10.2
- github.com/tamnd/any-cli v0.4.4
+ github.com/tamnd/any-cli v0.4.7
)
require (
diff --git a/go.sum b/go.sum
index 3919846..5c1b552 100644
--- a/go.sum
+++ b/go.sum
@@ -74,6 +74,12 @@ github.com/tamnd/any-cli v0.4.0 h1:ngyRJBvjZ2X1iBlwlmDLvY2S9aQWlDjVE7CiOwxtt5Y=
github.com/tamnd/any-cli v0.4.0/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk=
github.com/tamnd/any-cli v0.4.4 h1:mOo3JJ7M3ZWQtOocYVtMEN7Zhfc3ogVWcVrbarlt9jE=
github.com/tamnd/any-cli v0.4.4/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk=
+github.com/tamnd/any-cli v0.4.5 h1:dEeniLDoneCxK4A9SixIyND2xlS2A/RxUtRdqTWXlQw=
+github.com/tamnd/any-cli v0.4.5/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk=
+github.com/tamnd/any-cli v0.4.6 h1:5GHwOsr8Z9oRYCtFt49Q9bwUKf9NBqkhXUqfOB5So+4=
+github.com/tamnd/any-cli v0.4.6/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk=
+github.com/tamnd/any-cli v0.4.7 h1:aHjifufpIy0M4HQQo+Ex90M4xaHyYoAMNtNPqVWw8ug=
+github.com/tamnd/any-cli v0.4.7/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no=
github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM=
go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg=
From ed7a0a46b7a8c54dd08c887fcac322c0bc9bd0b2 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:28:11 +0700
Subject: [PATCH 13/21] gh: fix package URLs, and reword the reference errors
A container package is named after the repository and the thing inside
it, so its name carries a slash, and GitHub wants that slash as %2F.
Locate was writing it plain, which 404s, and the parser was taking the
last path segment as the name, which lost the first half. Both directions
round trip now.
The rest is wording. Every message here leads with a word rather than
with the reference it is about, because the renderer title-cases the
first token and a title-cased URL reads like the tool mangled the input.
An unknown kind now lists the kinds, which is the difference between an
error a reader can act on and one that sends them to the source, and
knownKind reads that same list so a kind cannot be accepted here and
left out of the list.
---
gh/uri.go | 70 ++++++++++++++++++++++++++++++++++---------------------
1 file changed, 44 insertions(+), 26 deletions(-)
diff --git a/gh/uri.go b/gh/uri.go
index 02d55b2..3ad1153 100644
--- a/gh/uri.go
+++ b/gh/uri.go
@@ -57,6 +57,17 @@ const (
KindEvent = "event"
)
+// Kinds is the whole set, in the order above, for help text and for the error a
+// bad kind produces. Listing them is the difference between an error a reader
+// can act on and one that sends them to the source.
+var Kinds = []string{
+ KindRepo, KindUser, KindOrg, KindIssue, KindPR, KindDiscussion,
+ KindCommit, KindBranch, KindTag, KindRelease, KindFile, KindTree,
+ KindLabel, KindMilestone, KindTopic, KindGist, KindPackage, KindAction,
+ KindWiki, KindAdvisory, KindCompare,
+ KindContributor, KindContribution, KindEvent,
+}
+
// Scheme is the URI scheme this package mints and dereferences.
const Scheme = "github"
@@ -159,21 +170,21 @@ func parseURI(s string) (kind, id, anchor string, err error) {
}
kind, id, ok := strings.Cut(rest, "/")
if !ok || kind == "" || id == "" {
- return "", "", "", errs.Usage("%q is not a %s:// URI", s, Scheme)
+ return "", "", "", errs.Usage("not a %s:// URI: %q", Scheme, s)
}
if !knownKind(kind) {
- return "", "", "", errs.Usage("%q is not a kind this tool knows", kind)
+ return "", "", "", errs.Usage("unknown kind %q; the kinds are %s", kind, strings.Join(Kinds, ", "))
}
return kind, strings.TrimSuffix(id, "/"), anchor, nil
}
+// knownKind reads the same list the error message prints, so a kind cannot be
+// accepted here and left out of the list a reader is shown.
func knownKind(k string) bool {
- switch k {
- case KindRepo, KindUser, KindOrg, KindIssue, KindPR, KindDiscussion, KindCommit,
- KindBranch, KindTag, KindRelease, KindFile, KindTree, KindLabel, KindMilestone,
- KindTopic, KindGist, KindPackage, KindAction, KindWiki, KindAdvisory, KindCompare,
- KindContributor, KindContribution, KindEvent:
- return true
+ for _, want := range Kinds {
+ if k == want {
+ return true
+ }
}
return false
}
@@ -184,7 +195,7 @@ func knownKind(k string) bool {
func parseURL(raw string) (kind, id, anchor string, err error) {
u, perr := url.Parse(raw)
if perr != nil {
- return "", "", "", errs.Usage("%q is not a URL: %v", raw, perr)
+ return "", "", "", errs.Usage("not a URL: %q, %v", raw, perr)
}
host := strings.ToLower(u.Host)
host = strings.TrimPrefix(host, "www.")
@@ -196,13 +207,13 @@ func parseURL(raw string) (kind, id, anchor string, err error) {
// /{owner}/{repo}/{ref}/{path...}
p := strings.Split(path, "/")
if len(p) < 4 {
- return "", "", "", errs.Usage("%q is not a raw file URL", raw)
+ return "", "", "", errs.Usage("not a raw file URL: %q", raw)
}
return KindFile, p[0] + "/" + p[1] + "@" + p[2] + "/" + strings.Join(p[3:], "/"), anchor, nil
case "gist.github.com", "gist.githubusercontent.com":
p := strings.Split(path, "/")
if len(p) == 0 || p[0] == "" {
- return "", "", "", errs.Usage("%q names no gist", raw)
+ return "", "", "", errs.Usage("no gist named in %q", raw)
}
// A gist URL is either /{id} or /{owner}/{id}. The id is the last
// segment that looks like one.
@@ -210,13 +221,13 @@ func parseURL(raw string) (kind, id, anchor string, err error) {
case "github.com", "codeload.github.com":
return classifyPath(path, anchor, raw)
default:
- return "", "", "", errs.Usage("%q is not a github.com URL", raw)
+ return "", "", "", errs.Usage("not a github.com URL: %q", raw)
}
}
func classifyPath(path, anchor, raw string) (kind, id, a string, err error) {
if path == "" {
- return "", "", "", errs.Usage("%q names no resource", raw)
+ return "", "", "", errs.Usage("no resource named in %q", raw)
}
p := strings.Split(path, "/")
@@ -240,7 +251,7 @@ func classifyPath(path, anchor, raw string) (kind, id, a string, err error) {
}
}
if reserved[p[0]] {
- return "", "", "", errs.Usage("%q is a github.com page, not a resource this tool reads", raw)
+ return "", "", "", errs.Usage("no resource behind %q; it is a github.com page, not a thing this tool reads", raw)
}
if len(p) == 1 {
return KindUser, p[0], anchor, nil
@@ -311,9 +322,12 @@ func classifyPath(path, anchor, raw string) (kind, id, a string, err error) {
}
return KindWiki, repo + "/Home", anchor, nil
case "pkgs":
- // /{owner}/{repo}/pkgs/{type}/{name}
+ // /{owner}/{repo}/pkgs/{type}/{name}, where the name is usually the
+ // repository and the thing inside it and so carries a %2F. Parsing
+ // decoded that back into a slash before the split, so the name is
+ // everything from the type onwards rather than the last segment.
if len(rest) >= 3 {
- return KindPackage, repo + "/" + rest[len(rest)-1], anchor, nil
+ return KindPackage, repo + "/" + strings.Join(rest[2:], "/"), anchor, nil
}
case "compare":
if len(rest) >= 2 {
@@ -341,10 +355,10 @@ func classifyBare(s string) (kind, id string, err error) {
}
if base, num, ok := strings.Cut(s, "#"); ok {
if !isNumber(num) {
- return "", "", errs.Usage("%q: the part after # must be a number", s)
+ return "", "", errs.Usage("the part after # must be a number, in %q", s)
}
if strings.Count(base, "/") != 1 {
- return "", "", errs.Usage("%q: a thread reference looks like owner/name#123", s)
+ return "", "", errs.Usage("not a thread reference: %q, which should look like owner/name#123", s)
}
// Bare owner/name#N is an issue, which is the same guess github.com
// makes: /issues/N redirects to /pull/N when N is a pull request.
@@ -353,7 +367,7 @@ func classifyBare(s string) (kind, id string, err error) {
if i := strings.Index(s, "@"); i >= 0 && strings.Count(s[:i], "/") == 1 {
repo, rev := s[:i], s[i+1:]
if rev == "" {
- return "", "", errs.Usage("%q: nothing after @", s)
+ return "", "", errs.Usage("nothing after the @ in %q", s)
}
if r, path, ok := strings.Cut(rev, "/"); ok {
return KindFile, repo + "@" + r + "/" + path, nil
@@ -368,7 +382,7 @@ func classifyBare(s string) (kind, id string, err error) {
switch strings.Count(s, "/") {
case 0:
if reserved[s] {
- return "", "", errs.Usage("%q is a github.com page, not an account", s)
+ return "", "", errs.Usage("no account behind %q; it is a github.com page, not a profile", s)
}
return KindUser, s, nil
case 1:
@@ -410,7 +424,7 @@ var routeWord = map[string]bool{
// safe to pipe back into the tool.
func Locate(kind, id string) (string, error) {
if id == "" {
- return "", errs.Usage("%s with no id", kind)
+ return "", errs.Usage("no id given for a %s", kind)
}
switch kind {
case KindRepo:
@@ -420,7 +434,7 @@ func Locate(kind, id string) (string, error) {
case KindIssue, KindPR, KindDiscussion:
repo, num, ok := strings.Cut(id, "#")
if !ok {
- return "", errs.Usage("%s id %q is missing its number", kind, id)
+ return "", errs.Usage("missing number: the %s id %q needs one", kind, id)
}
seg := map[string]string{KindIssue: "issues", KindPR: "pull", KindDiscussion: "discussions"}[kind]
return BaseURL + "/" + repo + "/" + seg + "/" + num, nil
@@ -439,13 +453,13 @@ func Locate(kind, id string) (string, error) {
case KindTag, KindRelease:
repo, tag, ok := cutRev(id)
if !ok {
- return "", errs.Usage("%s id %q is missing its tag", kind, id)
+ return "", errs.Usage("missing tag: the %s id %q needs one", kind, id)
}
return BaseURL + "/" + repo + "/releases/tag/" + tag, nil
case KindFile, KindTree:
repo, ref, path, ok := SplitPathID(id)
if !ok {
- return "", errs.Usage("%s id %q is not owner/name@ref/path", kind, id)
+ return "", errs.Usage("wrong shape: the %s id %q is not owner/name@ref/path", kind, id)
}
seg := "blob"
if kind == KindTree {
@@ -478,7 +492,11 @@ func Locate(kind, id string) (string, error) {
if !ok {
return "", errs.Usage("package id %q is not owner/name/package", id)
}
- return BaseURL + "/" + repo + "/pkgs/container/" + name, nil
+ // The name is escaped because a container package is usually called
+ // after the repository and the thing inside it, so it has a slash in
+ // it. GitHub wants that slash as %2F: the unescaped form 404s and the
+ // escaped one is the page.
+ return BaseURL + "/" + repo + "/pkgs/container/" + url.PathEscape(name), nil
case KindTopic:
return BaseURL + "/topics/" + id, nil
case KindAction:
@@ -513,7 +531,7 @@ func Locate(kind, id string) (string, error) {
}
return BaseURL + "/" + repo + "/compare/" + rng, nil
}
- return "", errs.Usage("%q is not a kind this tool knows", kind)
+ return "", errs.Usage("unknown kind %q; the kinds are %s", kind, strings.Join(Kinds, ", "))
}
// cutRev splits owner/name@rev. It looks for the `@` after the second slash so
From 2d4ff52c1416afbefe786b477a03cc424b58a282 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:28:19 +0700
Subject: [PATCH 14/21] gh: give every error message a word to lead with
The renderer capitalises the first token of an error. A message that
started with a path came back as Golang/Go/Blob/Master, which reads like
the tool broke the input rather than like the page was missing, and one
that started with the argument did the same to whatever was typed. So
every message now opens with a plain word and puts the reference after
it.
Two other things fell out of reading them all at once. Articles did not
agree: five of the twenty-four kinds start with a vowel, and "is a repo,
not a org" reads like nobody looked. aKind fixes that in the three places
it happened. And symbols now says the truth, which is that GitHub serves
no symbol list to a signed-out reader on any file tried, rather than
suggesting a retry that will not help.
doctor's help moved to doctor.go so that the one file allowed to name
GITHUB_TOKEN is the file whose job is to say it is ignored.
messages_test.go keeps this from drifting back. It parses every source
file in the module and fails a message that leads with a format verb, a
quote, or a capital, which are the three ways this gets written by
accident.
---
cli/files.go | 8 ++-
cli/page.go | 9 +++-
gh/client.go | 4 +-
gh/doctor.go | 13 +++++
gh/errors.go | 37 ++++++++------
gh/messages_test.go | 117 ++++++++++++++++++++++++++++++++++++++++++++
gh/ops.go | 81 +++++++++++++++++-------------
gh/people.go | 2 +-
8 files changed, 215 insertions(+), 56 deletions(-)
create mode 100644 gh/messages_test.go
diff --git a/cli/files.go b/cli/files.go
index d699bd7..97a354b 100644
--- a/cli/files.go
+++ b/cli/files.go
@@ -158,7 +158,7 @@ func (c *readmeCmd) run(ctx context.Context, args []string) error {
text = r.ReadmeHTML
}
if text == "" {
- return errs.NotFound("%s has no README", id)
+ return errs.NotFound("no README in %s", id)
}
_, err = io.WriteString(os.Stdout, text)
return err
@@ -282,5 +282,9 @@ func diffURL(args []string) (string, error) {
case gh.KindCommit, gh.KindCompare, gh.KindPR:
return gh.Locate(kind, id)
}
- return "", errs.Usage("%q is a %s; diff needs a commit, a pull request, a compare URL, or a repository with two refs", args[0], kind)
+ // Leads with a word, not with the argument. The renderer capitalises the
+ // first token of an error, and "golang/go" coming back as "Golang/Go" reads
+ // like the tool mangled the input rather than like the input was the wrong
+ // kind of thing.
+ return "", errs.Usage("cannot diff %q, which is a %s; diff needs a commit, a pull request, a compare URL, or a repository with two refs", args[0], kind)
}
diff --git a/cli/page.go b/cli/page.go
index 1c14fb2..d52acf5 100644
--- a/cli/page.go
+++ b/cli/page.go
@@ -102,9 +102,16 @@ func (c *pageCmd) run(ctx context.Context, args []string) error {
// GitHub's internal Relay identifiers, nobody knows them by heart,
// and a bare "not found" would send the reader off to dump the
// whole queries section to find out what to ask for.
- // The message leads with a word rather than the URL because the
+ // Both messages lead with a word rather than the URL because the
// error renderer capitalises what it starts with, and a
// title-cased URL reads as a typo.
+ if len(p.Queries) == 0 {
+ // Worth saying separately. Most pages preload nothing, so
+ // listing the names it has would be an empty list, and an empty
+ // list reads like the lookup broke rather than like the page
+ // carries no queries at all.
+ return errs.NotFound("no preloaded queries on %s at all; that is normal, only a few page kinds have them", url)
+ }
return errs.NotFound("no query named %q on %s; it has %s",
c.query, url, strings.Join(queryNames(p.Queries), ", "))
}
diff --git a/gh/client.go b/gh/client.go
index 8e2d202..8b776f6 100644
--- a/gh/client.go
+++ b/gh/client.go
@@ -181,7 +181,7 @@ func (c *Client) GetJSON(ctx context.Context, rawURL string, s Surface, v any) (
}
if v != nil {
if err := json.Unmarshal(resp.Body, v); err != nil {
- return resp, errs.New(errs.KindNetwork, "%s: %v", shortURL(rawURL), err)
+ return resp, errs.New(errs.KindNetwork, "cannot decode the json from %s: %v", shortURL(rawURL), err)
}
}
return resp, nil
@@ -397,7 +397,7 @@ func (c *Client) Poll(ctx context.Context, rawURL string, s Surface) (*Response,
wait *= 2
}
}
- return nil, errs.Unsupported("%s: github is still computing this statistic, try again shortly", shortURL(rawURL))
+ return nil, errs.Unsupported("still computing: github has not finished this statistic yet, try again shortly (%s)", shortURL(rawURL))
}
// --- URL building ---
diff --git a/gh/doctor.go b/gh/doctor.go
index 5dcd654..fbee05d 100644
--- a/gh/doctor.go
+++ b/gh/doctor.go
@@ -18,6 +18,19 @@ import (
// a person and by a script, and so the failing one can be picked out with the
// same --fields and -o json every other command takes.
+// doctorLong is the command's help. It lives here rather than beside the
+// registration because it names the token variables, and TestNoAuth wants every
+// mention of those names in the one file whose job is to talk about them.
+const doctorLong = "doctor answers the question people ask when a command comes back wrong: is\n" +
+ "it me, is it the network, or did GitHub change the page. It reads a small\n" +
+ "file to check reachability, a repository page to check that the embedded\n" +
+ "payload is still where every reader expects it, and the cache directory to\n" +
+ "check that it can be written.\n\n" +
+ "It also says out loud that GITHUB_TOKEN and GH_TOKEN are ignored, because a\n" +
+ "token in the environment does nothing here and the failure that causes is\n" +
+ "invisible: the tool works, it stays exactly as rate limited as before, and\n" +
+ "the obvious conclusion is that the token is wrong."
+
// Check is one diagnostic.
type Check struct {
Name string `json:"name" table:"check"`
diff --git a/gh/errors.go b/gh/errors.go
index b93d7e4..a316e89 100644
--- a/gh/errors.go
+++ b/gh/errors.go
@@ -23,6 +23,11 @@ import (
//
// Both mean "wrong surface", which is a thing the client can fix by trying the
// other one. Turning them into errors here would hide that.
+//
+// Every message here leads with a word rather than with the path it is about.
+// The renderer title-cases whatever a message starts with, and a path that
+// comes back as Golang/Go/Blob/Master reads like the tool mangled the input
+// rather than like the page was missing.
// statusError classifies a non-2xx response.
func statusError(rawURL string, status int, body []byte) error {
@@ -33,23 +38,23 @@ func statusError(rawURL string, status int, body []byte) error {
// public. Saying "pass a token" would be wrong: there is no token to
// pass. Saying what is actually true is more useful.
if isRateLimitBody(body) {
- return errs.RateLimited("%s: github is throttling anonymous reads, try again shortly", where)
+ return errs.RateLimited("github is throttling anonymous reads, try again shortly (%s)", where)
}
- return errs.NeedAuth("%s: not public, and this tool reads only public pages (use gh for the rest)", where)
+ return errs.NeedAuth("not public: %s, and this tool reads only public pages (use gh for the rest)", where)
case status == http.StatusNotFound:
- return errs.NotFound("%s: not found", where)
+ return errs.NotFound("not found: %s", where)
case status == http.StatusGone:
- return errs.NotFound("%s: gone", where)
+ return errs.NotFound("gone: %s", where)
case status == http.StatusTooManyRequests:
- return errs.RateLimited("%s: rate limited", where)
+ return errs.RateLimited("rate limited on %s", where)
case status == http.StatusUnavailableForLegalReasons:
- return errs.Unsupported("%s: unavailable for legal reasons (DMCA)", where)
+ return errs.Unsupported("unavailable for legal reasons (DMCA): %s", where)
case status == http.StatusBadRequest:
- return errs.Usage("%s: bad request", where)
+ return errs.Usage("bad request: %s", where)
case status >= 500:
- return errs.New(errs.KindNetwork, "%s: server error %d", where, status)
+ return errs.New(errs.KindNetwork, "server error %d on %s", status, where)
default:
- return errs.New(errs.KindGeneric, "%s: http %d", where, status)
+ return errs.New(errs.KindGeneric, "http %d on %s", status, where)
}
}
@@ -74,9 +79,9 @@ func wrapNetwork(rawURL string, err error) error {
}
var dnsErr *net.DNSError
if errors.As(err, &dnsErr) {
- return errs.New(errs.KindNetwork, "%s: cannot resolve %s", shortURL(rawURL), dnsErr.Name)
+ return errs.New(errs.KindNetwork, "cannot resolve %s, reading %s", dnsErr.Name, shortURL(rawURL))
}
- return errs.New(errs.KindNetwork, "%s: %v", shortURL(rawURL), err)
+ return errs.New(errs.KindNetwork, "reading %s: %v", shortURL(rawURL), err)
}
// shortURL trims the scheme and the host so an error message reads as a path.
@@ -95,13 +100,13 @@ func shortURL(raw string) string {
// code search, traffic, clones, referrers. It names what would be needed rather
// than being vague, because a vague "unsupported" wastes an afternoon.
func notPublic(what, why string) error {
- return errs.Unsupported("%s is not available without a session: %s", what, why)
+ return errs.Unsupported("not available without a session: %s, %s", what, why)
}
// usageBadID rejects a malformed identifier before a request goes out. Showing
// the expected shape saves the round trip and the 404 that would follow it.
func usageBadID(kind, got, want string) error {
- return errs.Usage("%q is not a %s, expected %s", got, kind, want)
+ return errs.Usage("expected a %s like %s, got %q", kind, want, got)
}
// structureChanged is the loud failure from doc 02 section 7: the page came
@@ -110,7 +115,7 @@ func usageBadID(kind, got, want string) error {
// zero exit code.
func structureChanged(what string) error {
return errs.New(errs.KindNetwork,
- "%s: the page structure changed, none of the expected data was there (run `github page %s` to see what arrived)",
+ "the page structure changed for %s, none of the expected data was there (run `github page %s` to see what arrived)",
what, what)
}
@@ -119,11 +124,11 @@ func structureChanged(what string) error {
// changed means the block is gone, bad payload means the block arrived and no
// longer parses, which is usually a type change on one field.
func badPayload(what string, err error) error {
- return errs.New(errs.KindNetwork, "%s: the payload did not decode: %v", what, err)
+ return errs.New(errs.KindNetwork, "the payload for %s did not decode: %v", what, err)
}
// noJSONHere is what a 410 means. It is separated out so the message can say
// the useful half: the data is reachable, just on a different surface.
func noJSONHere(rawURL string) error {
- return errs.Unsupported("%s serves no JSON; this is a page-only route", shortURL(rawURL))
+ return errs.Unsupported("no JSON at %s; this is a page-only route", shortURL(rawURL))
}
diff --git a/gh/messages_test.go b/gh/messages_test.go
new file mode 100644
index 0000000..242b804
--- /dev/null
+++ b/gh/messages_test.go
@@ -0,0 +1,117 @@
+package gh
+
+import (
+ "go/ast"
+ "go/parser"
+ "go/token"
+ "io/fs"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+)
+
+// messages_test.go guards a defect that is invisible in the source and obvious
+// on screen. The error renderer capitalises the first token of a message, so a
+// message that begins with the thing it is about comes back mangled:
+//
+// errs.NotFound("%s has no README", id) -> Gohugoio/Hugo has no README.
+// errs.Usage("%q is a %s, not a %s", ...) -> "Golang/Go" is a repo, not a user.
+//
+// A reader who sees that reasonably concludes the tool corrupted their input.
+// Every message here therefore leads with a plain lowercase word, and this test
+// says so, because the mistake is easy to make and impossible to see in review.
+
+// errorFuncs are the constructors whose first string argument is shown to a
+// person. errs.New takes a kind first, so its message is the second argument.
+var errorFuncs = map[string]int{
+ "Usage": 0,
+ "NotFound": 0,
+ "Unsupported": 0,
+ "NeedAuth": 0,
+ "RateLimited": 0,
+ "NoResults": 0,
+ "New": 1,
+}
+
+func TestErrorMessagesLeadWithAWord(t *testing.T) {
+ root := moduleRoot(t)
+ fset := token.NewFileSet()
+
+ err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ if name := d.Name(); name == ".git" || name == "bin" || name == "dist" || name == "docs" {
+ return fs.SkipDir
+ }
+ return nil
+ }
+ if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") {
+ return nil
+ }
+ rel, _ := filepath.Rel(root, path)
+ rel = filepath.ToSlash(rel)
+
+ file, perr := parser.ParseFile(fset, path, nil, 0)
+ if perr != nil {
+ t.Errorf("%s: %v", rel, perr)
+ return nil
+ }
+ ast.Inspect(file, func(n ast.Node) bool {
+ call, ok := n.(*ast.CallExpr)
+ if !ok {
+ return true
+ }
+ sel, ok := call.Fun.(*ast.SelectorExpr)
+ if !ok {
+ return true
+ }
+ pkg, ok := sel.X.(*ast.Ident)
+ if !ok || pkg.Name != "errs" {
+ return true
+ }
+ at, ok := errorFuncs[sel.Sel.Name]
+ if !ok || len(call.Args) <= at {
+ return true
+ }
+ lit, ok := call.Args[at].(*ast.BasicLit)
+ if !ok || lit.Kind != token.STRING {
+ return true
+ }
+ msg, uerr := strconv.Unquote(lit.Value)
+ if uerr != nil || msg == "" {
+ return true
+ }
+ if bad := leadsBadly(msg); bad != "" {
+ t.Errorf("%s:%d: errs.%s starts with %s: %q\nThe renderer capitalises the first token, so this reaches the reader looking like their input was mangled. Lead with a plain word instead.",
+ rel, fset.Position(lit.Pos()).Line, sel.Sel.Name, bad, msg)
+ }
+ return true
+ })
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+// leadsBadly names what is wrong with a message's first character, or returns
+// empty when there is nothing wrong. Three cases matter: a format verb, because
+// whatever fills it gets capitalised; a quote, because the capital lands inside
+// it; and an upper-case letter, because a message that already starts capital is
+// usually a proper noun that the renderer will then get wrong (GitHub, HTTP).
+func leadsBadly(msg string) string {
+ switch {
+ case strings.HasPrefix(msg, "%"):
+ return "a format verb"
+ case strings.HasPrefix(msg, `"`), strings.HasPrefix(msg, "'"), strings.HasPrefix(msg, "`"):
+ return "a quote"
+ case msg[0] >= 'A' && msg[0] <= 'Z':
+ return "a capital letter"
+ case strings.HasPrefix(msg, "http://"), strings.HasPrefix(msg, "https://"):
+ return "a URL"
+ }
+ return ""
+}
diff --git a/gh/ops.go b/gh/ops.go
index ebd7542..8b62c3e 100644
--- a/gh/ops.go
+++ b/gh/ops.go
@@ -58,7 +58,21 @@ func ResolveRef(want, input string) (string, error) {
return repo, nil
}
}
- return "", errs.Usage("%q is a %s, not a %s", input, kind, want)
+ return "", errs.Usage("wrong kind: %q is %s, not %s", input, aKind(kind), aKind(want))
+}
+
+// aKind puts the right article in front of a kind name. Five of the twenty-four
+// start with a vowel, and "not a org" in an error message reads like the tool
+// was written in a hurry.
+func aKind(kind string) string {
+ if kind == "" {
+ return "nothing"
+ }
+ switch kind[0] {
+ case 'a', 'e', 'i', 'o', 'u':
+ return "an " + kind
+ }
+ return "a " + kind
}
// guessed reports whether Classify was guessing rather than reading. Anything
@@ -92,7 +106,7 @@ func resolveThread(want, ref string, num int) (repo string, number int, err erro
// is a real error rather than a guess to be forgiven. A bare
// owner/name#123 is a guess: nothing in it says which of the two it is.
if kind != want && !guessed(ref, kind) {
- return "", 0, errs.Usage("%q is a %s, not a %s", ref, kind, want)
+ return "", 0, errs.Usage("wrong kind: %q is %s, not %s", ref, aKind(kind), aKind(want))
}
number, _ = strconv.Atoi(n)
return r, number, nil
@@ -102,7 +116,7 @@ func resolveThread(want, ref string, num int) (repo string, number int, err erro
return "", 0, err
}
if num <= 0 {
- return "", 0, errs.Usage("%s needs a number, either as a second argument or in the URL", want)
+ return "", 0, errs.Usage("no number given; %s needs one, either as a second argument or in the URL", want)
}
return repo, num, nil
}
@@ -117,7 +131,7 @@ func resolveRev(want, ref, rev string) (repo, out string, err error) {
}
if r, v, ok := cutRev(id); ok && rev == "" {
if kind != want && !guessed(ref, kind) {
- return "", "", errs.Usage("%q is a %s, not a %s", ref, kind, want)
+ return "", "", errs.Usage("wrong kind: %q is %s, not %s", ref, aKind(kind), aKind(want))
}
return r, v, nil
}
@@ -456,7 +470,7 @@ func resolveCompare(ref, base, head string) (string, string, string, error) {
if kind == KindCompare && base == "" {
repo, rng, ok := cutRev(id)
if !ok {
- return "", "", "", errs.Usage("%q is not a range", ref)
+ return "", "", "", errs.Usage("not a range: %q", ref)
}
// Three dots is the merge-base form and two is the direct diff.
// github.com accepts both and means different things by them, so the
@@ -466,7 +480,7 @@ func resolveCompare(ref, base, head string) (string, string, string, error) {
return repo, a, b, nil
}
}
- return "", "", "", errs.Usage("%q has no base...head in it", ref)
+ return "", "", "", errs.Usage("no base...head in %q", ref)
}
repo, err := ResolveRepo(ref)
if err != nil {
@@ -506,7 +520,7 @@ func (c *Client) fetchOne(ctx context.Context, kind, id string) (any, error) {
case KindIssue, KindPR, KindDiscussion:
repo, n, ok := SplitThreadID(id)
if !ok {
- return nil, errs.Usage("%q is not a thread id", id)
+ return nil, errs.Usage("not a thread id: %q", id)
}
num, _ := strconv.Atoi(n)
switch kind {
@@ -520,38 +534,38 @@ func (c *Client) fetchOne(ctx context.Context, kind, id string) (any, error) {
case KindCommit:
repo, sha, ok := cutRev(id)
if !ok {
- return nil, errs.Usage("%q is not a commit id", id)
+ return nil, errs.Usage("not a commit id: %q", id)
}
return c.CommitInfo(ctx, repo, sha, CommitInfoOptions{})
case KindRelease:
repo, tag, ok := cutRev(id)
if !ok {
- return nil, errs.Usage("%q is not a release id", id)
+ return nil, errs.Usage("not a release id: %q", id)
}
return c.Release(ctx, repo, tag, ReleaseOptions{Assets: true, Body: true})
case KindBranch, KindTag:
repo, name, ok := cutRev(id)
if !ok {
- return nil, errs.Usage("%q is not a %s id", id, kind)
+ return nil, errs.Usage("not a %s id: %q", kind, id)
}
return c.oneRef(ctx, kind, repo, name)
case KindCompare:
repo, rng, ok := cutRev(id)
if !ok {
- return nil, errs.Usage("%q is not a range", id)
+ return nil, errs.Usage("not a range: %q", id)
}
base, head, found := strings.Cut(rng, "...")
if !found {
base, head, found = strings.Cut(rng, "..")
}
if !found {
- return nil, errs.Usage("%q has no base...head in it", id)
+ return nil, errs.Usage("no base...head in %q", id)
}
return c.CompareRefs(ctx, repo, base, head, CompareOptions{Files: true})
case KindFile:
repo, ref, path, ok := SplitPathID(id)
if !ok {
- return nil, errs.Usage("%q is not a file id", id)
+ return nil, errs.Usage("not a file id: %q", id)
}
return c.Blob(ctx, repo, path, BlobOptions{Ref: ref})
case KindTopic:
@@ -582,7 +596,7 @@ func (c *Client) oneRef(ctx context.Context, kind, repo, name string) (*GitRef,
return nil, err
}
if found == nil {
- return nil, errs.NotFound("%s %s has no %s named %s", repo, kind, kind, name)
+ return nil, errs.NotFound("no %s named %s in %s", kind, name, repo)
}
return found, nil
}
@@ -773,7 +787,7 @@ func (c *Client) searchOne(ctx context.Context, typ, query string, limit int, em
case SearchCode:
return c.SearchCodeBy(ctx, query, limit, func(f File) error { return any1(&f) })
}
- return errs.Usage("%q is not a search type; the types are %s", typ, strings.Join(SearchTypes, ", "))
+ return errs.Usage("not a search type: %q; the types are %s", typ, strings.Join(SearchTypes, ", "))
}
// --- contents ---
@@ -824,11 +838,14 @@ func registerContentOps(app *kit.App) {
kit.Handle(app, kit.OpMeta{
Name: "symbols", Group: "contents",
Summary: "List the definitions GitHub extracted from a file",
- Long: "GitHub runs a symbol extractor over every blob it renders and ships the\n" +
- "result in the route payload. There is no unauthenticated REST equivalent\n" +
- "anywhere. The extractor is asynchronous, so an empty list can mean the\n" +
- "language is unsupported or that the analysis had not finished; the record\n" +
- "says which, and this command reports it rather than guessing.",
+ Long: "GitHub runs a symbol extractor over every blob it renders and used to\n" +
+ "ship the result in the route payload. There is no unauthenticated REST\n" +
+ "equivalent anywhere, which is why this command exists.\n\n" +
+ "As of now it will not return anything. The blob still says symbols are\n" +
+ "enabled and still renders the button, and the list behind it is empty for\n" +
+ "a signed-out reader on every file tried. This reports that rather than\n" +
+ "returning an empty list, and stays here because the field is still in the\n" +
+ "payload and may fill in again.",
Args: []kit.Arg{
{Name: "ref", Help: "owner/name, or a blob URL"},
{Name: "path", Help: "a file inside the repository", Optional: true},
@@ -883,14 +900,18 @@ func listSymbols(ctx context.Context, in symbolIn, emit func(*Symbol) error) err
if err != nil {
return err
}
- // The path goes in the middle of these sentences rather than at the front,
- // because the CLI title-cases the first word of an error and a path is the
- // one thing that must not be title-cased.
+ // Both messages lead with a plain word. The CLI title-cases whatever an
+ // error starts with, which turns a path into nonsense and, less obviously,
+ // turns GitHub into Github.
switch f.SymbolsStatus {
case "not_analyzed":
- return errs.Unsupported("GitHub does not extract symbols from the language %s is written in", path)
+ return errs.Unsupported("no symbols for %s: GitHub does not extract them from the language it is written in", path)
case "unavailable", "timed_out":
- return errs.Network("GitHub's symbol analysis for %s had not finished; ask again in a moment", path)
+ // Unsupported rather than a network kind, because asking again does not
+ // help. GitHub still renders the symbols button and still sets
+ // symbolsEnabled on the blob, and the list behind it comes back empty
+ // for a signed-out reader on every file tried.
+ return errs.Unsupported("no symbol list for %s: GitHub serves none to a signed-out reader, though it still offers the panel", path)
}
return emitEach(f.Symbols, emit)
}
@@ -1586,15 +1607,7 @@ func registerMetaOps(app *kit.App) {
kit.Handle(app, kit.OpMeta{
Name: "doctor", Group: "meta", List: true,
Summary: "Check the environment, the site, and the cache",
- Long: "doctor answers the question people ask when a command comes back wrong: is\n" +
- "it me, is it the network, or did GitHub change the page. It reads a small\n" +
- "file to check reachability, a repository page to check that the embedded\n" +
- "payload is still where every reader expects it, and the cache directory to\n" +
- "check that it can be written.\n\n" +
- "It also says out loud that GITHUB_TOKEN and GH_TOKEN are ignored, because\n" +
- "a token in the environment does nothing here and the failure that causes is\n" +
- "invisible: the tool works, it is just as rate limited as before, and the\n" +
- "obvious conclusion is that the token is wrong.",
+ Long: doctorLong,
}, runDoctor)
}
diff --git a/gh/people.go b/gh/people.go
index 23f7c42..2723ebe 100644
--- a/gh/people.go
+++ b/gh/people.go
@@ -621,7 +621,7 @@ func (c *Client) Activity(ctx context.Context, ref string, limit int, emit func(
}
}
if seen == 0 {
- return errs.NotFound("%s: the feed carried no entries", shortURL(u))
+ return errs.NotFound("empty feed: %s carried no entries", shortURL(u))
}
return nil
}
From d02c05230c7abbb9f146ba96a3007c94d81cff6b Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:28:41 +0700
Subject: [PATCH 15/21] gh: fail the build if a credential ever appears
The one promise this tool makes that a reader cannot check by using it is
that nothing here authenticates. You can see that a command works without
logging in; you cannot see that no path would send a credential if one
happened to be in the environment.
So it is asserted. Every source file is parsed with comments dropped and
checked for Authorization, GITHUB_TOKEN, GH_TOKEN, and api.github.com.
Comments are dropped because this file and the doctor both talk about
tokens at length, and a plain grep would have to be switched off the
first time someone wrote the rule down.
The allow list holds two files and a reason for each, and a second test
fails if an entry stops existing, so the list cannot quietly become a
place to hide things.
---
gh/noauth_test.go | 119 ++++++++++++++++++++++++++++++++++++++++++++++
1 file changed, 119 insertions(+)
create mode 100644 gh/noauth_test.go
diff --git a/gh/noauth_test.go b/gh/noauth_test.go
new file mode 100644
index 0000000..65e6ba2
--- /dev/null
+++ b/gh/noauth_test.go
@@ -0,0 +1,119 @@
+package gh
+
+import (
+ "bytes"
+ "go/parser"
+ "go/printer"
+ "go/token"
+ "io/fs"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+// noauth_test.go is the one promise this tool makes that a reader cannot check
+// by using it: that nothing here ever authenticates. A person can see that a
+// command works without logging in, but not that no code path would send a
+// credential if one happened to be around.
+//
+// So the promise is asserted instead. The test parses every source file with
+// comments dropped and fails on the words that would mean the promise was
+// broken. Comments are dropped because this file, the doctor, and the spec all
+// talk about tokens at length, and a grep that could not tell prose from code
+// would have to be switched off the first time someone documented the rule.
+
+var forbidden = []string{
+ "Authorization",
+ "GITHUB_TOKEN",
+ "GH_TOKEN",
+ "api.github.com",
+}
+
+// allowed lists the files that name a forbidden word in code for a reason. Each
+// one is here because saying the word is the point: doctor reads the
+// environment to warn that a token is ignored, and this test names all four.
+var allowed = map[string]string{
+ "gh/doctor.go": "reads the token variables by name to warn that they are ignored",
+ "gh/noauth_test.go": "is this test",
+}
+
+func TestNoAuth(t *testing.T) {
+ root := moduleRoot(t)
+ fset := token.NewFileSet()
+
+ err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error {
+ if err != nil {
+ return err
+ }
+ if d.IsDir() {
+ if name := d.Name(); name == ".git" || name == "bin" || name == "dist" || name == "docs" {
+ return fs.SkipDir
+ }
+ return nil
+ }
+ if !strings.HasSuffix(path, ".go") {
+ return nil
+ }
+ rel, _ := filepath.Rel(root, path)
+ rel = filepath.ToSlash(rel)
+ if _, ok := allowed[rel]; ok {
+ return nil
+ }
+
+ // Parsing without ParseComments and printing the result is how the
+ // comments come out: the printer only writes what the AST holds.
+ file, perr := parser.ParseFile(fset, path, nil, 0)
+ if perr != nil {
+ t.Errorf("%s: %v", rel, perr)
+ return nil
+ }
+ var code bytes.Buffer
+ if perr := (&printer.Config{Mode: printer.RawFormat}).Fprint(&code, fset, file); perr != nil {
+ t.Errorf("%s: %v", rel, perr)
+ return nil
+ }
+ for _, word := range forbidden {
+ if bytes.Contains(code.Bytes(), []byte(word)) {
+ t.Errorf("%s names %q in code. This tool reads public pages and never authenticates; if this is deliberate, the file needs a line in the allowed map saying why", rel, word)
+ }
+ }
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestNoAuthCoversItself keeps the allow list honest. A file that stops needing
+// its exemption should lose it, otherwise the list grows into a place to hide
+// things.
+func TestNoAuthCoversItself(t *testing.T) {
+ root := moduleRoot(t)
+ for rel, why := range allowed {
+ if why == "" {
+ t.Errorf("%s is exempt with no reason given", rel)
+ }
+ if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil {
+ t.Errorf("%s is exempt and does not exist: %v", rel, err)
+ }
+ }
+}
+
+func moduleRoot(t *testing.T) string {
+ t.Helper()
+ dir, err := os.Getwd()
+ if err != nil {
+ t.Fatal(err)
+ }
+ for {
+ if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil {
+ return dir
+ }
+ parent := filepath.Dir(dir)
+ if parent == dir {
+ t.Fatal("no go.mod above the test directory")
+ }
+ dir = parent
+ }
+}
From e85a209b57cf620e7c64b7eb1fb3cc15010fb5f9 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:28:50 +0700
Subject: [PATCH 16/21] gh: read a blob from the page, in one request
GitHub dropped the metadata block from the blob route JSON. The route now
answers with codeViewBlobRoute alone, so asking it for a file's language
gets an empty string and asking it for the size and the line counts gets
nothing at all. The live test caught this, which is what it is for.
The page still carries every block, so the page is now the read. That is
one request where the old path was two, since the route was never enough
on its own and the page was already being fetched for the styled source.
blobRoute is gone.
Symbols were a third request. The list used to arrive on the first read
sometimes and only on the second other times, so an unavailable one was
asked for again before it was believed. It no longer arrives at all: the
page still says symbolsEnabled and still renders the button, and the
block behind it is null on every file of every repository tried, on both
surfaces, with or without a cache, signed out. That retry was spent to
learn nothing, so it is gone too.
A payload with no blocks in it is now a structure-changed error rather
than an empty file, and the routes table says what the blob read actually
does, so `github routes` does not lie about it.
---
gh/contents.go | 81 ++++++++++++++++++-------------------------------
gh/live_test.go | 19 +++++++-----
gh/surface.go | 2 +-
3 files changed, 42 insertions(+), 60 deletions(-)
diff --git a/gh/contents.go b/gh/contents.go
index 7455ae9..2505439 100644
--- a/gh/contents.go
+++ b/gh/contents.go
@@ -232,32 +232,29 @@ func (c *Client) Blob(ctx context.Context, repo, path string, opts BlobOptions)
f.URL = blobURL(repo, ref, path)
f.RawURL = rawURL(repo, ref, path)
+ // One read, and it is the page.
+ //
+ // This used to be the route JSON with the page as a fallback, and it is not
+ // any more, because the route JSON stopped carrying the half that matters.
+ // Both halves are on the page: codeViewBlobRoute has the rendered view, the
+ // table of contents, and the symbol block, and codeViewBlobLayoutRoute has
+ // the file's own metadata, the language, the size, the line counts. The
+ // route JSON now answers with codeViewBlobRoute alone, so asking it for the
+ // language gets an empty string and asking it for both gets two requests
+ // where the page is one.
+ //
+ // Symbols used to be a third request. The list arrived sometimes on the
+ // first read and sometimes only on the second, so an unavailable one was
+ // asked for again before it was believed. It no longer arrives at all: the
+ // page still says symbolsEnabled and still renders the button, and the
+ // block behind it is null on every file of every repository tried, on both
+ // surfaces, with or without a cache, signed out. So that retry was spent to
+ // learn nothing, and it is gone.
url := blobURL(repo, ref, path)
- final, err := c.blobRoute(ctx, f, url)
- if err != nil {
+ if err := c.readBlobPage(ctx, f, opts.Styled); err != nil {
return nil, err
}
- f.addSource(final)
-
- // Symbols are served by a background analyser whose result is cached for a
- // short while, so the same URL answers with the symbol list one second and
- // null the next, on either surface, with any headers. Nothing about the
- // request changes it. So an unavailable list is retried: once on the page,
- // which is a different cache, and once more on the route with our own cache
- // entry dropped. Two extra requests is worth the difference between a
- // symbol list and silence, and after that the record says unavailable and
- // means it.
- if opts.Styled || f.SymbolsStatus == "unavailable" {
- if err := c.readBlobPage(ctx, f, opts.Styled); err == nil {
- f.addSource(url)
- }
- }
- if f.SymbolsStatus == "unavailable" {
- c.cacheDrop(url, SurfaceRouteJSON)
- if _, err := c.blobRoute(ctx, f, url); err != nil {
- return nil, err
- }
- }
+ f.addSource(url)
if opts.Content && !f.IsBinary {
b, err := c.Raw(ctx, repo, ref, path)
if err != nil {
@@ -277,28 +274,6 @@ func (c *Client) Blob(ctx context.Context, repo, path string, opts BlobOptions)
return f, nil
}
-// blobRoute fetches and decodes the render half of a blob into f, returning the
-// URL it ended up reading. It is a function rather than inline code because the
-// page fallback decodes the same block a second time.
-func (c *Client) blobRoute(ctx context.Context, f *File, url string) (string, error) {
- var env struct {
- Payload struct {
- Route json.RawMessage `json:"codeViewBlobRoute"`
- } `json:"payload"`
- }
- res, err := c.GetJSON(ctx, url, SurfaceRouteJSON, &env)
- if err != nil {
- return "", err
- }
- if len(env.Payload.Route) == 0 {
- return "", structureChanged(f.Repo + ":" + f.Path)
- }
- if err := decodeBlobRoute(f, env.Payload.Route); err != nil {
- return "", err
- }
- return res.FinalURL, nil
-}
-
// blobRouteData is the render half of a blob: what GitHub worked out about the file
// while displaying it. The bytes are not in here and that is deliberate on
// their side, not an omission on ours.
@@ -335,8 +310,8 @@ func decodeBlobRoute(f *File, raw json.RawMessage) error {
if err := json.Unmarshal(raw, &v); err != nil {
return badPayload(f.Path, err)
}
- // Assigned, not appended. This block gets decoded twice when the page
- // fallback runs, and appending would give a file two of every heading.
+ // Assigned, not appended, so a second decode of the same block replaces the
+ // headings rather than giving the file two of each.
f.TOC = nil
for _, h := range v.HeaderInfo.TOC {
f.TOC = append(f.TOC, Heading{Level: h.Level, Text: h.Text, Anchor: h.Anchor})
@@ -381,9 +356,10 @@ func decodeBlobRoute(f *File, raw json.RawMessage) error {
return nil
}
-// readBlobPage reads the page for the three things the route JSON does not
-// reliably give: the blob's own metadata, a symbol list that is actually there,
-// and, when styled is set, the per-line source with its highlight spans.
+// readBlobPage reads a blob page and decodes every block of it: the rendered
+// view and the symbols from codeViewBlobRoute, the file's own metadata from
+// codeViewBlobLayoutRoute, and, when styled is set, the per-line source with
+// its highlight spans.
//
// The styled key really does contain a dot in its name and really is not
// nested. It is payload["codeViewBlobLayoutRoute.StyledBlob"], one key, and a
@@ -400,7 +376,10 @@ func (c *Client) readBlobPage(ctx context.Context, f *File, styled bool) error {
if err := json.Unmarshal(embeddedPayload(res.Body), &env); err != nil {
return badPayload(f.Path, err)
}
- if raw, ok := env.Payload["codeViewBlobRoute"]; ok && f.SymbolsStatus != "ok" {
+ if len(env.Payload) == 0 {
+ return structureChanged(f.Repo + ":" + f.Path)
+ }
+ if raw, ok := env.Payload["codeViewBlobRoute"]; ok {
if err := decodeBlobRoute(f, raw); err != nil {
return err
}
diff --git a/gh/live_test.go b/gh/live_test.go
index 896c3e2..29261e8 100644
--- a/gh/live_test.go
+++ b/gh/live_test.go
@@ -14,8 +14,11 @@ import (
// for when you want to know whether a surface still looks the way the spec says
// it does.
//
-// `make fixtures` runs these with recording on, which is how the offline
-// scenario suite gets its data.
+// These are the tests that catch the failure this tool cannot survive: GitHub
+// moving something. Nothing offline can see that, because an offline test
+// checks the parser against bytes that were already parsed once. So the
+// assertions here are deliberately about shape rather than values. A star count
+// changes hourly and pinning one turns a test into a clock.
func liveClient(t *testing.T) *Client {
t.Helper()
@@ -564,11 +567,11 @@ func TestLiveContents(t *testing.T) {
if f.Lines == nil || *f.Lines < 100 {
t.Errorf("line count %v for a 13 KB file", f.Lines)
}
- // GitHub's symbol analyser answers null about half the time and the
- // same list a second later, on both surfaces, with any headers. Blob
- // retries twice, and past that the honest report is "unavailable"
- // rather than a hard failure here. not_analyzed for a Go file would be
- // a real change and does fail.
+ // GitHub's symbol analyser answers null on every file of every
+ // repository tried now, on both surfaces, signed out. So "unavailable"
+ // is the expected answer here rather than a failure, and if the block
+ // ever comes back this asserts it is shaped right. not_analyzed for a
+ // Go file would be a real change and does fail.
switch f.SymbolsStatus {
case "ok":
if len(f.Symbols) == 0 {
@@ -579,7 +582,7 @@ func TestLiveContents(t *testing.T) {
t.Errorf("symbol is half empty: %+v", s)
}
case "unavailable", "timed_out":
- t.Logf("symbols %s after three tries, the analyser was cold", f.SymbolsStatus)
+ t.Logf("symbols %s, the analyser did not answer", f.SymbolsStatus)
default:
t.Errorf("symbols status %q for a Go file", f.SymbolsStatus)
}
diff --git a/gh/surface.go b/gh/surface.go
index 8a0c0d3..bbdd552 100644
--- a/gh/surface.go
+++ b/gh/surface.go
@@ -30,7 +30,7 @@ type RouteInfo struct {
var Routes = []RouteInfo{
{"/{owner}/{repo}", "embedded", "route-json", "sidebarAbout lives only in the HTML payload"},
{"/{owner}/{repo}/tree/{ref}/{path}", "route-json", "embedded", ""},
- {"/{owner}/{repo}/blob/{ref}/{path}", "route-json", "raw", "metadata from the route, bytes from raw"},
+ {"/{owner}/{repo}/blob/{ref}/{path}", "embedded", "raw", "the route JSON dropped the metadata block, so the page is the read; bytes from raw"},
{"/{owner}/{repo}/branches", "route-json", "xhr", ""},
{"/{owner}/{repo}/refs", "xhr", "git", "names only, 6 KB against 588 KB"},
{"/{owner}/{repo}/commits/{ref}", "route-json", "feed", ""},
From 9f4bbbb75dabb252104b09e5ae7a6b62a0118f5a Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:28:56 +0700
Subject: [PATCH 17/21] gh: make -o url work, and narrow the author column
-o url printed nothing for every record. The renderer takes the URL from
the field tagged table:"-,url", and Base had a plain table:"-", so there
was no field to take. Nothing failed; it just printed empty lines.
Actor gets a String method while the file is open. Without it the
renderer falls back to JSON for a struct field, and a cell holding a
whole actor pushes every other column off the screen. A login is what an
author column is for, and the full record is still there in every other
format.
---
gh/base.go | 8 +++++++-
1 file changed, 7 insertions(+), 1 deletion(-)
diff --git a/gh/base.go b/gh/base.go
index fdd2bee..e98d115 100644
--- a/gh/base.go
+++ b/gh/base.go
@@ -28,7 +28,7 @@ type Base struct {
Kind string `json:"kind" table:"kind"`
ID string `json:"id" table:"id" kit:"id"`
URI string `json:"uri,omitempty" table:"-"`
- URL string `json:"url,omitempty" table:"-"`
+ URL string `json:"url,omitempty" table:"-,url"`
Sources []string `json:"sources,omitempty" table:"-"`
Via map[string]string `json:"via,omitempty" table:"-"`
Extra json.RawMessage `json:"extra,omitempty" table:"-"`
@@ -107,6 +107,12 @@ type Actor struct {
URI string `json:"uri,omitempty" table:"-"`
}
+// String is the login, which is what an author column in a table is for. The
+// renderer would otherwise fall back to JSON for a struct field, and a cell
+// holding the whole actor is wide enough to push every other column off the
+// screen. The full thing is still there in every other format.
+func (a Actor) String() string { return a.Login }
+
// actor builds an Actor from a login, filling the derived fields. An empty
// login gives an empty Actor rather than one with a URL to nowhere.
func actor(login string) Actor {
From 9114bcc8e0f4d7aa29aeadfcc3f505a87076e3f2 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:28:57 +0700
Subject: [PATCH 18/21] gh: refuse to hand back a page when a diff was asked
for
A .diff or .patch suffix on something that is not a change gets answered
with the page rather than a 404. /golang/go/pull/1000 is an issue, so its
.diff is the issue page, 200 and all, and the command was writing a
quarter of a megabyte of markup to the terminal or, worse, to the file
someone redirected it into. The mistake surfaced later as a patch that
would not apply.
---
gh/commit.go | 17 ++++++++++++++++-
1 file changed, 16 insertions(+), 1 deletion(-)
diff --git a/gh/commit.go b/gh/commit.go
index 3dd4779..91fa584 100644
--- a/gh/commit.go
+++ b/gh/commit.go
@@ -9,6 +9,7 @@ import (
"golang.org/x/net/html"
+ "github.com/tamnd/any-cli/kit/errs"
"github.com/tamnd/github-cli/pkg/gitproto"
"github.com/tamnd/github-cli/pkg/page"
)
@@ -1047,7 +1048,7 @@ func (c *Client) Patch(ctx context.Context, url string) (string, error) {
if err != nil {
return "", err
}
- return string(res.Body), nil
+ return plainText(res, url)
}
// Diff returns the unified diff, which is the patch without the commit
@@ -1057,6 +1058,20 @@ func (c *Client) Diff(ctx context.Context, url string) (string, error) {
if err != nil {
return "", err
}
+ return plainText(res, url)
+}
+
+// plainText refuses to hand back a web page.
+//
+// A .diff or .patch suffix on something that is not a change gets answered with
+// the page instead of a 404. /golang/go/pull/1000 is an issue, so its .diff is
+// the issue page, 200 and all. Without this the command writes a quarter of a
+// megabyte of markup to a terminal, or worse, to the file someone redirected it
+// into, and the mistake surfaces later as a patch that will not apply.
+func plainText(res *Response, url string) (string, error) {
+ if ct := res.Header.Get("Content-Type"); strings.Contains(ct, "text/html") {
+ return "", errs.NotFound("no diff at %s; github answered with a page, which means the reference names something that is not a change", shortURL(url))
+ }
return string(res.Body), nil
}
From 1fe692632727a8fd6ede6d858bccbd0cbd3d9eb8 Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:29:10 +0700
Subject: [PATCH 19/21] Point the release at the right names
The binary is github and the package is ./cmd/github, so the Makefile
builds those. It also grows a live target, because the suite that talks
to github.com is the only one that can tell you the site changed, and it
should not need remembering.
Three naming decisions in the goreleaser config, each written down with
its reason. The image is ghcr.io/tamnd/github-cli, not the short name,
because a short name in a user namespace belongs to whichever repository
pushed it first and the workflow token can only write packages linked to
its own repository, which is how hf-cli's release died at the last step.
The cask is github-cli because homebrew-cask already has a github, and it
is GitHub Desktop. And the cask strips the quarantine attribute on
install, since Gatekeeper kills an ad-hoc signed binary, which is what
every cross-compiled Go binary is.
---
.goreleaser.yaml | 25 +++++++++++++++++++++++--
Makefile | 15 ++++++++++++---
2 files changed, 35 insertions(+), 5 deletions(-)
diff --git a/.goreleaser.yaml b/.goreleaser.yaml
index b7072a2..f888ee5 100644
--- a/.goreleaser.yaml
+++ b/.goreleaser.yaml
@@ -71,8 +71,14 @@ nfpms:
dst: /usr/share/doc/github-cli/LICENSE
dockers_v2:
+ # The image is named for the repository, not for the binary. A short name in a
+ # user-owned GHCR namespace belongs to whichever repository pushed it first,
+ # and the workflow token only has write access to packages linked to the
+ # repository it is running in. A sibling project taking the short name is how
+ # a release fails at the last step with a 403 that reads like a permissions
+ # bug, which already happened once on hf-cli.
- images:
- - ghcr.io/tamnd/github
+ - ghcr.io/tamnd/github-cli
tags:
- "{{ .Version }}"
- latest
@@ -90,7 +96,13 @@ dockers_v2:
org.opencontainers.image.licenses: "Apache-2.0"
homebrew_casks:
- - name: github-cli-tamnd
+ # Pushed to the tap repository. It self-disables until
+ # HOMEBREW_TAP_GITHUB_TOKEN (a PAT with write access to tamnd/homebrew-tap) is
+ # set, so a tokenless release still writes the cask into dist for inspection.
+ #
+ # The cask is github-cli rather than github because homebrew-cask already has
+ # a cask called github, which is GitHub Desktop.
+ - name: github-cli
repository:
owner: tamnd
name: homebrew-tap
@@ -102,6 +114,15 @@ homebrew_casks:
commit_author:
name: Duc-Tam Nguyen
email: tamnd87@gmail.com
+ # Homebrew quarantines cask artifacts, and Gatekeeper kills a quarantined
+ # binary that is only ad-hoc signed, which is what a cross-compiled Go
+ # binary is. Strip the attribute at install so github runs on the first try.
+ hooks:
+ post:
+ install: |
+ if system_command("/usr/bin/xattr", args: ["-h"]).exit_status.zero?
+ system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/github"]
+ end
scoops:
- repository:
diff --git a/Makefile b/Makefile
index 1de66e5..1bf1ae9 100644
--- a/Makefile
+++ b/Makefile
@@ -1,7 +1,7 @@
# Build into bin/ (gitignored) so the binary never collides with the github/
# source package at the repo root.
-BINARY := bin/ghb
-PKG := ./cmd/ghb
+BINARY := bin/github
+PKG := ./cmd/github
VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev)
COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo none)
DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ)
@@ -10,7 +10,7 @@ LDFLAGS := -s -w \
-X github.com/tamnd/github-cli/cli.Commit=$(COMMIT) \
-X github.com/tamnd/github-cli/cli.Date=$(DATE)
-.PHONY: build install test vet fmt clean run
+.PHONY: build install test live vet fmt lint clean run
build:
@mkdir -p $(dir $(BINARY))
@@ -19,15 +19,24 @@ build:
install:
CGO_ENABLED=0 go install -trimpath -ldflags "$(LDFLAGS)" $(PKG)
+# The default run is offline and deterministic.
test:
go test ./...
+# live talks to github.com. It answers the one question no offline test can:
+# does the site still look the way the readers think it does.
+live:
+ GITHUB_LIVE=1 go test ./gh/ -run Live -count=1 -v
+
vet:
go vet ./...
fmt:
gofmt -w -s .
+lint:
+ golangci-lint run
+
clean:
rm -rf bin dist
From 64c773e8cc40e15c1ceaa3249ed9540560c1c7cb Mon Sep 17 00:00:00 2001
From: tamnd <1218621+tamnd@users.noreply.github.com>
Date: Sat, 25 Jul 2026 21:29:19 +0700
Subject: [PATCH 20/21] Rewrite the README, with a demo
The old one described a tool that shelled out to something else. This one
opens with what the thing actually is, shows a real session as a gif
recorded from the tape in docs/demo, and then answers the two questions
people arrive with: what can it read, and why is there no token.
The gif is generated, so the tape is in the repository and re-recording
is one command rather than a screen capture nobody can reproduce.
---
README.md | 342 +++++++++++++++++++++++++++++++++++-------
docs/demo/github.tape | 49 ++++++
docs/static/demo.gif | Bin 0 -> 459132 bytes
3 files changed, 338 insertions(+), 53 deletions(-)
create mode 100644 docs/demo/github.tape
create mode 100644 docs/static/demo.gif
diff --git a/README.md b/README.md
index a6a1b52..940ec58 100644
--- a/README.md
+++ b/README.md
@@ -1,9 +1,26 @@
-# github-cli
+# github
-A command-line for GitHub that reads public data by scraping HTML pages and
-Atom feeds. No API key required. No rate limit from the official REST API.
+[](https://github.com/tamnd/github-cli/actions/workflows/ci.yml)
+[](https://github.com/tamnd/github-cli/releases/latest)
+[](https://pkg.go.dev/github.com/tamnd/github-cli)
+[](https://goreportcard.com/report/github.com/tamnd/github-cli)
+[](./LICENSE)
-**Not affiliated with GitHub or Microsoft Corporation.**
+**github** reads github.com as data, with no token, ever.
+One pure-Go binary turns the site into typed records: every repository, user, organization, issue, pull request, discussion, commit, branch, tag, release, file, topic, gist, package, and marketplace action, each with a canonical `github://` address, every field its page stated, and typed edges to everything it names.
+Read one thing, list a million, walk the dependency graph, or export the lot as RDF.
+
+[Install](#install) • [Quick start](#quick-start) • [Read one thing](#read-one-thing) • [List and search](#list-and-search) • [Contents](#look-inside-a-repository) • [Graph](#walk-the-graph) • [Linked data](#export-linked-data) • [Output](#output) • [No token](#no-token) • [Serve](#serve-it) • [Driver](#use-it-as-a-resource-uri-driver)
+
+
+
+GitHub is already a knowledge graph that happens to be served as a website.
+Repositories depend on repositories, issues reference commits, commits belong to people, people belong to organizations, and every one of those relations is written down on a page somewhere.
+Most tools hand you back a page, or the subset of fields somebody decided you needed.
+`github` reads all of it, keeps all of it, and gives each entity an address.
+
+Not affiliated with GitHub or Microsoft.
+Full docs and guides live at **[tamnd.github.io/github-cli](https://tamnd.github.io/github-cli/)**.
## Install
@@ -11,80 +28,299 @@ Atom feeds. No API key required. No rate limit from the official REST API.
go install github.com/tamnd/github-cli/cmd/github@latest
```
-Or grab a prebuilt binary from the
-[releases](https://github.com/tamnd/github-cli/releases):
+Prefer a prebuilt binary?
+Grab an archive, a `.deb`/`.rpm`/`.apk`, or a signed checksum from [releases](https://github.com/tamnd/github-cli/releases).
+Or let a package manager handle it:
+
+```bash
+# Homebrew (macOS)
+brew install --cask tamnd/tap/github-cli
+
+# Scoop (Windows)
+scoop bucket add tamnd https://github.com/tamnd/scoop-bucket
+scoop install github-cli
+
+# apt (Debian, Ubuntu)
+curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg
+echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list
+sudo apt update && sudo apt install github-cli
+
+# dnf (Fedora, RHEL)
+sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo
+sudo dnf install github-cli
+
+# container
+docker run --rm ghcr.io/tamnd/github-cli:latest repo gohugoio/hugo
+```
+
+The binary is called `github`.
+It does not replace the official `gh`, which does the authenticated half of the site far better.
+This one does the public half without asking you to log in.
+
+## Quick start
+
+```bash
+github repo gohugoio/hugo # a record, not a page
+github owned torvalds -n 10 # a list, streamed
+github get https://github.com/golang/go/pull/1 # paste anything
+```
+
+`github get` takes a bare id, a URL you copied out of a browser, or a `github://` URI, works out what it points at, and reads it.
+Every other read command is the same thing with the kind already decided.
+
+Output adapts to where it goes: an aligned table on your terminal, JSONL the moment you pipe it somewhere.
+
+## Read one thing
+
+```bash
+github repo gohugoio/hugo # every field the page states
+github repo gohugoio/hugo --deep # plus what only the deferred fragments carry
+github user torvalds
+github org golang
+github issue golang/go 1234
+github pr golang/go 1
+github commit golang/go abc1234 # author, verification, changed files
+github release cli/cli # the latest one, or pass a tag
+github discussion vercel/next.js 12345
+github compare golang/go go1.22.0 go1.23.0
+```
+
+Records carry what the page carried, not a curated subset.
+A repository page states its head commit, its commit and release counts, its licence, its topics, its funding and citation flags, and its whole root tree, so the record does too, from one request.
+
+## List and search
```bash
-# Linux/macOS
-curl -sSL https://github.com/tamnd/github-cli/releases/latest/download/github_linux_amd64.tar.gz | tar xz
-./github --help
+github repos --language rust --sort stars -n 100
+github issues "repo:golang/go is:open label:NeedsInvestigation"
+github prs --owner golang
+github users --language go
+github search kubernetes # every entity kind at once
+github topics machine-learning
+github actions lint # the marketplace
+github trending --language go
```
-Or run the container image:
+Listing streams.
+`-n` stops early without fetching the next page, and no command holds a full result set in memory unless a format forces it to.
+
+`github code` is the one command that does not work: code search needs a signed-in session, and this tool has none.
+It says so rather than returning nothing.
+
+## Look inside a repository
```bash
-docker run --rm ghcr.io/tamnd/github:latest --help
+github tree golang/go src/net/http
+github tree gohugoio/hugo --recursive
+github cat gohugoio/hugo go.mod
+github blob golang/go src/runtime/proc.go
+github readme gohugoio/hugo
+github diff golang/go abc1234
+github archive golang/go master --format tar.gz
```
-## Commands
+`archive` is the one to reach for on a whole repository.
+One request to codeload streams the entire tree, where `tree --recursive` is one request per directory.
-| Command | Description | Source |
-|---------|-------------|--------|
-| `github trending` | Top trending repositories | HTML |
-| `github user ` | User profile | HTML |
-| `github repos ` | User's public repositories | HTML |
-| `github repo ` | Repository metadata | HTML |
-| `github commits ` | Recent commits | Atom feed |
-| `github releases ` | Releases | Atom feed |
-| `github tags ` | Tags | Atom feed |
-| `github issues ` | Issues | HTML |
-| `github pulls ` | Pull requests | HTML |
-| `github readme ` | README content | raw.githubusercontent.com |
-| `github file ` | Any file | raw.githubusercontent.com |
-| `github search ` | Search repositories | HTML |
-| `github followers ` | User followers | HTML |
-| `github following ` | Users followed by user | HTML |
-| `github stars ` | Starred repositories | HTML |
+There is also `github symbols`, which reads the code navigation index GitHub builds for every file.
+It currently returns nothing, and says so: the blob page still renders the symbols button, and the list behind it is empty for a signed-out reader.
+The command stays because the field is still in the payload and may fill in again.
-## Examples
+## History and people
```bash
-# Trending Go repos today
-github trending --lang go
+github commits golang/go -n 50
+github branches golang/go
+github tags golang/go
+github releases cli/cli
+github refs golang/go
+github timeline golang/go 1234 # everything that happened on one issue
+
+github owned torvalds # repositories as the profile shows them
+github stars torvalds
+github followers torvalds
+github members golang
+github contributions torvalds # the calendar, one record per day
+github gists torvalds
+github activity torvalds
+```
+
+## Walk the graph
-# User profile as JSON
-github user torvalds -o json
+```bash
+github graph golang/go # the node, its edges, and its facts
+github edges golang/go # just the edges
+github deps gohugoio/hugo # what it depends on, with versions and licences
+github dependents gohugoio/hugo # the repositories that depend on it
+github crawl golang/go --depth 2
+github crawl golang/go --depth 3 --dry-run # size the walk before running it
+github contributors golang/go
+github forks golang/go
+```
-# Recent commits on main
-github commits golang/go
+Edges come from five places: explicit ids, the embedded React payload, the atom feeds, parsed HTML, and text.
+Each edge records which one it came from, so a consumer can decide how much to trust it, and a walk can drop everything below a floor.
-# List open issues
-github issues golang/go
+`deps` and `dependents` read GitHub's dependency graph, which is the part of the site with no API at all.
+`deps` gives you the package, the version, the ecosystem, the manifest it came from, the licence, and whether it is direct or transitive.
-# Search for HTTP libraries
-github search "http client"
+## Export linked data
-# Fetch the README
-github readme torvalds/linux
+```bash
+github rdf gohugoio/hugo --format ttl
+github rdf gohugoio/hugo --format jsonld
+github export golang/go --depth 2 --format jsonl > go.jsonl
```
-## Output formats
+RDF comes out as N-Triples, Turtle, JSON-LD, or N-Quads, over `schema.org` where a term exists and a `gh:` namespace where none does.
+N-Triples and N-Quads stream, so exporting a large organization never needs the graph in memory.
-Every command supports `-o table|json|jsonl|csv|tsv|url` and `--fields`.
+## The page plane
+
+Every reader in this tool works from one extraction of the page, and `github page` prints that extraction whole:
```bash
-github trending -o jsonl | jq '.full_name'
-github repos torvalds --fields full_name,stars
+github page gohugoio/hugo # everything, organised
+github page gohugoio/hugo --section payload # just the embedded React payload
+github page gohugoio/hugo --section meta # just the og: and twitter: tags
+github page golang/go#1234 --query IssueViewerViewQuery
+github page https://github.com/trending --raw > trending.html
```
-## Notes
+This is the debugging tool.
+When a field comes back empty, `page` shows you the same view the reader had, so the answer is either "the page stopped carrying it" or "the selector is wrong", and you can tell which in one command.
+
+## Output
+
+Every command shares one contract: `-o table|markdown|list|json|jsonl|csv|tsv|url|raw`, `--fields` to pick columns, `--template` for a custom line, `-n` to limit.
+
+```bash
+github repos --language go --fields id,stars,forks
+github repos --language go --template '{{.id}} has {{.stars}} stars'
+github repo gohugoio/hugo -o json | jq .tree
+```
+
+`-o url` is the one the others are measured against.
+It prints one URL per record and nothing else, so this composes with no glue:
+
+```bash
+github owned torvalds -o url | xargs -n1 github get
+```
+
+Failures are typed too.
+Every surface exits 3 on an empty result, 4 when a page is not public, 5 on a rate limit, 6 on not found, 7 on unsupported, and 8 on a network failure, so a script can branch on the code without reading the message.
+
+## No token
+
+There is no API token in this tool and there will not be one.
+The unauthenticated REST API allows sixty requests an hour, which is not enough to read one organization, while the pages sit behind a CDN and are faster than the API even where the API would work.
+
+That is a promise you cannot check by using the tool.
+You can see that a command works without logging in, but not that no code path would send a credential if one happened to be lying around.
+So it is asserted instead: `gh/noauth_test.go` parses every source file with comments dropped and fails the build if the word `Authorization`, `GITHUB_TOKEN`, `GH_TOKEN`, or `api.github.com` shows up in code anywhere outside the one file whose job is to say those names out loud.
+
+If you do have a token in your environment, `github doctor` will tell you it is being ignored, because a token that does nothing looks exactly like a token that is wrong:
+
+```bash
+github doctor
+```
+
+`doctor` checks the environment, whether the site answers, whether the page still carries the payload every reader expects, whether the cache is writable, and what pacing this run is using.
+It is the first thing to reach for when something comes back wrong.
+
+The cost of no token is that this is read-only and public-only.
+For anything else, use the official [gh](https://cli.github.com).
+
+## Serve it
+
+The same operations are available over HTTP and as an MCP tool set for agents, with no extra code:
+
+```bash
+github serve --addr :7777 # every read verb becomes GET /v1/, streaming NDJSON
+github mcp # speak MCP over stdio
+```
+
+Arguments go in the query string, because most of them contain a slash and a path would swallow it:
+
+```bash
+curl 'localhost:7777/v1/repo?ref=gohugoio/hugo'
+curl 'localhost:7777/v1/blob?ref=gohugoio/hugo&path=go.mod'
+curl 'localhost:7777/v1/trending?language=go&limit=5'
+curl localhost:7777/v1/openapi.json
+```
+
+## Use it as a resource-URI driver
+
+`github` registers a `github` domain the way a program registers a database driver with `database/sql`.
+A host enables it with one blank import:
+
+```go
+import _ "github.com/tamnd/github-cli/gh"
+```
+
+Then [ant](https://github.com/tamnd/ant), or any program that links the package, dereferences `github://` URIs without knowing anything about the site:
+
+```bash
+ant get github://repo/gohugoio/hugo
+ant cat github://file/gohugoio/hugo@master/go.mod
+ant ls github://org/golang
+ant url github://pr/golang/go#1
+```
+
+## How it works
+
+One `kit.Handle` registration per operation, and every surface updates itself:
+
+```
+cmd/github/ thin main: hands cli.NewApp to kit.Run
+cli/ assembles the kit App and registers the byte-plane commands
+gh/ the library: client, records, graph, RDF, doctor, domain.go
+pkg/page/ one extraction of an HTML page, shared by every reader
+pkg/gitproto/ the git smart HTTP protocol, for the refs no page lists
+docs/ tago documentation site and the demo tape
+```
+
+That single declaration becomes a CLI command, an HTTP route, an MCP tool, and a URI dereference, so there is no second implementation to keep in step.
+
+Underneath, seven surfaces answer for different routes: the embedded React payload, the JSON a route returns when asked for JSON, the fragments a page defers to XHR, the search backend, the atom feeds, raw.githubusercontent.com, and the git protocol itself.
+`github routes` prints which surface answers for which route and which one it falls back to.
+
+Responses are cached on disk, keyed by surface and URL, for fifteen minutes.
+Anything addressed by a commit SHA is kept forever, because it cannot change.
+
+## Development
+
+```bash
+make build # ./bin/github
+make test # go test ./..., offline and deterministic
+make live # the smoke tests that actually talk to github.com
+make vet
+make lint
+```
+
+The offline tests check the parsers against bytes that were already parsed once, which is useful but cannot see the failure that matters: GitHub moving something.
+`make live` is what sees that, so the assertions there are about shape rather than values.
+A star count changes hourly, and pinning one turns a test into a clock.
+
+The demo above is a tape, not a screen recording.
+Regenerate it with [ascii-gif](https://github.com/tamnd/ascii-gif):
+
+```bash
+ascii-gif render docs/demo/github.tape -o docs/static/demo.gif
+```
+
+## Releasing
+
+Push a version tag and GitHub Actions runs GoReleaser, which builds the archives, Linux packages, the multi-arch GHCR image, checksums, SBOMs, and a cosign signature:
+
+```bash
+git tag -a v0.2.0 -m "v0.2.0"
+git push --tags
+```
-- HTML structure can change without notice. Parsers return empty strings on
- missing fields rather than crashing.
-- Search may return HTTP 429 from datacenter IPs. The binary exits with code 5
- when throttled. Add `--page 1` and wait a moment before retrying.
-- The default pacing is 500 ms between requests. Use `--delay` to adjust.
+The Homebrew and Scoop steps self-disable until their tokens exist, so a release works with no extra secrets.
## License
-Apache-2.0
+Apache-2.0.
+See [LICENSE](LICENSE).
diff --git a/docs/demo/github.tape b/docs/demo/github.tape
new file mode 100644
index 0000000..c680b37
--- /dev/null
+++ b/docs/demo/github.tape
@@ -0,0 +1,49 @@
+# Demo tape for github. Rendered with ascii-gif (github.com/tamnd/ascii-gif),
+# which supplies the window chrome and theme; this file is just the action.
+#
+# ascii-gif render docs/demo/github.tape -o docs/static/demo.gif
+#
+# github must be on PATH inside the recording shell, and github.com reachable.
+
+Hide
+Type "export PS1='$ ' PATH=$HOME/bin:$PATH && cd $(mktemp -d) && clear"
+Enter
+Show
+
+Sleep 800ms
+Type "github repo gohugoio/hugo --fields id,stars,forks,language"
+Sleep 700ms
+Enter
+Sleep 3.5s
+
+Type "clear"
+Enter
+Sleep 300ms
+Type "github owned torvalds --fields id,stars,forks,language -n 5"
+Sleep 700ms
+Enter
+Sleep 4s
+
+Type "clear"
+Enter
+Sleep 300ms
+Type "github deps gohugoio/hugo --fields package,version,ecosystem,license -n 6"
+Sleep 700ms
+Enter
+Sleep 4s
+
+Type "clear"
+Enter
+Sleep 300ms
+Type "github edges golang/go --fields predicate,object,source -n 6"
+Sleep 700ms
+Enter
+Sleep 4s
+
+Type "clear"
+Enter
+Sleep 300ms
+Type "github rdf gohugoio/hugo --format ttl | grep -m5 schema:"
+Sleep 700ms
+Enter
+Sleep 4.5s
diff --git a/docs/static/demo.gif b/docs/static/demo.gif
new file mode 100644
index 0000000000000000000000000000000000000000..ca9dc61745f1af73e6d70d7d0b2b5d10e3931e88
GIT binary patch
literal 459132
zcmeFYS5y=1)-IY#LhrqWBE5+Sp-Jc+0wP^LqzMQZ5fK3~p?5+LJyZcH0gN>vBV)*_1W1tBsti{IoQu|aGv4fk>%l*=HZdzWj({o
zBhAYz!OO3}&n?BzFUv0=A;2duARr?kASWQ8C?KdPfRGhHs0a$m3kt~z37tjo$|Ho8
z5h8Mkc6CwFvtnZB#l#iGB(%i26~y_}&d^9maz;o>ASET#C8e~a7-Xb{Ii+RJNz3cY
z%InHAD=I|mDJtkHD(WkqGgUgPaaKhAtg_}=Wdjv;^m&$Z=jC)&k>=DNHHBC;q#jc2
z5fW*kuA#54X`!X-sI6nCqjOQG{1QsTLRSxMV0zh*P1Q(D&B)l<*z~fgkb#*B+T7a9
z!raEf!rj8!+tSLx(&~y8pOH0>G1|t_#^Ji{*^3wLE?=_0a>*gkUeVs(u*crm&EEbm
zdwYK;`v4b1cNh2Du7BP7OXA{RI_@5S-S+SZ@wgi0DQWL@#@@?2%**?Z&(#Q@phVwG
zZ~UwR{jSIPqfh;>Ck6z?2HZ>uypa%ODso-L^SbZN8$!}IZYJIgO1OC^E97=+sFL?>
z#jAJ1Gw$3YgezVRznc?vsVXY2{9b%1HlY}&g~7!ZMY~|5lPY48h;i{najCWO$Q$uV
z#fhrHi4nPpDHX}tjj34=Qgd3;@>APJS9o1w4yre>S!I{N=kB
zlkdjozD&&QPR#y2F}FXnbU5dBZSLjLtHNgst3MXAN0);-UvHjnd^mmAWw4DO|4=^v
zarfZM6`!x^Ghd4rzr9{M`11X*o8=^K?q}ZXU)_LHOY>7}%hS~5({2FZ6d-a+Z)IqA
zQC}BrsH~s_1_A&8k?#-)1RxBMqy7%~eN6!HQ^0q)fC-66iGj1rxQ>v@`x98i%!*A~
zDh5+{&iOBov{VjfAoP<3Ok1l)v!yQ9yNCla(gfGMC5N>Zb3bE6qyG9@Wo2aD3>$GWO`ft7iB9WI^-xhJ}Z|)Ag61wKpzx
zVBU_En0GX-^n`tRzw)f3`OQ--m|n=DleE^K$S&*lytC!)FkZ~O)S|0(^BLjXwb#$P
z9&WuP>Zb@|FqNn5UH{=+4VXNNG{kJ38
zvVV>Dc755Js5CFLdfff(!~DZ*Z^j?@9R9u9pK>Z}{iOH%m+k2Xf4zM2_{U++myJn%
z2qb~|=*x#UFP}a=Jq6H6tzscuzNPwiT~K~~3ot$sxp~EaUKUNlC8^B$zD+~Et9_fUb+G+514Sdffj8hXeM`gJ
zFcHmqXFex^R~n50LJxWUHgg?)>NfLSLw7dw-Q%U-5g3Dr6exd{yHWPJg30LAOIgxe
z#UXQkTP0!d>b6QF7rQ7l@&u9WNXZ7WZt-1-`t6ET)%V+#cq5seO#WW-y1Bxyr0uHW
z(Dys{iSaV;YparvB(s%vCi|d#05}K^`gQvKYF*o$|A)rTcl95dBx-={tY?F|-#lm;
z8~e~QBJp6i^||U&EsGKk<$bR?c!F|wE*S73N+cGuUA{v10|%G&QVQxCuX*sPd$;|=
zryh7ZV7g&uaSPWa5QdWLcpb$bbWb!9^A~x&@7f*(F4?fxm(;f*1CM2irjudiCjAXd
z7(?hIs&m*9hqw{P{vZGX3jol%P5}GmI(9!lQyPw}>_5ffb>-d>9$L$_JjYYod9EVB
zYM(ehj>{{%vP4HA@K_Z4nVbm-ND?6bW!CX(D@ApmR+%f3n#l4Bfup$r8dpO
zqqi9s4l$JnsYhop`hc*lyZU4HH%~tvzQMv2!kXN^H~>X7GUP)QV=eHiT!(ER+jTNM
zZ{WIwT>&WeJ6kId0D)QIc)Kb;=(r^A@7p)mlZ)8t>A8Dl^ra8CJ$pU|<%{Y+I1)kY
zZH47y$x!a%pp*R($!4wex(^Pz^U3*-nPrsi_HYwzZ_CP@**FA%c?2A#%T*{7nKi{qYaNVCXCWH9)u+`Vz9Fl90ErX(T
z%6Rf&yo7qkpu8X984ARLxB#hYaECLL!;MY)LZ$b>oCGWbChy#WVkWlRyeG!yrR`i*epH7}^M6=+EX*VYU_7WzuOj2RD&4=&)1@VTIQ+;|jA
zk*2eW0q*JXpnV5b=QHEvWcgl(&wi;i7Q7TmhZ9^U7>YMsNpp^^6l_B7mS|pkC!Bpe
zhaS97kD6o%ZL1X6bRRy;vk&GtlpMQLs@#-+cV=rH!5FotuhOsvS1?%x4!vV;{kG`q
z6R~t(--ir@C~b2HVBH>;HrGCV5mcxV0vsX4GnWRuGQ8=s@NvAuLU0ja?FE8;y`WS(
z+Z}Y5zqg8M7RBU`9R&m;K#q?R1|+R+Bn|gNMByJPYD`l}wpb8+5d|e-QnHFpHY85(
z#nGXNH`05wD!qTcgwmm~g-a*z&Uef|yZ!pc&nXkUcvvM|fhB}f@P#^WJiC^V|8*`5tMd6?|BWKsgy2og;>7d;>=VMStkKUTX
zVYk@KR_IGj+nuJvC)3$~M=?ig0u_(?va|Q%ts~V~n~#QEviH+7PR@xaejnq{{+##n
zMDc9%_wlz`UrL$p%IPW|Pd&)`T4ViF(x&-%&N%B^Q^sSVD~dlBBQg)#U*2W8(fs3e
z3jXj3a}*G#c(VS1
zu*D`bA1yW#t-uo_36D{BjFFvZQzgWFh{kHn$K=LFU*L(Y=!?;FjOEjdH6p}vM#q}X
z$FlUrTJglugW_x*<7o8a>TE~qbDq27N2xEk)20ZKGmdG4
zx6|f%A{RT;9=}R^O^8_KNq?l4zQGgmE-bw~`!Ey18!D=vb=e_H!ta(;L6%8JmR#3Oh3{Dk+}X-qH&mRmX-;ovBMYu;bY<_(
zWnb6{MDgZ4Q_V5BbIs^ZPGw$>*^aNpLQcd%4m!!lRz26wKG&Ys%dsF=zCG8q>x$d=
zTt==uk1lsFr@X^kdA{my{#|)%vw1O-lbx$mc1@s!PgoDkupnAkZckb`@047nJ%X6L|~kkcCyF
z$u)Nhn+b*W(kTrKg^!L3Nl7WK>P0<{MeRE&RDDC~Eb28)ee%6%n5Vd}E_J}E_(fRp
z2yNO}SMl_G@wi{w1aHX#vSem7ZSGFV8$!vVbo%l_$>veX>SDvXR
z`<QweKtn7#uf813DUMTzJhX?QxY1N64F+5E;k-30KCzHvr
zNaXxZWJ%6s(p-=o<^0qQpA
zUU(h6u&%J1SiD%rbX-T|E3eR~=WwpC2`{fLtmo^lZ&)mEI<80XJ!sXaXmfrb5&odF
zu%f&Bf!yMQC&v|Jz6NEDh5_fwq3{M|VZ&H=<@3dc3d{t8#jRww*bKzC)D_Tf$2h1Ku7n_L&0J{?!@@ilvBG=FujIS6m|Eo?sSt~pt3
z4mxfI@ZU!wNv3M~G(W(
z^0(@#w2IRkNJO-X=d{ZFh~Qpo1q`+-MWB>5ANHRfwyHj`K40{3IQF5Ii?!yDhgVXoLzY~70Gxmes
z-H6UHug>HT3aLw-yPrGp4-~RAyB2af^XM&di@Kh*b`=-NmHy}&X6vqqxJPj5F1y`b
zYi?fE)7>%E-Q*%m;_rD=)!nva_AsJH-?yhbBD{C0ry{3^tciW9+55nucgWnduc$Yx
zwf8x_^!Sh7bhgJ+J=hSJ$Bc2k^O`M-J&(B}9=|@-3|rxUBB1$X<3|fL;)(6GC+}Ta
zcbA?>{dlt1)B0KSsRI3zgAbN7MNe}I*gN`H*l1~QPrAU^SIx1i$fp{SV>Aq7^~=&F4P(ubV=WeA4+X{^?T&Wzj&`Mv
z_PCBdmK%L~GD3Mi(qB9>7%(!dH8RRD@@#qdMZ@sR$l*zg;c0>4+1;U6y+aGBLrbnh
zD{@0`P6pSW556rP+zc4p(i+@h82qq2@Udaw@5q6Di-9i!1K)Q04}1H+r}qDF?f)s)
ze|pjfe9;Fk>4OIL!L<8m8T;s0D2$C1=DQSDOA5Onh4Uks`!Sg}jm&?UEGSPF{`pko
z#Z$48r)L76N@_oqW_&8U@*{5G6aeD05
zSs+dMxt1J>YwO}h_!)m~a|48wnuxF;8Y0H>>Y)lqPl+Z2HM_?;AFVwfdwD+o?tEg&
zeDdS@)Rp=4pYwRZg)HrboXZP&cNYjH3x$stidPm&e=ZOO7b~)OLBP{q9m{$x`>@rQVgLCqI|Sg3Eo{%LA8}hwd(qlq`=uUVgr^
zJpOZeLU3hDdu8VG%G}+R`I42z$1BS#E3fH(uB-~aUJn!m0)N{>PzcpIl52+C0|5b4
z#|TdSllrg!|AYS*M`8xB0&oC?sE23)(E07Lt&;E2u=5$z{`S}ygw2YNL_`NuXyrYY
zrO;x=sX{uo4AQouqghPW6?V13n|&GbPTk8#iV~E<^NQ=s3t9JHmcZ`N{C`{w3+k5;
z`R7ZV0#0cKOwb4z6Zd;**AYzlV52|U3&{Bl_+Cc?GlXIB
zy~wF62s(bfnud|qYNJHutG`_QrbP^LKx#-d5etIBm{b#Y>oUbAU;>{A!76dOTnb8g
z2Xc3U5zsS34S`GXsWU2G!CEr`et%a_-DAu$k#%6=Z4Q1t7O3B*3%QsU!{mFEum%wd
zUDW!%S03lu8F){%L9pvADDfU!*0$N9tYHjH61DU5Ek*HSQ$DA~Bv()#FvfIB*XZcB
zCp3w!bsJiZ0>kN)yto(f+bbl9S9;3IYD#w2H8)yzz5X_6;CvKU&rlAoHj1}1&)RzQkn1+@75;&4=Axt
zXEf>`FZo|!;Ye)}(SNtd?>@&0JlRB${bnp}
zIq^#MWFM1^0*C;6||Z()q;S=+US=?j|!j@;aftKVZF
zv@mR;Q3Lx2pE>Y~4E}ES!T8{*#?~)VPsra7+B=$p8jI8dH=gGDI9}*I+O4?rP=;H9
z)O7>KM=!VXtcmY1fd)#)Krm!-Vg%7}v0$HkX&+)*&mbKMco>XKI)k*9je-g${~
z?i)FF(WpMq4l5d40I4eQ38x<+>0(f#w0%%+JbDYN?1cxGC{NHz7IOlf(H0aO9tb~;
z1%TjUCp+7f_mUwyRha)P;PLqbE&c_02{gXDvo0ef5q);yU%bU(O6C12^p+-7zAWW>
zsZ8hj+V+UJxM(5W_+=ysm}}oVB84E>ZtCZOI0stnnI>>Jg#TiLq%8spgU;`r=#z0^
zI=9OKBdyOT>k26%OXjJs(Ox53P3Diu04>x<3-{5>knf&NXVR8117ppCv>Xv
zuAM%VXpwW5JaL2!{S<$!NR^RDX{tY5I?EH%*UjtXjCV)*y#%p|yC<5dogHU}?e58i
zU}MO{>iea?6N76lh!m!(%ju6kFq2rHDU2)~ew_CdW3!v^ai&^M>GZ-bt8Ye&{Zd=R
zay?KuC>O@FZ6@)4JcLD9s?%h4+*#Hn^Ap(-t97UoU@hQ#L|(&kx$U&EXJ`+JpXK%m
z{TUMj0sFD^u}2|9uL#7=s=WrnT-7FMnB9G`KsqnPYeSY2BO44McZ5|}YA!B1Niy6k
zL+{+2Z>6?48OaJ~*ShhH8Y}R6!8JaG=>kpI&;X!?>Vg-|sX>=$3b&nfOfIi0^-jwK
z@NVYY(#5h9PO+#BovXjZKyY-{&buN^6fLOWPXZ7^Eu{E=VE^Y2)bD(V??13luM8Im
zBYEi3GoF))>|E^|h~^VO&XXykMrn*68D&;&5ER(Ov7a7#6!EbfUoZEgqjF$w=tzzu
z>CUvsKpymNtG?8e?=l1ZDcOx*X(jhOFR&2@V@-W&NfXYj12cYUw+;UU9)
z`l_>b+6Vgw1#0s)Ao+ciwbQ(3TG+GdekLi9ZTxE;CEq<9K#
z!04pd2p-Ov9sr-*FpZIRXq8+2yqS<9<2`rn`+f`<22n?2QMy&*2`W0yCi)TvG&l}I
zm6M0)JDXCFFweueq1bq4n4{mdGqHf72c{4G&2MiFIDk8+cFdGl8>||Hg&bvX56jMX
zKH7e*kvvoF_9TjVRVM#K;zF_VzR;&1m(F?)hl<^Q(UY$XdzL25p0~X4;VPLx`}eZe
z!bceB(bOax!<{ob!Kqg0iPbf#1yJeqxAwtXEWmhy2W
zhUn$25xToOnYB2_H2=A$NJk2xZ#pM!Y_X2;zo@VBrL;^NK3Y!1fxz@=ch2zhv|bc2
z8&QQ2s{J^YJw}z$j;wrOM@|2@+97(eAVDhDQqw<=#6%19syRXqjRHd2w%1c+vk*2h
zQf?R^t=giLUOF-h0fL`WPYQ29C(V+HIN7dDH9aN(@rFjy-Z^NHIw;HV&ocd27M@A1
zpV)tH=UpBXlDjSwwku{%~R1Lw<6O*&xiO`zF*Qt2ry8b-v5*opJ
z;sY?FFvfX>_H!9c<43I>#9P{nT#Rh!cQajWa~wO9^dYe#?m7~RI^QW
zsTE3@e`Fu}q*mpKh)q@stqbvVtK+Tu`+ifb*3ll8R&HaA@nMuuPHuVyQ8IjsAXa8Vm
zBzMX^IJ@WuxtuH?@lDjM*irGdG6?aDce&H1T#xBgK(ERq*a=6^r(VyoeOAOILELn)
zCo%Ypahg1Ld?ZhH#V9=@p#FIhOYBG~`#l=Y#mripQWG&5@9@2T8d_%6lWZc2I;fQ@
z=ZfwGOV8r?+^YQEMg!Q5YXB4;!1;PxTK;jmHsYBAwKH`UfnVAg=h7WKs`I?@hr9O5
z^J75z{PDs^A&beqe107P{50V@CV-f<;0JPgSs0+oTYHaI)G04qySweY6jSiUup2}L
z+v)}e492xbK%AS4%NZ%d)roohXc|Xbt}FRv92LW?qp9K2eTGf>*Zee}s?2UGwEf%)
z-O!H=TH4VcpX$EO1Z;nt{z<}9j!y_?z+gb&_m8jM=qUbluys;&Cvr#3lRSy{nx)SR
z1$yBv>lV`L
z2i`f{cnpp~6vMj(y}9mm*<76|7F~vXT!*F+5zonlizl*F6NP?`Y*)(vUrF>{Y8QwC
z1gU!T^xt|kfyU$C6HM!Q-!o-)!%Wzy$2
zm0W+by9h19#T}R|b&BQkUrl<(qwH64zuZG4yLU~G$_Ig8{JfT6A{-O(s>izPa54O9g{3%z!^HNk
zFd7ge*msm?h56_kV;tw`?VW{UL6Z$d2+x&MBoo^v`$4q1VXLSJ-c<^-PA`!RTc5S8g0rmr|y=BCYxq0}txEEtGbl(p%d(bf=o0*YVC2#CqA|U4d_fdY`>2
zs|xNOl!fJLF=WQ?D#0bmi51TMl{)9|{knsLW0L`TrFZCZYR#>G-IRh~iAI2E6M`=J
z{VC4Is6{{XpHcB|BFFY`BBx6wavWi#3JUB$L=ML#?=drvuN+PiG0c*q-3P&Qq?yxQ
zyIMBQlg}znh0NQEjx$rq-YQBL2ZHg_jaT_iNt)0rt@~b4ip#u@loV%i?QWGeW8|L-
zIF~nCsCIoWOn!}wlQnGyo%e;8$6xLY+zjJ2=T=z2F$bOR`+3{FsW4=dP(?8HL5ciJDZ$>5CIF++M5J`0n6iO+5|uG|289cbtYygetU%?g8b?-qJrXP!8*vZQBT!57O>6W|)mTJ*DnvDLC=meHcjE3bcW>`gMj5{^f(5ld&l0xDrh
zuJJ;zwHRTZcM)V-F0Ct40dOaNOPYVB;`Blv$~9
zA;2!`@e@WtTJ_?t=EAJ8K<-WPw=qze1uFHo_qcN=&yi`0Y>K>s?iUR%CgKb^dV`<+
z9u@Dn*^zPCP#PsnjUgWgid+iW9^ER1a!imdE!o<1ZKZ#LL2z-ZYNHj!Q+a*1>hWZ?
zFc!k5D
z0sn@p%oPU7T=X7xoh@svP;QBMjwB^F(`>0rO)81_jc4gl+vwu&z_;cE4SHKNYK9V@
za{k_h=h7EfapF;0Hp%Q2y4JB~>ZJ@a6DV=d#lFCYK620E8BKHbdJ#6OV57ycxVPR7
zuId36Jnq!ZE_oRDi$1^U_5K90fL+8a4<3KrBAMe;fVt#wOe{<&xY|=SuH{)7a=|tP
z;P0X?!qFW~=U_$eS)|zE!;^N+rBk3zM+#FVf0+-+D+s%VHE&?12ib_AanM-F=NG3+&bv$9%-0}H0SWP~onvP{%3t16JMsrn4V}B~
z^}=fGe3b4=9in-GDPtV62nR_FGm)}a?Ze+9-K{3iUi
zdKXb4CgU^IVG%-I2y;`2&PugP-?IinS%~in1Gq?lJj?yB%r4@tsAzb=n$vv-K8(_2
znQ*N*d)7TGaaK5DS(^1uoa%ci!wl8q%*sDIVf+6mI3AHe_H(s8N<*FVfy&z$gDL6MyZR=4T#oF>YARnb0m6N!Z=AoUes$rxilwR
z7aB}7ZQEp^7f=zuyz@}VAc?U?x$*eR8y#BRx?`2_&wtsMm`5x|j;)P36?g=?eXB?8
z&ebDNXAL#mWSp4%4*DK{$ZlL9-5q{+%iW>zViHf)=Uct|e}fxy)LJc%MS^UKi>Flu
zX~X?Jx~lmz=87rTrWje37K;
z_kT<-_VM4+qJ43={Uqfl9fVgcXp>N0^Cd1`c`q0Rp~KKapM`mctJnW0cKkQ}5-3
zuhJt3UTmuRj*E$>Me;7n-XizMC-VzPp;0)hh%z>(i3`K_k2AALE$w@vJu1e7mt_Quh7jDQ#Fgsw)-WAv;vt`j5S
z<)T4*<=tw3OE4wr!W)B!Y_+hflZ+A2e}9SKmF*B7qPF_WQ>Nb+6>wOsxd#jRtawF$ApY#xZGN
zMHtkl(Ij0cbDsmUt*mFB4%41T`NpBhXMFIebNY+477W6z?nNfYGs?t&+QEpF{
z2DKnYU6>r-$t2LsP7H~pBBO^npa$wV>5ku&J3fjPcS+Nf{M%u}kimQm5Ivx(Pkmh)
zltYQ4LyRt*qk#n~t-?5P0IDPEO6-F+xJe@P8W3^*vh^HEGWbSfGEs(B)Qec(%)alx
z+tORl_a1&K_3YZ8GZ+l~!%X={u2C5I>3UlonyEnmW4<6@-=V#`HC
zOQ<%eHrI+T2Pl|5_qP5x27-P)_LBsN*2h`A9rI(Ixc%WxGDvv!v#M1>9D@KCaw?CB
z;pR$D*~Qb>d&E2{DL5D4h>_yg{QHHA(VV)QUVY$ZunC}L4OO+ehr8|Uf87*7d)pzF
zqzeai+NP^L8+^s)fb-jnQQh1K1_8moI0nte6KoOorMxH$9QM4zQ7Tm+%TeP78dVtN
z${8#QHAl|>)^Tqc#ZQ`DnH0DGKpHn0)!JfbX97F5tL<@v#ZY+7Y;G$O!=fTj$?ym0^FGS^~eS53rRh6i^%nwRAF>|>F11k#*NM1=2Hszs
zvjyYMoq;1`)nZ#osJI^0qF)d3xa%4YrXf|Mg9`b0XyQegD1e@%<~0n*%;5AV9s;O9
zO%?i(_{GZk;)sxMe>yM!j3fKj&sXGIJ@$HtNXH3@6&3Z)e5RF^AHLRbejW~;QrE8a
zcj&uDW$9Xv##V|;uJz}??uZC_(Ib2GN%~5}_xD$CUv(_Y3DXF8m;374djKR6R5}kf
zw}p|eyi^S%C*-?<&Xb{M{5st!FgEd)L}3f}&{BU^PEI{AjT8}w5;1Wn1DRag)@}=3
zvYWVc{86d=qJr~giN5o!2@Aw!{%xh64$aP628Q>@35dyZ#YUEe@||cJmG?Y)G*CNJ
z6wrZk2-aj1=gkr
z
z&betYzQm2sElM4+tQ&W&R+=gd9$d$*hAesAkeVP~>RcsMUw>M1aJ_@&>SlYKM~?K$
zZM0a$^^hf4C=GkSSj*BICVP$0s#KQOk=4+HTv$|?a*A}6IM77I4{@u=Q-s5d0+JTb
zj0Q3ZxMS_Wd3Mf*Y+CL_-OovcB}?R$!9|N03k=Xfo=YW58#xq1UD6k|C8t`>N>OAx
zIQ%v;jiN*t}CFj(iD^bh-nib#cQ}Z+D0^7|!deK8fOJIieJtVyPCN
zcp@sg$UdI@j=H4p=Tr1QGFzoT?&|69S%A=`hGm1ltvmhr2|BMM$qC{hvjBWKd?a(N
z70xZ?)<)RRdz{WJ7ToF-L(f!L+
z?OcSTzZ51eOH!39wu)7Gz4~H-4Q)7jHejhUpfSqtN0lem^!^0Ohv)1bR_6Ln!_E|!
zf@`ll7p_VyGm5{(cqY3Gxclu#DI>?XPD!Sh;2b$7sQWfm?Az~7v$ZHo6b9lp`q)H
zPWfe;*5+-$G$wy~B^*WHI&r=1)jPxpnoyZ|3TcbQoo5i)x4iek=z6&By%dt
z%*?Z_$K69n2Zv!Zz50cC(KRZGn5t*t2O9na(}$Zc!Zj5Fh@BK12+E`c=lFi}Z7ShZ
z8P3o`Hsm)e1<*UFovFG;H++pboYGM6one&Y3;T#&NZC9KYWjk
zqs?z9XcCB{YYUK7^9Ni@D^??4ScPHA!3dNTFbPZ!8mUNdW1p+WHxI
zKn&;@*$vU^72uBBSuzw-id>eHP>
zy+NNMo6A&1Pc?sWRoq@m((vS4V?L#^;e40ir{aehaB&my_w^M~=-{(GA_Uls?cmkj6A|9ffofF>yR4A
zc>lI1
z^+J{uuk%;;*{+yVbq^G}FNUL7I*iG+{Zk#->fA)4k=-NLiI{OK6_$w5kF%LhR6L0`74Y7uCst2m8lrN(7FU!~0M7=QMsE2EOY)l^$cMU8ABQKR&Jh5*dqwHk^6&
z=(`=0xKUnv;I*OcD1KhbqbhzdKUN9HGT&B-&}5LAx02_EY@WS%oLAyne4s(t
zWnYL5h76>^Z@*63Go7Ena3!_5USlQBdguyxv6Nw|rB*34=j^@2LJuI+2VL{>9%LHJ
zZ&LWQqUCjJ2(OVu&&7#Q&&+ZzF(y+VbU6Wl>|RQfxsH6dfwbIJ(tQPf6$S4In0RqX
zU5pP^zl&oL(WT;E0NY|Ljm)-zHshbxzVjbc;NQVIsQDi%Akqfn{q^&Q(j-xr1^)5;
zvg7rI=tNF=zUQb_uHm#9E?2zNlqeiTYpwh>3}`Tt2R{2cS@PzUF?!Y;T;jYBuNvp7
zDmtUdGdLQYMw-kCZ;pE+*zhE*kK?LA470&`zRMDoz-PGM%E3pTn(t7R!vvaKW#PtFxDbGN|q#z!7jmw4JRyyKYu!pHr4LwYt&Q^@
zWy1gD04mh?dg%(HV2H~>-3kawwfky{Lks<~R7DH7q2hSD8K@|pDj~uHd1b7ceAr=q
z@WmvS4~{LEsqZ7wTy)>0u1UYh+a&-oS9CHv7P_wLJ09Bd`8KK6)%*@4PkNdKQ
z;BtNb12TSk6{x(^enrEb&3h&naRzQPXqOO^V#Ck%vjeworL-~Tt+BUO@v2;w7d?O$
zyQt}ACNvXQc<`?J9vhR}iv6b?Q1s>LIhTlPoGz`jR11CH=DzzV!+p8{yomTqzg<0T
zDea@67rCcQScHiPL@acKyd7r|w(YD+?UD6lWWsU_0gcamGen=ut
z?ut_~#AuK;MV`wY3udX2y5g+88kVDLz2Lc#X(HMMzN9bY1!VhIpx^ComrdjJ8~%U$
zdzXVG(U%Z7YFvEvA8J`;8Xe_1B+X(NoK8#K#aWgH(TZZx_lu#HU$=Db9Pe!DB+zmE
zk^hDM0TTbJF(roa|Cs?^EXY0LU^MM-9e4u>X93asQ)d7uA%&eID8RRQBAOm?YI=mw
zg{xn+R&|`2aIfx9SAr@K3t~&o%Wx8i?+b&_~+>f%;vB30{F9hyD@W9
zv`cwS#chmQ1;>#0vp0}|cIir^D(M0QW^GQLv_+mzs+YviZ1bpn!Y6?X9fE@d53Ht`
zg?cled5P}N4L1Dbd;EDRMm4z1qUwTB%ipg^=t3RlOIn9U8$r$dQ!lJ0)YU%?6@9W-
zu>WyqCxY@t=x?FNx6?ihb|{|87vG&teY_c~z2>zBfRl6k(`EPS<}P`1kx3&$7wo=K
z;8+gjA^1*UyEFBv2~`4~V{M}VVN!r-`}db#Ef~V8Z@saO836zdm7H@ZA@W>%HlFHS
zkUr420neA2MnZMs#wl8kHh9R#&QW10&SBr<}T5uUISsEhq>l=S}bjOX)vBF{z5VN!NQ*XGP0$*$SJVlOOo>QFA{caAVK+9t^+&}WHV
z+Z@{uw&V+#x##V)*MD}0)Ss^1>o}c$w&L+zW)K@|2#PO#q=HV&x(#O{fMnQP#td?D
zfWRE^%ovaq9}x?sd2uGX%!rkeJ~(i~n>&QE
zsnpT`6#;sM;t$yAgPyY=cj@x=VtVe%zu!mWc9G8=%bL8Rfgc1l%K>1P93CICX|1w+fupDfE!a42bwQM1o6YxLS0V%iA!dUEEC4r3$&
z$e5@yZKd^5X?iTs{pit~+q?m1C@>L~RS<6?!WPcC1g#v1cj$7kfwOTa0e}#ORkFy(
z?N%UIzUJ#XitPlEAZRv0>=&8KsiXr+{+xHmE8N9k6E|i+X0nSUFarmZosv@yBJeyK
zY(}cERIjoGoAW^K&ZR)E(roKzh4dsBmrRpjybwC)a3z6o2Y?iV;dr%y0IOc
zwvp9Ng%%JkZ>p?b`%=x1#_pwxXP`z`38T1PfEi*Y^_xI5`>nk1M9
z13+xV_K(oM*PSm2{bq@ZQP~Lq(~998FQy|k0?==b{}+329u9Tiz5&lZW0)}*#xgP1
z5E??(%-DBh4M}5Pijbs8Gh-JTYY2@sv<*qxXOL`#NQ#QGwrW>*-QKV6dVW36?|6>m
zcO38g=ll2kH=obqj5oQ+;{#qx;wNu@yNTkkAKp->+Z
zSUjr?GN_i$ZlwgYhJ-ypz#=$2Fj<3|3PzK9#!EG*Ap~ntfXBABkGd6Rv1_!{&rOOs
zLqCyaN#~!;O`%i4{tt_qO)C=yB5V_UdctWZrUQX#adnP0Jjm535_rf=aJ$dI#(g)P
z5$jxxbX#M@@Zn0JO?wbG5lMWMRN9r(RQv|Gc^fI~CyPOLJ3pZQ7qw&hk!PUQpBIYz
z$E)?bZ#nVLl+ipA{;$4eLu9h#ETA-$5D2_>(N*zcKA?aBd=g3)k%C&mvzDdMUSi8ZSOB-#sE-taBEvch?MS(QK67^0FtR
z#&O3&nz}1At2*%58+_&`%{ym<4&nx)aOd0zGgm
z)GW3cH9VUX
zT5E3$2SZ(6-9HB>ZlJoVhNE#yP$HJOVv4ba@{XYJ7!0%W5dq)a*#f>G8N6Zvl4s
z{OsZsFfTnLyU)|Sgj?@ap-abZ+E&?PN6+&%qj+;TKyHg*@;1{JDwKb>3h7%Sb#TeM
zoK|jh-Pf(@&S=xIM~(x&9fS9SqfVx2wtajx#@gI@qiUoY$Za`BXq_ov&J?hUwgdjp
zZ@HN6ouNzi93Z#lvGU(r2H%MEqzftq2OHel5UA5v7^r@5~h!#`8Z#QiFd!bOPUh>YD|BL**<5zdxXID{#j6%{f34P0e*H!Qee8
zwf7`cp|Vu%H8wM3khNL7H#y7J8p;;mD>`)f!z>3b7ll^#{it-Wf_l6wH)1b4R{*~z
zpjj3W*`IG$_vBT}zA#TIaTp5}z)l77Z~7(XJNpzzAcEVxRe6`r@4Df?wK|=Bkqy~!
zgU;`2C@T%@J*8FL;JfwJO&?h4sWWulBtzK0ydw
z!&c44ye06D+}gJ7@WJwJr`+ET6x3b+YxTp02S!KujXQw+8#*K0kzA1Muz!RAybUum
zsX`KTPSk530KrQ7Q&<_2p`02a*-&MLSk1E+C(KCWFrZc8cHYY(TU+_%>i%1oGX0-d
zc%1>)|7jPo$N)g*3~8MKM{PBoM!EK;V#G0SF_gl&49F#e=z57+R>o0G)677r(^RVl
z1m6;G>tqeZN^H=WwL0f#lrY(jvb}skeo#%|!41S@f;K34YZy$x6snn(fKcdC*_Xq5d5@BFXcn&A@_Re-lb@Zp7-kzno3+p#B
z;7yLo)|Am^oG;>E%{LxRk;T*
zr4l{R{c_luM^$rR>SPMl=5L$sE(#VI{~*$ty!w0uZ-XE3xI4}NFv$tqG#NuqGV-+>
z68`x2Rxb6`mD#~H(2u0KzuqiAcpPTTM9#!Pu$CZ|uFK-$)`%5R7xFvIB#F-2Fiw#<
z+O7>vLcbWFID$y$Qov~Mu-OCKLJ|w8g8BwiM+u`y=?q<8Sd}RdJ*0SS5x3%%4~tk=
zAN~|00_sDyB^G85h2!Wz^bnvb2P*}y45#X{R)*muKz&$F$ckBj6Z%M9$9GA1h?K<~
z5?E)dViv0ae+V!Q1(>RnhtPjARsZ$rdhy4(|58vOoqv^rB-5XQ6H=zYdUqP;VvoeE
z>C>iXc;Ix5jFy6HXRCaV;M(8Xs*{w#_K0+BeLK5V*&<7U5)(ne^+3qaueloWW&>SL
z5*z$L?}RNz;lAzJ#yuX&4o#caBDl8qm`F)IzubA=httE{H4RifE!AFH)3<4y{Zjme
z#W7Ml?neyCS%1&i8L3V6oQNGaiAlWgixdt^msjQ^c&hznS_ykL{|ASkj+2Ws*z)4r
zKbX*rp7k6Uewv(iUFiw
z-hB=^k%FA00&uh)P@;)6TG$C+tNBt%5!P7OP2d#((K(2f&gykip`q3%TwrA9szNT(
z0PGX2?&UFJKNBESA>3;Y6+*xmIn)83I-ylcD$kU
z5uRv(XaE=}MtP_xr?Dc$Sz-a0TmoIpz6qa=D5;+Mjd6QZ$fMPX>y|JjnBGElmWn#9
zHrEa4aQ`S@tH4qJ?p1yrHTGX*GyocZiDe|#$>_i|L1_)&IgY&i?feG(9-e9&DN{(=
z#i;#3GYFZZi3ugE70Fv>lj2@za9hJn0YOc0QztkT3Vg?VyaiaNT$9u+_NX@1Fpn8m
z>pwJnHxuGmXS4h7ZZ&on!n*HF`cx>UtQ04G*FO)X&_oeb(~oc<&q=ndlhoW$jGm%6?Nz2hjO>W0o;o!NBhDi8n
z&rQKo18QG2K7{R>deT?2`*x+Fal-4qK90&?@9qiXGH+ugVAmj#J+`MJ`8||OEp=tIlVB2ag}0y$#(*RhiV{G?EKaL
zlf~h`J|l3*dW-Upiuor3;_!vxD4$ji#AWhK$e@a|1GE&b&rg;tSOE9~Vt^Py%g=}`NJ$R=8W(OFDH6P=
z$u1^UP+IMqchVJ7ccviqtiDDCa}yOKDcM)vi*0#W8YzDle=sK1?@Gcj;;d0|MJ_sO
z(>WznEr0q-$SEye@uer1!Pe>Fs*|g+^mg+P(&uQc@rY*k4bc3`fEfRdwQD5~u@w!G
zeW5!ty^iWBWyOeFkZ)}8J`hq)5M6_A3G57;ICo!x@KqQY_RQ5xl5p4{Hnpy5Ir4D;
zN&M~Zp>cxPo)LtUu|8Hm{8;q<_~j!T^~KUwppwRZ4w5tqG~&87C9Z+b0Y}P(ukbYH
zi&Kv(m)E-vqaj2=4Xv!fc<#1FQ~w??*sajSKu
zw{~!<##|xOheMtR-g%3Za8CMTU421zc)QPDWveRVq`!rXidFBH!Zi(OWKT6(v8BAd
zN#qIzL#dWnrrpz$9;xs#PT6Bvh}jY~)$ya&^RC8NyY&uPQmW?8PI~V^)aj&)&38`0
zlzA&X=X~Ue+rF<}JgT?bpM|-n6!awmkL244
zX7gI{htG?ktoru1jaJuYL7e!f-!%Hz?XT{)TkSof`XS)e}^J)-m?xWE>?6jeo~Bh?Cz1wp`rxmVvs5q%DNDQqtmUHQ1geqg-~f69s)z`S+PJP
zbODM%%zNfZfL!{z`-!N_OnqzDLv%xiz&40V%)vqeT_)>*6OYS=A-wz}_}EM9?uD>t
zG-Z$Nk$?knuPAA;a-Rg9iGZXF%$pH@IuJj8J%v6JN^z~uN+IKHkE-V^t@E9%nhZd}
zURxI0qJ}!2&sqLAQbGYZ|NrzNB7bA`bqYd$g=3Zalh)Smg6FIm_+2ZD4O*982ID5_
zb5xwN(>6b)I&fd9mLSWULsPMEgl%gZ@*!I=tGCGyG;!a{-DTnNw)dxarpdr4BvPapA)Omoa=958Ls^)48-P!jcf>qz*K;W8g1DUAvKt-$jhZT;4{#CB
z(@#qoN2rmB@F8u52c;4*B*Cq(GjVcu)jfu>C2`j~_ST7Xuoe9b0w7pj_RI_nd9k=e
zUouw;aW3wWdH>e{3Nu}}0la(u#U~4t>ioTuqrDDZ#P&|Lx{Rq3P6j5zBw)?qw${!q
z_8WD+dphsC#9NTl*X6{f3)RQW_f&Doqu#FxGYi1SM0RIT>!x9g(9?HBS^G
zv{w0+V1y)vqVlA^&N9yK;_VZF#@a$qDVG?Stv3%UdXnnnlOUEo+YT@eKuO8h^GgKP
zx=ab^QUf+lGPBe!@Z#*{qbO{NT$N=N9E_BB0AL)xb72^yDhC|*GJSQI71C~n2Njhp
z-jEg@Yg6nT&|Bu_zE`xq)Rg$JZn64M+<2*d@C|{k|EVT$f{;F&ql70#fMFhy5eq7yuZt8Rgt!qQrscWAih34rH_RP$!_=Jtl7D>lprCet@kmC^6~L7IGT`
z$i=fF*bPouaIG{(|A+3O3e8;-%HABjThZq17FZ2v%spRZU)%_{cy9QFc|*&`Z|)J>
zPllv6`W^bX8&EtWYcDRxs&4PlecbZnl6>^oX_;La-uCxf{LjYr-@`_>{FW}ueGJw1kTQuSY$Ztxt@VKTnQQe@%!deo`ut&gvv
zkpD4{QU3UweqlDW?QeK)rT$AmAj<9mrV7=jN}_7Mzu>un_sHaEfQgd2j^mk0l3-gz
zmSnwOGgsLn$<3$_ZwSD1&F4K}QulS5S57DMfj
z#K`fOnpr4PNTWNYl9E75m-grlA2o+(!Y`S0AA+eBaaj73Z9XSdUeg|wum}nwVAc{6
z9>K7?AaVCZ{kdI9=07A0IS7Jza^J&+Jh$zgGV`o6eq<;N>g#pe^&ywnP-$*^MA1KgJA}1(*7S3M;Ue8Ym<@@XE&Ysx6*a
zED*0OCizj!3kg}AML467QeLH$V~MX3W0F(_{IrZzYuRX+nh(#($n{U4?)<+G9V^)M
zOVwWW8>mZ5RIZPbdLRToih?^ynpn*3X$RWN}By(PFV^hs)burf}Gbz6NykHU`SITl0RWgGE;TO!t(5
zA2uyB26ypu@ae(BX&(?fjXu>f?T)SKT{pJrh`muOY}T6RS3ep^4ZOv$kh~lLd*Mb`
z-0|Ywl&RVm7pP3-ijC1!pDP2Lb#Tjx{m-;N4J&M?aG2O<#X%UQT(%oF`0bAe1)gG2
zZ488LUjfGWb^RE2vz9MJE-qNS>lO^aU=*3G&MvfqJDz1Lp=8OyMeGxwpr9b@6t4La
zomYg=WizcGXnIcCirUgg-~u56MebV;^QZ`sN&P^}xe~(>_@;-?3o1X4N1U>`B!nwG
z0b`Q+D*vQ=)M`@^P6dW;h!}~Pa4?KwNzxAX9C#)guAxSl`Gp{A6Jav#B<)BBC~=
z+Q>B^i$&`4Jk^V~FF4a4YBUO%%m3$1@cWPJ!JWy3to>pe_rWQEV=$J^v;u_1X^FE>
zZ(_s+S(K($B}-8$lSq_dj8bE+vQ%90>SidaRL)u}XL)6sB`E6d-Ps>Er&4AjKQK8n
zs^VN_A`|zqtV?ZBP>^hWcvTG6P+@5i`1tb0>U)^tmW;EBHN3{QP{H``r#?q8kB1zF
zIPLa)Wz`wBj6GZ-9x>F+df-MB>^yy&4^^YFxo-5SO9urt<{fly&;9Nl|$r1X{7b^n(sv>2NmR}wgJY=o=9hXs7efvE@ytqrPjxbQG|jU>T-+;3K9Bx&??OJ@j(kgd
zdC!(xU`_D@JGR61NSV6x1`sG5sx9%<5^Bc-AochNh?0MS)b7b+1GLOs0Zd3O6(Rd6%iW&Phnt7HhEhVi3IptNwG2OaX+1
z+j4D31FZyY@saVX^$^rC2~B+V);4*|B2_b$J^d^5{bg{^dwaa!hf*Hn6Yi
z(;f+-Q97*L>6FP=kvLMCbG+JJKF_{FHXo(QoFTRv=~!hz39Sv47_;^)uI)?Une
z+zj1*AoK@^lh+Z~9Jji{S8|XB0ta5?Eq&l^=*)!?!)eqm0W50lo}V#|DG@x+rYaP%
z7=SJP{k>@i`5yU%Az;Fc?K7HT7zufY~KJ^_feR4#6i;Kd?$z0
zw_^$5k}G;|FyL86ssM&)a)()>C56fLmJ&-|nZsfsaMXBT$(u<5ft>qY#HGm{o2-8d
z1ZT_KD51-*zwjXkfVqtEvOyVE)5x3gBoJ**N|VcPf(&Z7Ik}44K8)xg9zYndyG6|^
zX*{SrgG?2%Rc@%JO4|YeNn);em&3K$`bVK+@o8g#?mS?)5D4o~+Gpd-=w6o#G?W+w
z1II3n$H{?#4De8}m2-SBmBavKfk1NzMn!jgU9gOw_zJFqN&&3QT77?5#o-CHB~BXF
z$h_vy@Au#z-|s)YG8MYGt{S7&m^#9>)=yZ04;cpp<72P@8_ycGD&2pqW+rmDEY<{}^oTySb%3=FHYW>A)
zP9J#u=783$ep%Ckqyul6r*=QTc9=EcB`i~^#k;26>AG;SLu#Yy@o(05Bv1-N>Ec>b
z51z-RV=vw)H0O(P`VNl8@hD8BRO8s$u&Z>G>T>aiqsnaCT3FP}hEO&FA6)BXjC+)7
z6Y#?7VkK3Ex1P)szuJSkscc(nB}0#)97Yglcu09)15am#!|cke2e(kgDFb$}I@`w&
zy9-R$Q{FY9*TNNPV)VE{;Dq~?q{V_k_wvOXrL1^d75l1ja`$?wYEU@W6bri2c)InaoD$F#`TO{h=Cb=s{44Kt-<5&@sy2Zqb5
zL|6G0-LQZbcsLP#AMK?cyqp#xMe{uq!h*kTUdN1yh6@e(vs+9MMFi)S1|%
zAJjHv|3D*;@krHW(`5VEX<2Doxb0Hl_QRl%rati@G~rDj7>fz#)mCRDOukz7Oy6{T
z0R$Gv4R_Cvf%4L1T@;f?C^8T0>jafDgWH^ppcC8uLQJYi_fh5SpE^J>A)QxyEHVyD
zPUww5w@uFqXq+PJSV^a^y7mNr*{>@>#Y(cxMRk>K0W7Q|$RnRv<;8{x+mZp@^M=H7
zmt6s;6yrR4)3Z&1eE*Q+>is-?2~=}sSTnq_Ike)0BN>beP60s##PB?yu*3zxKN6UM
z-}sX%;pk-HR0RlGG*w%9f$?yw6j8h=ie%!uY^DIHh0Km;{HX&g{o}L$4dqRLKkUvb
zzo5KC&MzphKyhsME=WyNKpX3SPn1`(DnxAjNL$9$_7spe`%(%E8*P*{tro7S%GV5M
z+Ij>h+Ob(s0eM{j%<=dc3N{4@<)#43LIZc9w5EMI_evnD7gElU+`1_O1{wuWctiQc
zn5oUZLd8r7f;=LhQfs9@ZUqj!
zG@kvs%JRuBN6bYk+~i>pu-rjCSXe^D5wGYdL+5sDT1rIJ7no*o`Ns?+I8-of-OCFD
z+yU@yI^|<#B=*QXxPJ>srR-aX8oS)!FvYcWXATg#MM}e;vh`VC?%({bOE1;JbQLeJ
zJik+EF-Kf9+
zI*`TBz{vaRjY()tcZ8p#8>q&-*<;$WW_|h7NCxq8%(WQ3Ai~*<^t7T3L-w=eTqNm$?#}NS#@L<_L$ZH-7-WvnO$I)4fK3Oe
z4<%?fEaYp+Oonb2P}JwHmtiDjZFvt4M`G&v0adluuFm$$Iz|KZ!E~E8~y^OL~ZncaGW!Jo#a2scCeV+Q!ODC|qF$gh>Xg
z{8as?)M_(PyJruAPr#MXv$Zf?z%nA1);>`msIajceq0vd&`%7o`87ouzD*9bEKrx3
z%sKI{=GqS5Cfjqpuosbkws37fFUG&6dO(i;#^{1N-`ADlR%t&mdQGiWDhU?}WH~*z
zVabc-$9un5>Vs!_wE+{Jv_X~>c_6rNbm0z@B%$%BzeojzF@rC7*tIX3)tkAC9uEI_
zvcE`e%bWNs^3CJT&W4HiAC8B+F?S74T+1nW2;nzYqG*M{lGuZ;=h(-{vTiOFWH9Bwr9WkY~V#cu@1z9Oo@@nSxQi=qtdDdlFJN8
zhQeokqxQ_XE}K-*NLp8@=9I3jU_z7M*Lv3N98tOf5InfkD6SF9nSO{HOgo8?3Fdog
zN#dn-=^lN3#n0g)B(!e~o#JpgiY=rn*&GnDJ&V9eVUv`wlGgL>>GoqPdHHCgs}n&!
z;4~f+`6BH?qo$uh)WjL53+)`6flHdZ)WgLzC3SA!VQqMxeyVMTUoL#4Z^j&jHC#Tk
zp-a98DI^kI-WK2dEfs-8^6F1y+-AUpBz0iDLsK&dL|1Fcjk2W0H^-qTZBA|*foxxx
zGz;%=*?1WIY4D1>ai(+iU&yzo&VP{@zVZDa|NVo#Tc7?&`nKT8MCfq@CHK+oMmM3tt{kcF+z0`_;s$-Bs<2vDDzL4e
zTHN;p+T`&KcqDL!Ufj)?rhLJ|=pB;W0UyOaD)U_IRPw71pVcM`hFqAwyElOY^?Wk$
z31rmizmPLhmA+=7GV!e`YCaj)blIr(?jkuk{er|kQo7H{l6|+PWs=ce$;)g!xOeE3
zY|>7v;4+F_BHPMnw#khFhWEX5GA!@pdLDSH-NZ+A6;>ChVvQ){6nd&)*`8L
zxg7ZBSM5d4#;?4eXCkHc7?koDA}3N<=E_A!MB!Rrrk+zT}USEoY@RuH>S!rr7T0H3wARH*N)
z&BuPDHn%$46HuL9-~MmUz!Tu^{O-7a0W)aU@3&fHJ$EmblnFfwJOh%m>()I(ymx&J
zFv%d8JHmXpl;%PeIpYKQW?UQr*O+8oX19cte(6{jTKg?E*ywm8T+KvGE%c)%}x>(0NCleIvOra
z0Afnrgmkxy-&|68H@XHI#!9UFMATCXmGq}X&ZggXS4axlH@=*W%zE>z;p^KE$i7$B
z%M22`JEOqONo!#y=oiSB%R_vAT
z$7jngomU*=(V2JLjkij;+?ZjtEk}sP;|?VsWc9v%qY_89T!fySSJhAz}5kl5xMxKkX?(Q
zpVB7GWcD=Aqx>417mjZ`ySz~GXHWg|e|`T{EIy&od*bCaVrrWzvVIeR$pM_cWYZa~)3J;g#n1&Rl(6_nc%S1q(eENjiYUab`{K(rlK
zb!$Lw`N(+j{*P
zm@T-PhUj|k_UWP5IWG)SMt2+B)OZewk#30u`gflzernRmGoTZDO_Th=nVJ=vewW|g
zZYQsmX?@pr#F@jrG7L(vIB`zbb@QC^^2b;kmH!-bUicc+98E$|DK;d=x0y
zv)no-IRuR&=P|jHRCH*1rxjRyqPTkqFPQALi-g|8sYsUx=i{-0Jy4
zg81rSRKYr8hsX7CVpS}TY3!4RiBKDdi{a6KPtn;d)+sdbfAPkahAQl1)f1of6|33!
z5b1ridyB(hDa!ZVE8w0C%vqp55TbXqd+Dk4=wTiVhrPE6Zw(7SeZNl
z(>sSe9_d@41Ln?XiqtCh7a6G0D|7{8c}|!TcZT5YT*g0$9X$48D2ob@w3R;{t!`+B
zRmCL-@R#a3V-I%XGC
zW3p_T>z|q}y#Ac0#U{t0lqQOwJNhp@4jr>>)a9Po`np}wKkpG^mWt3J9uSX?{0fL*
z6nk8=HeQ&C1FD5LmkHCXhK*Yy(^i;A)&ab*Au)zW75J7Y>h>*9i4PUFWsD+?W0%H)
zUtMl@FO~@qKjlEc$K;zPf4&9ybi35>RFWgNPcuuUCA>CPgMFVP7fhQ@gVF4H4P_D|
zyiBpjJ_Yr_KDrlxW$zHPxxNix=j9TmMQw?WfY=j7VpCq8i9Fk~$?EE7_5TpTprHRF
za9F=7Kz;`_{dZZw0NGmrguew{A8;Rt?n=)V*A#`d(%D^k7~>RtgmyP8S5>M-s6URL
z0Z|DW_-K1!0a=(HudhKGUF?HoIlnL=%+|r{Aa)CQqYqEr>$Fa3Rk3Gx<(=NP6Cr+u
zy>#=4H0Oz5y{^jGu{8rq$s>t*$0o>yiN}+znKAOJT|k
zNy%D%wzM9_Hp};3Yj=5siADk$jz8ynm~L|4q2sNYsw_&K8>G4Os>e3{qvC$UuguVG
zQzYQXOY+gVg50RbA73FJrk_a6BUT%ya6WEW?VSe=8e`y&RU(6KmwQFOvn+x+Ot)qUV{qeT=AVOs
zWsKJgI8+x=fLDk8z`(j-yQXWOSjU>iSCgm)buqlBfC3Q9z?K`cwryp_4*bbB@BGP;pn=>G}t|JO!<@W)aA
z(+E%njekXYY!v(34$yCN(q%udM|yA}C?<64o)}7-67s0PR{dlHd1#-oP^aWrsOE_d
z9|Wz}y?{Zv3jq4ui-vP7st%)@haYVoh9x^j^1@~hdZQt
zC4oELUGP@_&ZY!TP&@mj{S#v6hs@55<+zX?ds~->xk51UI~5`Gy~(Yjee810uudcI
zhOv1d-XAAr{UNx2G7xR~>da$T+6*O~lzrmLxh0a*<+wXKKD9>i3bs*~8qH{o*gl!7
ziJ!apyx70ZKd^+I$JIU}3*^Kmzk2yvL0w}2vVh0{A3hUhGeaR-u2K<_0=7PfV4?JC
z#^(Ua_*?7`(692xvHb>agTIJ$
zcYsL$$6b$X(wlBtkzfju*4dXDig~_}oVAeQy=Ra-RjEVQcx@rnB5lH>t9pz^+Y``T
z3x2CP43|_qkp2ZsgEN^1nQwHBVmS!7Ir^Ycx@sOo0FsV9yZz%560hOO7}e^6uH8LyWr!{8r7siAlRda$;yzK@8DB}iNGL5mb{+_77V6O#=i7~@u3p<
zV(>A{LuCkxBGMl0C>s(JQVMZ8m~Z-1boQP36^s(ph#Nc+(<>0RL6CvF=b|4%F$X87
zEG(X^)*PWC#3Dk))``{B+H8NIwks%oYpP23iVUZg#;v&6DzG_=B?o-fUm)?>k&!hu
zUVjRlb%DF^pCCT<_iHVt69eRL03iM!Ia2pcAb%rtdUB;wNQAhJrO`=O>dzE6Y4KB|
z)ps2ic8M;t`(QD6!eH16^lq;T94V>4PO369g!N+=ESuj`C5*{R!8LQudD+NqJ8zf}
zE;Lb~XM3rUAyHHMQKwQ2c&PX@&L!~3hC#2pkJ{sw2Oev$xU1i;vC?Cx0z`?iwOxE4
zxHsP;4-#lsQ6DJ}eIDP(QBCBK+^_n&qk^Bb!SZ6w(1wOyG42d7q8_kU=~z
zI1;$AIU`LN5iRI&;p(~2o;9*Y#;tr&A;G@Aa$~iR3||%z<7_7g{-(PV5(ZDmME?*r
zc=i5vQ5|9I@O9@exw9u-hNo(1wQKa_C-+bGU)p)tEvochEPN$!Sml9yX13`BqsE^V
za+%FX3b)VNN^CT_Pmwq}?FtKh@~LsmT`rBo0Au!$0NEj+k$|TwXL-9wjE3xTGZ`%o
zqKUvS-$NiZ0@uf8{lTV!i9K}`8I6@&|ExPWfj4ZRC^@co`s;Eu7@orN*_8sw+1LGW
zc7$MDSXCUvd%QaOv)yR5Y&s3^3bgyI>LitxCZD7FkN$7t6$)?%{^gVV<)$a<4`uz6
z==M6GnM0l7rAi9Kt#YBYx3jzz7~C>#MI^>pG9_8nwf@#oWx?v=+BoGt0SVh)%d&tB
zr&4*F7t>#7=dej;)GPR`*mZ%kYD~-wJr$v5SAq`l2EWD
zJQ>TAyr3U_g6lECy6DQKMbVDJr?ogbnmY}GZVnvW6!tnlH9zGzx}Gr$S4^0c{JJvQ
z=x4AyPWD0j<%wrvNTDWF(AGhVRD`adHum+Fwh{W8v?}Ssq&xGH%sDRfhh)UPdlCfI
zrE7Vk%uCo(A>q9H9ah@3srspfB}2dEo-oO^wQN65oX+e(6k3P{DOtvTlU>{GN`-2r
z@3bmmcexooW8c~M&`i*K2BP%ZG1Uev$ar^Wk1)(u_V8ML4G1L|T!S&b6I|H8`Bkz?
zCL}^Y)niMrDw~Q>>jP5W-u2XB)Gid1d73sDzCR``ic?|=MBrH{1-dH#uI_eAgGcs2
z@;Vg-^NHbspg1xE636Wx62OlHZ`dwp%Sk21QK;QzcZ!P!!T50%I%2cPI8XdxaowQW
zIqz;~U0;v=#lB2c9vDftk4aOTEG}|3oSbn(g5^I34+9ZyMr!#xRJnYb)iV7y5FMy+
zTNpw4$``s+4@l5j?y@WacECj)3`G#>JV=zd5(Iqh^NUUA;rJMEjn+bt)^rk`=N}
zAgCJNp~n`xDx?w7>=`(nct*NZe4=z#YxMw`j0%ykivX9}LyX^y$
z+LvPsp)yzZ99MU_m1}n92eIOP_)vvw+>^f~KL+#b^-9lXR_O2UJhwSaUvnz##rO|N?A~+FZV=dI>Wcf++0~P&!X)Qaz}+dX
zXgfEMo(jKy#?$yMbA~GMsJ{I^wU4lcZe
zXEXgN0!rfw|Acm9+jvD59e~FPo@U1@3oevefNWoe&pVOUZIU|I)KXD<3uCB=y&cU9
z)s)xG+be)4#=Ih>MU7n}F5VLUZ!`M;3Ke($#2kN&Z~>t^67hS4tHQvk0!_mbQ4(2;xD4dM+Kr@9Va$Z*8MG5$$UzJEDpJp$oA
zD%q0Z@Sr1p>IFV)>O_oE}fdDd?WW{WRZ$s
zb9`UPqU^zPLxT9x1hl!YAT@9%HI>jjzAOpJF0pX}}eE#RcL3*xBxx
z3A>~gSO__j84F_dP#jv(Cx)3NXG=o_9=ZDC0+=c)8DlPLDA}++(WQ7hnfa_sp+|t<
z-QzCpaE8bBKefnU8qyF*eBiM8?vCY9lTW&25l2H{zCgG%2drp6RH4JPiSLC$Py{?Z
z99*qj(NseiidZNmSgJ4A12ccB_q7oF2gnnmmJc|b_xjH;MNzhVDyNQekfN3tE1vuf~S|b&n?*!Kz?_22;Z(4>3J?$@}2S!ODAk1bt!IU
zU8JK53~qr1Oom|Qgm;q3L(P}4MkoZ|w(ps5h#p=6Nde2uq
zC?URAVtN@Rf&^!oYQ&Z&UHjEuo`eai5nC@<3uGSCPr5LB5V!goSk@)$2rk8zW+E_=
z_3Fo+?xvlw-@zhVTT`LrCj%ks6mn{5d?EPieP4G{Dx7_tYD8J#AG;N&0I|i>;DTa=p
zE0=>J>sy@+CX_DuMc%pG4ibOF9dlGE|K5c)`~+^?xi^joV!_G(RFw;Ps`dQ=Ptr}5
z<$)v4<-rxmg{?5-v2bnv;cEwBKMM*73
zYr>&%yf`2h7*uPfKj%o`iP^PIqNo70{v6jGtKT{9Nv0M_aVOK0D+-jT&q(ez%
zNv%X8&KCk=kyEwb_|5QnqtUaS2#3f2CfGX!+?Ib0Cw}7+w8k%T(qE*H6f?s=3KP>x
z28I)7AyWFNo-Eh8>-pH%k1K$fsQy&6>oz~1(fL7^c8Iy3RNS0XD%$16M_bN<6|hDG
zh(rP}T34z2pvUQ4KwVXB@|Y>ljJ#W;yUAX)Y37*r?k8P77S_Gp8RvECK7aqhAt~;e
zA6ULuvi(G%Ff~*3EFcp)7ByS%^iF>6c63@h}Ipv@;D&&_XhYAtq4{$Af`6
z)7qKQ`|M|gu_Y9`iyF;q6_>fKInp@YyK83%Vn9r{vw0
z$R2}`tJSs==cJ$r`X4NTbCdiVTIxx}u%jeU;pJ;COlD)-H>IDD6I^}2InnY$%Wt5E)czpHov
zZ3IeQb-KxNb4b~aV;(;&X@wZg
zh}=2aC&7xYQ2JnWBM;Qf~cYy`szkvX7IW9)Ppj%N*b%
z`)99X_&<9zNPJ+gqq<)oPTnGQO<|kIQp__R5W(g@p6IG@J5NsAepPklscpWbML0=q
zvBy47#UOfF{L#h)K`~2SjAe(lkdmD;m(!v$L@V+i_G6^l+d=fTEZ=lpWa^X
zj6wa6B=`Tr6VwL9fnp&=KO_IwS39oAp>PBsZUI>B*qwosv(L4St-X~axfxgk5?gmW
zpJbLJWEWT8U#RNR;@BV8Fjz|7HIQq!uklWW!4}4d4>ydDQ=*|Z*ta*0*4Y-R?;L1v
z9y>*=v#+wh*mD20_l3yE0~b#}I8X1+@r-1p4Ob(nA9-C?LuHp@o)1(xTsrsoYT}0%
zj|VTEe|nPvowBkh%wcRJFxP+uc%J8?i!E)-)}dh%DR;q
z#w2h#OUc)PEP2v>;^V*4Bd}$CD81^FC0|L3vs=udKWSdfHZ~|&sF;2Y
z=s&|-FDcgG9%U~v4r;wvf^Q*fQ>ldHSyo-yi|mj_u=+qmrB#a265MzX7~BxpiEC;)
zBdZpcakgu1G2+aUp8d-c_0h3Q6hi%#7iW2eWzWy?OSkTBefBWtOi6vHwi@+%%avE{
zV$t?z6L0$`Rkb}rm%nTtX?&z|Frwsy1a+Z$D(dw4#^Y}~mwRPG(_hrey!Cm1<+x_z
z`_iLczR%Icd#TIlJ?E~^Bm879J^wnjlmF4ElIwz#Nyc@3@U7J5f3YT>?|P>9*DH!x
z)^1te1{#QkAlMAWU4NNkbmppb{k6EVg^4q51Dn^Tbhn?<~F9EdOf%1y?ba;^g3=dAncCr;NsyOw0X+qq9u{&{Vk7H;W9w0{C)9p>b`d)FgYb@gd^M^YO?(ga5ZM1dx>NiG(x>Dhc8MqU@wMcuZl6)Ao)*tz1
z1g5as5N7FRF>w$9)oQZTYZ=W;H)@wIk*q}}XbWYk2NAB)%?WU{)mE^UkHid9Ub+Rl
zpByh4u*85f=vkTD+G_?--G|5fQg>>^OAa(KhN#K%{Kp-cQm<<}W%Eh8XY)(rR%uf^&>O=e9r1?tI*
zR6Z2J+^Td@x_gF+FgJ2*XsN5|PE3UvT_WGFLcv4~x}7!!fG$SNz_#5wy$LRn>OB1s
zvn7z~aK8p2&UJ)hP!^jdeD6iRdM0Fqf%s?EAel`(^nj!_-Gugk^zDtO(#u
zljG+b#m^uTImqK$?KSsNT^=_Tx3Wg=aMnb@!h68Sl0QDbvf3HF%?J$Jm+*YUkG?ni
z{~E>Y@Y@F^zHTb5-zvM$Kj_+5ZnW7=C*B#dL`K9YPh-
zm_BxJvKG97djjpu-kCfDm5ry(P#HALYyVWF0-c)jm#ctGlgJv1MSqu?G-RdL^JPN^
ziHY(!xbsj<7i_cph13tRL8$l{(!w6gtqXqhVHy-DrO9&J>Zah=&p5~D<(u}U5<*0-
zF|N#hiaY+!HTzA=%vJw$@n%c!^7VQ+;V;CC@l#0|2T4+LFFUQ-%K?G}4-)!oRc4b-
z-m%E-!lp{zaP7`smV5rLQ|68`Gra-g;YYmT49WHI|3yuT9?CzvqtoV@%&6@kNXSD|S;6CVzH^(Fa!iuoW;
zoW<<=7(@XTF(4pH3yOfCya5yt
zTSP>_78De*MMYk0Z*G43?lblt_x^M4xci=Q?ipu{{Qcxf=3H~lHP@V<@5%?~T_Se5
zjH5=AOP8rswBorYeN;}~U4sQ_y+65UpD_)jPXU$jvtKn~iv=0HP!zlun@fYmP$DKo
zR9ApWxPW4)AKFU;-}vToCc$$Nltv;UDx0yjLNCvS$j^k-KpG@VgmHet*a?rM7Z`p}
z)zPFuNWKbDLa%`ppN)IA1wfw8+OKM8uR=kvf!IkP_95C5f{|#%aT$FiXmT17j#O
zA82J@pIk*-;y~s!NC@4h$yaBw%JEWl);BTyVE2~wH15|E*|SDS5`!Jt2OxmF#!;{(
z0g3Iw!gzV$C&-@Zv=Fhsij<#lV2tfS4e-peRWZ?3E6AAlb%wehfthfk9Ke=Wc6@xI
zFv91Us~^SkAx~(LYedw`6Mp#;_=dMf^-ORi349o?aGD6#Ab=Gq^2vXvt&tQ?(}99j
zAcPl97()%jLFiRT&4sKrKGxr>V4(Z(!o!F#-~5&fJ`NJNnIj4()I}#iUVcC7ii244
znMdFt4F=0;$(*}G$3cXOO32Ghx)>D1cjL
z(`6S7Yf`nSlZ3(;T-4|j0L#yd>cP5)mZkG;3-%q0T$Mi&$HkC9S2K$gDvM@0d$g+8
z<4Lsnri$5H0qYo`&5o%O9#VopSp9%M@=>@&+vZgOQY9QlA1sm5ATffH(?(}ro*BVE$Y56xkwv1<+9JlSg
zB!4|BZI2-3;AqjerNB&I9UlD1H+o=_0F1#M3sL52gu=O8m%ZFmWv4QhDbZ2^jasM>
zLt#GVMas_rYDC0@xN=#9Iw^+0>7XFr^c_v7^{TS&hM~Cpvs^yHP;~q~N#UDRfOVv8
z3#C-v5`b-eVjHJndqt?tKjg>N>6@LB8sSj9$e08iY3`Ft2G0C7N!;m`D%PC<$%X@f_invH;OzI2SL+;BYl86gWJ=G?xR*mXICG*
z4S9}>E)(SZpkuaCKte|p=QbEe2lUMw)Fh07F!{NisIxR=43Ph3s0{oiVli-o$}0TH
z6-RtM{T@Tf^8fX
z9^jxNHK=7mp1TmNIZOB2nKnR!jT)R78O=+WGgiZa2ZRSh4q|etjz#{4&ksCmRPh*lsD_?@9(~^GZUd_d0~z
z?WrkWqqlTky?6@;Rh4)}nj2qn(7}met%N-q)e2WsDVijZ3a=T8qIqoqMe!Nx6DSG4
z+ySUDD)x=h(V=c|tt6^rJUQ4rDKrFcmPXzUA
z!gTmJ*r-~ek|Hl|Mqb1Req3s^l)xo`!U~{plLC)sw@V2Oi`+yWIm)lQ_J^#^{|?Y9
zjAfc4-;xv#wPLpsA~kE`zIZFN5`$O#Jq@U*Hf5apUG0ebf(8ch&Zg
z5y(RlvP?+rn?P~7b`Ez~Wy|pcQbPMl;$W}}$P