diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index a9d2589..8530157 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -26,14 +26,14 @@ jobs: build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: submodules: true # Sitemap lastmod comes from the latest content commit. fetch-depth: 0 - name: Checkout tago - uses: actions/checkout@v6.0.2 + uses: actions/checkout@v7.0.0 with: repository: tamnd/tago path: .tago-src @@ -107,7 +107,7 @@ jobs: group: cloudflare-pages-github-cli cancel-in-progress: true steps: - - uses: actions/checkout@v6.0.2 + - uses: actions/checkout@v7.0.0 with: fetch-depth: 1 sparse-checkout: scripts/ diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b450097..f888ee5 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -3,22 +3,22 @@ # entries for the package managers (Homebrew, Scoop). `git tag vX.Y.Z && git # push --tags` fans out to all of them through .github/workflows/release.yml. # -# Publish steps that push to a repository you do not own yet (the Homebrew tap, +# Publish steps that push to a repository we do not own yet (the Homebrew tap, # the Scoop bucket) self-disable when their token is absent. A release with no # extra secrets still produces every downloadable artifact and the container # image; each manager lights up the moment its repository and token exist. version: 2 -project_name: ghb +project_name: github-cli before: hooks: - go mod download builds: - - id: ghb - binary: ghb - main: ./cmd/ghb + - id: github + binary: github + main: ./cmd/github env: - CGO_ENABLED=0 flags: @@ -43,7 +43,7 @@ builds: archives: - id: default - name_template: "ghb_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}" + name_template: "github_{{ .Version }}_{{ .Os }}_{{ .Arch }}{{ with .Arm }}v{{ . }}{{ end }}" format_overrides: - goos: windows formats: [zip] @@ -53,12 +53,12 @@ archives: nfpms: - id: linux-packages - package_name: ghb + package_name: github-cli file_name_template: "{{ .ConventionalFileName }}" vendor: tamnd homepage: https://github.com/tamnd/github-cli maintainer: Duc-Tam Nguyen - description: Browse GitHub repositories, users and releases + description: "A command-line for GitHub: scrapes HTML, no API key needed." license: Apache-2.0 formats: - deb @@ -68,11 +68,17 @@ nfpms: section: utils contents: - src: ./LICENSE - dst: /usr/share/doc/ghb/LICENSE + dst: /usr/share/doc/github-cli/LICENSE dockers_v2: + # The image is named for the repository, not for the binary. A short name in a + # user-owned GHCR namespace belongs to whichever repository pushed it first, + # and the workflow token only has write access to packages linked to the + # repository it is running in. A sibling project taking the short name is how + # a release fails at the last step with a 403 that reads like a permissions + # bug, which already happened once on hf-cli. - images: - - ghcr.io/tamnd/ghb + - ghcr.io/tamnd/github-cli tags: - "{{ .Version }}" - latest @@ -82,7 +88,7 @@ dockers_v2: - linux/arm64 labels: org.opencontainers.image.title: "{{ .ProjectName }}" - org.opencontainers.image.description: "Browse GitHub repositories, users and releases" + org.opencontainers.image.description: "A command-line for GitHub: scrapes HTML, no API key needed." org.opencontainers.image.url: "https://github.com/tamnd/github-cli" org.opencontainers.image.source: "https://github.com/tamnd/github-cli" org.opencontainers.image.version: "{{ .Version }}" @@ -90,18 +96,33 @@ dockers_v2: org.opencontainers.image.licenses: "Apache-2.0" homebrew_casks: - - name: ghb + # Pushed to the tap repository. It self-disables until + # HOMEBREW_TAP_GITHUB_TOKEN (a PAT with write access to tamnd/homebrew-tap) is + # set, so a tokenless release still writes the cask into dist for inspection. + # + # The cask is github-cli rather than github because homebrew-cask already has + # a cask called github, which is GitHub Desktop. + - name: github-cli repository: owner: tamnd name: homebrew-tap token: '{{ envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}' directory: Casks homepage: https://github.com/tamnd/github-cli - description: Browse GitHub repositories, users and releases + description: "A command-line for GitHub: scrapes HTML, no API key needed." skip_upload: '{{ if envOrDefault "HOMEBREW_TAP_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}' commit_author: name: Duc-Tam Nguyen email: tamnd87@gmail.com + # Homebrew quarantines cask artifacts, and Gatekeeper kills a quarantined + # binary that is only ad-hoc signed, which is what a cross-compiled Go + # binary is. Strip the attribute at install so github runs on the first try. + hooks: + post: + install: | + if system_command("/usr/bin/xattr", args: ["-h"]).exit_status.zero? + system_command "/usr/bin/xattr", args: ["-dr", "com.apple.quarantine", "#{staged_path}/github"] + end scoops: - repository: @@ -109,7 +130,7 @@ scoops: name: scoop-bucket token: '{{ envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}' homepage: https://github.com/tamnd/github-cli - description: Browse GitHub repositories, users and releases + description: "A command-line for GitHub: scrapes HTML, no API key needed." license: Apache-2.0 skip_upload: '{{ if envOrDefault "SCOOP_BUCKET_GITHUB_TOKEN" "" }}false{{ else }}true{{ end }}' commit_author: @@ -155,15 +176,6 @@ changelog: - "^ci:" - Merge pull request - Merge branch - groups: - - title: Features - regexp: '^.*?feat(\(.+\))??!?:.+$' - order: 0 - - title: Fixes - regexp: '^.*?fix(\(.+\))??!?:.+$' - order: 1 - - title: Other - order: 999 release: github: diff --git a/Dockerfile b/Dockerfile index 1fe69d4..c7db790 100644 --- a/Dockerfile +++ b/Dockerfile @@ -12,10 +12,10 @@ ARG TARGETPLATFORM # ca-certificates for HTTPS; tzdata for sane timestamps. RUN apk add --no-cache ca-certificates tzdata \ - && adduser -D -H -u 10001 ghb + && adduser -D -H -u 10001 github -COPY $TARGETPLATFORM/ghb /usr/bin/ghb +COPY $TARGETPLATFORM/github /usr/bin/github -USER ghb +USER github -ENTRYPOINT ["/usr/bin/ghb"] +ENTRYPOINT ["/usr/bin/github"] diff --git a/Makefile b/Makefile index 1de66e5..1bf1ae9 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,7 @@ # Build into bin/ (gitignored) so the binary never collides with the github/ # source package at the repo root. -BINARY := bin/ghb -PKG := ./cmd/ghb +BINARY := bin/github +PKG := ./cmd/github VERSION := $(shell git describe --tags --always --dirty 2>/dev/null || echo dev) COMMIT := $(shell git rev-parse --short HEAD 2>/dev/null || echo none) DATE := $(shell date -u +%Y-%m-%dT%H:%M:%SZ) @@ -10,7 +10,7 @@ LDFLAGS := -s -w \ -X github.com/tamnd/github-cli/cli.Commit=$(COMMIT) \ -X github.com/tamnd/github-cli/cli.Date=$(DATE) -.PHONY: build install test vet fmt clean run +.PHONY: build install test live vet fmt lint clean run build: @mkdir -p $(dir $(BINARY)) @@ -19,15 +19,24 @@ build: install: CGO_ENABLED=0 go install -trimpath -ldflags "$(LDFLAGS)" $(PKG) +# The default run is offline and deterministic. test: go test ./... +# live talks to github.com. It answers the one question no offline test can: +# does the site still look the way the readers think it does. +live: + GITHUB_LIVE=1 go test ./gh/ -run Live -count=1 -v + vet: go vet ./... fmt: gofmt -w -s . +lint: + golangci-lint run + clean: rm -rf bin dist diff --git a/README.md b/README.md index 2992bd7..940ec58 100644 --- a/README.md +++ b/README.md @@ -1,63 +1,326 @@ -# ghb +# github -Browse GitHub repositories, users and releases +[![ci](https://github.com/tamnd/github-cli/actions/workflows/ci.yml/badge.svg)](https://github.com/tamnd/github-cli/actions/workflows/ci.yml) +[![Release](https://img.shields.io/github/v/release/tamnd/github-cli)](https://github.com/tamnd/github-cli/releases/latest) +[![Go Reference](https://pkg.go.dev/badge/github.com/tamnd/github-cli.svg)](https://pkg.go.dev/github.com/tamnd/github-cli) +[![Go Report Card](https://goreportcard.com/badge/github.com/tamnd/github-cli)](https://goreportcard.com/report/github.com/tamnd/github-cli) +[![License](https://img.shields.io/github/license/tamnd/github-cli)](./LICENSE) -`ghb` is a single pure-Go binary. It speaks to github over plain -HTTPS, shapes the responses into clean records, and pipes into the rest of your -tools. No API key, nothing to run alongside it. +**github** reads github.com as data, with no token, ever. +One pure-Go binary turns the site into typed records: every repository, user, organization, issue, pull request, discussion, commit, branch, tag, release, file, topic, gist, package, and marketplace action, each with a canonical `github://` address, every field its page stated, and typed edges to everything it names. +Read one thing, list a million, walk the dependency graph, or export the lot as RDF. + +[Install](#install) • [Quick start](#quick-start) • [Read one thing](#read-one-thing) • [List and search](#list-and-search) • [Contents](#look-inside-a-repository) • [Graph](#walk-the-graph) • [Linked data](#export-linked-data) • [Output](#output) • [No token](#no-token) • [Serve](#serve-it) • [Driver](#use-it-as-a-resource-uri-driver) + +![github reading a repository, listing an account, resolving a dependency graph, printing graph edges, and emitting schema.org triples](docs/static/demo.gif) + +GitHub is already a knowledge graph that happens to be served as a website. +Repositories depend on repositories, issues reference commits, commits belong to people, people belong to organizations, and every one of those relations is written down on a page somewhere. +Most tools hand you back a page, or the subset of fields somebody decided you needed. +`github` reads all of it, keeps all of it, and gives each entity an address. + +Not affiliated with GitHub or Microsoft. +Full docs and guides live at **[tamnd.github.io/github-cli](https://tamnd.github.io/github-cli/)**. ## Install ```bash -go install github.com/tamnd/github-cli/cmd/ghb@latest +go install github.com/tamnd/github-cli/cmd/github@latest ``` -Or grab a prebuilt binary from the [releases](https://github.com/tamnd/github-cli/releases), or run -the container image: +Prefer a prebuilt binary? +Grab an archive, a `.deb`/`.rpm`/`.apk`, or a signed checksum from [releases](https://github.com/tamnd/github-cli/releases). +Or let a package manager handle it: ```bash -docker run --rm ghcr.io/tamnd/ghb:latest --help +# Homebrew (macOS) +brew install --cask tamnd/tap/github-cli + +# Scoop (Windows) +scoop bucket add tamnd https://github.com/tamnd/scoop-bucket +scoop install github-cli + +# apt (Debian, Ubuntu) +curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg +echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list +sudo apt update && sudo apt install github-cli + +# dnf (Fedora, RHEL) +sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo +sudo dnf install github-cli + +# container +docker run --rm ghcr.io/tamnd/github-cli:latest repo gohugoio/hugo ``` -## Usage +The binary is called `github`. +It does not replace the official `gh`, which does the authenticated half of the site far better. +This one does the public half without asking you to log in. + +## Quick start ```bash -ghb --help -ghb version +github repo gohugoio/hugo # a record, not a page +github owned torvalds -n 10 # a list, streamed +github get https://github.com/golang/go/pull/1 # paste anything ``` -This is a fresh scaffold. The command tree starts with `version`; build out the -real commands in `cli/` on top of the `github` library package. +`github get` takes a bare id, a URL you copied out of a browser, or a `github://` URI, works out what it points at, and reads it. +Every other read command is the same thing with the kind already decided. -## Development +Output adapts to where it goes: an aligned table on your terminal, JSONL the moment you pipe it somewhere. +## Read one thing + +```bash +github repo gohugoio/hugo # every field the page states +github repo gohugoio/hugo --deep # plus what only the deferred fragments carry +github user torvalds +github org golang +github issue golang/go 1234 +github pr golang/go 1 +github commit golang/go abc1234 # author, verification, changed files +github release cli/cli # the latest one, or pass a tag +github discussion vercel/next.js 12345 +github compare golang/go go1.22.0 go1.23.0 ``` -cmd/ghb/ thin main, wires cli.Root into fang -cli/ the cobra command tree -github/ the library: HTTP client and data models -docs/ tago documentation site + +Records carry what the page carried, not a curated subset. +A repository page states its head commit, its commit and release counts, its licence, its topics, its funding and citation flags, and its whole root tree, so the record does too, from one request. + +## List and search + +```bash +github repos --language rust --sort stars -n 100 +github issues "repo:golang/go is:open label:NeedsInvestigation" +github prs --owner golang +github users --language go +github search kubernetes # every entity kind at once +github topics machine-learning +github actions lint # the marketplace +github trending --language go +``` + +Listing streams. +`-n` stops early without fetching the next page, and no command holds a full result set in memory unless a format forces it to. + +`github code` is the one command that does not work: code search needs a signed-in session, and this tool has none. +It says so rather than returning nothing. + +## Look inside a repository + +```bash +github tree golang/go src/net/http +github tree gohugoio/hugo --recursive +github cat gohugoio/hugo go.mod +github blob golang/go src/runtime/proc.go +github readme gohugoio/hugo +github diff golang/go abc1234 +github archive golang/go master --format tar.gz ``` +`archive` is the one to reach for on a whole repository. +One request to codeload streams the entire tree, where `tree --recursive` is one request per directory. + +There is also `github symbols`, which reads the code navigation index GitHub builds for every file. +It currently returns nothing, and says so: the blob page still renders the symbols button, and the list behind it is empty for a signed-out reader. +The command stays because the field is still in the payload and may fill in again. + +## History and people + +```bash +github commits golang/go -n 50 +github branches golang/go +github tags golang/go +github releases cli/cli +github refs golang/go +github timeline golang/go 1234 # everything that happened on one issue + +github owned torvalds # repositories as the profile shows them +github stars torvalds +github followers torvalds +github members golang +github contributions torvalds # the calendar, one record per day +github gists torvalds +github activity torvalds +``` + +## Walk the graph + +```bash +github graph golang/go # the node, its edges, and its facts +github edges golang/go # just the edges +github deps gohugoio/hugo # what it depends on, with versions and licences +github dependents gohugoio/hugo # the repositories that depend on it +github crawl golang/go --depth 2 +github crawl golang/go --depth 3 --dry-run # size the walk before running it +github contributors golang/go +github forks golang/go +``` + +Edges come from five places: explicit ids, the embedded React payload, the atom feeds, parsed HTML, and text. +Each edge records which one it came from, so a consumer can decide how much to trust it, and a walk can drop everything below a floor. + +`deps` and `dependents` read GitHub's dependency graph, which is the part of the site with no API at all. +`deps` gives you the package, the version, the ecosystem, the manifest it came from, the licence, and whether it is direct or transitive. + +## Export linked data + +```bash +github rdf gohugoio/hugo --format ttl +github rdf gohugoio/hugo --format jsonld +github export golang/go --depth 2 --format jsonl > go.jsonl +``` + +RDF comes out as N-Triples, Turtle, JSON-LD, or N-Quads, over `schema.org` where a term exists and a `gh:` namespace where none does. +N-Triples and N-Quads stream, so exporting a large organization never needs the graph in memory. + +## The page plane + +Every reader in this tool works from one extraction of the page, and `github page` prints that extraction whole: + +```bash +github page gohugoio/hugo # everything, organised +github page gohugoio/hugo --section payload # just the embedded React payload +github page gohugoio/hugo --section meta # just the og: and twitter: tags +github page golang/go#1234 --query IssueViewerViewQuery +github page https://github.com/trending --raw > trending.html +``` + +This is the debugging tool. +When a field comes back empty, `page` shows you the same view the reader had, so the answer is either "the page stopped carrying it" or "the selector is wrong", and you can tell which in one command. + +## Output + +Every command shares one contract: `-o table|markdown|list|json|jsonl|csv|tsv|url|raw`, `--fields` to pick columns, `--template` for a custom line, `-n` to limit. + +```bash +github repos --language go --fields id,stars,forks +github repos --language go --template '{{.id}} has {{.stars}} stars' +github repo gohugoio/hugo -o json | jq .tree +``` + +`-o url` is the one the others are measured against. +It prints one URL per record and nothing else, so this composes with no glue: + +```bash +github owned torvalds -o url | xargs -n1 github get +``` + +Failures are typed too. +Every surface exits 3 on an empty result, 4 when a page is not public, 5 on a rate limit, 6 on not found, 7 on unsupported, and 8 on a network failure, so a script can branch on the code without reading the message. + +## No token + +There is no API token in this tool and there will not be one. +The unauthenticated REST API allows sixty requests an hour, which is not enough to read one organization, while the pages sit behind a CDN and are faster than the API even where the API would work. + +That is a promise you cannot check by using the tool. +You can see that a command works without logging in, but not that no code path would send a credential if one happened to be lying around. +So it is asserted instead: `gh/noauth_test.go` parses every source file with comments dropped and fails the build if the word `Authorization`, `GITHUB_TOKEN`, `GH_TOKEN`, or `api.github.com` shows up in code anywhere outside the one file whose job is to say those names out loud. + +If you do have a token in your environment, `github doctor` will tell you it is being ignored, because a token that does nothing looks exactly like a token that is wrong: + +```bash +github doctor +``` + +`doctor` checks the environment, whether the site answers, whether the page still carries the payload every reader expects, whether the cache is writable, and what pacing this run is using. +It is the first thing to reach for when something comes back wrong. + +The cost of no token is that this is read-only and public-only. +For anything else, use the official [gh](https://cli.github.com). + +## Serve it + +The same operations are available over HTTP and as an MCP tool set for agents, with no extra code: + +```bash +github serve --addr :7777 # every read verb becomes GET /v1/, streaming NDJSON +github mcp # speak MCP over stdio +``` + +Arguments go in the query string, because most of them contain a slash and a path would swallow it: + +```bash +curl 'localhost:7777/v1/repo?ref=gohugoio/hugo' +curl 'localhost:7777/v1/blob?ref=gohugoio/hugo&path=go.mod' +curl 'localhost:7777/v1/trending?language=go&limit=5' +curl localhost:7777/v1/openapi.json +``` + +## Use it as a resource-URI driver + +`github` registers a `github` domain the way a program registers a database driver with `database/sql`. +A host enables it with one blank import: + +```go +import _ "github.com/tamnd/github-cli/gh" +``` + +Then [ant](https://github.com/tamnd/ant), or any program that links the package, dereferences `github://` URIs without knowing anything about the site: + +```bash +ant get github://repo/gohugoio/hugo +ant cat github://file/gohugoio/hugo@master/go.mod +ant ls github://org/golang +ant url github://pr/golang/go#1 +``` + +## How it works + +One `kit.Handle` registration per operation, and every surface updates itself: + +``` +cmd/github/ thin main: hands cli.NewApp to kit.Run +cli/ assembles the kit App and registers the byte-plane commands +gh/ the library: client, records, graph, RDF, doctor, domain.go +pkg/page/ one extraction of an HTML page, shared by every reader +pkg/gitproto/ the git smart HTTP protocol, for the refs no page lists +docs/ tago documentation site and the demo tape +``` + +That single declaration becomes a CLI command, an HTTP route, an MCP tool, and a URI dereference, so there is no second implementation to keep in step. + +Underneath, seven surfaces answer for different routes: the embedded React payload, the JSON a route returns when asked for JSON, the fragments a page defers to XHR, the search backend, the atom feeds, raw.githubusercontent.com, and the git protocol itself. +`github routes` prints which surface answers for which route and which one it falls back to. + +Responses are cached on disk, keyed by surface and URL, for fifteen minutes. +Anything addressed by a commit SHA is kept forever, because it cannot change. + +## Development + +```bash +make build # ./bin/github +make test # go test ./..., offline and deterministic +make live # the smoke tests that actually talk to github.com +make vet +make lint +``` + +The offline tests check the parsers against bytes that were already parsed once, which is useful but cannot see the failure that matters: GitHub moving something. +`make live` is what sees that, so the assertions there are about shape rather than values. +A star count changes hourly, and pinning one turns a test into a clock. + +The demo above is a tape, not a screen recording. +Regenerate it with [ascii-gif](https://github.com/tamnd/ascii-gif): + ```bash -make build # ./bin/ghb -make test # go test ./... -make vet # go vet ./... +ascii-gif render docs/demo/github.tape -o docs/static/demo.gif ``` ## Releasing -Push a version tag and GitHub Actions runs GoReleaser, which builds the -archives, Linux packages, the multi-arch GHCR image, checksums, SBOMs, and a -cosign signature: +Push a version tag and GitHub Actions runs GoReleaser, which builds the archives, Linux packages, the multi-arch GHCR image, checksums, SBOMs, and a cosign signature: ```bash -git tag v0.1.0 +git tag -a v0.2.0 -m "v0.2.0" git push --tags ``` -The Homebrew and Scoop steps self-disable until their tokens exist, so the first -release works with no extra secrets. +The Homebrew and Scoop steps self-disable until their tokens exist, so a release works with no extra secrets. ## License -Apache-2.0. See [LICENSE](LICENSE). +Apache-2.0. +See [LICENSE](LICENSE). diff --git a/cli/cmd_releases.go b/cli/cmd_releases.go deleted file mode 100644 index 19e895f..0000000 --- a/cli/cmd_releases.go +++ /dev/null @@ -1,26 +0,0 @@ -package cli - -import ( - "github.com/spf13/cobra" -) - -func (a *App) releasesCmd() *cobra.Command { - return &cobra.Command{ - Use: "releases ", - Short: "List releases for a repository", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - owner, repo, err := splitOwnerRepo(args[0]) - if err != nil { - return codeError(exitUsage, err) - } - n := a.effectiveLimit(20) - a.progressf("fetching releases for %s/%s...", owner, repo) - releases, err := a.client.Releases(cmd.Context(), owner, repo, n) - if err != nil { - return mapFetchErr(err) - } - return a.renderOrEmpty(releases, len(releases)) - }, - } -} diff --git a/cli/cmd_repo.go b/cli/cmd_repo.go deleted file mode 100644 index cbbb04b..0000000 --- a/cli/cmd_repo.go +++ /dev/null @@ -1,36 +0,0 @@ -package cli - -import ( - "fmt" - "strings" - - "github.com/spf13/cobra" -) - -func (a *App) repoCmd() *cobra.Command { - return &cobra.Command{ - Use: "repo ", - Short: "Show a single repository", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - owner, repo, err := splitOwnerRepo(args[0]) - if err != nil { - return codeError(exitUsage, err) - } - a.progressf("fetching repo %s/%s...", owner, repo) - r, err := a.client.GetRepo(cmd.Context(), owner, repo) - if err != nil { - return mapFetchErr(err) - } - return a.render(r) - }, - } -} - -func splitOwnerRepo(s string) (owner, repo string, err error) { - parts := strings.SplitN(s, "/", 2) - if len(parts) != 2 || parts[0] == "" || parts[1] == "" { - return "", "", fmt.Errorf("argument must be owner/repo, got %q", s) - } - return parts[0], parts[1], nil -} diff --git a/cli/cmd_search.go b/cli/cmd_search.go deleted file mode 100644 index 639e6f9..0000000 --- a/cli/cmd_search.go +++ /dev/null @@ -1,36 +0,0 @@ -package cli - -import ( - "github.com/spf13/cobra" - "github.com/tamnd/github-cli/github" -) - -func (a *App) searchCmd() *cobra.Command { - var ( - lang string - sort string - ) - cmd := &cobra.Command{ - Use: "search ", - Short: "Search GitHub repositories", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - n := a.effectiveLimit(20) - opts := github.SearchRepoOptions{ - Query: args[0], - Language: lang, - Sort: sort, - Limit: n, - } - a.progressf("searching repositories for %q...", args[0]) - repos, err := a.client.SearchRepos(cmd.Context(), opts) - if err != nil { - return mapFetchErr(err) - } - return a.renderOrEmpty(repos, len(repos)) - }, - } - cmd.Flags().StringVar(&lang, "lang", "", "language filter (e.g. Go, Python)") - cmd.Flags().StringVar(&sort, "sort", "stars", "sort order: stars|forks|updated|help-wanted-issues") - return cmd -} diff --git a/cli/cmd_trending.go b/cli/cmd_trending.go deleted file mode 100644 index b28af4d..0000000 --- a/cli/cmd_trending.go +++ /dev/null @@ -1,34 +0,0 @@ -package cli - -import ( - "github.com/spf13/cobra" - "github.com/tamnd/github-cli/github" -) - -func (a *App) trendingCmd() *cobra.Command { - var ( - lang string - days int - ) - cmd := &cobra.Command{ - Use: "trending", - Short: "Show trending repositories (proxy via search API)", - RunE: func(cmd *cobra.Command, _ []string) error { - n := a.effectiveLimit(25) - opts := github.TrendingOptions{ - Language: lang, - Days: days, - Limit: n, - } - a.progressf("fetching trending repositories (last %d days)...", days) - repos, err := a.client.Trending(cmd.Context(), opts) - if err != nil { - return mapFetchErr(err) - } - return a.renderOrEmpty(repos, len(repos)) - }, - } - cmd.Flags().StringVar(&lang, "lang", "", "language filter (e.g. Go, Python)") - cmd.Flags().IntVar(&days, "days", 7, "time window in days (7, 30, or 365)") - return cmd -} diff --git a/cli/cmd_user.go b/cli/cmd_user.go deleted file mode 100644 index 28329ed..0000000 --- a/cli/cmd_user.go +++ /dev/null @@ -1,37 +0,0 @@ -package cli - -import ( - "github.com/spf13/cobra" -) - -func (a *App) userCmd() *cobra.Command { - var repos bool - cmd := &cobra.Command{ - Use: "user ", - Short: "Show a GitHub user profile", - Args: cobra.ExactArgs(1), - RunE: func(cmd *cobra.Command, args []string) error { - username := args[0] - a.progressf("fetching user %q...", username) - user, err := a.client.GetUser(cmd.Context(), username) - if err != nil { - return mapFetchErr(err) - } - if err := a.render(user); err != nil { - return err - } - if repos { - n := a.effectiveLimit(10) - a.progressf("fetching top %d repos for %q...", n, username) - repoList, err := a.client.UserRepos(cmd.Context(), username, n) - if err != nil { - return mapFetchErr(err) - } - return a.render(repoList) - } - return nil - }, - } - cmd.Flags().BoolVar(&repos, "repos", false, "also list the user's top repos") - return cmd -} diff --git a/cli/errors.go b/cli/errors.go deleted file mode 100644 index 7546fe4..0000000 --- a/cli/errors.go +++ /dev/null @@ -1,15 +0,0 @@ -package cli - -import ( - "errors" - - "github.com/tamnd/github-cli/github" -) - -func isNotFound(err error) bool { - return errors.Is(err, github.ErrNotFound) -} - -func isRateLimit(err error) bool { - return errors.Is(err, github.ErrRateLimit) -} diff --git a/cli/export.go b/cli/export.go new file mode 100644 index 0000000..8b17e1c --- /dev/null +++ b/cli/export.go @@ -0,0 +1,118 @@ +package cli + +import ( + "bufio" + "context" + "encoding/json" + "io" + "os" + + "github.com/tamnd/any-cli/kit" + "github.com/tamnd/any-cli/kit/errs" + "github.com/tamnd/github-cli/gh" +) + +// export.go writes a whole graph to one file. Everything it does can be had by +// redirecting `github crawl` or `github rdf`, and it exists because what people +// want at the end of a walk is one file they can load somewhere else, named once +// rather than assembled out of two commands and a shell operator. + +type exportCmd struct { + format string + out string + depth int + follow []string + minTrust string + limit int +} + +func newExportCmd() kit.Command { + c := &exportCmd{} + return kit.Command{ + Use: "export ", + Short: "Write a whole graph to one file", + Long: "export walks from the seed and writes the result in one go. The formats are\n" + + "jsonl (one node, edge, or fact per line), json (a single object), and the four\n" + + "RDF serialisations nt, ttl, jsonld, and nq.\n\n" + + "Without --out it writes to stdout, which makes it a drop-in for a pipeline.", + Group: "graph", + Args: kit.ExactArgs(1), + Flags: c.flags, + Run: c.run, + } +} + +func (c *exportCmd) flags(f *kit.FlagSet) { + f.StringVar(&c.format, "format", "jsonl", "jsonl, json, nt, ttl, jsonld, or nq") + f.StringVarP(&c.out, "out", "O", "", "write here instead of stdout") + f.IntVar(&c.depth, "depth", 1, "walk this many edges out") + f.StringSliceVar(&c.follow, "follow", nil, "predicates to follow (default: the structural ones)") + f.StringVar(&c.minTrust, "min-trust", gh.DefaultMinTrust, "drop edges below this rule") + f.IntVar(&c.limit, "limit", 0, "stop after this many nodes") +} + +func (c *exportCmd) run(ctx context.Context, args []string) error { + cl, err := clientFrom(ctx) + if err != nil { + return err + } + kind, id, g, err := buildGraph(ctx, cl, args[0], graphWalk{ + depth: c.depth, + follow: c.follow, + minTrust: c.minTrust, + limit: c.limit, + }) + if err != nil { + return err + } + gh.SortEdges(g.Edges) + + out := io.Writer(os.Stdout) + if c.out != "" { + f, err := os.Create(c.out) + if err != nil { + return err + } + defer func() { _ = f.Close() }() + out = f + } + w := bufio.NewWriter(out) + defer func() { _ = w.Flush() }() + + switch c.format { + case "jsonl": + return writeJSONL(w, g) + case "json": + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(g) + case gh.FormatNT, gh.FormatTurtle, gh.FormatJSONLD, gh.FormatNQuads: + graph, _ := gh.Locate(kind, id) + return gh.WriteRDF(w, g, gh.RDFOptions{Format: c.format, Graph: graph}) + default: + return errs.Usage("unknown --format %q", c.format) + } +} + +// writeJSONL puts the nodes first and everything that points at them after, so a +// reader building an index in one pass never sees an edge before both of its +// ends. +func writeJSONL(w io.Writer, g *gh.Graph) error { + enc := json.NewEncoder(w) + for i := range g.Nodes { + if err := enc.Encode(&g.Nodes[i]); err != nil { + return err + } + } + for i := range g.Edges { + if err := enc.Encode(&g.Edges[i]); err != nil { + return err + } + } + for i := range g.Facts { + if err := enc.Encode(&g.Facts[i]); err != nil { + return err + } + } + return nil +} diff --git a/cli/files.go b/cli/files.go new file mode 100644 index 0000000..97a354b --- /dev/null +++ b/cli/files.go @@ -0,0 +1,290 @@ +package cli + +import ( + "context" + "fmt" + "io" + "os" + + "github.com/tamnd/any-cli/kit" + "github.com/tamnd/any-cli/kit/errs" + "github.com/tamnd/github-cli/gh" +) + +// files.go holds the commands that emit bytes rather than records. They are the +// reason the tool is useful for actual file work and not only metadata, and they +// are escape hatches precisely because a file is not a record: piping a tarball +// through a JSON renderer would be a mistake in every direction. + +// clientFrom reaches the one client kit built for this run. Escape-hatch +// commands do not get the kit:"inject" treatment, so they ask for it here, and +// asking here means they share the run's pacing and cache with every operation. +func clientFrom(ctx context.Context) (*gh.Client, error) { + st := kit.FromContext(ctx) + if st == nil { + return nil, errs.New(errs.KindGeneric, "no run state on the context") + } + v, err := st.Client(ctx) + if err != nil { + return nil, err + } + c, ok := v.(*gh.Client) + if !ok { + return nil, errs.New(errs.KindGeneric, "the run has no github client") + } + return c, nil +} + +type catCmd struct{ ref string } + +func newCatCmd() kit.Command { + c := &catCmd{} + return kit.Command{ + Use: "cat ", + Short: "Write one file from a repository to stdout", + Long: "cat streams the bytes straight through raw.githubusercontent.com, so a\n" + + "large file costs no memory and is never written to the cache. A blob URL\n" + + "works as a single argument, since it already names the path and the ref.", + Group: "contents", + Args: kit.RangeArgs(1, 2), + Flags: c.flags, + Run: c.run, + } +} + +func (c *catCmd) flags(f *kit.FlagSet) { + f.StringVar(&c.ref, "rev", "", "branch, tag, or commit sha") +} + +func (c *catCmd) run(ctx context.Context, args []string) error { + cl, err := clientFrom(ctx) + if err != nil { + return err + } + repo, ref, path, err := resolvePath(args, c.ref) + if err != nil { + return err + } + if path == "" { + return errs.Usage("cat needs a path, either as a second argument or in the URL") + } + _, err = cl.Download(ctx, repo, ref, path, os.Stdout) + return err +} + +// resolvePath is the byte-plane twin of the resolver the operations use: it +// accepts a blob URL that carries everything, or a repository and a path. +func resolvePath(args []string, rev string) (repo, ref, path string, err error) { + arg := "" + if len(args) > 1 { + arg = args[1] + } + kind, id, err := gh.Classify(args[0]) + if err != nil { + return "", "", "", err + } + if kind == gh.KindFile || kind == gh.KindTree { + if r, v, p, ok := gh.SplitPathID(id); ok { + if rev != "" { + v = rev + } + if arg != "" { + p = arg + } + return r, v, p, nil + } + } + repo, err = gh.ResolveRepo(args[0]) + if err != nil { + return "", "", "", err + } + return repo, rev, arg, nil +} + +type readmeCmd struct { + ref string + html bool +} + +func newReadmeCmd() kit.Command { + c := &readmeCmd{} + return kit.Command{ + Use: "readme ", + Short: "Write a repository's README to stdout", + Long: "readme prints the rendered README as text. GitHub renders it server side,\n" + + "so what comes back is what the page shows, badges resolved and relative\n" + + "links rewritten. Use --html for the markup the page carries.", + Group: "contents", + Args: kit.ExactArgs(1), + Flags: c.flags, + Run: c.run, + } +} + +func (c *readmeCmd) flags(f *kit.FlagSet) { + f.StringVar(&c.ref, "rev", "", "branch, tag, or commit sha") + f.BoolVar(&c.html, "html", false, "print the rendered markup instead of the text") +} + +func (c *readmeCmd) run(ctx context.Context, args []string) error { + cl, err := clientFrom(ctx) + if err != nil { + return err + } + id, err := gh.ResolveRepo(args[0]) + if err != nil { + return err + } + // A ref makes this a file read rather than a repository read, because the + // repository page only ever renders the default branch's README. + if c.ref != "" { + r, err := cl.Repo(ctx, id, gh.RepoOptions{}) + if err != nil { + return err + } + path := r.ReadmePath + if path == "" { + path = "README.md" + } + _, err = cl.Download(ctx, id, c.ref, path, os.Stdout) + return err + } + r, err := cl.Repo(ctx, id, gh.RepoOptions{Readme: true}) + if err != nil { + return err + } + text := r.ReadmeText + if c.html { + text = r.ReadmeHTML + } + if text == "" { + return errs.NotFound("no README in %s", id) + } + _, err = io.WriteString(os.Stdout, text) + return err +} + +type archiveCmd struct { + format string + out string +} + +func newArchiveCmd() kit.Command { + c := &archiveCmd{} + return kit.Command{ + Use: "archive [ref]", + Short: "Download a repository as a tarball or a zip", + Long: "One request to codeload gets the whole tree. For anything past a few\n" + + "directories this beats walking `github tree --recursive`, which is one\n" + + "request per directory. Nothing is buffered: the stream goes straight to\n" + + "the file or to stdout.", + Group: "contents", + Args: kit.RangeArgs(1, 2), + Flags: c.flags, + Run: c.run, + } +} + +func (c *archiveCmd) flags(f *kit.FlagSet) { + f.StringVar(&c.format, "format", "tar.gz", "tar.gz or zip") + f.StringVarP(&c.out, "output", "o", "", "write here instead of stdout") +} + +func (c *archiveCmd) run(ctx context.Context, args []string) error { + cl, err := clientFrom(ctx) + if err != nil { + return err + } + repo, err := gh.ResolveRepo(args[0]) + if err != nil { + return err + } + ref := "" + if len(args) > 1 { + ref = args[1] + } + body, err := cl.Archive(ctx, repo, ref, c.format) + if err != nil { + return err + } + defer body.Close() + + w := io.Writer(os.Stdout) + if c.out != "" { + f, err := os.Create(c.out) + if err != nil { + return err + } + defer f.Close() + w = f + } + _, err = io.Copy(w, body) + return err +} + +type diffCmd struct{ patch bool } + +func newDiffCmd() kit.Command { + c := &diffCmd{} + return kit.Command{ + Use: "diff [base] [head]", + Short: "Write a commit's or a range's diff to stdout", + Long: "diff takes a commit reference for one commit's changes, or a repository\n" + + "and two refs for a range. --patch gives the git-format-patch mailbox\n" + + "instead, which carries the author, the date, and the message of every\n" + + "commit and applies cleanly with git am.", + Group: "contents", + Args: kit.RangeArgs(1, 3), + Flags: c.flags, + Run: c.run, + } +} + +func (c *diffCmd) flags(f *kit.FlagSet) { + f.BoolVar(&c.patch, "patch", false, "the format-patch mailbox rather than the plain diff") +} + +func (c *diffCmd) run(ctx context.Context, args []string) error { + cl, err := clientFrom(ctx) + if err != nil { + return err + } + url, err := diffURL(args) + if err != nil { + return err + } + fetch := cl.Diff + if c.patch { + fetch = cl.Patch + } + text, err := fetch(ctx, url) + if err != nil { + return err + } + _, err = io.WriteString(os.Stdout, text) + return err +} + +// diffURL turns the two shapes into the one page URL both diffs hang off. +func diffURL(args []string) (string, error) { + if len(args) >= 3 { + repo, err := gh.ResolveRepo(args[0]) + if err != nil { + return "", err + } + return fmt.Sprintf("%s/%s/compare/%s...%s", gh.BaseURL, repo, args[1], args[2]), nil + } + kind, id, err := gh.Classify(args[0]) + if err != nil { + return "", err + } + switch kind { + case gh.KindCommit, gh.KindCompare, gh.KindPR: + return gh.Locate(kind, id) + } + // Leads with a word, not with the argument. The renderer capitalises the + // first token of an error, and "golang/go" coming back as "Golang/Go" reads + // like the tool mangled the input rather than like the input was the wrong + // kind of thing. + return "", errs.Usage("cannot diff %q, which is a %s; diff needs a commit, a pull request, a compare URL, or a repository with two refs", args[0], kind) +} diff --git a/cli/output.go b/cli/output.go deleted file mode 100644 index 06d10a2..0000000 --- a/cli/output.go +++ /dev/null @@ -1,25 +0,0 @@ -package cli - -import ( - "io" - - "github.com/tamnd/github-cli/pkg/render" -) - -// Format aliases so command code reads cleanly. -type Format = render.Format - -const ( - FormatTable = render.FormatTable - FormatJSON = render.FormatJSON - FormatJSONL = render.FormatJSONL - FormatCSV = render.FormatCSV - FormatTSV = render.FormatTSV - FormatURL = render.FormatURL - FormatRaw = render.FormatRaw -) - -// NewRenderer builds a renderer writing to w. -func NewRenderer(w io.Writer, format Format, fields []string, noHeader bool, tmpl string) *render.Renderer { - return render.New(w, format, fields, noHeader, tmpl) -} diff --git a/cli/page.go b/cli/page.go new file mode 100644 index 0000000..d52acf5 --- /dev/null +++ b/cli/page.go @@ -0,0 +1,177 @@ +package cli + +import ( + "bufio" + "context" + "encoding/json" + "os" + "sort" + "strings" + + "github.com/tamnd/any-cli/kit" + "github.com/tamnd/any-cli/kit/errs" + "github.com/tamnd/github-cli/gh" + "github.com/tamnd/github-cli/pkg/page" +) + +// page.go is the 1:1 view. Everything else in this tool decides what matters on +// a page and throws the rest away; this command throws nothing away, which makes +// it three things at once. +// +// It is the escape hatch. A consumer who wants a field no record models yet can +// have it today instead of waiting for a release. +// +// It is the debugging tool. When a record comes back thin the first question is +// always whether the data was missing from the page or dropped by the decoder, +// and this is the only way to tell the two apart. +// +// It is how a recorded fixture is read back, since a fixture is the bytes and +// nothing else. +// +// It is a byte-plane command because its output is one document, not a stream of +// records, and pretending otherwise would put a table renderer in front of a +// GraphQL response. + +type pageCmd struct { + section string + query string + raw bool + compact bool +} + +func newPageCmd() kit.Command { + c := &pageCmd{} + return kit.Command{ + Use: "page ", + Short: "Print everything a page carries, organised", + Long: "page fetches one page and prints the whole extraction as JSON: the React\n" + + "route payload, the preloaded Relay queries, GitHub's own schema.org block,\n" + + "the ld+json, the og: and twitter: meta, the microdata, and the deferred\n" + + "fragments the page names for itself.\n\n" + + "--section narrows it to one of payload, queries, structured_data,\n" + + "linked_data, partials, meta, microdata, or fragments. --query prints one\n" + + "preloaded query result by name, which is where issue and pull request pages\n" + + "keep everything. With no argument, --query lists the names.\n\n" + + "--raw writes the original markup instead, which is what you want when the\n" + + "question is about the HTML rather than about the data in it.", + Group: "meta", + Args: kit.ExactArgs(1), + Flags: c.flags, + Run: c.run, + } +} + +func (c *pageCmd) flags(f *kit.FlagSet) { + f.StringVar(&c.section, "section", "", "print one section only") + f.StringVar(&c.query, "query", "", "print one preloaded query by name (empty lists them)") + f.BoolVar(&c.raw, "raw", false, "print the original markup instead of the extraction") + f.BoolVar(&c.compact, "compact", false, "one line of JSON rather than indented") +} + +func (c *pageCmd) run(ctx context.Context, args []string) error { + cl, err := clientFrom(ctx) + if err != nil { + return err + } + url, err := pageURL(args[0]) + if err != nil { + return err + } + p, err := cl.Page(ctx, url) + if err != nil { + return err + } + + w := bufio.NewWriter(os.Stdout) + defer func() { _ = w.Flush() }() + + if c.raw { + _, err := w.Write(p.HTML) + return err + } + + enc := json.NewEncoder(w) + if !c.compact { + enc.SetIndent("", " ") + } + + if c.query != "" { + q, ok := p.Queries[c.query] + if !ok { + // The names are the useful half of this failure. Query names are + // GitHub's internal Relay identifiers, nobody knows them by heart, + // and a bare "not found" would send the reader off to dump the + // whole queries section to find out what to ask for. + // Both messages lead with a word rather than the URL because the + // error renderer capitalises what it starts with, and a + // title-cased URL reads as a typo. + if len(p.Queries) == 0 { + // Worth saying separately. Most pages preload nothing, so + // listing the names it has would be an empty list, and an empty + // list reads like the lookup broke rather than like the page + // carries no queries at all. + return errs.NotFound("no preloaded queries on %s at all; that is normal, only a few page kinds have them", url) + } + return errs.NotFound("no query named %q on %s; it has %s", + c.query, url, strings.Join(queryNames(p.Queries), ", ")) + } + return enc.Encode(q) + } + if c.section != "" { + v, err := section(p, c.section) + if err != nil { + return err + } + return enc.Encode(v) + } + return enc.Encode(p) +} + +// pageURL turns anything a person might paste into the page to fetch. A full +// URL is taken as given, including the parts of the site that name no entity, +// like /trending and /explore, because the debugging tool is least useful on +// exactly the pages the model does not cover yet. +func pageURL(ref string) (string, error) { + if strings.HasPrefix(ref, "http://") || strings.HasPrefix(ref, "https://") { + return ref, nil + } + kind, id, err := gh.Classify(ref) + if err != nil { + return "", err + } + return gh.Locate(kind, id) +} + +func section(p *page.Page, name string) (any, error) { + switch strings.ToLower(strings.TrimSpace(name)) { + case "payload": + return p.Payload, nil + case "queries": + return p.Queries, nil + case "structured_data", "structured-data", "structured": + return p.StructuredData, nil + case "linked_data", "linked-data", "ld", "ld+json": + return p.LinkedData, nil + case "partials": + return p.Partials, nil + case "meta": + return p.Meta, nil + case "microdata": + return p.Microdata, nil + case "fragments": + return p.Fragments, nil + default: + return nil, errs.Usage("unknown --section %q: payload, queries, structured_data, linked_data, partials, meta, microdata, or fragments", name) + } +} + +// queryNames is sorted because the map order would otherwise change between two +// runs against the same bytes, and this output gets diffed. +func queryNames(q map[string]json.RawMessage) []string { + out := make([]string, 0, len(q)) + for k := range q { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/cli/rdf.go b/cli/rdf.go new file mode 100644 index 0000000..acdf944 --- /dev/null +++ b/cli/rdf.go @@ -0,0 +1,121 @@ +package cli + +import ( + "bufio" + "context" + "os" + + "github.com/tamnd/any-cli/kit" + "github.com/tamnd/github-cli/gh" +) + +// rdf.go holds the linked-data output. It is a byte-plane command for the same +// reason cat is: N-Triples and Turtle are not records, they are a serialisation +// with their own rules, and putting them through the record renderer would +// produce something that is neither. + +type rdfCmd struct { + format string + graph string + depth int + follow []string + minTrust string + limit int +} + +func newRDFCmd() kit.Command { + c := &rdfCmd{} + return kit.Command{ + Use: "rdf ", + Short: "Write an entity as RDF triples", + Long: "rdf serialises one entity, its edges, and its literals. N-Triples is the\n" + + "default because it streams line by line, so a deep walk never needs the whole\n" + + "graph in memory; Turtle and JSON-LD do need it and are slower on large graphs\n" + + "for that reason.\n\n" + + "Subjects are the github.com URLs rather than the github:// URIs, so the output\n" + + "is dereferenceable by anything on the web. The URI is kept as a gh:uri\n" + + "literal, so nothing is lost.\n\n" + + "With --depth it walks first and serialises the whole result, which is how you\n" + + "get a loadable dataset rather than one subject.", + Group: "graph", + Args: kit.ExactArgs(1), + Flags: c.flags, + Run: c.run, + } +} + +func (c *rdfCmd) flags(f *kit.FlagSet) { + f.StringVar(&c.format, "format", gh.FormatNT, "nt, ttl, jsonld, or nq") + f.StringVar(&c.graph, "graph", "", "the named graph for nq output (default: the entity URL)") + f.IntVar(&c.depth, "depth", 0, "walk this many edges out before serialising") + f.StringSliceVar(&c.follow, "follow", nil, "predicates to follow when walking") + f.StringVar(&c.minTrust, "min-trust", gh.DefaultMinTrust, "drop edges below this rule") + f.IntVar(&c.limit, "limit", 0, "stop a walk after this many nodes") +} + +func (c *rdfCmd) run(ctx context.Context, args []string) error { + cl, err := clientFrom(ctx) + if err != nil { + return err + } + kind, id, g, err := buildGraph(ctx, cl, args[0], graphWalk{ + depth: c.depth, + follow: c.follow, + minTrust: c.minTrust, + limit: c.limit, + }) + if err != nil { + return err + } + gh.SortEdges(g.Edges) + + w := bufio.NewWriter(os.Stdout) + defer func() { _ = w.Flush() }() + + graph := c.graph + if graph == "" { + graph, _ = gh.Locate(kind, id) + } + return gh.WriteRDF(w, g, gh.RDFOptions{Format: c.format, Graph: graph}) +} + +// graphWalk is the set of knobs rdf and export share. They are the same walk +// with a different writer on the end, so the flags are declared twice and read +// once. +type graphWalk struct { + depth int + follow []string + minTrust string + limit int +} + +// buildGraph resolves a reference and returns either the one entity or the whole +// walk, depending on depth. Both come back as a Graph, so the serialisers never +// need to know which it was. +// +// It holds the result in memory, which is the price of the formats that cannot +// stream. `github crawl` is the streaming answer for a walk too big for this. +func buildGraph(ctx context.Context, cl *gh.Client, ref string, o graphWalk) (string, string, *gh.Graph, error) { + kind, id, g, err := cl.GraphOfRef(ctx, ref) + if err != nil { + return "", "", nil, err + } + if o.depth <= 0 { + return kind, id, g, nil + } + g = &gh.Graph{} + err = cl.Crawl(ctx, gh.URI(kind, id), gh.CrawlOptions{ + Depth: o.depth, + Follow: o.follow, + MinTrust: o.minTrust, + Limit: o.limit, + }, gh.CrawlSink{ + Node: func(n *gh.Node) error { g.AddNode(*n); return nil }, + Edge: func(e *gh.Edge) error { g.Edges = append(g.Edges, *e); return nil }, + Fact: func(f *gh.Fact) error { g.Facts = append(g.Facts, *f); return nil }, + }) + if err != nil { + return "", "", nil, err + } + return kind, id, g, nil +} diff --git a/cli/root.go b/cli/root.go index 015de9b..e50961c 100644 --- a/cli/root.go +++ b/cli/root.go @@ -1,13 +1,10 @@ -// Package cli builds the ghb command tree on top of the github library. +// Package cli assembles the github command tree from the gh domain on top of +// the any-cli/kit framework. package cli import ( - "fmt" - "os" - - "github.com/mattn/go-isatty" - "github.com/spf13/cobra" - "github.com/tamnd/github-cli/github" + "github.com/tamnd/any-cli/kit" + "github.com/tamnd/github-cli/gh" ) // Build metadata, set via -ldflags at release time. @@ -17,139 +14,32 @@ var ( Date = "unknown" ) -// exit codes. -const ( - exitError = 1 - exitUsage = 2 - exitNoData = 3 -) - -// ExitError carries a process exit code up to main. -type ExitError struct { - Code int - Err error -} - -func (e *ExitError) Error() string { - if e.Err != nil { - return e.Err.Error() - } - return fmt.Sprintf("exit %d", e.Code) -} - -func (e *ExitError) Unwrap() error { return e.Err } - -func codeError(code int, err error) error { return &ExitError{Code: code, Err: err} } - -// App holds shared state threaded through every command. -type App struct { - client *github.Client - cfg github.Config - - output string - fields []string - noHeader bool - template string - limit int - quiet bool -} - -// Root builds the root command and its subtree. -func Root() *cobra.Command { - app := &App{cfg: github.DefaultConfig()} - - root := &cobra.Command{ - Use: "ghb", - Short: "Browse GitHub repositories, users and releases", - Long: `ghb reads public GitHub data through the GitHub REST API v3. -No authentication is required. Returns records as table, JSON, JSONL, -CSV, TSV, or URLs. - -ghb is an independent tool and is not affiliated with GitHub or Microsoft.`, - SilenceUsage: true, - SilenceErrors: true, - PersistentPreRunE: func(cmd *cobra.Command, _ []string) error { - return app.setup() - }, - } - - pf := root.PersistentFlags() - pf.StringVarP(&app.output, "output", "o", "auto", "output: table|json|jsonl|csv|tsv|url|raw (auto=table on TTY, jsonl piped)") - pf.StringSliceVar(&app.fields, "fields", nil, "comma-separated columns to include") - pf.BoolVar(&app.noHeader, "no-header", false, "omit the header row in table/csv/tsv") - pf.StringVar(&app.template, "template", "", "Go text/template applied per record") - pf.IntVarP(&app.limit, "limit", "n", 0, "limit number of records (0 = command default)") - pf.BoolVarP(&app.quiet, "quiet", "q", false, "suppress progress on stderr") - - pf.DurationVar(&app.cfg.Rate, "delay", app.cfg.Rate, "minimum spacing between requests") - pf.DurationVar(&app.cfg.Timeout, "timeout", app.cfg.Timeout, "per-request timeout") - pf.IntVar(&app.cfg.Retries, "retries", app.cfg.Retries, "retry attempts on 429/5xx") - pf.StringVar(&app.cfg.UserAgent, "user-agent", app.cfg.UserAgent, "User-Agent sent with each request") - - root.AddCommand( - app.searchCmd(), - app.repoCmd(), - app.trendingCmd(), - app.userCmd(), - app.releasesCmd(), - newVersionCmd(), - ) - return root -} - -func (a *App) setup() error { - if a.output == "" || a.output == "auto" { - if isatty.IsTerminal(os.Stdout.Fd()) { - a.output = string(FormatTable) - } else { - a.output = string(FormatJSONL) - } - } - if !Format(a.output).Valid() { - return codeError(exitUsage, fmt.Errorf("unknown output format %q", a.output)) - } - a.client = github.NewClient(a.cfg) - return nil -} - -func (a *App) render(records any) error { - r := NewRenderer(os.Stdout, Format(a.output), a.fields, a.noHeader, a.template) - return r.Render(records) -} - -func (a *App) renderOrEmpty(records any, n int) error { - if err := a.render(records); err != nil { - return err - } - if n == 0 { - return codeError(exitNoData, nil) - } - return nil -} - -func (a *App) progressf(format string, args ...any) { - if a.quiet { - return - } - _, _ = fmt.Fprintf(os.Stderr, format+"\n", args...) -} - -func mapFetchErr(err error) error { - if err == nil { - return nil - } - if isNotFound(err) { - return codeError(exitNoData, err) - } - if isRateLimit(err) { - return codeError(exitError, err) - } - return codeError(exitError, err) -} - -func (a *App) effectiveLimit(def int) int { - if a.limit > 0 { - return a.limit - } - return def +// NewApp assembles the kit application from the gh domain. The domain's +// Register installs the client factory and every operation, so the binary and a +// multi-domain host (ant, which blank-imports the package) share one source of +// truth. kit.Run turns the App into the CLI, plus the serve and mcp surfaces and +// the typed-error-to-exit-code mapping. +// +// To add a command, declare it in gh/ops.go with kit.Handle and it appears here +// automatically. Reach for app.AddCommand only for a verb that does not fit the +// emit-records shape, the way the byte-plane commands below do not. +func NewApp() *kit.App { + id := gh.Domain{}.Info().Identity + id.Version = Version + + // WithDefaults is how the site's own baseline reaches the resolved config. + // Without it the run would use the framework's numbers, which are tuned for + // an API with a published rate limit rather than for a CDN. + app := kit.New(id, kit.WithDefaults(gh.DomainDefaults)) + (gh.Domain{}).Register(app) + + app.AddCommand(newVersionCmd()) + app.AddCommand(newCatCmd()) + app.AddCommand(newReadmeCmd()) + app.AddCommand(newArchiveCmd()) + app.AddCommand(newDiffCmd()) + app.AddCommand(newPageCmd()) + app.AddCommand(newRDFCmd()) + app.AddCommand(newExportCmd()) + return app } diff --git a/cli/version.go b/cli/version.go index bdcd7ab..54a0fd3 100644 --- a/cli/version.go +++ b/cli/version.go @@ -1,27 +1,41 @@ package cli import ( + "context" "fmt" + "os" "runtime" - "github.com/spf13/cobra" + "github.com/tamnd/any-cli/kit" ) -func newVersionCmd() *cobra.Command { - var short bool - cmd := &cobra.Command{ +// versionCmd is an escape-hatch command: it prints build info rather than +// emitting records, so it does not fit the operation shape. fang also wires +// --version from the App identity; this adds a `version` subcommand and a +// --short form for scripts. +type versionCmd struct{ short bool } + +func newVersionCmd() kit.Command { + v := &versionCmd{} + return kit.Command{ Use: "version", Short: "Print version information", - RunE: func(c *cobra.Command, _ []string) error { - if short { - _, _ = fmt.Fprintln(c.OutOrStdout(), Version) - return nil - } - _, _ = fmt.Fprintf(c.OutOrStdout(), "ghb %s (commit %s, built %s, %s/%s, %s)\n", - Version, Commit, Date, runtime.GOOS, runtime.GOARCH, runtime.Version()) - return nil - }, + Args: kit.NoArgs, + Flags: v.flags, + Run: v.run, + } +} + +func (v *versionCmd) flags(f *kit.FlagSet) { + f.BoolVar(&v.short, "short", false, "print just the version number") +} + +func (v *versionCmd) run(_ context.Context, _ []string) error { + if v.short { + _, _ = fmt.Fprintln(os.Stdout, Version) + return nil } - cmd.Flags().BoolVar(&short, "short", false, "print just the version number") - return cmd + _, _ = fmt.Fprintf(os.Stdout, "github %s (commit %s, built %s, %s/%s, %s)\n", + Version, Commit, Date, runtime.GOOS, runtime.GOARCH, runtime.Version()) + return nil } diff --git a/cmd/ghb/main.go b/cmd/ghb/main.go deleted file mode 100644 index 3ab444c..0000000 --- a/cmd/ghb/main.go +++ /dev/null @@ -1,27 +0,0 @@ -// Command ghb is a single-binary command line for github. -package main - -import ( - "context" - "os" - "os/signal" - "syscall" - - "github.com/charmbracelet/fang" - "github.com/tamnd/github-cli/cli" -) - -func main() { - ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM) - defer stop() - - root := cli.Root() - // fang gives styled help, errors, and shell completion for free; the command - // tree and its exit-code mapping stay in the cli package. - if err := fang.Execute(ctx, root, - fang.WithVersion(cli.Version), - fang.WithNotifySignal(os.Interrupt, syscall.SIGTERM), - ); err != nil { - os.Exit(1) - } -} diff --git a/cmd/github/main.go b/cmd/github/main.go new file mode 100644 index 0000000..6fba252 --- /dev/null +++ b/cmd/github/main.go @@ -0,0 +1,14 @@ +// Command github is a command-line for GitHub that scrapes HTML pages and +// Atom feeds. No API key or authentication is required. +// +// github is an independent tool and is not affiliated with GitHub or Microsoft. +package main + +import ( + "github.com/tamnd/any-cli/kit" + "github.com/tamnd/github-cli/cli" +) + +func main() { + kit.Main(cli.NewApp()) +} diff --git a/docs/content/_index.md b/docs/content/_index.md index 6c9181c..bec2440 100644 --- a/docs/content/_index.md +++ b/docs/content/_index.md @@ -1,26 +1,63 @@ --- -title: "ghb" -description: "Browse GitHub repositories, users and releases" -heroTitle: "github, from the command line" -heroLead: "Browse GitHub repositories, users and releases One pure-Go binary, no API key, output that pipes into the rest of your tools." +title: "github" +description: "Read github.com as data, with no API token." +heroTitle: "GitHub, as records" +heroLead: "Every repository, user, organization, issue, pull request, discussion, commit, branch, tag, release, file, topic, gist, package, and marketplace action, with every field its source returned, a canonical github:// address, and edges to everything it names." heroPrimaryURL: "/getting-started/quick-start/" heroPrimaryText: "Get started" --- -Browse GitHub repositories, users and releases +`github` is one pure-Go binary. +It reads github.com over plain HTTPS, shapes what comes back into typed records, and prints output that pipes into the rest of your tools. + +There is no API token anywhere in it and there will not be one. ```bash -ghb --help # the command tree -ghb version # build info +github repo gohugoio/hugo # every field the page states +github owned torvalds -n 10 # an account's repositories +github graph golang/go # the node and its edges +github rdf gohugoio/hugo --format ttl # schema.org triples ``` -This site is a starting point. As you add commands, document each one under -[guides](/guides/) and keep the [CLI reference](/reference/cli/) in step with -the real flag set. +Output adapts to where it goes: an aligned table on your terminal, JSONL the moment you pipe it somewhere. + +![github reading a repository, listing an account, showing dependencies, printing graph edges, and emitting schema.org triples](/demo.gif) + +## Why no token + +The unauthenticated REST API allows sixty requests an hour, which is not enough to read one organization. +The pages sit behind a CDN, so they are faster than the API even where the API would work, and they carry more: trending, contribution calendars, dependency graphs, and download counts have no API route at all. + +So this tool reads the same JSON the site's own browser code reads, plus the Atom feeds, the search JSON, the raw host, and the git protocol. +The cost is that it is read-only and public-only. +For anything else, use the official [gh](https://cli.github.com). + +## What makes it different + +- **Six surfaces, one record.** A repository record is assembled from the embedded React payload, the route JSON, a deferred XHR fragment, and the search index, and it records which URL each field came from. +- **It is a graph.** Repository references, mentions, closing keywords, forks, and dependencies all become typed edges, each one carrying the source that stated it. +- **It exports linked data.** N-Triples, Turtle, JSON-LD, and N-Quads, keyed on the same `github://` addresses. +- **Nothing is hidden.** `github page` prints the whole extraction of any URL, so a field no record models yet is still reachable today. + +## Two ways to use it + +- **As a command** for reading GitHub by hand or in a script. + Start with the [quick start](/getting-started/quick-start/). +- **As a resource-URI driver** so a host like [ant](https://github.com/tamnd/ant) can address the site as `github://` URIs and follow links across sites. + See [resource URIs](/guides/resource-uris/). + +Both are the same code: one operation, declared once, is a CLI command, an HTTP route, an MCP tool, and a URI dereference. + +Not affiliated with GitHub or Microsoft. +It reads the public site the way any reader does. ## Where to go next -- New here? Read the [introduction](/getting-started/introduction/), then the - [quick start](/getting-started/quick-start/). -- Installing? See [installation](/getting-started/installation/). -- Need every flag? The [CLI reference](/reference/cli/) is the full surface. +- New here? + Read the [introduction](/getting-started/introduction/), then the [quick start](/getting-started/quick-start/). +- Installing? + See [installation](/getting-started/installation/). +- Doing a specific job? + The [guides](/guides/) are task-first. +- Need every flag? + The [CLI reference](/reference/cli/) is the full surface. diff --git a/docs/content/getting-started/_index.md b/docs/content/getting-started/_index.md index b7fa0ac..1416798 100644 --- a/docs/content/getting-started/_index.md +++ b/docs/content/getting-started/_index.md @@ -1,12 +1,9 @@ --- title: "Getting started" linkTitle: "Getting started" -description: "Install ghb, learn the model, and run your first command." +description: "Install github, learn the model, and run your first command." weight: 10 featured: true --- -Three short pages: the [introduction](/getting-started/introduction/) for how -ghb is put together, [installation](/getting-started/installation/) to -get the binary, and the [quick start](/getting-started/quick-start/) to run -something real. +Three short pages: the [introduction](/getting-started/introduction/) for how `github` is put together, [installation](/getting-started/installation/) to get the binary, and the [quick start](/getting-started/quick-start/) to run something real. diff --git a/docs/content/getting-started/installation.md b/docs/content/getting-started/installation.md index 7dee68a..ccae32a 100644 --- a/docs/content/getting-started/installation.md +++ b/docs/content/getting-started/installation.md @@ -1,45 +1,80 @@ --- title: "Installation" -description: "Install ghb from a release, with go install, or from source." +description: "Install github from a release, a package manager, go install, or source." weight: 20 --- -## Prebuilt binaries - -Every [release](https://github.com/tamnd/github-cli/releases) carries archives for Linux, macOS, -and Windows on amd64 and arm64, plus deb, rpm, and apk packages for Linux. -Download, unpack, put `ghb` on your `PATH`, done. The `checksums.txt` -on each release is signed with keyless [cosign](https://docs.sigstore.dev/) if -you want to verify before running. +The binary is called `github`. +It does not replace the official `gh`, which does the authenticated half of the site far better. +This one does the public half without asking you to log in. ## With Go ```bash -go install github.com/tamnd/github-cli/cmd/ghb@latest +go install github.com/tamnd/github-cli/cmd/github@latest ``` -That puts `ghb` in `$(go env GOPATH)/bin`, which is `~/go/bin` unless -you moved it. Make sure that directory is on your `PATH`. +That puts `github` in `$(go env GOPATH)/bin`, which is `~/go/bin` unless you moved it. +Make sure that directory is on your `PATH`. -## From source +## Package managers ```bash -git clone https://github.com/tamnd/github-cli -cd github-cli -make build # produces ./bin/ghb -./bin/ghb version +# Homebrew (macOS) +brew install --cask tamnd/tap/github-cli + +# Scoop (Windows) +scoop bucket add tamnd https://github.com/tamnd/scoop-bucket +scoop install github-cli + +# apt (Debian, Ubuntu) +curl -fsSL https://tamnd.github.io/linux-repo/gpg.key | sudo gpg --dearmor -o /usr/share/keyrings/tamnd.gpg +echo "deb [signed-by=/usr/share/keyrings/tamnd.gpg] https://tamnd.github.io/linux-repo/apt stable main" | sudo tee /etc/apt/sources.list.d/tamnd.list +sudo apt update && sudo apt install github-cli + +# dnf (Fedora, RHEL) +sudo dnf config-manager --add-repo https://tamnd.github.io/linux-repo/dnf/tamnd.repo +sudo dnf install github-cli +``` + +The Homebrew formula is a cask rather than a formula because `brew install github` already means GitHub Desktop. + +## Prebuilt binaries + +Every [release](https://github.com/tamnd/github-cli/releases) carries archives for Linux, macOS, Windows, and FreeBSD on amd64 and arm64, plus deb, rpm, and apk packages for Linux. +Download, unpack, put `github` on your `PATH`, done. + +Each release also ships an SBOM and a `checksums.txt` signed with keyless [cosign](https://docs.sigstore.dev/), if you want to verify before running: + +```bash +cosign verify-blob checksums.txt \ + --certificate checksums.txt.pem \ + --signature checksums.txt.sig \ + --certificate-identity-regexp 'https://github\.com/tamnd/github-cli/.*' \ + --certificate-oidc-issuer https://token.actions.githubusercontent.com ``` ## Container image ```bash -docker run --rm ghcr.io/tamnd/ghb:latest --help +docker run --rm ghcr.io/tamnd/github-cli:latest repo gohugoio/hugo +``` + +## From source + +```bash +git clone https://github.com/tamnd/github-cli +cd github-cli +make build # produces ./bin/github +./bin/github version ``` ## Checking the install ```bash -ghb version +github version +github doctor ``` -prints the version and exits. +`doctor` is the more useful of the two. +It reports whether the site answers, whether the page still carries the payload every reader expects, whether the cache is writable, what pacing this run is using, and whether you have a token set that this tool is about to ignore. diff --git a/docs/content/getting-started/introduction.md b/docs/content/getting-started/introduction.md index f567d6f..5b31299 100644 --- a/docs/content/getting-started/introduction.md +++ b/docs/content/getting-started/introduction.md @@ -1,29 +1,51 @@ --- title: "Introduction" -description: "What ghb is and how it is put together." +description: "What github is, why it uses no token, and how it is put together." weight: 10 --- -Browse GitHub repositories, users and releases +`github` is a single binary that reads github.com over plain HTTPS and gives back records rather than pages. +There is nothing to sign up for and nothing to run alongside it. -ghb is a single binary. It speaks to github over plain HTTPS, -shapes the responses into clean records, and gets out of your way. There is -nothing to sign up for and nothing to run alongside it. +## Why there is no token -## How it is built +The unauthenticated REST API allows sixty requests an hour. +That is not enough to read one organization, let alone crawl a graph, and the moment a tool needs a token it needs a place to put it, a way to rotate it, and a story about what happens when it leaks. + +The public site has none of those problems. +It sits behind a CDN, so it answers faster than the API does even for the things the API covers, and it covers more: trending, contribution calendars, the dependency graph, release download counts, and the marketplace have no API route at all. + +So this tool sends no `Authorization` header, never reads `GITHUB_TOKEN` or `GH_TOKEN`, and never touches `api.github.com`. +A test in the package parses every source file and fails the build if any of those four strings appear outside the doctor that warns about them. + +The cost is that this is read-only and public-only. +For anything else, use the official [gh](https://cli.github.com). + +## Seven surfaces -- A **library package** (`github`) holds the HTTP client and the typed - data models. It paces requests, sets an honest User-Agent, and retries the - transient failures any public site throws under load. -- A **command tree** (`cli`) wraps the library in subcommands with shared - output formats and flags. -- One **`cmd/ghb`** entry point ties them together. +GitHub serves the same facts several different ways, and no single one of them is complete. +`github` reads all of them and treats the choice as a routing problem: + +| Surface | What it is | +|---|---| +| `html` | The rendered page, and the React payload embedded in it | +| `route-json` | The same route asked for with `Accept: application/json`, which is what the site's own navigation uses | +| `xhr` | Fragments the page defers and fetches separately | +| `search` | The search backend, which answers in JSON for ten types | +| `feed` | The Atom feeds for commits, releases, tags, wikis, and accounts | +| `raw` | `raw.githubusercontent.com` and `codeload.github.com`, for file bytes and archives | +| `git` | The dumb HTTP git protocol, for a complete ref list in one request | + +`github routes` prints the whole mapping, including which surface is primary for each route and which one it falls back to. + +## How it is built -## Scope +- The **`gh` package** holds the HTTP client, the surface router, the response cache, and the typed records. + It paces requests, sets an honest User-Agent, retries the transient failures any public site throws, and records for every field which URL stated it. +- The **`pkg/page` package** does the extraction: the React payload, the preloaded Relay queries, schema.org blocks, ld+json, og and twitter meta, microdata, and the deferred fragments a page names for itself. +- The **`cli` package** wraps operations in subcommands. +- One **`cmd/github`** entry point ties them together. -ghb is a read-only client over data github already serves -publicly. It reads that data and shapes it for you. That narrow scope keeps it a -single small binary with no database, no daemon, and no setup. +Operations are declared once with [kit](https://github.com/tamnd/any-cli) and appear as a CLI command, an HTTP route, an MCP tool, and a `github://` URI type without any per-surface code. -Next: [install it](/getting-started/installation/), then take the -[quick start](/getting-started/quick-start/). +Next: [install it](/getting-started/installation/), then take the [quick start](/getting-started/quick-start/). diff --git a/docs/content/getting-started/quick-start.md b/docs/content/getting-started/quick-start.md index 015e730..a4834cc 100644 --- a/docs/content/getting-started/quick-start.md +++ b/docs/content/getting-started/quick-start.md @@ -1,19 +1,97 @@ --- title: "Quick start" -description: "Run your first ghb command." +description: "Read a repository, list an account, walk the graph, export triples." weight: 30 --- -Once `ghb` is on your `PATH`: +Once `github` is on your `PATH`, there is nothing to configure. +No login, no token, no config file. + +## Read one thing + +```bash +github repo gohugoio/hugo +github user torvalds +github org golang +github issue golang/go 1 +github pr kubernetes/kubernetes 100000 +github release gohugoio/hugo # the latest one +github commit golang/go 3e5c0f3 +``` + +Or let it work out what a reference names: + +```bash +github get gohugoio/hugo +github get https://github.com/golang/go/issues/1 +github get github://pr/golang/go#1 +``` + +A reference can be `owner/name`, a github.com URL, or a `github://` URI, on every command that takes one. +`github url ` parses one and tells you what it decided, which is the fastest way to check a shape: + +```console +$ github url gohugoio/hugo +{"kind":"repo","id":"gohugoio/hugo","uri":"github://repo/gohugoio/hugo","url":"https://github.com/gohugoio/hugo"} +``` + +## List things + +```bash +github owned torvalds -n 10 # an account's repositories +github releases gohugoio/hugo +github commits golang/go -n 20 +github contributors gohugoio/hugo +github stars torvalds +github members golang +github trending --language go --since weekly +``` + +## Search ```bash -ghb --help # see the command tree -ghb version # build info +github repos "language:go stars:>10000" --sort stars +github issues "repo:golang/go is:open label:NeedsInvestigation" +github users "location:vietnam followers:>500" +github search "site generator" # every type at once ``` -This is a fresh scaffold, so the command tree is just `version` for now. Add -your first real command in `cli/`, build on the `github` library package, -and document it here. +## Look inside a repository + +```bash +github tree gohugoio/hugo # the root +github tree gohugoio/hugo commands --recursive +github cat gohugoio/hugo go.mod # bytes to stdout +github readme gohugoio/hugo +github diff golang/go 3e5c0f3 +github archive gohugoio/hugo --format zip -o hugo.zip +``` + +## Walk the graph + +```bash +github graph gohugoio/hugo # the node, its edges, and its facts +github edges golang/go # edges only +github crawl golang/go --depth 2 --kinds repo,user +github rdf gohugoio/hugo --format ttl +``` + +## Where output goes + +A table when you are looking at it, JSONL the moment you pipe it: + +```bash +github releases gohugoio/hugo -n 5 # a table +github releases gohugoio/hugo -n 5 | jq -r .tag # JSONL +github releases gohugoio/hugo --fields tag,author,published +``` + +## When something looks wrong + +```bash +github doctor # environment, site, page shape, cache, pacing +github routes # which surface answers for which route +github page gohugoio/hugo # everything the page carries, organised +``` -A good first command usually fetches one thing and prints it as JSON, so the -output pipes straight into `jq` and the rest of your tools. +Next: the [guides](/guides/) are task-first, and the [CLI reference](/reference/cli/) is the whole surface. diff --git a/docs/content/guides/_index.md b/docs/content/guides/_index.md index ae79973..9a18c4b 100644 --- a/docs/content/guides/_index.md +++ b/docs/content/guides/_index.md @@ -1,11 +1,17 @@ --- title: "Guides" linkTitle: "Guides" -description: "Task-oriented walkthroughs for the things people do with ghb." +description: "Task-oriented walkthroughs for the things people do with github." weight: 20 featured: true --- -Each guide is built around a job rather than a command. Add one per real task as -you build out ghb. They assume you have run the -[quick start](/getting-started/quick-start/). +Each guide is built around a job rather than a command. +They assume you have run the [quick start](/getting-started/quick-start/). + +- [Walk the graph](/guides/graph/) — URIs, edges, trust, and crawling GitHub as a knowledge graph. +- [Export linked data](/guides/linked-data/) — N-Triples, Turtle, JSON-LD, and N-Quads out of the site. +- [Look inside a repository](/guides/contents/) — trees, files, diffs, archives, and what symbols will and will not do. +- [Read what the API will not tell you](/guides/pages/) — the page plane, `--deep`, and `github page`. +- [Resource URIs](/guides/resource-uris/) — use `github` as a driver so a host program can address the site as `github://` URIs. +- [Add a command](/guides/adding-a-command/) — declare an operation once and get a command, a route, an MCP tool, and a URI type. diff --git a/docs/content/guides/adding-a-command.md b/docs/content/guides/adding-a-command.md new file mode 100644 index 0000000..3db5a50 --- /dev/null +++ b/docs/content/guides/adding-a-command.md @@ -0,0 +1,189 @@ +--- +title: "Add a command" +description: "Declare an operation once and get a command, a route, an MCP tool, and a URI type." +weight: 60 +--- + +Everything `github` does is one `kit.Handle` registration in `gh/ops.go`, backed by a client method and a record type. +Add those three pieces and every surface updates itself. + +## 1. The record + +Records live in the `gh` package, one file per family. +Every one embeds `Base`, which carries the kind, the id, the two addresses, the sources it was read from, the tier each field came from, and `Extra`: + +```go +type Widget struct { + Base + Name string `json:"name" table:"name"` + Owner string `json:"owner,omitempty" table:"owner"` + Stars *int `json:"stars,omitempty" table:"stars,num"` +} +``` + +Two habits matter here. + +A count is a pointer, because zero and absent are different answers and a table that shows `0` for "GitHub did not say" is lying. + +And every decoder ends with `decodeExtra`, which returns the keys the struct did not claim: + +```go +w.setIdentity(KindWidget, id) +w.addSource(resp.URL) +w.addExtra("widgetPayload", decodeExtra(raw, &w)) +``` + +That is the rule the whole tool is built on: decode what you model, keep what you do not. +Without it, a field GitHub adds next month is a field nobody ever sees. +With it, `Extra` names the block it came from and the field shows up the first time anyone looks. + +## 2. The client method + +```go +func (c *Client) Widget(ctx context.Context, id string) (*Widget, error) { + var w Widget + res, err := c.GetJSON(ctx, BaseURL+"/"+id+"/widget", SurfaceRouteJSON, &w) + if err != nil { + return nil, err + } + w.setIdentity(KindWidget, id) + w.addSource(res.URL) + return &w, nil +} +``` + +`Get`, `GetJSON`, `GetHTML`, `Page`, `Stream`, and `Poll` are the whole client surface, and every one of them handles pacing, retries, the response cache, and the status-to-error mapping, so a method that goes through them inherits the whole policy. +`Poll` is the one for a route that answers 202 while GitHub computes, which is how the contributor graph works. +Which surface you reach for is the real decision, and `github routes` is the record of every one already made. +See [the page plane](/guides/pages/) for what each surface is. + +## 3. The operation + +An input struct declares the arguments and flags by reflection, and the handler emits records: + +```go +type widgetIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or any URL from the repository"` +} + +func getWidget(ctx context.Context, in widgetIn, emit func(*Widget) error) error { + id, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + w, err := in.C.Widget(ctx, id) + if err != nil { + return err + } + return emit(w) +} +``` + +Then register it in the right `register*Ops` function: + +```go +kit.Handle(app, kit.OpMeta{ + Name: "widget", Group: "discover", URIType: KindWidget, Single: true, Resolver: true, + Summary: "Read one widget", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, +}, getWidget) +``` + +That is the whole change. +The operation is now: + +```bash +github widget golang/go # the command +curl 'localhost:7777/v1/widget?ref=golang/go' # the route, under serve +ant get github://widget/golang/go # the URI dereference, via a host +``` + +It is also an MCP tool under `github mcp`, with the summary as its description and the input struct as its schema. + +## Input struct tags + +| Tag | Meaning | +|---|---| +| `kit:"arg"` | a positional argument | +| `kit:"arg,variadic"` | the rest of the positionals | +| `kit:"flag"` | a flag, named from the field in snake case | +| `kit:"flag,name=min-trust"` | a flag with the name spelled out | +| `kit:"flag,short=r"` | a flag with a short form | +| `kit:"flag,inherit"` | a flag the app already defines globally, like `Limit` | +| `kit:"inject"` | filled in by the app, which is how the client arrives | + +Alongside them, `help:"..."`, `default:"..."`, and `enum:"a,b,c"` do what they look like. + +## Resolver ops and list ops + +Two fields shape how a host treats an operation: + +- **`Single: true`** with **`Resolver: true`** marks the canonical one-record fetch for a `URIType`. + It answers `ant get`. +- **`List: true`** marks a member-lister for a parent resource. + It answers `ant ls`, and should emit records that are themselves addressable so every member is a URI a host can follow. + +## Reference resolution + +Handlers do not parse references themselves. +`ResolveRepo` takes anything that names a repository, including a file URL or an issue URL, and gives back `owner/name`. +`ResolveRef(kind, input)` does the same for any other kind. + +Both are strict about explicit input and forgiving about guesses. +`github org golang` works because a bare word is a guess that yields to the command. +`github org https://github.com/golang/go` fails, because that URL states its kind and the kind is wrong. + +## Errors + +Return the kinds from `kit/errs` and every surface reports the same outcome with the same exit code: + +```go +return errs.Usage("not a widget id: %q", id) +return errs.NotFound("no widget in %s", repo) +``` + +Start the message with a plain lowercase word. +The renderer capitalises the first token, so a message that leads with its argument comes back as `"Golang/Go" is a repo` and reads like the tool mangled the input. +`gh/messages_test.go` parses every source file and fails on messages that lead with a quote, a format verb, a URL, or a capital, because the mistake is invisible in review and obvious on screen. + +## Add it to the tests + +The offline suite has no network in it. +What guards a new command is a live test in `gh/live_test.go`, which runs only under `GITHUB_LIVE=1`: + +```go +func TestLiveWidget(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + w, err := c.Widget(ctx, "gohugoio/hugo") + if err != nil { + t.Fatal(err) + } + if w.Name == "" { + t.Error("name missing, the widget payload did not decode") + } + if len(w.Extra) > 0 { + t.Errorf("unmodelled keys: %s", w.Extra) + } +} +``` + +Assert shape, not values. +A star count changes hourly and pinning one turns a test into a clock. +The `Extra` check is the one that earns its keep: it is what tells you GitHub added a field. + +```bash +make test # offline, the default +make live # against github.com +``` + +If a new command touches the page plane, add a check to `github doctor` too, so a reader whose command came back empty can find out why without reading the source. + +## Commands that write bytes + +An operation that produces a file or a stream rather than records is not a `kit.Handle`. +Use `kit.Command` and register it in `cli/`, the way `cat`, `readme`, `diff`, `archive`, `rdf`, `export`, and `page` do. +Those are the escape hatch, and there are only a handful of them for a reason: a record is almost always the better answer. diff --git a/docs/content/guides/contents.md b/docs/content/guides/contents.md new file mode 100644 index 0000000..f04f39c --- /dev/null +++ b/docs/content/guides/contents.md @@ -0,0 +1,99 @@ +--- +title: "Look inside a repository" +description: "Trees, files, diffs, archives, and the one command that no longer works." +weight: 30 +--- + +Every command here takes a repository, a github.com URL, or a `github://` URI, and every one of them accepts `--rev` to read a branch, a tag, or a commit sha instead of the default branch. + +## The tree + +```bash +github tree gohugoio/hugo # the root +github tree gohugoio/hugo commands # one directory +github tree gohugoio/hugo --recursive # the whole thing +github tree gohugoio/hugo --rev v0.164.0 +``` + +`--recursive` is one request per directory, so on anything large it is worth reaching for `github archive` instead and reading the tarball locally. + +Ids carry the commit, not the branch, which is what makes them stable: + +``` +gohugoio/hugo@7df45f615a8ce8611a2288b5ce23eb81a6158b18/commands +``` + +## One file + +`blob` is the metadata: size, language, whether it is binary, whether it is LFS, whether GitHub thinks it is generated, and the raw URL: + +```bash +github blob gohugoio/hugo go.mod +github blob gohugoio/hugo go.mod --content # the bytes, as a field +github blob gohugoio/hugo go.mod --styled # the rendered lines and highlight spans +``` + +`cat` is the bytes and nothing else, streamed: + +```bash +github cat gohugoio/hugo go.mod +github cat gohugoio/hugo commands/commands.go --rev v0.164.0 +github cat https://github.com/gohugoio/hugo/blob/master/go.mod +``` + +`cat` goes straight through `raw.githubusercontent.com`, so a large file costs no memory and is never cached. + +## The README + +```bash +github readme gohugoio/hugo # rendered text +github readme gohugoio/hugo --html # the markup the page carries +``` + +GitHub renders the README server-side, so what comes back is what the page shows: badges resolved, relative links rewritten. + +## Diffs + +```bash +github diff golang/go 6db72bb # one commit +github diff gohugoio/hugo v0.163.0 v0.164.0 # a range +github diff https://github.com/gohugoio/hugo/pull/13000 # a pull request +github diff golang/go 6db72bb --patch # the format-patch mailbox +``` + +`--patch` gives you something `git am` will apply, with the author, date, and message of every commit in it. + +`github compare` is the record form of the same thing, if you want the list of changed files rather than the diff text: + +```bash +github compare gohugoio/hugo v0.163.0 v0.164.0 +``` + +## The whole thing at once + +```bash +github archive gohugoio/hugo -o hugo.tar.gz +github archive gohugoio/hugo v0.164.0 --format zip -o hugo.zip +github archive gohugoio/hugo | tar tz | head +``` + +One request to codeload gets the entire tree. +For anything past a few directories this beats walking `github tree --recursive`. + +## Symbols, and why they do not work + +`github symbols` asks GitHub for the definitions it extracted from a file. +GitHub still runs that extractor, still sets `symbolsEnabled` on every blob, and still renders the button in the page. +It no longer serves the list to a signed-out reader: the `symbols` key comes back null on every file of every repository tried, on both the page and the route JSON, with or without a cache. + +The command stays because there is no unauthenticated REST equivalent anywhere, so if that ever comes back this is where it will show up. +Until then it reports that plainly rather than pretending the file has no symbols: + +```console +$ github symbols gohugoio/hugo commands/commands.go + ERROR + + No symbol list for commands/commands.go: GitHub serves none to a signed-out reader, though it still offers the panel. +``` + +Next: [read what the API will not tell you](/guides/pages/). diff --git a/docs/content/guides/graph.md b/docs/content/guides/graph.md new file mode 100644 index 0000000..c789f2c --- /dev/null +++ b/docs/content/guides/graph.md @@ -0,0 +1,129 @@ +--- +title: "Walk the graph" +description: "How github turns the site into nodes and edges, and how to crawl them." +weight: 10 +--- + +GitHub is already a graph. +A repository is owned by an org, forked from another repository, written in a language, and tagged with topics. +An issue is authored by a user, labelled, assigned, and closed by a pull request that targets a branch that points at a commit. +`github` reads all of that and gives you nodes and edges. + +## Addresses + +Every entity has one canonical address, and every command accepts it: + +``` +github://repo/gohugoio/hugo +github://user/torvalds +github://org/golang +github://issue/golang/go#1 +github://pr/golang/go#1 +github://commit/golang/go@3e5c0f3 +github://file/gohugoio/hugo@master/go.mod +github://release/gohugoio/hugo@v0.164.0 +github://topic/static-site-generator +github://gist/aaronsw/1234567 +``` + +`github url ` turns anything into one: + +```console +$ github url https://github.com/golang/go/issues/1 +{"kind":"issue","id":"golang/go#1","uri":"github://issue/golang/go#1","url":"https://github.com/golang/go/issues/1"} +``` + +The same three forms work as input everywhere: `owner/name`, a github.com URL, or a `github://` URI. + +## One node and its edges + +```console +$ github graph gohugoio/hugo +{"uri":"github://repo/gohugoio/hugo","kind":"repo","id":"gohugoio/hugo","label":"gohugoio/hugo","url":"https://github.com/gohugoio/hugo"} +{"subject":"github://repo/gohugoio/hugo","predicate":"ownedBy","object":"github://org/gohugoio","source":"id"} +{"subject":"github://repo/gohugoio/hugo","predicate":"hasTopic","object":"github://topic/blog-engine","source":"payload"} +``` + +`github edges ` is the same walk with the node and the facts left out, which is what you want when you are building a graph rather than reading one: + +```bash +github edges golang/go --predicate hasTopic +``` + +## The vocabulary + +Every edge this tool emits uses a predicate from a fixed set. +Adding a relation means adding a constant first, so the vocabulary never drifts. + +| Group | Predicates | +|---|---| +| Ownership | `ownedBy`, `memberOf`, `partOf`, `belongsToPackage` | +| Derivation | `forkOf`, `templateOf`, `mirrorOf`, `dependsOn`, `usedBy` | +| Authorship | `authoredBy`, `committedBy`, `contributedTo`, `assignedTo`, `reviewedBy`, `reviewRequestedFrom`, `mergedBy` | +| Reference | `references`, `closes`, `closedBy`, `duplicateOf`, `subIssueOf`, `linkedTo`, `targetsBranch`, `fromBranch`, `pointsAt`, `parentOf` | +| Classification | `hasTopic`, `hasLabel`, `inMilestone`, `writtenIn`, `licensedUnder`, `relatedTopic` | +| Social | `starredBy`, `follows`, `sponsors`, `reactedWith` | + +`writtenIn` and `licensedUnder` take a bare string rather than a URI, because a language and a licence are not github.com entities. + +Facts are the literal properties that are not edges at all: `name`, `description`, `homepage`, `created`, `updated`, `stars`, `forks`, `watchers`, `commits`, `avatar`, `state`, `count`. + +## Where an edge came from + +Every edge records the kind of evidence behind it, because not all of them are equally solid: + +| Source | Means | +|---|---| +| `id` | Derived from the identifier itself, so it cannot be wrong: `gohugoio/hugo` is owned by `gohugoio` | +| `payload` | Stated in the embedded React payload or the route JSON | +| `feed` | Stated in an Atom feed | +| `html` | Read out of the rendered markup | +| `text` | Inferred from prose, such as a `#123` in an issue body | + +`--min-trust` drops everything below a level, so a pipeline that cannot tolerate a guess asks for `payload` and gets only what the site stated as data: + +```bash +github edges golang/go --min-trust payload +``` + +The default is `html`, which keeps everything except prose inference. + +## Crawling + +```bash +github crawl golang/go --depth 2 +github crawl golang/go --depth 2 --kinds repo,user --limit 200 +github crawl golang/go --depth 1 --edges-only +``` + +The default follow set is `partOf`, `ownedBy`, `forkOf`, `hasTopic`, `authoredBy`. +It deliberately leaves out `references`, `starredBy`, `follows`, `dependsOn`, and `usedBy`, because each of those turns a bounded walk into an unbounded one. +Ask for them by name when you want them: + +```bash +github crawl golang/go --depth 2 --follow dependsOn,usedBy +github crawl golang/go --depth 1 --follow all +``` + +`--dry-run` prints the estimate and stops, which is worth doing before any crawl of something popular: + +```bash +github crawl kubernetes/kubernetes --depth 3 --dry-run +``` + +## Keeping it + +Any command can tee its records into a store as it runs, so a crawl doubles as an import: + +```bash +github crawl golang/go --depth 2 --db out.db +github crawl golang/go --depth 2 --db 'postgres://localhost/graph' +``` + +Or write a whole subgraph to one file: + +```bash +github export golang/go --depth 2 --out go.jsonl +``` + +Next: [export it as linked data](/guides/linked-data/). diff --git a/docs/content/guides/linked-data.md b/docs/content/guides/linked-data.md new file mode 100644 index 0000000..5a3df3f --- /dev/null +++ b/docs/content/guides/linked-data.md @@ -0,0 +1,88 @@ +--- +title: "Export linked data" +description: "RDF, Turtle, JSON-LD, and N-Quads out of github.com." +weight: 20 +--- + +Once the site is nodes and edges, serialising it as RDF is a formatting problem. +`github` writes four serialisations, all keyed on github.com URLs, so a triple this tool emits joins with a triple anyone else emits about the same page. + +## One entity + +```bash +github rdf gohugoio/hugo # N-Triples, the default +github rdf gohugoio/hugo --format ttl # Turtle +github rdf gohugoio/hugo --format jsonld # JSON-LD +github rdf gohugoio/hugo --format nq # N-Quads +``` + +Turtle is the readable one: + +```turtle + + a schema:SoftwareSourceCode ; + a doap:Project ; + rdfs:label "gohugoio/hugo" ; + schema:keywords ; +``` + +## The vocabulary + +Established vocabularies first, a private namespace only where nothing fits: + +| Prefix | Namespace | +|---|---| +| `schema:` | `https://schema.org/` | +| `doap:` | `http://usefulinc.com/ns/doap#` | +| `foaf:` | `http://xmlns.com/foaf/0.1/` | +| `rdf:`, `rdfs:`, `xsd:` | the RDF core | +| `gh:` | `https://github.com/ns#`, for the things GitHub has that nobody standardised | + +A repository is a `schema:SoftwareSourceCode` and a `doap:Project`. +An issue is a `gh:Issue` and a `schema:DiscussionForumPosting`. +A user is a `schema:Person`, an org is a `schema:Organization`, a release is a `schema:SoftwareApplication`. + +The double typing is deliberate. +The schema.org type is what a general consumer already understands, and the `gh:` type is what a GitHub-aware consumer needs to tell an issue apart from a discussion. + +## More than one entity + +`--depth` walks before serialising, so a single document can carry a subgraph: + +```bash +github rdf golang/go --depth 2 --format ttl > go.ttl +github rdf golang/go --depth 2 --follow dependsOn --format nt | wc -l +``` + +The same trust filter applies here as in `github edges`, which matters more for RDF than for anything else, since a triple carries no evidence with it: + +```bash +github rdf golang/go --min-trust payload --format ttl +``` + +For a whole crawl, `github export` writes the same formats in one go: + +```bash +github export golang/go --depth 2 --format ttl --out go.ttl +github export golang/go --depth 2 --format nq --out go.nq +``` + +## Named graphs + +N-Quads puts each triple in a named graph, and the default name is the entity's own URL, so you can tell which read produced which statement after merging several: + +```bash +github rdf gohugoio/hugo --format nq +github rdf gohugoio/hugo --format nq --graph https://example.org/import/2026-07-25 +``` + +## Loading it somewhere + +Every format here is standard, so anything that reads RDF will take it: + +```bash +github export golang/go --depth 2 --format ttl --out go.ttl +# then load go.ttl into Oxigraph, Jena, GraphDB, Blazegraph, rdflib, whatever +``` + +Next: [look inside a repository](/guides/contents/). diff --git a/docs/content/guides/pages.md b/docs/content/guides/pages.md new file mode 100644 index 0000000..39d3462 --- /dev/null +++ b/docs/content/guides/pages.md @@ -0,0 +1,105 @@ +--- +title: "Read what the API will not tell you" +description: "The page plane, --deep, and github page." +weight: 40 +--- + +Some of the most useful things on GitHub have no API route at all: trending, contribution calendars, the dependency graph, release download counts, the marketplace, and the language histogram. +Others are served to a browser and not to `/api`. + +This is not a scraping problem, because GitHub does not render its data as prose. +A modern GitHub page carries a React payload as JSON in a `") + if j < 0 { + return []byte("{}") + } + return []byte(strings.TrimSpace(rest[:j])) +} + +// Head issues a HEAD and returns the response headers. It exists for one +// question, "how big is this file", which is worth asking without downloading +// the answer. +func (c *Client) Head(ctx context.Context, url string) (http.Header, error) { + if err := c.acquire(ctx); err != nil { + return nil, err + } + defer c.release() + c.pace(ctx) + + req, err := c.newRequest(ctx, url, SurfaceRaw) + if err != nil { + return nil, err + } + req.Method = http.MethodHead + res, err := c.HTTP.Do(req) + if err != nil { + return nil, wrapNetwork(url, err) + } + _ = res.Body.Close() + if res.StatusCode < 200 || res.StatusCode >= 300 { + return nil, statusError(url, res.StatusCode, nil) + } + return res.Header, nil +} diff --git a/gh/crawl.go b/gh/crawl.go new file mode 100644 index 0000000..8322773 --- /dev/null +++ b/gh/crawl.go @@ -0,0 +1,313 @@ +package gh + +import ( + "context" + "fmt" + "strings" + + "github.com/tamnd/any-cli/kit/errs" +) + +// crawl.go walks the graph. The algorithm is a breadth-first frontier with a +// visited set and a hard budget, and that is all it should ever be: the +// interesting decisions here are about what not to follow and when to stop, not +// about traversal. +// +// The walk is sequential. Doc 04 section 5 gives the crawler a Concurrency +// field, and it would be a lie in this client: every request already queues +// through one rate limiter, so N workers would only queue N deep behind the same +// pacer while making the output order unpredictable. If the pacer ever grows a +// real parallel mode, this is the place to add the workers. + +// CrawlOptions bounds a walk. The budgets are the point of the struct. A tool +// that can accidentally send a million requests at somebody else's servers +// should be hard to point that way by accident. +type CrawlOptions struct { + Depth int + Follow []string + Kinds []string + MinTrust string + Limit int + + NodesOnly bool + EdgesOnly bool +} + +// CrawlSink receives what the walk finds. Emission is streaming: a crawl of a +// large organization must never need the whole graph in memory, and a crawl that +// is interrupted has already emitted everything it found. +type CrawlSink struct { + Node func(*Node) error + Edge func(*Edge) error + Fact func(*Fact) error +} + +// defaults fills in the spec's defaults. Depth 1 and the structural follow set +// are the safe walk: references, stars, follows, and the two dependency +// predicates fan out without bound, so each has to be asked for by name. +func (o *CrawlOptions) defaults() { + if o.Depth <= 0 { + o.Depth = 1 + } + if o.MinTrust == "" { + o.MinTrust = DefaultMinTrust + } + if len(o.Follow) == 0 { + o.Follow = DefaultFollow + } +} + +// followSet accepts a predicate written either way, so --follow ownedBy and +// --follow gh:ownedBy mean the same thing. The word "all" turns the filter off. +func followSet(follow []string) map[string]bool { + out := map[string]bool{} + for _, f := range follow { + for _, part := range strings.Split(f, ",") { + part = strings.TrimSpace(part) + if part == "" { + continue + } + if part == "all" { + return nil + } + out[strings.TrimPrefix(part, "gh:")] = true + } + } + return out +} + +// kindSet is the same idea for --kinds. An empty set expands every kind. +func kindSet(kinds []string) map[string]bool { + out := map[string]bool{} + for _, k := range kinds { + for _, part := range strings.Split(k, ",") { + part = strings.TrimSpace(part) + if part != "" { + out[part] = true + } + } + } + return out +} + +// splitURI takes a github:// URI apart. The crawler holds URIs rather than +// records, which is what keeps its memory proportional to the number of nodes +// seen and not to their size. +func splitURI(uri string) (kind, id string, ok bool) { + k, i, _, err := parseURI(uri) + if err != nil { + return "", "", false + } + return k, i, true +} + +// Crawl walks outward from a seed reference. Nodes, edges, and facts come out as +// they are discovered, and the walk stops cleanly at either bound and reports +// what it had rather than failing. +func (c *Client) Crawl(ctx context.Context, seed string, o CrawlOptions, sink CrawlSink) error { + o.defaults() + kind, id, err := Classify(seed) + if err != nil { + return err + } + allow := followSet(o.Follow) + expand := kindSet(o.Kinds) + + type item struct { + uri string + depth int + } + start := URI(kind, id) + visited := map[string]bool{start: true} + frontier := []item{{start, 0}} + nodes := 0 + + for len(frontier) > 0 { + cur := frontier[0] + frontier = frontier[1:] + + // The limit is checked before the fetch, which is what makes it a + // budget rather than a suggestion. + if o.Limit > 0 && nodes >= o.Limit { + return nil + } + if err := ctx.Err(); err != nil { + return wrapNetwork("", err) + } + curKind, curID, ok := splitURI(cur.uri) + if !ok { + continue + } + rec, err := c.fetchOne(ctx, curKind, curID) + if err != nil { + // One unreachable node must not end a walk that has already + // produced useful output. A deleted repository, a kind this tool + // cannot dereference yet, and a page that has moved are all normal + // mid-crawl, and the alternative is a two-hour walk that throws + // away its results on the last hop. + if softSkip(err) { + continue + } + return err + } + node, edges, facts := Extract(rec) + if node.URI == "" { + continue + } + nodes++ + if !o.EdgesOnly && sink.Node != nil { + n := node + if err := sink.Node(&n); err != nil { + return err + } + } + edges = FilterTrust(edges, o.MinTrust) + if !o.NodesOnly { + for i := range edges { + if sink.Edge != nil { + e := edges[i] + if err := sink.Edge(&e); err != nil { + return err + } + } + } + for i := range facts { + if sink.Fact != nil { + f := facts[i] + if err := sink.Fact(&f); err != nil { + return err + } + } + } + } + if cur.depth >= o.Depth { + continue + } + for _, e := range edges { + // A language or a licence is a bare string with no page behind it, + // so it is an edge but never a target. + if !strings.HasPrefix(e.Object, Scheme+"://") { + continue + } + if len(allow) > 0 && !allow[e.Predicate] { + continue + } + objKind, _, ok := splitURI(e.Object) + if !ok { + continue + } + if len(expand) > 0 && !expand[objKind] { + continue + } + // Cycles are normal on this graph. A fork points at its parent and + // the parent's fork list points back, and the visited set is the + // only defence that needs. + if visited[e.Object] { + continue + } + visited[e.Object] = true + frontier = append(frontier, item{e.Object, cur.depth + 1}) + } + } + return nil +} + +// softSkip reports whether an error is one node's problem rather than the +// walk's. Not found, needs a login, and a kind that has no reader yet are all +// "skip this one", and anything else stops the crawl. +func softSkip(err error) bool { + switch errs.KindOf(err) { + case errs.KindNotFound, errs.KindNeedAuth, errs.KindUnsupported: + return true + default: + return false + } +} + +// CrawlPlan is what --dry-run answers with. It is a record rather than a line +// on stderr so the answer goes through the same renderer, formats, and pipes as +// every other command, and so a script can size a walk without reading prose. +type CrawlPlan struct { + Base + + Seed string `json:"seed" table:"seed"` + Depth int `json:"depth" table:"depth"` + Nodes int `json:"nodes" table:"nodes"` + + Note string `json:"note" table:"note"` +} + +// Estimate reads the seed and reports what one more level would cost. It is +// deliberately a lower bound and the note says so: the first level is countable +// because the seed's edges are in hand, and everything past it depends on a +// branching factor that cannot be seen from here without doing the walk. +func (c *Client) Estimate(ctx context.Context, seed string, o CrawlOptions) (*CrawlPlan, error) { + o.defaults() + kind, id, err := Classify(seed) + if err != nil { + return nil, err + } + rec, err := c.fetchOne(ctx, kind, id) + if err != nil { + return nil, err + } + _, edges, _ := Extract(rec) + edges = FilterTrust(edges, o.MinTrust) + allow := followSet(o.Follow) + expand := kindSet(o.Kinds) + + seen := map[string]bool{URI(kind, id): true} + next := 0 + for _, e := range edges { + if !strings.HasPrefix(e.Object, Scheme+"://") || seen[e.Object] { + continue + } + if len(allow) > 0 && !allow[e.Predicate] { + continue + } + objKind, _, ok := splitURI(e.Object) + if !ok || (len(expand) > 0 && !expand[objKind]) { + continue + } + seen[e.Object] = true + next++ + } + nodes := 1 + next + if o.Limit > 0 && nodes > o.Limit { + nodes = o.Limit + } + note := fmt.Sprintf("at least %d nodes through depth 1, about one request each", nodes) + if o.Depth > 1 { + note += fmt.Sprintf(", and more at depth %d depending on how the next level branches", o.Depth) + } + plan := &CrawlPlan{Seed: seed, Depth: o.Depth, Nodes: nodes, Note: note} + plan.setIdentity(kind, id) + return plan, nil +} + +// GraphOf builds the node, edges, and facts for one entity. `github graph`, +// `github edges`, and `github rdf` all call it, so the three never disagree +// about what an entity's edges are. +func (c *Client) GraphOf(ctx context.Context, kind, id string) (*Graph, error) { + rec, err := c.fetchOne(ctx, kind, id) + if err != nil { + return nil, err + } + g := &Graph{} + g.Add(rec) + return g, nil +} + +// GraphOfRef is GraphOf for a reference that has not been classified yet, and it +// reports the kind it turned out to be so a caller can say what it read. +func (c *Client) GraphOfRef(ctx context.Context, ref string) (string, string, *Graph, error) { + kind, id, err := Classify(ref) + if err != nil { + return "", "", nil, err + } + g, err := c.GraphOf(ctx, kind, id) + if err != nil { + return "", "", nil, err + } + return kind, id, g, nil +} diff --git a/gh/deps.go b/gh/deps.go new file mode 100644 index 0000000..4539fba --- /dev/null +++ b/gh/deps.go @@ -0,0 +1,263 @@ +package gh + +import ( + "context" + "net/url" + "strconv" + "strings" + + "golang.org/x/net/html" + + "github.com/tamnd/github-cli/pkg/page" +) + +// deps.go reads the two dependency graph pages. They are the most valuable +// keyless surface on the site and the least reliable one: the graph is opt-in +// per repository, the rows are prose, and a package GitHub cannot resolve to a +// repository is a name and nothing else. +// +// Both pages are read rather than one, because they are not two views of the +// same list. Dependencies come from the manifests in this repository and +// dependents come from every other repository's manifests, so neither can be +// derived from the other. +// +// The two pagers disagree, which is why there are two of them here. The +// dependency list is a Rails pager with ?page=N and a rel="next" anchor, and the +// dependents list is a cursor in a button. + +// Dependencies lists what a repository declares in its manifests. +// +// A repository with the dependency graph switched off answers with a page and +// no rows, which is an empty list rather than an error: the difference between +// "nothing to report" and "not enabled" is not on the page, so claiming to know +// which one it is would be making it up. +func (c *Client) Dependencies(ctx context.Context, repo string, limit int, emit func(Dependency) error) error { + if _, _, ok := SplitRepo(repo); !ok { + return usageBadID("repository", repo, "owner/name") + } + base := repoSubURL(repo, "network/dependencies") + fetch := func(ctx context.Context, token string) ([]Dependency, string, error) { + u := base + if n := pageToken(token); n > 1 { + u = query(u, "page", strconv.Itoa(n)) + } + doc, final, err := c.rowPage(ctx, u, page.DependencyRow) + if err != nil { + return nil, "", err + } + if doc == nil { + return nil, "", structureChanged(repo + " dependencies") + } + var out []Dependency + for _, row := range page.FindAll(doc, page.BoxRow) { + d, ok := dependencyRow(row, repo, final) + if ok { + out = append(out, d) + } + } + return out, railsNext(doc, token), nil + } + return paginate(ctx, limit, fetch, emit) +} + +// rowPage reads one page of a dependency graph listing. +// +// It exists because GitHub answers a cursor page with a 200, the right title, +// the right chrome, and no rows at all, often enough to matter. An empty page is +// indistinguishable from the end of the list, so the walk stops early and +// reports a third of the dependents as the whole set. Asking a second time gets +// the rows, so the read is repeated once before an empty page is believed. +// +// Dropping the cached copy first is the part that matters. Without it the retry +// reads the same empty bytes back and the wrong answer sticks for the life of +// the entry, which is how this was found: --no-cache returned two hundred rows +// and the cached run returned sixty, over and over. +// +// The cost is one extra request for a repository whose dependency graph really +// is empty, which is the right trade: a repository with the graph switched off +// is cheap to ask twice, and silently reporting an empty list for one with +// thousands of dependents is not recoverable by the caller. +func (c *Client) rowPage(ctx context.Context, u string, rows page.Sel) (*html.Node, string, error) { + doc, final, n, err := c.readRows(ctx, u, rows) + if err != nil || n > 0 { + return doc, final, err + } + c.cacheDrop(u, SurfaceHTML) + doc, final, n, err = c.readRows(ctx, u, rows) + if err != nil { + return nil, "", err + } + // An empty page that stays empty is not worth keeping either. The next run + // would read it back and stop in the same place without ever asking again. + if n == 0 { + c.cacheDrop(u, SurfaceHTML) + } + return doc, final, nil +} + +func (c *Client) readRows(ctx context.Context, u string, rows page.Sel) (*html.Node, string, int, error) { + res, err := c.GetHTML(ctx, u) + if err != nil { + return nil, "", 0, err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return nil, res.FinalURL, 0, nil + } + return doc, res.FinalURL, len(page.FindAll(doc, rows)), nil +} + +// dependencyRow reads one manifest entry. The interesting half is the line under +// the package name, which is one span holding the ecosystem, the manifest, who +// detected it and when, and sometimes the licence, separated by middots. +func dependencyRow(row *html.Node, repo, source string) (Dependency, bool) { + box := page.Find(row, page.DependencyRow) + if box == nil { + return Dependency{}, false + } + name := page.Find(box, page.DependencyName) + if name == nil { + return Dependency{}, false + } + d := Dependency{Repo: repo, Package: strings.TrimSpace(page.Text(name))} + if d.Package == "" { + return Dependency{}, false + } + if a := page.Find(box, page.DependencyLink); a != nil { + if p := hrefPath(page.Attr(a, "href")); strings.Count(p, "/") == 1 { + d.SourceRepo = p + d.setIdentity(KindRepo, p) + } + } + if v := page.Find(box, page.DependencyVersion); v != nil { + d.Version = strings.TrimSpace(page.Text(v)) + } + if r := page.Find(box, page.DependencyRelation); r != nil { + d.Relationship = strings.ToLower(strings.TrimSpace(page.Text(r))) + } + if m := page.Find(row, page.DependencyManifest); m != nil { + d.Manifest = strings.TrimSpace(page.Text(m)) + if m.Parent != nil { + d.Ecosystem, d.License = manifestLine(page.Text(m.Parent), d.Manifest) + } + } + d.addSource(source) + return d, true +} + +// manifestLine takes the middot-separated line apart. The ecosystem is always +// first and the licence, when there is one, is always last; the middle is the +// manifest name and the detection note, neither of which needs splitting out +// here. +// +// The note comes in two wordings, "Detected by dependabot on " and +// "Detected automatically on ", and when there is no licence the note is +// what sits last, so both prefixes have to be recognised or the date ends up +// filed as the licence. +func manifestLine(text, manifest string) (ecosystem, license string) { + var parts []string + for _, p := range strings.Split(text, "·") { + if p = strings.TrimSpace(p); p != "" { + parts = append(parts, p) + } + } + if len(parts) == 0 { + return "", "" + } + ecosystem = parts[0] + last := parts[len(parts)-1] + if last != ecosystem && last != manifest && !strings.HasPrefix(last, "Detected ") { + license = last + } + return ecosystem, license +} + +// Dependents lists the repositories that depend on this one. +// +// The list is ordered by stars and it is long: a popular library has tens of +// thousands of rows at thirty a page, so --limit is the flag that matters here +// and the walk stops the moment it is reached. +func (c *Client) Dependents(ctx context.Context, repo string, limit int, emit func(Dependent) error) error { + if _, _, ok := SplitRepo(repo); !ok { + return usageBadID("repository", repo, "owner/name") + } + base := repoSubURL(repo, "network/dependents") + fetch := func(ctx context.Context, token string) ([]Dependent, string, error) { + u := base + if token != "" { + u = query(u, "dependents_after", token) + } + doc, final, err := c.rowPage(ctx, u, page.DependentRow) + if err != nil { + return nil, "", err + } + if doc == nil { + return nil, "", structureChanged(repo + " dependents") + } + var out []Dependent + for _, row := range page.FindAll(doc, page.DependentRow) { + d, ok := dependentRow(row, repo, final) + if ok { + out = append(out, d) + } + } + return out, dependentsCursor(doc), nil + } + return paginate(ctx, limit, fetch, emit) +} + +// dependentRow reads one dependent. Owner and name are two anchors rather than +// one, the same shape the fork list uses, so the id is assembled. +func dependentRow(row *html.Node, repo, source string) (Dependent, bool) { + link := page.Find(row, page.DependentRepo) + if link == nil { + return Dependent{}, false + } + id := hrefPath(page.Attr(link, "href")) + owner, _, ok := SplitRepo(id) + if !ok { + return Dependent{}, false + } + d := Dependent{Repo: repo, Dependent: id, Owner: owner} + d.setIdentity(KindRepo, id) + if u := page.Find(row, page.DependentUser); u != nil { + if login := hrefPath(page.Attr(u, "href")); login != "" { + d.Owner = login + } + } + if img := page.Find(row, page.Sel{Tag: "img", Class: "avatar"}); img != nil { + d.AvatarURL = page.Attr(img, "src") + } + d.Stars = iconCount(row, page.DependentStars) + d.Forks = iconCount(row, page.DependentForks) + d.addSource(source) + return d, true +} + +// iconCount reads the number beside an icon. The icon is what says which count +// it is, because the two spans are otherwise identical. +func iconCount(row *html.Node, sel page.Sel) *int { + el := page.Find(row, sel) + if el == nil { + return nil + } + if n, _, ok := page.CountIn(page.Text(el)); ok { + return intp(n) + } + return nil +} + +// dependentsCursor pulls the opaque cursor out of the Next button. There is no +// page number on this listing and no total to count against, so the cursor the +// page hands back is the only way forward. +func dependentsCursor(doc *html.Node) string { + a := page.Find(doc, page.DependentNext) + if a == nil { + return "" + } + u, err := url.Parse(page.Attr(a, "href")) + if err != nil { + return "" + } + return u.Query().Get("dependents_after") +} diff --git a/gh/discover.go b/gh/discover.go new file mode 100644 index 0000000..87bbb7b --- /dev/null +++ b/gh/discover.go @@ -0,0 +1,578 @@ +package gh + +import ( + "context" + "encoding/json" + "strconv" + "strings" + "time" + + "golang.org/x/net/html" + + "github.com/tamnd/github-cli/pkg/page" +) + +// discover.go reads the pages that answer "what is out there": trending, topic +// pages, fork networks, and repository statistics. +// +// Trending is the clearest case for this whole tool. There is no JSON version +// of it anywhere, with a token or without, so a page decoder is not a fallback +// here, it is the only implementation that can exist. + +// TrendingOptions are the three knobs the trending page has. +type TrendingOptions struct { + // Since is daily, weekly, or monthly. Empty means daily, which is what the + // page defaults to. + Since string + // Language filters by the language slug in the URL, not by a query. + Language string + // SpokenLanguage is the natural-language filter, a two-letter code. + SpokenLanguage string + Limit int +} + +// Trending lists the trending repositories. Rank is the position on the page, +// which is the only ordering the surface has and is worth keeping, since the +// list has no other stable key. +func (c *Client) Trending(ctx context.Context, opts TrendingOptions, emit func(Trending) error) error { + u := trendingURL("", opts) + res, err := c.GetHTML(ctx, u) + if err != nil { + return err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return structureChanged("trending") + } + period := firstNonEmpty(opts.Since, "daily") + rank := 0 + for _, row := range page.FindAll(doc, page.TrendingRow) { + t, ok := trendingRow(row, period, res.FinalURL) + if !ok { + continue + } + rank++ + t.Rank = rank + if err := emit(t); err != nil { + return err + } + if opts.Limit > 0 && rank >= opts.Limit { + return nil + } + } + if rank == 0 { + return structureChanged("trending") + } + return nil +} + +// TrendingDevelopers lists the trending developers, each with the repository +// the page picked out for them. +func (c *Client) TrendingDevelopers(ctx context.Context, opts TrendingOptions, emit func(Account) error) error { + u := trendingURL("developers", opts) + res, err := c.GetHTML(ctx, u) + if err != nil { + return err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return structureChanged("trending developers") + } + seen := 0 + for _, row := range page.FindAll(doc, page.Sel{Tag: "article", Class: "Box-row"}) { + a, ok := trendingDev(row, res.FinalURL) + if !ok { + continue + } + if err := emit(a); err != nil { + return err + } + seen++ + if opts.Limit > 0 && seen >= opts.Limit { + return nil + } + } + if seen == 0 { + return structureChanged("trending developers") + } + return nil +} + +// trendingURL builds the trending address. The language is a path segment and +// the period is a query parameter, which is the site's own split and not one +// worth normalising away. +func trendingURL(section string, opts TrendingOptions) string { + u := BaseURL + "/trending" + if section != "" { + u += "/" + section + } else if opts.Language != "" { + u += "/" + strings.ToLower(opts.Language) + } + var kv []string + if opts.Since != "" { + kv = append(kv, "since", opts.Since) + } + if opts.SpokenLanguage != "" { + kv = append(kv, "spoken_language_code", opts.SpokenLanguage) + } + if len(kv) == 0 { + return u + } + return query(u, kv...) +} + +// trendingRow reads one card. The three counts on it are the same shape and +// only their link tells them apart: stargazers, forks, and the period figure, +// which has no link at all. +func trendingRow(row *html.Node, period, source string) (Trending, bool) { + h := page.Find(row, page.Sel{Tag: "h2"}) + if h == nil { + return Trending{}, false + } + a := page.Find(h, page.Sel{Tag: "a", Attr: "href"}) + if a == nil { + return Trending{}, false + } + id := hrefPath(page.Attr(a, "href")) + owner, name, ok := SplitRepo(id) + if !ok { + return Trending{}, false + } + t := Trending{Period: period} + t.Owner, t.Name = owner, name + t.setIdentity(KindRepo, id) + t.addSource(source) + + if p := page.Find(row, page.Sel{Tag: "p"}); p != nil { + t.Description = page.Text(p) + } + if l := page.Find(row, page.Sel{Attr: "itemprop", AttrValue: "programmingLanguage"}); l != nil { + t.Language = page.Text(l) + } + if col := page.Find(row, page.Sel{Class: "repo-language-color"}); col != nil { + t.LanguageColor = styleColor(page.Attr(col, "style")) + } + for _, link := range page.FindAll(row, page.Sel{Tag: "a", Attr: "href"}) { + n, _, ok := page.ParseCompactCount(page.Text(link)) + if !ok { + continue + } + switch href := page.Attr(link, "href"); { + case strings.HasSuffix(href, "/stargazers"): + t.Stars = intp(n) + case strings.HasSuffix(href, "/forks"): + t.Forks = intp(n) + } + } + if s := page.Find(row, page.Sel{Class: "float-sm-right"}); s != nil { + if n, _, ok := page.CountIn(page.Text(s)); ok { + t.StarsInPeriod = intp(n) + } + } + for _, img := range page.FindAll(row, page.Sel{Tag: "img", Class: "avatar-user"}) { + login := strings.TrimPrefix(page.Attr(img, "alt"), "@") + if login == "" { + continue + } + who := actor(login) + who.AvatarURL = page.Attr(img, "src") + t.BuiltBy = append(t.BuiltBy, who) + } + return t, true +} + +// trendingDev reads one developer card. The popular repository on it is a +// pointer, not a record: it has a name and a description and nothing else, so +// it goes into PinnedRepos where the profile's own picks go. +func trendingDev(row *html.Node, source string) (Account, bool) { + link := page.Find(row, page.Sel{Tag: "h1", Class: "h3"}) + if link == nil { + return Account{}, false + } + nameLink := page.Find(link, page.Sel{Tag: "a", Attr: "href"}) + if nameLink == nil { + return Account{}, false + } + login := hrefPath(page.Attr(nameLink, "href")) + if login == "" || strings.Contains(login, "/") { + return Account{}, false + } + a := Account{Login: login, Type: "User", Name: page.Text(nameLink)} + a.setIdentity(KindUser, login) + a.addSource(source) + if a.Name == a.Login { + a.Name = "" + } + if img := page.Find(row, page.Sel{Tag: "img", Class: "avatar-user"}); img != nil { + a.AvatarURL = page.Attr(img, "src") + } + if h := page.Find(row, page.Sel{Tag: "h1", Class: "h4"}); h != nil { + if repo := page.Find(h, page.Sel{Tag: "a", Attr: "href"}); repo != nil { + if id := hrefPath(page.Attr(repo, "href")); strings.Count(id, "/") == 1 { + a.PinnedRepos = append(a.PinnedRepos, id) + } + } + } + return a, true +} + +// --- topic pages --- + +// TopicPage reads one topic. The search result for a topic carries the name and +// a short blurb; the page carries the long description, the logo, who created +// the thing, when it was released, the Wikipedia link, and the related topics, +// which is most of what makes a topic worth having a record for. +func (c *Client) TopicPage(ctx context.Context, slug string) (*Topic, error) { + slug = strings.Trim(slug, "/") + if slug == "" || strings.Contains(slug, "/") { + return nil, usageBadID("topic", slug, "a topic slug") + } + res, err := c.GetHTML(ctx, BaseURL+"/topics/"+slug) + if err != nil { + return nil, err + } + p := page.Extract(res.FinalURL, res.Body) + doc := p.Doc() + if doc == nil { + return nil, structureChanged(slug) + } + t := &Topic{Name: slug} + t.setIdentity(KindTopic, slug) + t.addSource(res.FinalURL) + t.GitHubURL = t.URL + + if h := page.Find(doc, page.Sel{Tag: "h1", Class: "h1"}); h != nil { + t.DisplayName = page.Text(h) + } + if trigger := page.Find(doc, page.Sel{Tag: "topic-feeds-toast-trigger"}); trigger != nil { + t.DisplayName = firstNonEmpty(page.Attr(trigger, "data-topic-display-name"), t.DisplayName) + } + if md := page.Find(doc, page.MarkdownBody); md != nil { + t.DescriptionHTML = page.OuterHTML(md) + t.Description = page.BlockText(md) + // The page has one description where the search result has two. The + // first paragraph is the same string the short one would be, so it is + // filled from here rather than left empty for no reason. + t.ShortDescription, _, _ = strings.Cut(t.Description, "\n") + } + if img := page.Find(doc, page.Sel{Tag: "img", Attr: "alt", AttrSuffix: " logo"}); img != nil { + t.LogoURL = page.Attr(img, "src") + } + if w := page.Find(doc, page.TopicWikipedia); w != nil { + t.WikipediaURL = page.Attr(w, "href") + } + t.CreatedBy = labelledText(doc, "Created by") + t.Released = labelledText(doc, "Released") + if n := page.Find(doc, page.Sel{Tag: "h2", Class: "h3"}); n != nil { + // "Here are 89,195 public repositories matching this topic..." + if count, _, ok := page.CountIn(strings.TrimPrefix(page.Text(n), "Here are ")); ok { + t.AppliedCount = intp(count) + } + } + for _, dd := range page.FindAll(doc, page.Sel{Tag: "dd"}) { + if n, _, ok := page.ParseCompactCount(strings.TrimSuffix(page.Text(dd), " followers")); ok && + strings.HasSuffix(page.Text(dd), "followers") { + t.StargazerCount = intp(n) + } + } + for _, rel := range relatedTopics(doc, slug) { + t.Related = append(t.Related, rel) + } + if t.DisplayName == "" && t.Description == "" { + return nil, structureChanged(slug) + } + return t, nil +} + +// relatedTopics reads the sidebar's related topics. +// +// They are not in a container. The heading and the links are siblings, and the +// same link class is on every topic chip of every repository in the result +// list below, so scoping by class alone pulls in a few hundred unrelated +// topics. The heading is the only boundary the markup gives, so the walk +// starts there and stops at the next heading. +func relatedTopics(doc *html.Node, slug string) []string { + var head *html.Node + for _, h := range page.FindAll(doc, page.Sel{Tag: "h2"}) { + if page.Text(h) == "Related topics" { + head = h + break + } + } + if head == nil { + return nil + } + var out []string + for n := head.NextSibling; n != nil; n = n.NextSibling { + if n.Type == html.ElementNode && (n.Data == "h2" || n.Data == "h3") { + break + } + for _, a := range page.FindAll(n, page.Sel{Tag: "a", Class: "topic-tag-link"}) { + rel := strings.TrimPrefix(hrefPath(page.Attr(a, "href")), "topics/") + if rel != "" && rel != slug && !contains(out, rel) { + out = append(out, rel) + } + } + } + return out +} + +// labelledText reads the value beside a muted label in the topic sidebar. The +// label is a span inside the paragraph and the value is the rest of it, which +// is the only structure the markup offers. +func labelledText(doc *html.Node, label string) string { + for _, p := range page.FindAll(doc, page.Sel{Tag: "p"}) { + span := page.Find(p, page.Sel{Tag: "span", Class: "color-fg-muted"}) + if span == nil || page.Text(span) != label { + continue + } + return strings.TrimSpace(strings.TrimPrefix(page.Text(p), label)) + } + return "" +} + +// --- fork networks --- + +// Forks lists the public forks of a repository. The page is the only keyless +// source: the network graph route needs a session and the search index does not +// model the parent link. +func (c *Client) Forks(ctx context.Context, repo string, limit int, emit func(Repo) error) error { + if _, _, ok := SplitRepo(repo); !ok { + return usageBadID("repository", repo, "owner/name") + } + base := repoSubURL(repo, "forks") + fetch := func(ctx context.Context, token string) ([]Repo, string, error) { + u := base + if n := pageToken(token); n > 1 { + u = query(u, "page", strconv.Itoa(n)) + } + res, err := c.GetHTML(ctx, u) + if err != nil { + return nil, "", err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return nil, "", structureChanged(repo + " forks") + } + var out []Repo + for _, row := range page.FindAll(doc, page.BoxRow) { + f, ok := forkRow(row, repo, res.FinalURL) + if ok { + out = append(out, f) + } + } + return out, railsNext(doc, token), nil + } + return paginate(ctx, limit, fetch, emit) +} + +// forkRow reads one row of a fork list. The owner and the name are separate +// links, so the id is assembled rather than read off one href. +func forkRow(row *html.Node, parent, source string) (Repo, bool) { + h := page.Find(row, page.Sel{Tag: "h2"}) + if h == nil { + return Repo{}, false + } + var owner, name string + for _, a := range page.FindAll(h, page.Sel{Tag: "a", Attr: "href"}) { + p := hrefPath(page.Attr(a, "href")) + switch { + case owner == "" && !strings.Contains(p, "/"): + owner = p + case strings.Count(p, "/") == 1: + owner, name, _ = SplitRepo(p) + } + } + if owner == "" || name == "" { + return Repo{}, false + } + id := owner + "/" + name + r := Repo{Owner: owner, Name: name, IsFork: true, ForkOf: parent} + r.setIdentity(KindRepo, id) + r.addSource(source) + for _, a := range page.FindAll(row, page.Sel{Tag: "a", Attr: "href"}) { + n, _, ok := page.ParseCompactCount(page.Text(a)) + if !ok { + continue + } + switch href := page.Attr(a, "href"); { + case strings.HasSuffix(href, "/stargazers"): + r.Stars = intp(n) + case strings.HasSuffix(href, "/forks"): + r.Forks = intp(n) + } + } + if t := page.Find(row, page.RelTimeEl); t != nil { + r.PushedAt = parseTime(page.Attr(t, "datetime")) + } + return r, true +} + +// --- statistics --- + +// Contributors reads the contributor graph's own data route. +// +// The route answers 202 with an empty body while GitHub computes the numbers, +// which is normal rather than an error and is why this polls. A large +// repository takes a few seconds the first time and is instant afterwards. +func (c *Client) Contributors(ctx context.Context, repo string, opts ContributorOptions, emit func(Contributor) error) error { + if _, _, ok := SplitRepo(repo); !ok { + return usageBadID("repository", repo, "owner/name") + } + u := repoSubURL(repo, "graphs/contributors-data") + res, err := c.Poll(ctx, u, SurfaceXHR) + if err != nil { + return err + } + var raw []contributorData + if err := json.Unmarshal(res.Body, &raw); err != nil { + return badPayload(shortURL(u), err) + } + if len(raw) == 0 { + return structureChanged(repo + " contributors") + } + // The route answers in ascending order of contribution, which is the + // reverse of what anyone asking for contributors wants. + seen := 0 + for i := len(raw) - 1; i >= 0; i-- { + rec := raw[i].contributor(repo, res.FinalURL, opts.Weeks) + if err := emit(rec); err != nil { + return err + } + seen++ + if opts.Limit > 0 && seen >= opts.Limit { + return nil + } + } + return nil +} + +// ContributorOptions is what to do with the week array. +type ContributorOptions struct { + // Weeks keeps the per-week breakdown. It is off by default because the + // route sends every week since the repository began for every contributor, + // which on an old project is a few hundred entries each and megabytes of + // mostly zeroes for an answer whose question was "who wrote this". + Weeks bool + Limit int +} + +type contributorData struct { + Author *struct { + ID int `json:"id"` + Login string `json:"login"` + Avatar string `json:"avatar"` + Path string `json:"path"` + } `json:"author"` + Total int `json:"total"` + Weeks []struct { + W int64 `json:"w"` + A int `json:"a"` + D int `json:"d"` + C int `json:"c"` + } `json:"weeks"` +} + +// contributor folds the week array into the record. The array is six hundred +// entries for an old repository and nearly all of them are zero, so the +// summable fields are summed here and the first and last weeks with any +// activity are kept as dates, which is what a table can show. +func (d contributorData) contributor(repo, source string, keepWeeks bool) Contributor { + rec := Contributor{Repo: repo, Commits: intp(d.Total)} + if d.Author != nil { + rec.Login = d.Author.Login + rec.AvatarURL = d.Author.Avatar + if d.Author.ID > 0 { + rec.DatabaseID = intp(d.Author.ID) + } + } + rec.setIdentity(KindContributor, repo+"@"+rec.Login) + rec.addSource(source) + + adds, dels := 0, 0 + for _, w := range d.Weeks { + adds += w.A + dels += w.D + if w.A == 0 && w.D == 0 && w.C == 0 { + continue + } + at := time.Unix(w.W, 0).UTC() + if rec.FirstWeek == nil { + first := at + rec.FirstWeek = &first + } + last := at + rec.LastWeek = &last + if keepWeeks { + rec.Weeks = append(rec.Weeks, ContributorWeek{Week: at, Additions: w.A, Deletions: w.D, Commits: w.C}) + } + } + rec.Additions = intp(adds) + rec.Deletions = intp(dels) + return rec +} + +// Languages reports the language histogram as one record per language. The +// numbers are on the repository record already; this exists because "what is +// this written in, in what proportion" is a question worth one command rather +// than a field selector on another one. +// Languages reports the language breakdown, largest first. +// +// This reads the sidebar fragment rather than a whole repository page, because +// the fragment is where the numbers are and it is 3 KB where the page is 300. +// The numbers are percentages: GitHub computes byte counts and publishes only +// the proportions, so a byte count is not something this can report honestly. +func (c *Client) Languages(ctx context.Context, repo string, emit func(LanguageShare) error) error { + sb, err := c.sidebar(ctx, repo) + if err != nil { + return err + } + langs := sb.langs() + if len(langs) == 0 { + return structureChanged(repo + " languages") + } + source := repoSubURL(repo, "_sidebar") + for _, l := range langs { + share := LanguageShare{ + Repo: repo, + Language: l.Name, + Percent: l.Percentage, + Color: l.Color, + } + share.setIdentity(KindRepo, repo) + share.addSource(source) + if err := emit(share); err != nil { + return err + } + } + return nil +} + +// Stats is the counts in one record. Everything in it is already on the +// repository record; the point is a record with nothing else in it, so +// `github stats x -o json` is a thing you can diff week to week. +func (c *Client) Stats(ctx context.Context, repo string) (*RepoStats, error) { + // Deep, because the contributor and dependent counts are behind their own + // fragments and a counts record missing two of the counts is not worth + // having. + r, err := c.Repo(ctx, repo, RepoOptions{Deep: true}) + if err != nil { + return nil, err + } + s := &RepoStats{ + Repo: repo, + Stars: r.Stars, + Forks: r.Forks, + Watchers: r.Watchers, + OpenIssues: r.OpenIssues, + Commits: r.CommitCount, + Releases: r.ReleaseCount, + Tags: r.TagCount, + Contributors: r.ContributorCount, + Dependents: r.DependentCount, + PushedAt: r.PushedAt, + } + s.setIdentity(KindRepo, repo) + s.addSource(r.Sources...) + return s, nil +} diff --git a/gh/doctor.go b/gh/doctor.go new file mode 100644 index 0000000..fbee05d --- /dev/null +++ b/gh/doctor.go @@ -0,0 +1,191 @@ +package gh + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strings" + "time" + + "github.com/tamnd/github-cli/pkg/page" +) + +// doctor.go answers the question people ask when a command comes back wrong: is +// it me, is it the network, or did GitHub change the page? +// +// Every check is a record rather than a paragraph, so the answer can be read by +// a person and by a script, and so the failing one can be picked out with the +// same --fields and -o json every other command takes. + +// doctorLong is the command's help. It lives here rather than beside the +// registration because it names the token variables, and TestNoAuth wants every +// mention of those names in the one file whose job is to talk about them. +const doctorLong = "doctor answers the question people ask when a command comes back wrong: is\n" + + "it me, is it the network, or did GitHub change the page. It reads a small\n" + + "file to check reachability, a repository page to check that the embedded\n" + + "payload is still where every reader expects it, and the cache directory to\n" + + "check that it can be written.\n\n" + + "It also says out loud that GITHUB_TOKEN and GH_TOKEN are ignored, because a\n" + + "token in the environment does nothing here and the failure that causes is\n" + + "invisible: the tool works, it stays exactly as rate limited as before, and\n" + + "the obvious conclusion is that the token is wrong." + +// Check is one diagnostic. +type Check struct { + Name string `json:"name" table:"check"` + Status string `json:"status" table:"status"` + Detail string `json:"detail" table:"detail"` +} + +// The three states a check can be in. Warn exists because most of what goes +// wrong here is survivable: a token in the environment, a cache that cannot be +// written, a page that parsed but looks thinner than it should. +const ( + StatusOK = "ok" + StatusWarn = "warn" + StatusFail = "fail" +) + +// tokenVars are the variables people expect to matter and which do not. They are +// checked by name and never read for their value: this file will not put a +// credential in a record, and there is nothing here that would use one. +var tokenVars = []string{"GITHUB_TOKEN", "GH_TOKEN", "GITHUB_API_TOKEN", "GH_ENTERPRISE_TOKEN"} + +// Doctor runs the checks in order and emits one record each. It stops for +// nothing: a failed reachability check makes the page check fail too, and seeing +// both is more useful than seeing the first one alone. +func (c *Client) Doctor(ctx context.Context, emit func(*Check) error) error { + for _, ck := range []func(context.Context) *Check{ + c.checkAuthEnv, + c.checkReach, + c.checkPagePlane, + c.checkCache, + c.checkPacing, + } { + if err := emit(ck(ctx)); err != nil { + return err + } + } + return nil +} + +// checkAuthEnv is the one people need and do not know to ask for. A token in the +// environment does nothing here, and the failure it causes is invisible: the +// tool works, it is just as rate limited as it was before, and the obvious +// conclusion is that the token is wrong. +func (c *Client) checkAuthEnv(context.Context) *Check { + var set []string + for _, v := range tokenVars { + if os.Getenv(v) != "" { + set = append(set, v) + } + } + if len(set) == 0 { + return &Check{Name: "auth", Status: StatusOK, + Detail: "no token in the environment, which is what this tool wants"} + } + return &Check{Name: "auth", Status: StatusWarn, + Detail: fmt.Sprintf("%s is set and ignored: github reads public pages and never sends an Authorization header, so a token changes nothing here. Use gh for the authenticated API", strings.Join(set, " and "))} +} + +// checkReach is one small request to the site. robots.txt is the right target: +// it is a few hundred bytes, it is not behind any of the machinery this tool +// reads, and it comes back the same for everyone. +func (c *Client) checkReach(ctx context.Context) *Check { + start := time.Now() + res, err := c.Get(ctx, BaseURL+"/robots.txt", SurfaceRaw) + if err != nil { + return &Check{Name: "reach", Status: StatusFail, + Detail: fmt.Sprintf("cannot read %s: %v", BaseURL, err)} + } + return &Check{Name: "reach", Status: StatusOK, + Detail: fmt.Sprintf("%s answered %d in %s", BaseURL, res.Status, time.Since(start).Round(time.Millisecond))} +} + +// checkPagePlane reads a repository page and looks for the embedded React +// payload. This is the check that catches the failure this tool cannot survive: +// GitHub reorganising the page. Every structureChanged error in the package +// starts here, so when one fires, this says whether the whole plane moved or +// only the one selector. +func (c *Client) checkPagePlane(ctx context.Context) *Check { + p, err := c.Page(ctx, BaseURL+"/golang/go") + if err != nil { + return &Check{Name: "page", Status: StatusFail, + Detail: fmt.Sprintf("cannot read a repository page: %v", err)} + } + switch { + case p.Plane == page.PlaneReact && len(p.Payload) > 0: + return &Check{Name: "page", Status: StatusOK, + Detail: fmt.Sprintf("the react payload is where it should be, %d keys in %d bytes", len(p.Payload), p.Bytes)} + case len(p.Microdata) > 0 || len(p.Meta) > 0: + return &Check{Name: "page", Status: StatusWarn, + Detail: "no react payload, but the meta and microdata are readable: the records will be thinner than they should be. Run github page golang/go to see what came back"} + default: + return &Check{Name: "page", Status: StatusFail, + Detail: "a repository page carried nothing this understands. Either the request was intercepted or the page changed shape. Run github page golang/go --raw to see the bytes"} + } +} + +// checkCache reports what is on disk and, more to the point, whether it can be +// written. A read-only cache directory turns every run into a cold one, which +// looks like the site being slow rather than like a local problem. +func (c *Client) checkCache(context.Context) *Check { + if c.NoCache { + return &Check{Name: "cache", Status: StatusWarn, + Detail: "the cache is off for this run, so every request goes to the network"} + } + if c.CacheDir == "" { + return &Check{Name: "cache", Status: StatusWarn, Detail: "no cache directory is configured"} + } + if err := os.MkdirAll(c.CacheDir, 0o755); err != nil { + return &Check{Name: "cache", Status: StatusFail, + Detail: fmt.Sprintf("cannot create %s: %v", c.CacheDir, err)} + } + probe := filepath.Join(c.CacheDir, ".doctor") + if err := os.WriteFile(probe, []byte("ok"), 0o644); err != nil { + return &Check{Name: "cache", Status: StatusFail, + Detail: fmt.Sprintf("%s is not writable: %v", c.CacheDir, err)} + } + _ = os.Remove(probe) + + n, bytes := cacheSize(c.CacheDir) + return &Check{Name: "cache", Status: StatusOK, + Detail: fmt.Sprintf("%s holds %d entries, %s, kept for %s", c.CacheDir, n, humanBytes(bytes), c.CacheTTL)} +} + +// checkPacing prints the numbers a run is using. It is not a test of anything; +// it is here because "why is this slow" and "why did I get rate limited" are +// both answered by these four values and neither is visible otherwise. +func (c *Client) checkPacing(context.Context) *Check { + return &Check{Name: "pacing", Status: StatusOK, + Detail: fmt.Sprintf("%s between requests across %d workers, %s timeout, %d retries, user agent %q", + c.Rate, c.Workers, c.HTTP.Timeout, c.Retries, c.UserAgent)} +} + +func cacheSize(dir string) (entries int, bytes int64) { + _ = filepath.WalkDir(dir, func(_ string, d os.DirEntry, err error) error { + if err != nil || d.IsDir() { + return nil //nolint:nilerr // a directory that cannot be walked is reported as empty + } + if info, err := d.Info(); err == nil { + entries++ + bytes += info.Size() + } + return nil + }) + return entries, bytes +} + +func humanBytes(n int64) string { + const unit = 1024 + if n < unit { + return fmt.Sprintf("%d B", n) + } + div, exp := int64(unit), 0 + for m := n / unit; m >= unit; m /= unit { + div *= unit + exp++ + } + return fmt.Sprintf("%.1f %cB", float64(n)/float64(div), "KMGT"[exp]) +} diff --git a/gh/domain.go b/gh/domain.go new file mode 100644 index 0000000..a7f58b0 --- /dev/null +++ b/gh/domain.go @@ -0,0 +1,143 @@ +package gh + +import ( + "context" + "net/http" + "path/filepath" + "time" + + "github.com/tamnd/any-cli/kit" +) + +// domain.go is the seam between this library and the kit framework. It declares +// what the site is called, how its addresses are parsed, and how a client is +// built from the resolved config. Every verb a person can type is registered in +// ops.go; nothing else in the package imports kit. + +// Domain is the kit driver for github.com. A blank import of this package +// enables it in any multi-domain host, the way a database driver registers +// itself, and the same Domain builds the single github binary. +type Domain struct{} + +func init() { kit.Register(Domain{}) } + +// Info names the domain and every hostname that means it. The extra hosts are +// not decoration: a pasted raw.githubusercontent.com or gist.github.com link is +// a github address and has to resolve here rather than fall through as an +// unknown site. +func (Domain) Info() kit.DomainInfo { + return kit.DomainInfo{ + Scheme: Scheme, + Aliases: []string{"gh"}, + Hosts: []string{ + "github.com", "www.github.com", + "raw.githubusercontent.com", "gist.github.com", + "gist.githubusercontent.com", "codeload.github.com", + }, + Identity: kit.Identity{ + Binary: "github", + Short: "Read all of GitHub as structured data, with no token", + Long: "github reads github.com and gives back records rather than pages.\n\n" + + "Every repository, user, organization, issue, pull request, discussion,\n" + + "commit, branch, tag, release, file, topic, gist, package, and marketplace\n" + + "action has a canonical github:// address, a typed record carrying every\n" + + "field its source returned, and edges to the other things it names.\n\n" + + "There is no API token anywhere in this tool and there will not be one.\n" + + "The unauthenticated REST API allows sixty requests an hour, which is not\n" + + "enough to read one organization, while the pages are behind a CDN and are\n" + + "faster than the API even where the API would work. The cost is that this\n" + + "is read-only and public-only. For anything else, use the official gh.", + Site: BaseURL, + Repo: "https://github.com/tamnd/github-cli", + }, + } +} + +// Classify satisfies kit.Resolver, so a URI typed at a multi-domain host and one +// typed at github are read by the same parser. +func (Domain) Classify(input string) (uriType, id string, err error) { + return Classify(input) +} + +// Locate satisfies kit.Resolver: the https location of one resource. +func (Domain) Locate(uriType, id string) (string, error) { + return Locate(uriType, id) +} + +// DomainDefaults overlays this site's baseline onto the framework's. GitHub +// publishes no rate limit for the pages, so these are chosen to be quieter than +// a person browsing with a few tabs open: eight requests a second across four +// workers. +func DomainDefaults(c *kit.Config) { + c.Rate = Defaults.Rate + c.Retries = Defaults.Retries + c.Workers = Defaults.Workers + c.Timeout = Defaults.Timeout +} + +// flags holds the domain's own global flags. kit resolves the framework globals +// (--limit, --rate, --timeout, --no-cache) itself; these are the ones only this +// tool has, and they are read once when the client is built. +// +// Package-level state is the framework's contract here: GlobalFlags binds to the +// domain's variables and the client factory reads them, and there is exactly one +// run per process. +var flags struct { + deep bool + jobs int + cache string +} + +// Register installs the client factory, the domain globals, and every +// operation. It does no I/O and is deterministic, so a host can call it at +// startup. +func (d Domain) Register(app *kit.App) { + app.SetClient(newClientFor) + app.GlobalFlags(bindFlags) + registerOps(app) +} + +func bindFlags(f *kit.FlagSet) { + f.BoolVar(&flags.deep, "deep", false, "also fetch the fragments a page defers, and merge what only they carry") + f.IntVarP(&flags.jobs, "jobs", "j", 0, "concurrent requests (0 = the default 4)") + f.StringVar(&flags.cache, "cache", "", "response cache directory (default under the data dir)") +} + +// newClientFor builds the one client a run shares. Every command reaches it +// through a kit:"inject" field, so pacing and the cache are shared across a +// whole pipeline rather than per command. +func newClientFor(_ context.Context, cfg kit.Config) (any, error) { + conf := Defaults + if cfg.UserAgent != "" { + conf.UserAgent = cfg.UserAgent + } + if cfg.Rate > 0 { + conf.Rate = cfg.Rate + } + if cfg.Retries > 0 { + conf.Retries = cfg.Retries + } + if cfg.Workers > 0 { + conf.Workers = cfg.Workers + } + if cfg.Timeout > 0 { + conf.Timeout = cfg.Timeout + } + conf.NoCache = cfg.NoCache + conf.CacheDir = filepath.Join(cfg.CacheDir, "http") + if flags.cache != "" { + conf.CacheDir = flags.cache + } + if flags.jobs > 0 { + conf.Workers = flags.jobs + } + conf.Deep = flags.deep + + c := NewClient(conf) + c.HTTP = &http.Client{Timeout: conf.Timeout} + return c, nil +} + +// timeoutFor is the per-command ceiling for the walks that can run long, used +// by the byte-plane commands that do not get a kit-managed context. +const timeoutFor = 10 * time.Minute diff --git a/gh/errors.go b/gh/errors.go new file mode 100644 index 0000000..a316e89 --- /dev/null +++ b/gh/errors.go @@ -0,0 +1,134 @@ +package gh + +import ( + "context" + "errors" + "net" + "net/http" + "strings" + + "github.com/tamnd/any-cli/kit/errs" +) + +// errors.go is the one place an HTTP status becomes a program outcome. The CLI +// exit code, the HTTP response under `github serve`, and the MCP error object +// all read the kind decided here, so a 404 means the same thing on every +// surface. +// +// Two of GitHub's status codes are routing decisions rather than failures and +// are handled by the caller, not here: +// +// 406 this route does not serve JSON; ask for HTML instead +// 410 there is no JSON at this address at all +// +// Both mean "wrong surface", which is a thing the client can fix by trying the +// other one. Turning them into errors here would hide that. +// +// Every message here leads with a word rather than with the path it is about. +// The renderer title-cases whatever a message starts with, and a path that +// comes back as Golang/Go/Blob/Master reads like the tool mangled the input +// rather than like the page was missing. + +// statusError classifies a non-2xx response. +func statusError(rawURL string, status int, body []byte) error { + where := shortURL(rawURL) + switch { + case status == http.StatusUnauthorized || status == http.StatusForbidden: + // This tool sends no credential, so a 401 or a 403 means the page is not + // public. Saying "pass a token" would be wrong: there is no token to + // pass. Saying what is actually true is more useful. + if isRateLimitBody(body) { + return errs.RateLimited("github is throttling anonymous reads, try again shortly (%s)", where) + } + return errs.NeedAuth("not public: %s, and this tool reads only public pages (use gh for the rest)", where) + case status == http.StatusNotFound: + return errs.NotFound("not found: %s", where) + case status == http.StatusGone: + return errs.NotFound("gone: %s", where) + case status == http.StatusTooManyRequests: + return errs.RateLimited("rate limited on %s", where) + case status == http.StatusUnavailableForLegalReasons: + return errs.Unsupported("unavailable for legal reasons (DMCA): %s", where) + case status == http.StatusBadRequest: + return errs.Usage("bad request: %s", where) + case status >= 500: + return errs.New(errs.KindNetwork, "server error %d on %s", status, where) + default: + return errs.New(errs.KindGeneric, "http %d on %s", status, where) + } +} + +// isRateLimitBody spots the throttle page GitHub serves as a 403 when a client +// asks for too much too fast. It is the same status as a private repository +// and the body is the only way to tell them apart. +func isRateLimitBody(body []byte) bool { + s := strings.ToLower(string(body)) + if len(s) > 4096 { + s = s[:4096] + } + return strings.Contains(s, "rate limit") || strings.Contains(s, "abuse detection") || + strings.Contains(s, "too many requests") +} + +// wrapNetwork turns a transport failure into the network kind, keeping the +// distinction between "the name does not resolve" and "the page said no", +// because those two send a person to very different places. +func wrapNetwork(rawURL string, err error) error { + if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { + return err + } + var dnsErr *net.DNSError + if errors.As(err, &dnsErr) { + return errs.New(errs.KindNetwork, "cannot resolve %s, reading %s", dnsErr.Name, shortURL(rawURL)) + } + return errs.New(errs.KindNetwork, "reading %s: %v", shortURL(rawURL), err) +} + +// shortURL trims the scheme and the host so an error message reads as a path. +// The host is the same for every message in this tool, so printing it in every +// message is noise. +func shortURL(raw string) string { + s := strings.TrimPrefix(raw, "https://") + s = strings.TrimPrefix(s, "http://") + if i := strings.Index(s, "?"); i > 0 { + s = s[:i] + } + return s +} + +// notPublic is the message for the surfaces that exist but need a session: +// code search, traffic, clones, referrers. It names what would be needed rather +// than being vague, because a vague "unsupported" wastes an afternoon. +func notPublic(what, why string) error { + return errs.Unsupported("not available without a session: %s, %s", what, why) +} + +// usageBadID rejects a malformed identifier before a request goes out. Showing +// the expected shape saves the round trip and the 404 that would follow it. +func usageBadID(kind, got, want string) error { + return errs.Usage("expected a %s like %s, got %q", kind, want, got) +} + +// structureChanged is the loud failure from doc 02 section 7: the page came +// back fine but carried none of the blocks the decoder knows. That is different +// from a missing optional field, and it must not return an empty record with a +// zero exit code. +func structureChanged(what string) error { + return errs.New(errs.KindNetwork, + "the page structure changed for %s, none of the expected data was there (run `github page %s` to see what arrived)", + what, what) +} + +// badPayload is a decode failure on a block that was there. It is separate from +// structureChanged because the two send you to different places: structure +// changed means the block is gone, bad payload means the block arrived and no +// longer parses, which is usually a type change on one field. +func badPayload(what string, err error) error { + return errs.New(errs.KindNetwork, "the payload for %s did not decode: %v", what, err) +} + +// noJSONHere is what a 410 means. It is separated out so the message can say +// the useful half: the data is reachable, just on a different surface. +func noJSONHere(rawURL string) error { + return errs.Unsupported("no JSON at %s; this is a page-only route", shortURL(rawURL)) +} diff --git a/gh/gh.go b/gh/gh.go new file mode 100644 index 0000000..57e077a --- /dev/null +++ b/gh/gh.go @@ -0,0 +1,56 @@ +// Package gh reads github.com without a token. +// +// Every byte this package fetches is a byte a logged-out browser would get: +// public HTML, the JSON those pages ship inside themselves, the JSON their own +// front end asks for, Atom feeds, and the git smart protocol. There is no REST +// client here and there never will be one. The unauthenticated REST API allows +// sixty requests an hour, which is not enough to read one organization, and the +// pages are behind a CDN, which makes them faster than the API even when the +// API would work. +// +// The consequence is a read-only tool. It cannot see a private repository and +// it cannot write anything. For that, use the official gh. +package gh + +import "time" + +// The hosts. All five are public and none of them accept a credential from us. +const ( + BaseURL = "https://github.com" + RawURL = "https://raw.githubusercontent.com" + CodeLoad = "https://codeload.github.com" + GistURL = "https://gist.github.com" + GistRaw = "https://gist.githubusercontent.com" + AvatarURL = "https://avatars.githubusercontent.com" + OpenGraph = "https://opengraph.githubassets.com" +) + +// UserAgentBase is the honest half of the User-Agent. The version is appended +// at runtime. It is deliberately not configurable: making it configurable would +// be making impersonation a feature, and a tool that reads only public pages +// has no reason to hide. +const UserAgentBase = "github-cli" + +// Defaults are the pacing numbers every command starts from. GitHub publishes +// no rate limit for the pages, so these are chosen to be quieter than a person +// browsing with a few tabs open: eight requests a second across four workers. +var Defaults = Config{ + Rate: 125 * time.Millisecond, + Retries: 4, + Workers: 4, + Timeout: 30 * time.Second, +} + +// Config is the resolved per-run configuration. It carries no credential field +// because there is no credential. +type Config struct { + UserAgent string + Rate time.Duration + Retries int + Workers int + Timeout time.Duration + CacheDir string + NoCache bool + CacheTTL time.Duration + Deep bool +} diff --git a/gh/graph.go b/gh/graph.go new file mode 100644 index 0000000..03cb55e --- /dev/null +++ b/gh/graph.go @@ -0,0 +1,949 @@ +package gh + +import ( + "sort" + "strconv" + "strings" + "time" +) + +// graph.go turns records into triples. github.com is already a knowledge graph: +// a repository names its owner, its topics, its licence, and the repository it +// was forked from; a pull request names the issue it closes and the branch it +// targets; a commit names its parents. This file reads those declarations off a +// record and emits them as typed edges. +// +// It is pure. No network, no client, no ordering dependency, which is what makes +// the whole graph plane testable against a fixture and what makes +// `github edges golang/go#1 --min-trust id` answer without a request. + +// Node is one entity. It is deliberately thin: the label and the two addresses +// and nothing else, because the full record is one `github get` away by URI and +// duplicating it here would make a crawl of ten thousand nodes unprintable. +type Node struct { + URI string `json:"uri" table:"uri"` + Kind string `json:"kind" table:"kind"` + ID string `json:"id" table:"id"` + Label string `json:"label,omitempty" table:"label,truncate"` + URL string `json:"url,omitempty" table:"url,url"` +} + +// Edge is one directed, typed relation between two entities. +// +// There is no inverse flag. Every predicate has exactly one direction, and +// where the inverse is what you want, the edge is emitted with the other node as +// its subject rather than with a flag saying to read it backwards. +type Edge struct { + Subject string `json:"subject" table:"subject"` + Predicate string `json:"predicate" table:"predicate"` + Object string `json:"object" table:"object"` + Source string `json:"source" table:"source"` + Weight *int `json:"weight,omitempty" table:"weight"` + At *time.Time `json:"at,omitempty" table:"-"` +} + +// Fact is a literal statement about a node: a star count, a description, a +// timestamp. +// +// It is a separate type from Edge on purpose. Edge.Object is a URI and every +// consumer of the graph is entitled to treat it as one, so putting "12000" in +// that field to carry a star count would break each of them for the sake of +// saving a struct. RDF emits both; `github edges` emits only edges, which is why +// its output reads as relations rather than as a flattened record. +type Fact struct { + Subject string `json:"subject" table:"subject"` + Predicate string `json:"predicate" table:"predicate"` + Value string `json:"value" table:"value,truncate"` + Datatype string `json:"datatype,omitempty" table:"-"` +} + +// The five extraction rules, in descending order of trust. Every edge carries +// the one that produced it, which is the field a consumer uses to decide how +// much to believe. +const ( + // SrcID is derived from the id structure alone. No fetch, always correct. + SrcID = "id" + // SrcPayload is an explicit reference in a JSON payload or a Relay result. + SrcPayload = "payload" + // SrcFeed is an explicit reference in an Atom feed. + SrcFeed = "feed" + // SrcHTML was parsed out of rendered markup with a selector. Good, and it + // degrades to a missing edge rather than a wrong one when a template moves. + SrcHTML = "html" + // SrcText is a pattern matched in free text: #42, a bare SHA. Heuristic, and + // dropped by the default --min-trust. + SrcText = "text" +) + +// trustRank orders the rules. Higher is more trustworthy. +var trustRank = map[string]int{ + SrcID: 4, + SrcPayload: 3, + SrcFeed: 2, + SrcHTML: 1, + SrcText: 0, +} + +// DefaultMinTrust keeps everything except free-text guesses. +const DefaultMinTrust = SrcHTML + +// TrustAtLeast reports whether a source meets a floor. An unknown floor lets +// everything through rather than silently dropping the whole graph, and an +// unknown source is treated as the weakest thing there is. +func TrustAtLeast(source, min string) bool { + floor, ok := trustRank[min] + if !ok { + return true + } + return trustRank[source] >= floor +} + +// TrustLevels is the accepted set, for help text and for validation. +var TrustLevels = []string{SrcID, SrcPayload, SrcFeed, SrcHTML, SrcText} + +// The predicate vocabulary. This is the complete set: an edge this tool emits +// has its predicate here, and adding a relation means adding a constant first. +const ( + // Ownership and membership. + PredOwnedBy = "ownedBy" + PredMemberOf = "memberOf" + PredPartOf = "partOf" + PredBelongsToPackage = "belongsToPackage" + + // Derivation. The edges that make a graph worth walking. + PredForkOf = "forkOf" + PredTemplateOf = "templateOf" + PredMirrorOf = "mirrorOf" + PredDependsOn = "dependsOn" + PredUsedBy = "usedBy" + + // Authorship and activity. + PredAuthoredBy = "authoredBy" + PredCommittedBy = "committedBy" + PredContributedTo = "contributedTo" + PredAssignedTo = "assignedTo" + PredReviewedBy = "reviewedBy" + PredReviewRequestedFrom = "reviewRequestedFrom" + PredMergedBy = "mergedBy" + + // Reference. + PredReferences = "references" + PredCloses = "closes" + PredClosedBy = "closedBy" + PredDuplicateOf = "duplicateOf" + PredSubIssueOf = "subIssueOf" + PredLinkedTo = "linkedTo" + PredTargetsBranch = "targetsBranch" + PredFromBranch = "fromBranch" + PredPointsAt = "pointsAt" + PredParentOf = "parentOf" + + // Classification. + PredHasTopic = "hasTopic" + PredHasLabel = "hasLabel" + PredInMilestone = "inMilestone" + PredWrittenIn = "writtenIn" + PredLicensedUnder = "licensedUnder" + PredRelatedTopic = "relatedTopic" + + // Social. Opt-in everywhere, because the star list of a popular repository + // is thousands of pages and nobody wants that by accident. + PredStarredBy = "starredBy" + PredFollows = "follows" + PredSponsors = "sponsors" + PredReactedWith = "reactedWith" +) + +// The literal predicates. These name Fact rows rather than edges. +const ( + FactName = "name" + FactDescription = "description" + FactHomepage = "homepage" + FactCreated = "created" + FactUpdated = "updated" + FactStars = "stars" + FactForks = "forks" + FactWatchers = "watchers" + FactCommits = "commits" + FactURI = "uri" + FactAvatar = "avatar" + FactState = "state" + FactCount = "count" +) + +// SocialPredicates are the ones a command has to be asked for by name. +var SocialPredicates = []string{PredStarredBy, PredFollows, PredSponsors, PredReactedWith} + +// DefaultFollow is the crawler's follow set. It deliberately excludes +// references, starredBy, follows, dependsOn, and usedBy: those five turn a +// bounded walk into an unbounded one, and each has to be asked for by name. +var DefaultFollow = []string{PredPartOf, PredOwnedBy, PredForkOf, PredHasTopic, PredAuthoredBy} + +// LiteralPredicates are the two whose object is a bare string rather than a +// URI, because a language and a licence are not github.com entities. RDF gives +// them synthetic IRIs in the gh: namespace; `github edges` prints them as they +// are written on the page. +var LiteralPredicates = map[string]bool{ + PredWrittenIn: true, + PredLicensedUnder: true, + PredReactedWith: true, +} + +// --- the builder --- + +// builder accumulates one node with its edges and facts while an extractor +// walks a record. +type builder struct { + node Node + edges []Edge + facts []Fact +} + +// start sets the node. Every extractor calls it first, and nothing is emitted +// for a record whose identity did not resolve. +func (b *builder) start(kind, id, label, url string) { + if id == "" { + return + } + if label == "" { + label = id + } + if url == "" { + if u, err := Locate(kind, id); err == nil { + url = u + } + } + b.node = Node{URI: URI(kind, id), Kind: kind, ID: id, Label: label, URL: url} +} + +// to emits an edge from this node to another entity named by kind and id. +func (b *builder) to(pred, objKind, objID, source string) { + if objID == "" { + return + } + b.toURI(pred, URI(objKind, objID), source) +} + +// toURI is to for an object whose URI is already built. +func (b *builder) toURI(pred, objURI, source string) { + if b.node.URI == "" || objURI == "" { + return + } + b.edges = append(b.edges, Edge{Subject: b.node.URI, Predicate: pred, Object: objURI, Source: source}) +} + +// toURL emits an edge to whatever a github.com URL names. Relay results carry +// links rather than ids for linked pull requests, duplicates, and cross +// references, and classification is exactly the function that turns one into +// the other. +func (b *builder) toURL(pred, rawURL, source string) { + if uri := uriOfURL(rawURL); uri != "" { + b.toURI(pred, uri, source) + } +} + +// raw emits an edge whose object is a bare string rather than a URI: a +// language, a licence, a reaction. +func (b *builder) raw(pred, value, source string) { + if b.node.URI == "" || value == "" { + return + } + b.edges = append(b.edges, Edge{Subject: b.node.URI, Predicate: pred, Object: value, Source: source}) +} + +// from emits an edge whose subject is not this node. A contributor edge points +// at the repository rather than away from it, and inverting it to make this node +// the subject would be a lie about which way the relation runs. +func (b *builder) from(subjURI, pred, objURI, source string) { + if subjURI == "" || objURI == "" { + return + } + b.edges = append(b.edges, Edge{Subject: subjURI, Predicate: pred, Object: objURI, Source: source}) +} + +// weigh attaches a count to the last edge appended. It is separate from the +// emitters so the common case stays a one-liner. +func (b *builder) weigh(n *int) { + if n == nil || len(b.edges) == 0 { + return + } + b.edges[len(b.edges)-1].Weight = n +} + +// when attaches a time to the last edge appended. +func (b *builder) when(t *time.Time) { + if t == nil || len(b.edges) == 0 { + return + } + b.edges[len(b.edges)-1].At = t +} + +// fact records a literal. An empty value is skipped, because "this repository +// has no description" is better said by the absence of a statement than by an +// empty one. +func (b *builder) fact(pred, value, datatype string) { + if b.node.URI == "" || value == "" { + return + } + b.facts = append(b.facts, Fact{Subject: b.node.URI, Predicate: pred, Value: value, Datatype: datatype}) +} + +// num records a count. A nil count is a count the surface did not state, which +// is not the same as zero and does not become a statement. +func (b *builder) num(pred string, n *int) { + if n == nil { + return + } + b.fact(pred, strconv.Itoa(*n), TypeInteger) +} + +func (b *builder) at(pred string, t *time.Time) { + if t == nil || t.IsZero() { + return + } + b.fact(pred, t.UTC().Format(time.RFC3339), TypeDateTime) +} + +// actorEdge emits an edge to a person. The actor's own type is used when it says +// one, so a bot or an organization does not silently become a user. +func (b *builder) actorEdge(pred string, a Actor, source string) { + if a.Login == "" { + return + } + b.to(pred, actorKind(a), a.Login, source) +} + +func actorKind(a Actor) string { + if strings.EqualFold(a.Type, "Organization") { + return KindOrg + } + return KindUser +} + +// uriOfURL classifies a github.com URL into a URI, and answers empty for +// anything that is not one. Extractors use it rather than Classify directly so +// a link to an external site drops out instead of producing an error nobody can +// act on. +func uriOfURL(raw string) string { + if raw == "" { + return "" + } + kind, id, err := Classify(raw) + if err != nil { + return "" + } + return URI(kind, id) +} + +// --- extraction --- + +// Extract turns one record into its node, its edges, and its facts. A record +// kind it does not know produces an empty node, which every caller treats as +// nothing to say rather than as an error. +func Extract(rec any) (Node, []Edge, []Fact) { + b := &builder{} + switch r := rec.(type) { + case *Repo: + b.repo(r) + case *Trending: + b.trending(r) + case *Account: + b.account(r) + case *Org: + b.org(r) + case *Issue: + b.issue(r) + case *PullRequest: + b.pull(r) + case *Discussion: + b.discussion(r) + case *Thread: + b.thread(r) + case *Commit: + b.commit(r) + case *GitRef: + b.gitRef(r) + case *Release: + b.release(r) + case *Topic: + b.topic(r) + case *Package: + b.pkg(r) + case *WikiPage: + b.wiki(r) + case *Gist: + b.gist(r) + case *File: + b.file(r) + case *TreeEntry: + b.treeEntry(r) + case *Contributor: + b.contributor(r) + case *Dependency: + b.dependency(r) + case *Dependent: + b.dependent(r) + case *LanguageShare: + b.languageShare(r) + case *RepoStats: + b.stats(r) + default: + return Node{}, nil, nil + } + return b.node, b.edges, b.facts +} + +// repo is the centre of the graph. Everything else hangs off a repository, and +// most of what a walk finds interesting is stated on this one record. +func (b *builder) repo(r *Repo) { + id := r.ID + if id == "" && r.Owner != "" && r.Name != "" { + id = r.Owner + "/" + r.Name + } + b.start(KindRepo, id, id, r.URL) + + // The owner comes from the id, which is why this edge costs nothing. Which + // of the two account kinds it is comes from the page, so a repository read + // from a surface that did not say defaults to user and is corrected the + // moment the owner itself is fetched. + if r.Owner != "" { + b.to(PredOwnedBy, ownerKind(r), r.Owner, SrcID) + } + // ForkOf is the "Forked from" line in the header, and it is the only place + // any keyless surface names the parent. templateOf and mirrorOf have their + // constants in the vocabulary and no producer here, because the page states + // that a repository is a template or a mirror without ever naming what it + // was generated from or what it mirrors. + b.to(PredForkOf, KindRepo, r.ForkOf, SrcHTML) + + for _, t := range r.Topics { + b.to(PredHasTopic, KindTopic, t, SrcPayload) + } + b.raw(PredLicensedUnder, r.License, SrcHTML) + b.languages(r) + + for i := range r.Tree { + e := &r.Tree[i] + if e.URI != "" { + b.from(e.URI, PredPartOf, b.node.URI, SrcID) + } + } + + b.fact(FactName, id, "") + b.fact(FactDescription, r.Description, "") + b.fact(FactHomepage, r.Homepage, "") + b.num(FactStars, r.Stars) + b.num(FactForks, r.Forks) + b.num(FactWatchers, r.Watchers) + b.num(FactCommits, r.CommitCount) + b.at(FactCreated, r.CreatedAt) + b.at(FactUpdated, firstSetTime(r.PushedAt, r.UpdatedAt)) + b.fact(FactAvatar, r.OwnerAvatarURL, "") + b.fact(FactURI, b.node.URI, "") +} + +// ownerKind decides between a user and an organization. IsOrgOwned is set by +// the page template, which is the only surface that states it without a token. +func ownerKind(r *Repo) string { + if r.IsOrgOwned { + return KindOrg + } + return KindUser +} + +// languages emits one writtenIn edge per language, weighted by the percentage +// the histogram gave. +// +// The source is honest about where the number came from: Via records +// sidebar-percent when the histogram was read from the deferred sidebar +// fragment, which is a payload, and anything else was read off the language bar +// in the markup. +func (b *builder) languages(r *Repo) { + source := SrcHTML + if r.Via["languages"] == "sidebar-percent" { + source = SrcPayload + } + if len(r.Languages) == 0 { + b.raw(PredWrittenIn, r.Language, source) + return + } + for _, name := range sortedLanguages(r.Languages) { + b.raw(PredWrittenIn, name, source) + if n := r.Languages[name]; n > 0 { + b.weigh(intp(int(n))) + } + } +} + +// sortedLanguages orders a histogram by share and then by name, so two runs over +// the same repository produce the same edge order. +func sortedLanguages(m map[string]int64) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.SliceStable(out, func(i, j int) bool { + if m[out[i]] != m[out[j]] { + return m[out[i]] > m[out[j]] + } + return out[i] < out[j] + }) + return out +} + +// trending is a repository plus the people the trending page credits. +func (b *builder) trending(t *Trending) { + b.repo(&t.Repo) + for _, a := range t.BuiltBy { + if a.Login != "" { + b.from(URI(actorKind(a), a.Login), PredContributedTo, b.node.URI, SrcHTML) + } + } +} + +func (b *builder) account(a *Account) { + kind := KindUser + if strings.EqualFold(a.Type, "Organization") { + kind = KindOrg + } + b.start(kind, a.Login, firstNonEmpty(a.Name, a.Login), a.URL) + + for _, org := range a.Organizations { + b.to(PredMemberOf, KindOrg, org, SrcHTML) + } + // A pinned repository the person does not own is pinned work they + // contributed to, and this tool cannot tell which from the profile alone. + // Only the ones whose id carries this login become ownership edges; the rest + // are left out rather than asserted wrongly. + for _, repo := range a.PinnedRepos { + if owner, _, ok := SplitRepo(repo); ok && strings.EqualFold(owner, a.Login) { + b.from(URI(KindRepo, repo), PredOwnedBy, b.node.URI, SrcHTML) + } + } + + b.fact(FactName, firstNonEmpty(a.Name, a.Login), "") + b.fact(FactDescription, a.Bio, "") + b.fact(FactHomepage, a.Website, "") + b.fact(FactAvatar, a.AvatarURL, "") + b.at(FactCreated, a.CreatedAt) + b.fact(FactURI, b.node.URI, "") + if a.Followers != nil { + b.fact("followers", strconv.Itoa(*a.Followers), TypeInteger) + } +} + +func (b *builder) org(o *Org) { + b.account(&o.Account) + // The node kind comes from the template that answered, and this one is the + // organization template, so it is an organization whatever the account + // record's Type string says. + if b.node.URI != "" { + b.node.Kind = KindOrg + b.node.URI = URI(KindOrg, o.Login) + } + for _, m := range o.Members { + b.from(URI(KindUser, m), PredMemberOf, b.node.URI, SrcHTML) + } + if o.MemberCount != nil { + b.fact("members", strconv.Itoa(*o.MemberCount), TypeInteger) + } + // TopTopics and TopLanguages are aggregates over the organization's + // repositories rather than properties of the organization, so they produce + // no hasTopic or writtenIn edge here. The repositories state their own. +} + +// thread covers what issues, pull requests, and discussions share. +func (b *builder) thread(t *Thread) { + kind := t.Kind + if kind == "" { + kind = KindIssue + } + b.start(kind, t.ID, t.Title, t.URL) + + if t.Repo != "" { + b.to(PredPartOf, KindRepo, t.Repo, SrcID) + } + b.actorEdge(PredAuthoredBy, t.Author, SrcPayload) + for _, l := range t.Labels { + if l.Name != "" && t.Repo != "" { + b.to(PredHasLabel, KindLabel, t.Repo+"/"+l.Name, SrcPayload) + } + } + if m := t.Milestone; m != nil && m.Number != nil && t.Repo != "" { + b.to(PredInMilestone, KindMilestone, t.Repo+"/"+strconv.Itoa(*m.Number), SrcPayload) + } + for _, a := range t.Assignees { + b.actorEdge(PredAssignedTo, a, SrcPayload) + } + for _, r := range t.Reactions { + b.raw(PredReactedWith, strings.ToLower(r.Content), SrcPayload) + b.weigh(intp(r.Count)) + } + + b.fact(FactName, t.Title, "") + b.fact(FactState, t.State, "") + b.at(FactCreated, t.CreatedAt) + b.at(FactUpdated, t.UpdatedAt) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) issue(i *Issue) { + b.thread(&i.Thread) + b.toURL(PredDuplicateOf, i.DuplicateOf, SrcPayload) + for _, u := range i.LinkedPRs { + b.toURL(PredLinkedTo, u, SrcPayload) + } + for _, u := range i.ClosedByPRs { + b.toURL(PredClosedBy, u, SrcPayload) + } + // SubIssueTotal is a count and not a list, so subIssueOf has no producer on + // this record. The timeline carries the parent, which is a separate read. +} + +func (b *builder) pull(p *PullRequest) { + b.thread(&p.Thread) + if p.Repo != "" { + b.to(PredTargetsBranch, KindBranch, refID(p.Repo, p.BaseRef), SrcPayload) + b.to(PredFromBranch, KindBranch, refID(p.Repo, p.HeadRef), SrcPayload) + } + if p.MergedBy != nil { + b.actorEdge(PredMergedBy, *p.MergedBy, SrcPayload) + } + for _, a := range p.ReviewRequests { + b.actorEdge(PredReviewRequestedFrom, a, SrcPayload) + } + for _, u := range p.ClosesIssues { + b.toURL(PredCloses, u, SrcPayload) + } +} + +// refID builds owner/name@ref, and answers empty for an empty ref so that a +// pull request read from a surface that did not state its base does not point +// at a branch called nothing. +func refID(repo, ref string) string { + if repo == "" || ref == "" { + return "" + } + return repo + "@" + ref +} + +func (b *builder) discussion(d *Discussion) { + b.thread(&d.Thread) + // The answer's author is not the discussion's author and there is no + // predicate for "answered by" in the vocabulary, so the fact records who it + // was rather than inventing one. + if d.AnswerAuthor != nil && d.AnswerAuthor.Login != "" { + b.fact("answeredBy", d.AnswerAuthor.Login, "") + } +} + +func (b *builder) commit(c *Commit) { + id := c.ID + if id == "" && c.Repo != "" && c.SHA != "" { + id = c.Repo + "@" + c.SHA + } + b.start(KindCommit, id, c.Subject, c.URL) + + if c.Repo != "" { + b.to(PredPartOf, KindRepo, c.Repo, SrcID) + for _, p := range c.Parents { + // The arrow reads "that commit is the parent of this one", which is + // why the parent is the subject and not the object. + b.from(URI(KindCommit, c.Repo+"@"+p), PredParentOf, b.node.URI, SrcPayload) + } + } + for _, a := range c.Authors { + b.actorEdge(PredAuthoredBy, a, SrcPayload) + } + if c.Committer != nil { + b.actorEdge(PredCommittedBy, *c.Committer, SrcPayload) + } + // GitHub resolved these references itself when it rendered the message, so + // they are payload rather than the text rule that would find the same #N in + // the raw subject line. + for _, ref := range c.IssueRefs { + b.toURL(PredReferences, ref.URL, SrcPayload) + } + + b.fact(FactName, c.Subject, "") + b.at(FactCreated, firstSetTime(c.AuthoredAt, c.CommittedAt)) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) gitRef(r *GitRef) { + kind := r.Type + if kind != KindTag { + kind = KindBranch + } + id := r.ID + if id == "" { + id = refID(r.Repo, r.Name) + } + b.start(kind, id, r.Name, r.URL) + + if r.Repo != "" { + b.to(PredPartOf, KindRepo, r.Repo, SrcID) + // A ref read from the git protocol carries its object name, which is the + // one edge in this whole file that comes from git rather than from + // github.com. + if r.SHA != "" { + b.to(PredPointsAt, KindCommit, r.Repo+"@"+firstNonEmpty(r.PeeledSHA, r.SHA), SrcPayload) + } + } + if r.Author != nil { + b.actorEdge(PredAuthoredBy, *r.Author, SrcHTML) + } + b.fact(FactName, r.Name, "") + b.at(FactCreated, r.AuthoredAt) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) release(r *Release) { + id := r.ID + if id == "" { + id = refID(r.Repo, r.Tag) + } + b.start(KindRelease, id, firstNonEmpty(r.Title, r.Tag), r.URL) + + if r.Repo != "" { + b.to(PredPartOf, KindRepo, r.Repo, SrcID) + b.to(PredPointsAt, KindCommit, refID(r.Repo, r.CommitSHA), SrcHTML) + } + if r.Author != nil { + // The releases listing comes from the Atom feed, where the author is an + // element rather than a selector. + b.actorEdge(PredAuthoredBy, *r.Author, SrcFeed) + } + b.fact(FactName, firstNonEmpty(r.Title, r.Tag), "") + b.at(FactCreated, r.PublishedAt) + b.at(FactUpdated, r.UpdatedAt) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) topic(t *Topic) { + b.start(KindTopic, firstNonEmpty(t.ID, t.Name), firstNonEmpty(t.DisplayName, t.Name), t.URL) + for _, rel := range t.Related { + b.to(PredRelatedTopic, KindTopic, rel, SrcHTML) + } + b.fact(FactName, firstNonEmpty(t.DisplayName, t.Name), "") + b.fact(FactDescription, firstNonEmpty(t.ShortDescription, t.Description), "") + b.num(FactStars, t.StargazerCount) + b.num(FactCount, t.AppliedCount) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) pkg(p *Package) { + id := p.ID + if id == "" && p.Repo != "" { + id = p.Repo + "/" + p.Name + } + b.start(KindPackage, id, p.Name, p.URL) + // The direction is the vocabulary's: the package is the subject and the + // repository it was published from is the object. + b.to(PredBelongsToPackage, KindRepo, p.Repo, SrcPayload) + for _, t := range p.Topics { + b.to(PredHasTopic, KindTopic, t, SrcPayload) + } + b.fact(FactName, p.Name, "") + b.fact(FactDescription, p.Summary, "") + b.at(FactUpdated, p.UpdatedAt) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) wiki(w *WikiPage) { + id := w.ID + if id == "" && w.Repo != "" { + id = w.Repo + "/" + firstNonEmpty(w.Path, w.Title) + } + b.start(KindWiki, id, w.Title, w.URL) + b.to(PredPartOf, KindRepo, w.Repo, SrcID) + if w.Author != nil { + b.actorEdge(PredAuthoredBy, *w.Author, SrcHTML) + } + b.fact(FactName, w.Title, "") + b.at(FactUpdated, w.UpdatedAt) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) gist(g *Gist) { + b.start(KindGist, g.ID, firstNonEmpty(g.Description, g.ID), g.URL) + b.to(PredOwnedBy, KindUser, g.Owner, SrcHTML) + for _, f := range g.Files { + b.raw(PredWrittenIn, f.Language, SrcHTML) + } + b.fact(FactDescription, g.Description, "") + b.num(FactStars, g.Stars) + b.num(FactForks, g.Forks) + b.at(FactCreated, g.CreatedAt) + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) file(f *File) { + b.start(KindFile, f.ID, f.Path, f.URL) + b.to(PredPartOf, KindRepo, f.Repo, SrcID) + b.raw(PredWrittenIn, f.Language, SrcPayload) + b.fact(FactName, f.Path, "") + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) treeEntry(t *TreeEntry) { + kind := KindFile + if strings.Contains(t.Type, "directory") { + kind = KindTree + } + b.start(kind, t.ID, t.Path, t.URL) + b.to(PredPartOf, KindRepo, t.Repo, SrcID) + b.fact(FactName, t.Path, "") + b.fact(FactURI, b.node.URI, "") +} + +// contributor is the one weighted authorship edge, and the weight is what makes +// `github edges --predicate contributedTo` a ranked list rather than a set. +func (b *builder) contributor(c *Contributor) { + b.start(KindUser, c.Login, c.Login, BaseURL+"/"+c.Login) + if c.Repo != "" { + b.toURI(PredContributedTo, URI(KindRepo, c.Repo), SrcPayload) + b.weigh(c.Commits) + b.when(c.LastWeek) + } + b.fact(FactURI, b.node.URI, "") +} + +func (b *builder) languageShare(l *LanguageShare) { + if l.Repo == "" { + return + } + b.start(KindRepo, l.Repo, l.Repo, "") + b.raw(PredWrittenIn, l.Language, SrcPayload) +} + +// dependency and dependent are the two halves of the same relation read off two +// different pages. Both put the page's repository on the subject side, so a +// dependency row from hugo says hugo dependsOn chroma and a dependent row from +// hugo says hugo usedBy someone. A package GitHub could not resolve to a +// repository has nothing to point at and produces no node. +func (b *builder) dependency(d *Dependency) { + if d.SourceRepo == "" || d.Repo == "" { + return + } + b.start(KindRepo, d.SourceRepo, d.SourceRepo, d.URL) + b.from(URI(KindRepo, d.Repo), PredDependsOn, b.node.URI, SrcHTML) +} + +func (b *builder) dependent(d *Dependent) { + if d.Dependent == "" || d.Repo == "" { + return + } + b.start(KindRepo, d.Dependent, d.Dependent, d.URL) + b.from(URI(KindRepo, d.Repo), PredUsedBy, b.node.URI, SrcHTML) + if d.Owner != "" { + b.to(PredOwnedBy, KindUser, d.Owner, SrcID) + } + b.num(FactStars, d.Stars) + b.num(FactForks, d.Forks) +} + +func (b *builder) stats(s *RepoStats) { + if s.Repo == "" { + return + } + b.start(KindRepo, s.Repo, s.Repo, "") + b.num(FactStars, s.Stars) + b.num(FactForks, s.Forks) + b.num(FactWatchers, s.Watchers) + b.num(FactCommits, s.Commits) + b.at(FactUpdated, s.PushedAt) + b.fact(FactURI, b.node.URI, "") +} + +func firstSetTime(ts ...*time.Time) *time.Time { + for _, t := range ts { + if t != nil && !t.IsZero() { + return t + } + } + return nil +} + +// --- the materialised graph --- + +// Graph is a set of nodes, edges, and facts held in memory. The streaming +// commands never build one; `github graph` for a single entity, `github rdf` for +// the buffered serialisations, and the tests all want the whole thing in hand. +type Graph struct { + Nodes []Node `json:"nodes"` + Edges []Edge `json:"edges"` + Facts []Fact `json:"facts,omitempty"` +} + +// Add folds a record into the graph, skipping a node already present so that a +// repeat visit does not duplicate it. +func (g *Graph) Add(rec any) { + node, edges, facts := Extract(rec) + if node.URI == "" { + return + } + g.AddNode(node) + g.Edges = append(g.Edges, edges...) + g.Facts = append(g.Facts, facts...) +} + +// AddNode adds one node if its URI is new. +func (g *Graph) AddNode(n Node) { + if n.URI == "" { + return + } + for _, have := range g.Nodes { + if have.URI == n.URI { + return + } + } + g.Nodes = append(g.Nodes, n) +} + +// Targets returns the object URIs reachable under an allowed predicate set, +// which is what the crawler walks. A bare-string object is never a target: +// there is no page for a language. +func (g *Graph) Targets(allow map[string]bool) []string { + var out []string + seen := map[string]bool{} + for _, e := range g.Edges { + if !strings.HasPrefix(e.Object, Scheme+"://") { + continue + } + if len(allow) > 0 && !allow[e.Predicate] { + continue + } + if !seen[e.Object] { + seen[e.Object] = true + out = append(out, e.Object) + } + } + return out +} + +// FilterTrust drops the edges below a floor, in place. +func FilterTrust(edges []Edge, min string) []Edge { + out := edges[:0] + for _, e := range edges { + if TrustAtLeast(e.Source, min) { + out = append(out, e) + } + } + return out +} + +// SortEdges gives an export a stable order, which is what makes a diff of two +// runs readable. +func SortEdges(edges []Edge) { + sort.SliceStable(edges, func(i, j int) bool { + a, b := edges[i], edges[j] + if a.Subject != b.Subject { + return a.Subject < b.Subject + } + if a.Predicate != b.Predicate { + return a.Predicate < b.Predicate + } + return a.Object < b.Object + }) +} diff --git a/gh/live_test.go b/gh/live_test.go new file mode 100644 index 0000000..29261e8 --- /dev/null +++ b/gh/live_test.go @@ -0,0 +1,1037 @@ +package gh + +import ( + "context" + "encoding/json" + "os" + "strings" + "testing" + "time" +) + +// live_test.go talks to github.com. It is skipped unless GITHUB_LIVE=1, so the +// normal test run stays offline and deterministic, and this is what you reach +// for when you want to know whether a surface still looks the way the spec says +// it does. +// +// These are the tests that catch the failure this tool cannot survive: GitHub +// moving something. Nothing offline can see that, because an offline test +// checks the parser against bytes that were already parsed once. So the +// assertions here are deliberately about shape rather than values. A star count +// changes hourly and pinning one turns a test into a clock. + +func liveClient(t *testing.T) *Client { + t.Helper() + if os.Getenv("GITHUB_LIVE") != "1" { + t.Skip("set GITHUB_LIVE=1 to run against github.com") + } + cfg := Defaults + cfg.CacheDir = t.TempDir() + return NewClient(cfg) +} + +func TestLiveRepo(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + r, err := c.Repo(ctx, "gohugoio/hugo", RepoOptions{}) + if err != nil { + t.Fatal(err) + } + if r.Owner != "gohugoio" || r.Name != "hugo" { + t.Fatalf("identity: %+v", r.Base) + } + // Each of these comes from a different block of the page, so between them + // they say the whole merge worked and not just one decoder. + if r.DefaultBranch == "" { + t.Error("default branch missing, codeViewLayoutRoute did not decode") + } + if r.Stars == nil { + t.Error("stars missing, sidebarAbout did not decode") + } + if r.HeadSHA == "" { + t.Error("head sha missing, codeViewRepoRoute did not decode") + } + if len(r.Tree) == 0 { + t.Error("tree empty") + } + if r.License == "" { + t.Error("licence missing, the octicon-law selector stopped matching") + } + if r.Language == "" { + t.Error("language missing, both the language bar and the search fallback came up empty") + } + if len(r.Extra) > 0 { + // Not a failure by itself, but it is how a new upstream field announces + // itself, so it gets printed. + t.Logf("unmodelled keys: %s", string(r.Extra)) + } + out, _ := json.MarshalIndent(r, "", " ") + t.Logf("%s", out) +} + +// TestLiveAccount covers both profile templates. sindresorhus has every vcard +// row a user profile can have except email, torvalds has achievements and no +// bio, and golang is the organization template, so between the three every +// branch of the decoder runs. +func TestLiveAccount(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) + defer cancel() + + t.Run("user", func(t *testing.T) { + a, err := c.Account(ctx, "sindresorhus") + if err != nil { + t.Fatal(err) + } + if a.Type != "User" { + t.Fatalf("type %q, the vcard template stopped matching", a.Type) + } + if a.Name == "" { + t.Error("name missing, p-name stopped matching") + } + if a.Bio == "" { + t.Error("bio missing, data-bio-text stopped matching") + } + if a.Website == "" { + t.Error("website missing, the itemprop=url row stopped matching") + } + if len(a.SocialLinks) == 0 { + t.Error("social links missing, the itemprop=social rows stopped matching") + } + if a.Followers == nil { + t.Error("followers missing, the tab=followers link stopped matching") + } + if a.RepoCount == nil { + t.Error("repo count missing, the tab counters stopped matching") + } + if a.DatabaseID == nil { + t.Error("database id missing, the avatar URL shape changed") + } + if len(a.PinnedRepos) == 0 { + t.Error("pinned repos missing") + } + if len(a.Organizations) == 0 { + t.Error("organizations missing, the hovercard-type hook changed") + } + logExtra(t, "user", a.Extra) + }) + + t.Run("user_achievements", func(t *testing.T) { + a, err := c.Account(ctx, "torvalds") + if err != nil { + t.Fatal(err) + } + if a.Company == "" { + t.Error("company missing, the worksFor row stopped matching") + } + if a.Location == "" { + t.Error("location missing, the homeLocation row stopped matching") + } + if len(a.Achievements) == 0 { + t.Error("achievements missing") + } + logExtra(t, "torvalds", a.Extra) + }) + + t.Run("org", func(t *testing.T) { + o, err := c.Org(ctx, "golang") + if err != nil { + t.Fatal(err) + } + if o.Type != "Organization" { + t.Fatalf("type %q, the orghead template stopped matching", o.Type) + } + if o.Name == "" { + t.Error("name missing, the orghead h1 stopped matching") + } + if o.Bio == "" { + t.Error("description missing, the muted sibling stopped matching") + } + if o.Website == "" { + t.Error("website missing, itemprop=url stopped matching") + } + if o.Followers == nil { + t.Error("followers missing, the /followers link stopped matching") + } + if len(o.Members) == 0 { + t.Error("members strip empty, member-avatar stopped matching") + } + // The organization counters are empty spans marked "Not available" + // without a session. If one ever arrives populated the tool should + // start reading it, so this asserts the absence rather than ignoring it. + if o.RepoCount != nil { + t.Errorf("repo count %d arrived on an org page, the counters are no longer session-gated", *o.RepoCount) + } + logExtra(t, "org", o.Extra) + }) + + t.Run("org_rejects_user", func(t *testing.T) { + if _, err := c.Org(ctx, "torvalds"); err == nil { + t.Error("Org accepted a user profile") + } + }) +} + +// TestLiveThread covers the three thread surfaces, which is really three +// unrelated decoders sharing a record. golang/go#1 is the oldest issue on the +// site and has a milestone, a label, and eighty-eight timeline events, so it +// exercises every branch of the Relay decoder at once. +func TestLiveThread(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + t.Run("issue", func(t *testing.T) { + iss, err := c.Issue(ctx, "golang/go", 1) + if err != nil { + t.Fatal(err) + } + if iss.Kind != KindIssue || iss.Number != 1 { + t.Fatalf("identity: %+v", iss.Base) + } + if iss.Title == "" { + t.Error("title missing, the issue node stopped decoding") + } + if iss.Body == "" { + t.Error("body missing") + } + if iss.State != "closed" { + t.Errorf("state %q, expected the lowercased enum", iss.State) + } + if iss.Author.Login == "" { + t.Error("author missing") + } + if len(iss.Labels) == 0 { + t.Error("labels missing, the label edges moved") + } + if iss.Milestone == nil { + t.Error("milestone missing") + } + if iss.NodeID == "" || iss.DatabaseID == nil { + t.Error("ids missing") + } + logExtra(t, "issue", iss.Extra) + }) + + t.Run("issue_timeline", func(t *testing.T) { + n, comments := 0, 0 + err := c.Timeline(ctx, "golang/go", 1, 20, func(it TimelineItem) error { + n++ + if it.Type == "" { + t.Error("timeline item with no type") + } + if it.Type == "issue_comment" { + comments++ + if it.Body == "" { + t.Error("comment with no body") + } + } + logExtra(t, "timeline "+it.Type, it.Extra) + return nil + }) + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatal("no timeline items, frontTimelineItems stopped decoding") + } + if comments == 0 { + t.Error("no comments in the timeline, the IssueComment shape changed") + } + }) + + t.Run("pull", func(t *testing.T) { + pr, err := c.PullRequest(ctx, "cli/cli", 9000) + if err != nil { + t.Fatal(err) + } + if pr.Kind != KindPR || pr.Number != 9000 { + t.Fatalf("identity: %+v", pr.Base) + } + if pr.State != "merged" || !pr.Merged { + t.Errorf("state %q merged %v, the route stopped reporting the merge", pr.State, pr.Merged) + } + if pr.BaseRef == "" || pr.HeadRef == "" { + t.Error("refs missing") + } + if pr.HeadOID == "" { + t.Error("head sha missing") + } + if pr.MergedBy == nil { + t.Error("merged-by missing") + } + // The body only ever comes from the hovercard. If it is empty the + // fragment stopped answering, and the pull request record loses the one + // field the route cannot supply. + if pr.Body == "" { + t.Error("body missing, the hovercard fragment stopped answering") + } + logExtra(t, "pull", pr.Extra) + }) + + t.Run("pull_commits", func(t *testing.T) { + n := 0 + err := c.PullCommits(ctx, "cli/cli", 9000, 10, func(cm Commit) error { + n++ + if cm.SHA == "" || cm.Subject == "" { + t.Errorf("thin commit: %+v", cm) + } + if len(cm.Authors) == 0 { + t.Errorf("%s has no authors", cm.SHA) + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatal("no commits, commitGroups stopped decoding") + } + }) + + t.Run("discussion", func(t *testing.T) { + d, err := c.Discussion(ctx, "google/docsy-example", 479) + if err != nil { + t.Fatal(err) + } + if d.Repo != "google/docsy-example" { + t.Errorf("repo %q, the sidebar data-url shape changed", d.Repo) + } + if d.Title == "" { + t.Error("title missing") + } + if d.BodyHTML == "" { + t.Error("body missing, the QAPage block stopped carrying text") + } + if !d.IsAnswered { + t.Error("answered flag missing, both the pill and acceptedAnswer stopped matching") + } + if d.Category == "" { + t.Error("category missing, the sidebar link stopped matching") + } + if d.Upvotes == nil { + t.Error("upvotes missing") + } + if d.NodeID == "" { + t.Error("node id missing, data-gid stopped matching") + } + logExtra(t, "discussion", d.Extra) + }) + + t.Run("org_discussion", func(t *testing.T) { + // An organization discussion is served from /orgs/{login}/ and the + // record still has to name the repository that owns it. + d, err := c.Discussion(ctx, "community", 1) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(d.Repo, "/") { + t.Errorf("repo %q is not owner/name", d.Repo) + } + if d.Author.Login == "" { + t.Error("author missing") + } + if len(d.Labels) == 0 { + t.Error("labels missing") + } + }) +} + +// TestLiveSearch walks every search type that works without a session. It is +// one test rather than nine because the value is in the comparison: when one +// type changes shape and the other eight do not, the failure says so. +func TestLiveSearch(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + + // Each case asserts the one field that proves the decoder ran rather than + // just that a result came back. + cases := []struct { + typ string + query string + run func(context.Context, string) (int, string, error) + }{ + {SearchRepos, "hugo", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchRepositories(ctx, q, 5, func(r Repo) error { + n++ + if r.Owner == "" || r.Name == "" { + bad = "identity empty" + } + if r.Stars == nil { + bad = "stars nil for " + r.ID + } + logExtra(t, "repo "+r.ID, r.Extra) + return nil + }) + return n, bad, err + }}, + {SearchIssues, "repo:golang/go generics", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchIssuesAndPulls(ctx, q, SearchIssues, 5, func(th Thread) error { + n++ + if th.Number == 0 || th.Repo == "" { + bad = "identity empty" + } + if th.Author.Login == "" { + bad = "author empty, author_name moved" + } + logExtra(t, "issue "+th.ID, th.Extra) + return nil + }) + return n, bad, err + }}, + {SearchPulls, "repo:golang/go generics", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchIssuesAndPulls(ctx, q, SearchPulls, 5, func(th Thread) error { + n++ + if th.Kind != KindPR { + bad = th.ID + " is not classified as a pull request" + } + return nil + }) + return n, bad, err + }}, + {SearchUsers, "torvalds", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchAccounts(ctx, q, 5, func(a Account) error { + n++ + if a.Login == "" { + bad = "login empty" + } + logExtra(t, "user "+a.ID, a.Extra) + return nil + }) + return n, bad, err + }}, + {SearchCommits, "repo:golang/go fix", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchCommitsBy(ctx, q, 5, func(cm Commit) error { + n++ + if cm.SHA == "" { + bad = "sha empty" + } + if cm.Subject == "" { + bad = "subject empty for " + cm.SHA + } + logExtra(t, "commit "+cm.SHA, cm.Extra) + return nil + }) + return n, bad, err + }}, + {SearchDiscussions, "hugo", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchDiscussionsBy(ctx, q, 5, func(d Discussion) error { + n++ + if d.Number == 0 { + bad = "number zero" + } + // hl_title arrives entity-escaped here and nowhere else, so a + // stray / means stripHL stopped unescaping. + if strings.Contains(d.Title, "&#") { + bad = "title still escaped: " + d.Title + } + logExtra(t, "discussion "+d.ID, d.Extra) + return nil + }) + return n, bad, err + }}, + {SearchTopics, "go", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchTopicsBy(ctx, q, 5, func(tp Topic) error { + n++ + if tp.Name == "" { + bad = "name empty" + } + logExtra(t, "topic "+tp.ID, tp.Extra) + return nil + }) + return n, bad, err + }}, + {SearchPackages, "hugo", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchPackagesBy(ctx, q, 5, func(pk Package) error { + n++ + if pk.Name == "" { + bad = "name empty" + } + if pk.Type == "" { + bad = "type empty for " + pk.Name + } + logExtra(t, "package "+pk.ID, pk.Extra) + return nil + }) + return n, bad, err + }}, + {SearchWikis, "hugo", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchWikisBy(ctx, q, 5, func(w WikiPage) error { + n++ + if w.Repo == "" { + bad = "repo empty" + } + logExtra(t, "wiki "+w.ID, w.Extra) + return nil + }) + return n, bad, err + }}, + {SearchMarket, "lint", func(ctx context.Context, q string) (int, string, error) { + n, bad := 0, "" + err := c.SearchMarketplace(ctx, q, 5, func(a Action) error { + n++ + if a.Slug == "" { + bad = "slug empty" + } + logExtra(t, "action "+a.ID, a.Extra) + return nil + }) + return n, bad, err + }}, + } + + for _, tc := range cases { + t.Run(tc.typ, func(t *testing.T) { + n, bad, err := tc.run(ctx, tc.query) + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatalf("no results for %q, the type or the envelope changed", tc.query) + } + if bad != "" { + t.Error(bad) + } + }) + } +} + +// TestLiveContents covers the tree route, the blob route, and raw bytes. The +// three are one test because the interesting question is whether the metadata +// and the bytes still agree with each other. +func TestLiveContents(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) + defer cancel() + + t.Run("tree", func(t *testing.T) { + var got []TreeEntry + err := c.Tree(ctx, "cli/cli", "pkg", TreeOptions{}, func(e TreeEntry) error { + got = append(got, e) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) < 5 { + t.Fatalf("only %d entries under pkg/", len(got)) + } + e := got[0] + if e.Name == "" || !strings.HasPrefix(e.Path, "pkg/") { + t.Errorf("path is not repository-relative: %+v", e) + } + if e.Type == "" { + t.Error("contentType missing") + } + // An empty ref means HEAD in the URL and a resolved SHA on the record. + if len(e.Ref) != 40 { + t.Errorf("ref %q is not a resolved commit", e.Ref) + } + logExtra(t, "tree", e.Extra) + }) + + t.Run("tree_recursive", func(t *testing.T) { + n := 0 + err := c.Tree(ctx, "cli/cli", "pkg/iostreams", TreeOptions{Recursive: true, Limit: 12}, func(TreeEntry) error { + n++ + return nil + }) + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatal("recursive walk emitted nothing") + } + }) + + t.Run("blob", func(t *testing.T) { + f, err := c.Blob(ctx, "cli/cli", "pkg/iostreams/iostreams.go", BlobOptions{}) + if err != nil { + t.Fatal(err) + } + if f.Language != "Go" { + t.Errorf("language %q, the blob layout route stopped decoding", f.Language) + } + if f.Lines == nil || *f.Lines < 100 { + t.Errorf("line count %v for a 13 KB file", f.Lines) + } + // GitHub's symbol analyser answers null on every file of every + // repository tried now, on both surfaces, signed out. So "unavailable" + // is the expected answer here rather than a failure, and if the block + // ever comes back this asserts it is shaped right. not_analyzed for a + // Go file would be a real change and does fail. + switch f.SymbolsStatus { + case "ok": + if len(f.Symbols) == 0 { + t.Fatal("status ok with no symbols") + } + s := f.Symbols[0] + if s.Name == "" || s.Kind == "" || s.ExtentEnd == 0 { + t.Errorf("symbol is half empty: %+v", s) + } + case "unavailable", "timed_out": + t.Logf("symbols %s, the analyser did not answer", f.SymbolsStatus) + default: + t.Errorf("symbols status %q for a Go file", f.SymbolsStatus) + } + logExtra(t, "blob", f.Extra) + }) + + t.Run("blob_markdown", func(t *testing.T) { + f, err := c.Blob(ctx, "cli/cli", "README.md", BlobOptions{Content: true}) + if err != nil { + t.Fatal(err) + } + if len(f.TOC) == 0 { + t.Error("no table of contents on a rendered markdown file") + } + if f.RichText == "" { + t.Error("no rendered html") + } + if !strings.Contains(f.Content, "gh") { + t.Errorf("content does not look like the readme: %.60q", f.Content) + } + if f.Via["content"] != "raw" { + t.Errorf("content provenance is %q", f.Via["content"]) + } + }) + + t.Run("raw", func(t *testing.T) { + b, err := c.Raw(ctx, "cli/cli", "trunk", "go.mod") + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(string(b), "module ") { + t.Errorf("go.mod does not start with a module line: %.40q", b) + } + }) +} + +// TestLiveHistory covers the six surfaces the history layer reads. They are one +// test because they are one question asked six ways, and when GitHub changes a +// payload it is usually the disagreement between two of them that shows it. +func TestLiveHistory(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 180*time.Second) + defer cancel() + + t.Run("commits", func(t *testing.T) { + var got []Commit + err := c.Commits(ctx, "cli/cli", CommitOptions{Limit: 40}, func(cm Commit) error { + got = append(got, cm) + return nil + }) + if err != nil { + t.Fatal(err) + } + // A page is 35, so 40 proves the cursor came back and was accepted. + if len(got) != 40 { + t.Fatalf("walked %d commits, want the limit of 40", len(got)) + } + cm := got[0] + if len(cm.SHA) != 40 { + t.Errorf("sha %q", cm.SHA) + } + if cm.Subject == "" { + t.Error("subject empty, shortMessage is often null and the markdown fallback did not run") + } + if cm.AuthoredAt == nil { + t.Error("no authored date") + } + if len(cm.Authors) == 0 { + t.Error("no authors") + } + if cm.DateGroup == "" { + t.Error("no day heading, commitGroups lost its title") + } + if cm.ID != "cli/cli@"+cm.SHA { + t.Errorf("id %q", cm.ID) + } + logExtra(t, "commit", cm.Extra) + }) + + t.Run("commits_filtered", func(t *testing.T) { + // The filters go to GitHub, so a path that exists and an author who + // touched it should come back non-empty and every record should be on + // that path. + n := 0 + err := c.Commits(ctx, "cli/cli", CommitOptions{Path: "go.mod", Limit: 5}, func(Commit) error { + n++ + return nil + }) + if err != nil { + t.Fatal(err) + } + if n == 0 { + t.Fatal("no commits touched go.mod, the path filter is being dropped") + } + }) + + t.Run("commit", func(t *testing.T) { + cm, err := c.CommitInfo(ctx, "cli/cli", "trunk", CommitInfoOptions{Files: true}) + if err != nil { + t.Fatal(err) + } + if len(cm.SHA) != 40 { + t.Errorf("a branch name did not resolve to a sha: %q", cm.SHA) + } + if cm.Additions == nil || cm.Deletions == nil { + t.Error("headerInfo did not decode") + } + if len(cm.Files) == 0 { + t.Fatal("no files, diffEntryData did not decode") + } + f := cm.Files[0] + if f.Path == "" || f.Status == "" { + t.Errorf("half a file change: %+v", f) + } + if len(cm.Parents) == 0 { + t.Error("no parents on a commit that is not the root") + } + logExtra(t, "commit info", cm.Extra) + }) + + t.Run("verify", func(t *testing.T) { + // GitHub signs every commit it makes itself, so a merge on cli/cli is + // the reliable case. This is the only surface that says so. + var head []*Commit + err := c.Commits(ctx, "cli/cli", CommitOptions{Limit: 5}, func(cm Commit) error { + head = append(head, &cm) + return nil + }) + if err != nil { + t.Fatal(err) + } + if err := c.VerifyCommits(ctx, "cli/cli", head); err != nil { + t.Fatal(err) + } + signed := 0 + for _, cm := range head { + if cm.Verification != "" { + signed++ + } + } + if signed == 0 { + t.Error("commit search reported verification on none of five commits") + } + }) + + t.Run("branches", func(t *testing.T) { + var got []GitRef + err := c.Branches(ctx, "cli/cli", RefOptions{Limit: 10}, func(r GitRef) error { + got = append(got, r) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) == 0 { + t.Fatal("no branches") + } + // The page carries what the protocol cannot: who last pushed and when. + hasAuthor := false + for _, r := range got { + if r.Type != "branch" { + t.Errorf("type %q on a branch", r.Type) + } + if r.Author != nil && r.AuthoredAt != nil { + hasAuthor = true + } + } + if !hasAuthor { + t.Error("no branch carried an author, which is the only reason to read the page") + } + if got[0].Repo != "cli/cli" { + t.Errorf("repo %q", got[0].Repo) + } + }) + + t.Run("branches_complete", func(t *testing.T) { + // The advertisement has no cap, so it should beat the page's list and + // every entry should carry a SHA. + var got []GitRef + err := c.Branches(ctx, "cli/cli", RefOptions{Complete: true}, func(r GitRef) error { + got = append(got, r) + return nil + }) + if err != nil { + t.Fatal(err) + } + if len(got) < 5 { + t.Fatalf("the advertisement gave %d branches", len(got)) + } + def := 0 + for _, r := range got { + if len(r.SHA) != 40 { + t.Errorf("%s has sha %q", r.Name, r.SHA) + } + if r.IsDefault { + def++ + } + } + if def != 1 { + t.Errorf("%d branches claim to be the default", def) + } + }) + + t.Run("tags", func(t *testing.T) { + var got []GitRef + err := c.Tags(ctx, "cli/cli", RefOptions{Limit: 50}, func(r GitRef) error { + got = append(got, r) + return nil + }) + if err != nil { + t.Fatal(err) + } + // The point of reading the protocol by default is that neither the feed + // nor the page gives more than ten. + if len(got) != 50 { + t.Fatalf("%d tags, want 50 from a repository with hundreds", len(got)) + } + annotated := 0 + for _, r := range got { + if r.Type != "tag" { + t.Errorf("type %q on a tag", r.Type) + } + if r.PeeledSHA != "" { + annotated++ + } + } + t.Logf("%d of %d tags are annotated", annotated, len(got)) + }) + + t.Run("refs", func(t *testing.T) { + // No limit: it is one response either way, and a limit here truncates + // inside the branch list and never reaches the tags. + heads, tags := 0, 0 + err := c.Refs(ctx, "cli/cli", RefOptions{}, func(r GitRef) error { + switch r.Type { + case "branch": + heads++ + case "tag": + tags++ + } + return nil + }) + if err != nil { + t.Fatal(err) + } + if heads == 0 || tags == 0 { + t.Errorf("refs gave %d branches and %d tags, it should give both", heads, tags) + } + }) + + t.Run("default_branch", func(t *testing.T) { + // symref=HEAD comes free with the advertisement and should agree with + // the repository page, which reads it from a completely different place. + name, err := c.DefaultBranch(ctx, "cli/cli") + if err != nil { + t.Fatal(err) + } + if name != "trunk" { + t.Errorf("default branch %q, want trunk", name) + } + }) + + t.Run("releases", func(t *testing.T) { + var got []Release + err := c.Releases(ctx, "cli/cli", ReleaseOptions{Limit: 15, Body: true}, func(r Release) error { + got = append(got, r) + return nil + }) + if err != nil { + t.Fatal(err) + } + // Ten a page, so fifteen proves rel="next" was found and followed. + if len(got) != 15 { + t.Fatalf("%d releases, want the limit of 15", len(got)) + } + latest := 0 + for _, r := range got { + if r.Tag == "" { + t.Errorf("release with no tag: %+v", r.Base) + } + if r.IsLatest { + latest++ + } + } + if latest != 1 { + t.Errorf("%d releases are labelled Latest", latest) + } + if got[0].PublishedAt == nil { + t.Error("no publish date on the newest release") + } + if got[0].Body == "" { + t.Error("no release notes with Body set") + } + logExtra(t, "release", got[0].Extra) + }) + + t.Run("release_assets", func(t *testing.T) { + rel, err := c.Release(ctx, "cli/cli", "v2.63.2", ReleaseOptions{Assets: true, Body: true}) + if err != nil { + t.Fatal(err) + } + if rel.Tag != "v2.63.2" { + t.Errorf("tag %q", rel.Tag) + } + if rel.Title == "" { + t.Error("no title") + } + if rel.Author == nil { + t.Error("no publisher, the byline link did not match") + } + if rel.PublishedAt == nil { + t.Error("no publish date") + } + if rel.Body == "" { + t.Error("no release notes") + } + // The commit the tag points at is the one thing the per-tag page has + // that a list entry does not. + if len(rel.CommitSHA) != 40 { + t.Errorf("commit sha %q", rel.CommitSHA) + } + if len(rel.Assets) < 5 { + t.Fatalf("%d assets, the expanded_assets fragment did not decode", len(rel.Assets)) + } + a := rel.Assets[0] + if a.Name == "" || a.URL == "" { + t.Errorf("half an asset: %+v", a) + } + if a.SizeDisplay == "" { + t.Error("no size on an asset") + } + if a.Label == "" { + t.Error("no label, the row's first truncated span stopped matching") + } + if a.UpdatedAt == nil { + t.Error("no upload time on an asset") + } + // Download counts are gone for a logged-out client. If they ever come + // back this logs it rather than failing. + if a.DownloadCount != nil { + t.Logf("download counts are being served again: %d", *a.DownloadCount) + } + }) + + t.Run("release_latest", func(t *testing.T) { + // "latest" is a redirect, so this proves the decoder reads whatever it + // lands on rather than the tag it was handed. It is also the release + // that carries digests: GitHub started attaching them recently and old + // releases do not have them. + rel, err := c.Release(ctx, "cli/cli", "latest", ReleaseOptions{Assets: true}) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(rel.Tag, "v") { + t.Errorf("tag %q, the redirect target did not decode", rel.Tag) + } + if !rel.IsLatest { + t.Error("the latest release is not labelled Latest") + } + if len(rel.Assets) == 0 { + t.Fatal("no assets on the latest release") + } + digests := 0 + for _, a := range rel.Assets { + if strings.HasPrefix(a.Digest, "sha256:") { + digests++ + } + } + if digests == 0 { + t.Error("no sha256 digests, which is the one thing that replaced download counts") + } + }) + + t.Run("compare", func(t *testing.T) { + cmp, err := c.CompareRefs(ctx, "cli/cli", "v2.63.1", "v2.63.2", CompareOptions{Files: true}) + if err != nil { + t.Fatal(err) + } + if cmp.CommitCount == 0 { + t.Fatal("no commits in the range, the mailbox did not split") + } + if cmp.FileCount == 0 { + t.Fatal("no files in the range") + } + if cmp.Additions == 0 && cmp.Deletions == 0 { + t.Error("a release range with no line changes") + } + first := cmp.Commits[0] + if len(first.SHA) != 40 || first.Subject == "" || first.AuthoredAt == nil { + t.Errorf("half a commit from the patch: %+v", first.Base) + } + // The mailbox has names and emails, not logins, except where the email + // is a noreply address. At least one of these should be. + logins := 0 + for _, cm := range cmp.Commits { + for _, a := range cm.Authors { + if a.Login != "" { + logins++ + } + } + } + t.Logf("%d of %d commits gave a login through a noreply address", logins, cmp.CommitCount) + if cmp.ID != "cli/cli@v2.63.1...v2.63.2" { + t.Errorf("id %q", cmp.ID) + } + }) + + t.Run("diff", func(t *testing.T) { + // The diff is the patch without the mail headers, so it should be + // smaller and it should not carry a From line. + url := BaseURL + "/cli/cli/compare/v2.63.1...v2.63.2" + diff, err := c.Diff(ctx, url) + if err != nil { + t.Fatal(err) + } + if !strings.HasPrefix(diff, "diff --git ") { + t.Errorf("a diff should start with a diff header: %.60q", diff) + } + patch, err := c.Patch(ctx, url) + if err != nil { + t.Fatal(err) + } + if len(patch) <= len(diff) { + t.Errorf("patch %d bytes is not bigger than diff %d bytes", len(patch), len(diff)) + } + }) +} + +// TestLiveCodeSearchStaysRefused guards the one search type that answers 200 +// with nothing. If GitHub ever opens it up this test fails, which is the +// notification to go implement it. +func TestLiveCodeSearchStaysRefused(t *testing.T) { + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + var env searchEnvelope + if _, err := c.GetJSON(ctx, searchURL("func main", SearchCode, 1), SurfaceSearch, &env); err != nil { + t.Fatal(err) + } + if len(env.Payload.Results) > 0 { + t.Errorf("code search returned %d results without a session, it can be implemented now", + len(env.Payload.Results)) + } +} + +func logExtra(t *testing.T, what string, extra json.RawMessage) { + t.Helper() + if len(extra) > 0 { + t.Logf("%s unmodelled: %s", what, string(extra)) + } +} diff --git a/gh/merge.go b/gh/merge.go new file mode 100644 index 0000000..fd2861e --- /dev/null +++ b/gh/merge.go @@ -0,0 +1,249 @@ +package gh + +import ( + "encoding/json" + "fmt" + "reflect" + "sort" + "strings" +) + +// merge.go combines the several surfaces that answer for one record. +// +// The order is explicit and lives at the call site, one ordered list per record +// type, because a merge that cannot say which surface won is a merge that +// cannot be debugged. What lives here is only the per-field mechanics that all +// of them share: +// +// - a source that did not return a field never clears it, so merging is +// driven by a populated check and never by struct assignment, +// - Sources accumulates in merge order, so provenance is the merge history, +// - when two surfaces give different non-empty values for one field, the +// later one wins and the disagreement is recorded in Extra["_conflict"]. +// +// That last rule is the interesting one. The scenario suite asserts _conflict +// is empty, so two surfaces disagreeing becomes a named test failure and gets +// looked at, instead of being quietly averaged into a number nobody can trace. + +// mergeInto copies every populated field of src into dst, returning the fields +// where the two disagreed. dst and src must be pointers to the same struct +// type. +// +// Booleans are a known soft spot: a false bool is indistinguishable from an +// unset one, so a false never overwrites a true. Every bool in the model is +// phrased so that false is the safe default (IsFork, IsArchived, HasWiki), which +// makes that the right behaviour rather than a compromise. +func mergeInto(dst, src any) map[string][]string { + dv := reflect.ValueOf(dst) + sv := reflect.ValueOf(src) + if dv.Kind() != reflect.Pointer || sv.Kind() != reflect.Pointer { + return nil + } + if dv.Type() != sv.Type() { + return nil + } + conflicts := map[string][]string{} + mergeStruct(dv.Elem(), sv.Elem(), "", conflicts) + if len(conflicts) == 0 { + return nil + } + return conflicts +} + +func mergeStruct(dst, src reflect.Value, prefix string, conflicts map[string][]string) { + t := dst.Type() + for i := range t.NumField() { + f := t.Field(i) + if !f.IsExported() { + continue + } + d, s := dst.Field(i), src.Field(i) + + // Base is the envelope, not data. Its fields have their own merge rules + // and are handled by mergeBase. + if f.Anonymous && f.Type == reflect.TypeOf(Base{}) { + mergeBase(d.Addr().Interface().(*Base), s.Addr().Interface().(*Base)) + continue + } + // Any other embedded struct is part of the record: Thread inside Issue, + // Repo inside Trending, Account inside Org. + if f.Anonymous && f.Type.Kind() == reflect.Struct { + mergeStruct(d, s, prefix, conflicts) + continue + } + + name := jsonName(f) + if name == "-" { + continue + } + if prefix != "" { + name = prefix + "." + name + } + if !populated(s) { + continue + } + if populated(d) && !reflect.DeepEqual(d.Interface(), s.Interface()) { + conflicts[name] = []string{display(d), display(s)} + } + d.Set(s) + } +} + +// mergeBase keeps the identity of the first source and unions the provenance. +// A later surface never renames a record: if the id changed, the merge was +// between two different things and the caller made a mistake. +func mergeBase(dst, src *Base) { + if dst.Kind == "" { + dst.Kind = src.Kind + } + if dst.ID == "" { + dst.ID = src.ID + } + if dst.URI == "" { + dst.URI = src.URI + } + if dst.URL == "" { + dst.URL = src.URL + } + dst.addSource(src.Sources...) + for field, tier := range src.Via { + recordVia(dst, field, tier) + } + dst.Extra = mergeExtra(dst.Extra, src.Extra) +} + +// mergeExtra unions two unmodelled-key sets. Both are keys nobody claimed, so +// there is nothing smarter to do than keep them all; a key present in both +// keeps the later value, matching the field rule above. +func mergeExtra(dst, src json.RawMessage) json.RawMessage { + if len(src) == 0 { + return dst + } + if len(dst) == 0 { + return src + } + var a, b map[string]json.RawMessage + if json.Unmarshal(dst, &a) != nil || json.Unmarshal(src, &b) != nil { + return dst + } + for k, v := range b { + a[k] = v + } + out, err := json.Marshal(a) + if err != nil { + return dst + } + return out +} + +// recordConflicts writes the disagreements into Extra["_conflict"]. It is a +// reserved key and the suite asserts it is absent, which is the whole point: +// this is a tripwire, not a feature. +func recordConflicts(b *Base, conflicts map[string][]string) { + if len(conflicts) == 0 { + return + } + m := map[string]json.RawMessage{} + if len(b.Extra) > 0 { + _ = json.Unmarshal(b.Extra, &m) + } + existing := map[string][]string{} + if raw, ok := m["_conflict"]; ok { + _ = json.Unmarshal(raw, &existing) + } + for k, v := range conflicts { + existing[k] = v + } + raw, err := json.Marshal(existing) + if err != nil { + return + } + m["_conflict"] = raw + out, err := json.Marshal(m) + if err != nil { + return + } + b.Extra = out +} + +// recordVia notes which extraction tier produced a field. It is what tells you +// that a field which used to arrive from a JSON payload is now arriving from a +// class selector, which is the early warning that something moved. +func recordVia(b *Base, field, tier string) { + if b.Via == nil { + b.Via = map[string]string{} + } + b.Via[field] = tier +} + +// populated is the "did this surface actually say something" test. Absent is +// not zero anywhere in this tool, and this function is where that rule is +// enforced for the merge. +func populated(v reflect.Value) bool { + switch v.Kind() { + case reflect.Pointer, reflect.Interface, reflect.Map, reflect.Slice: + return !v.IsNil() && (v.Kind() != reflect.Slice && v.Kind() != reflect.Map || v.Len() > 0) + case reflect.String: + return v.String() != "" + case reflect.Bool: + return v.Bool() + case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: + return v.Int() != 0 + case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64: + return v.Uint() != 0 + case reflect.Float32, reflect.Float64: + return v.Float() != 0 + case reflect.Struct: + return !v.IsZero() + default: + return !v.IsZero() + } +} + +func jsonName(f reflect.StructField) string { + tag := f.Tag.Get("json") + name, _, _ := strings.Cut(tag, ",") + if name == "" { + return f.Name + } + return name +} + +// display renders a value for a conflict entry. It stays short: the point is to +// let a human see which two surfaces disagreed, not to reproduce the payload. +func display(v reflect.Value) string { + if v.Kind() == reflect.Pointer && !v.IsNil() { + v = v.Elem() + } + switch v.Kind() { + case reflect.String: + return truncate(v.String(), 120) + case reflect.Slice, reflect.Map, reflect.Struct: + raw, err := json.Marshal(v.Interface()) + if err != nil { + return fmt.Sprint(v.Interface()) + } + return truncate(string(raw), 200) + default: + return fmt.Sprint(v.Interface()) + } +} + +func truncate(s string, n int) string { + if len(s) <= n { + return s + } + return s[:n] + "..." +} + +// sortedKeys keeps merge and conflict output deterministic. Map order is random +// in Go, and a record whose field order changes between runs makes every diff +// of two outputs useless. +func sortedKeys[V any](m map[string]V) []string { + out := make([]string, 0, len(m)) + for k := range m { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/gh/messages_test.go b/gh/messages_test.go new file mode 100644 index 0000000..242b804 --- /dev/null +++ b/gh/messages_test.go @@ -0,0 +1,117 @@ +package gh + +import ( + "go/ast" + "go/parser" + "go/token" + "io/fs" + "path/filepath" + "strconv" + "strings" + "testing" +) + +// messages_test.go guards a defect that is invisible in the source and obvious +// on screen. The error renderer capitalises the first token of a message, so a +// message that begins with the thing it is about comes back mangled: +// +// errs.NotFound("%s has no README", id) -> Gohugoio/Hugo has no README. +// errs.Usage("%q is a %s, not a %s", ...) -> "Golang/Go" is a repo, not a user. +// +// A reader who sees that reasonably concludes the tool corrupted their input. +// Every message here therefore leads with a plain lowercase word, and this test +// says so, because the mistake is easy to make and impossible to see in review. + +// errorFuncs are the constructors whose first string argument is shown to a +// person. errs.New takes a kind first, so its message is the second argument. +var errorFuncs = map[string]int{ + "Usage": 0, + "NotFound": 0, + "Unsupported": 0, + "NeedAuth": 0, + "RateLimited": 0, + "NoResults": 0, + "New": 1, +} + +func TestErrorMessagesLeadWithAWord(t *testing.T) { + root := moduleRoot(t) + fset := token.NewFileSet() + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if name := d.Name(); name == ".git" || name == "bin" || name == "dist" || name == "docs" { + return fs.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") || strings.HasSuffix(path, "_test.go") { + return nil + } + rel, _ := filepath.Rel(root, path) + rel = filepath.ToSlash(rel) + + file, perr := parser.ParseFile(fset, path, nil, 0) + if perr != nil { + t.Errorf("%s: %v", rel, perr) + return nil + } + ast.Inspect(file, func(n ast.Node) bool { + call, ok := n.(*ast.CallExpr) + if !ok { + return true + } + sel, ok := call.Fun.(*ast.SelectorExpr) + if !ok { + return true + } + pkg, ok := sel.X.(*ast.Ident) + if !ok || pkg.Name != "errs" { + return true + } + at, ok := errorFuncs[sel.Sel.Name] + if !ok || len(call.Args) <= at { + return true + } + lit, ok := call.Args[at].(*ast.BasicLit) + if !ok || lit.Kind != token.STRING { + return true + } + msg, uerr := strconv.Unquote(lit.Value) + if uerr != nil || msg == "" { + return true + } + if bad := leadsBadly(msg); bad != "" { + t.Errorf("%s:%d: errs.%s starts with %s: %q\nThe renderer capitalises the first token, so this reaches the reader looking like their input was mangled. Lead with a plain word instead.", + rel, fset.Position(lit.Pos()).Line, sel.Sel.Name, bad, msg) + } + return true + }) + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// leadsBadly names what is wrong with a message's first character, or returns +// empty when there is nothing wrong. Three cases matter: a format verb, because +// whatever fills it gets capitalised; a quote, because the capital lands inside +// it; and an upper-case letter, because a message that already starts capital is +// usually a proper noun that the renderer will then get wrong (GitHub, HTTP). +func leadsBadly(msg string) string { + switch { + case strings.HasPrefix(msg, "%"): + return "a format verb" + case strings.HasPrefix(msg, `"`), strings.HasPrefix(msg, "'"), strings.HasPrefix(msg, "`"): + return "a quote" + case msg[0] >= 'A' && msg[0] <= 'Z': + return "a capital letter" + case strings.HasPrefix(msg, "http://"), strings.HasPrefix(msg, "https://"): + return "a URL" + } + return "" +} diff --git a/gh/noauth_test.go b/gh/noauth_test.go new file mode 100644 index 0000000..65e6ba2 --- /dev/null +++ b/gh/noauth_test.go @@ -0,0 +1,119 @@ +package gh + +import ( + "bytes" + "go/parser" + "go/printer" + "go/token" + "io/fs" + "os" + "path/filepath" + "strings" + "testing" +) + +// noauth_test.go is the one promise this tool makes that a reader cannot check +// by using it: that nothing here ever authenticates. A person can see that a +// command works without logging in, but not that no code path would send a +// credential if one happened to be around. +// +// So the promise is asserted instead. The test parses every source file with +// comments dropped and fails on the words that would mean the promise was +// broken. Comments are dropped because this file, the doctor, and the spec all +// talk about tokens at length, and a grep that could not tell prose from code +// would have to be switched off the first time someone documented the rule. + +var forbidden = []string{ + "Authorization", + "GITHUB_TOKEN", + "GH_TOKEN", + "api.github.com", +} + +// allowed lists the files that name a forbidden word in code for a reason. Each +// one is here because saying the word is the point: doctor reads the +// environment to warn that a token is ignored, and this test names all four. +var allowed = map[string]string{ + "gh/doctor.go": "reads the token variables by name to warn that they are ignored", + "gh/noauth_test.go": "is this test", +} + +func TestNoAuth(t *testing.T) { + root := moduleRoot(t) + fset := token.NewFileSet() + + err := filepath.WalkDir(root, func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + if d.IsDir() { + if name := d.Name(); name == ".git" || name == "bin" || name == "dist" || name == "docs" { + return fs.SkipDir + } + return nil + } + if !strings.HasSuffix(path, ".go") { + return nil + } + rel, _ := filepath.Rel(root, path) + rel = filepath.ToSlash(rel) + if _, ok := allowed[rel]; ok { + return nil + } + + // Parsing without ParseComments and printing the result is how the + // comments come out: the printer only writes what the AST holds. + file, perr := parser.ParseFile(fset, path, nil, 0) + if perr != nil { + t.Errorf("%s: %v", rel, perr) + return nil + } + var code bytes.Buffer + if perr := (&printer.Config{Mode: printer.RawFormat}).Fprint(&code, fset, file); perr != nil { + t.Errorf("%s: %v", rel, perr) + return nil + } + for _, word := range forbidden { + if bytes.Contains(code.Bytes(), []byte(word)) { + t.Errorf("%s names %q in code. This tool reads public pages and never authenticates; if this is deliberate, the file needs a line in the allowed map saying why", rel, word) + } + } + return nil + }) + if err != nil { + t.Fatal(err) + } +} + +// TestNoAuthCoversItself keeps the allow list honest. A file that stops needing +// its exemption should lose it, otherwise the list grows into a place to hide +// things. +func TestNoAuthCoversItself(t *testing.T) { + root := moduleRoot(t) + for rel, why := range allowed { + if why == "" { + t.Errorf("%s is exempt with no reason given", rel) + } + if _, err := os.Stat(filepath.Join(root, filepath.FromSlash(rel))); err != nil { + t.Errorf("%s is exempt and does not exist: %v", rel, err) + } + } +} + +func moduleRoot(t *testing.T) string { + t.Helper() + dir, err := os.Getwd() + if err != nil { + t.Fatal(err) + } + for { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir + } + parent := filepath.Dir(dir) + if parent == dir { + t.Fatal("no go.mod above the test directory") + } + dir = parent + } +} diff --git a/gh/ops.go b/gh/ops.go new file mode 100644 index 0000000..8b62c3e --- /dev/null +++ b/gh/ops.go @@ -0,0 +1,1638 @@ +package gh + +import ( + "context" + "strconv" + "strings" + + "github.com/tamnd/any-cli/kit" + "github.com/tamnd/any-cli/kit/errs" +) + +// ops.go is the table of contents for the whole tool. Every verb a person can +// type is registered here and nowhere else, so the answer to "what can github +// do" is one file, and each registration is simultaneously a CLI subcommand, an +// HTTP route under `github serve`, and an MCP tool under `github mcp`. +// +// The handlers are thin on purpose. Anything with a decision in it belongs in +// the library next to the data it decides about; what is left here is reference +// resolution and one call. + +func registerOps(app *kit.App) { + registerReadOps(app) + registerSearchOps(app) + registerContentOps(app) + registerHistoryOps(app) + registerPeopleOps(app) + registerDiscoverOps(app) + registerGraphOps(app) + registerMetaOps(app) +} + +// --- reference resolution --- + +// ResolveRef resolves any accepted reference to the id for the kind a command +// names. +// +// Classify has two guesses in it: one bare word is a user, and one bare +// owner/name is a repository. Those are guesses because a pure function cannot +// tell a user from an organization or a repository from anything else without +// asking, so a guess yields to the command that names its own kind while an +// explicit URL or URI does not. That is what makes `github org golang` work and +// `github org https://github.com/golang/go` fail with a message that says why. +func ResolveRef(want, input string) (string, error) { + kind, id, err := Classify(input) + if err != nil { + return "", err + } + if kind == want { + return id, nil + } + if guessed(input, kind) { + return id, nil + } + // A sub-resource names its repository, so a file URL or an issue URL is a + // fine way to refer to the repository it lives in. + if want == KindRepo { + if repo, ok := RepoOf(kind, id); ok { + return repo, nil + } + } + return "", errs.Usage("wrong kind: %q is %s, not %s", input, aKind(kind), aKind(want)) +} + +// aKind puts the right article in front of a kind name. Five of the twenty-four +// start with a vowel, and "not a org" in an error message reads like the tool +// was written in a hurry. +func aKind(kind string) string { + if kind == "" { + return "nothing" + } + switch kind[0] { + case 'a', 'e', 'i', 'o', 'u': + return "an " + kind + } + return "a " + kind +} + +// guessed reports whether Classify was guessing rather than reading. Anything +// with a scheme in it was read: github.com states the kind in the path and a +// github:// URI states it outright. +func guessed(input, kind string) bool { + if strings.Contains(input, "://") { + return false + } + return kind == KindUser || kind == KindRepo || kind == KindIssue +} + +// ResolveRepo resolves any reference to the repository it belongs to. Commands +// that work on a repository use it so that a pasted file URL, issue URL, or +// release URL all name the repository they are part of, which is what makes +// `github commits ` work. +func ResolveRepo(input string) (string, error) { + return ResolveRef(KindRepo, input) +} + +// resolveThread resolves the two ways to name an issue, pull request, or +// discussion: a repository and a number, or one URL that already carries both. +// The second form is the one people have in their clipboard. +func resolveThread(want, ref string, num int) (repo string, number int, err error) { + kind, id, err := Classify(ref) + if err != nil { + return "", 0, err + } + if r, n, ok := SplitThreadID(id); ok && num == 0 { + // An issue URL and a pull URL are different paths, so a mismatch here + // is a real error rather than a guess to be forgiven. A bare + // owner/name#123 is a guess: nothing in it says which of the two it is. + if kind != want && !guessed(ref, kind) { + return "", 0, errs.Usage("wrong kind: %q is %s, not %s", ref, aKind(kind), aKind(want)) + } + number, _ = strconv.Atoi(n) + return r, number, nil + } + repo, err = ResolveRepo(ref) + if err != nil { + return "", 0, err + } + if num <= 0 { + return "", 0, errs.Usage("no number given; %s needs one, either as a second argument or in the URL", want) + } + return repo, num, nil +} + +// resolveRev resolves the two ways to name a thing that hangs off a git +// revision: a repository and a rev, or one URL carrying both. rev may be empty +// when the caller has a default for it. +func resolveRev(want, ref, rev string) (repo, out string, err error) { + kind, id, err := Classify(ref) + if err != nil { + return "", "", err + } + if r, v, ok := cutRev(id); ok && rev == "" { + if kind != want && !guessed(ref, kind) { + return "", "", errs.Usage("wrong kind: %q is %s, not %s", ref, aKind(kind), aKind(want)) + } + return r, v, nil + } + repo, err = ResolveRepo(ref) + if err != nil { + return "", "", err + } + return repo, rev, nil +} + +// resolvePath resolves a repository and a path inside it, from either a blob or +// tree URL or a repository plus a path argument. +func resolvePath(ref, path, rev string) (repo, outRef, outPath string, err error) { + kind, id, err := Classify(ref) + if err != nil { + return "", "", "", err + } + if kind == KindFile || kind == KindTree { + r, v, p, ok := SplitPathID(id) + if ok { + if rev != "" { + v = rev + } + if path != "" { + p = path + } + return r, v, p, nil + } + } + repo, err = ResolveRepo(ref) + if err != nil { + return "", "", "", err + } + return repo, rev, strings.TrimPrefix(path, "/"), nil +} + +// emitEach sends a slice one record at a time. The client methods that return a +// slice do so because one document holds the whole answer; the command surface +// still wants records, and a single document is never large enough to be worth +// streaming through a channel. +func emitEach[T any](items []T, emit func(*T) error) error { + for i := range items { + if err := emit(&items[i]); err != nil { + return err + } + } + return nil +} + +// byValue adapts a client method that emits values to a handler that emits +// pointers. The client methods emit values because a record built inside a +// pager has no reason to escape to the heap; kit wants a pointer because the +// record may be rendered, stored, and serialised after the call returns. +func byValue[T any](emit func(*T) error) func(T) error { + return func(v T) error { return emit(&v) } +} + +// --- reading one thing --- + +type repoIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, a github.com URL, or a github:// URI"` + Readme bool `kit:"flag" help:"include the rendered README, which is most of the bytes"` + Deep bool `kit:"flag" help:"also fetch the language histogram and the dependent count"` +} + +type nameIn struct { + C *Client `kit:"inject"` + Name string `kit:"arg" help:"a login, a profile URL, or a github:// URI"` +} + +type threadIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a full thread URL that already carries the number"` + Num int `kit:"arg" help:"the number, when the reference does not carry one"` +} + +type bareRefIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"any github reference"` +} + +func registerReadOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "repo", Group: "read", Single: true, URIType: KindRepo, Resolver: true, + Summary: "Read one repository with every field the page states", + Long: "The repository page carries more than the API does: the sidebar's about\n" + + "block, the topic list, the license name as rendered, the release the page\n" + + "is pointing at, and the counts for stars, forks, watchers, and open issues.", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, URL, or github:// URI"}}, + }, getRepo) + + kit.Handle(app, kit.OpMeta{ + Name: "user", Group: "read", Single: true, URIType: KindUser, Resolver: true, + Summary: "Read one user profile", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, getUser) + + kit.Handle(app, kit.OpMeta{ + Name: "org", Group: "read", Single: true, URIType: KindOrg, Resolver: true, + Summary: "Read one organization", + Long: "An organization renders a different template from a user, so this returns\n" + + "the fields only that template has. For a name whose kind nobody knows yet,\n" + + "use `github user`, which reads whichever template answers and reports the\n" + + "kind it found.", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, getOrg) + + kit.Handle(app, kit.OpMeta{ + Name: "issue", Group: "read", Single: true, URIType: KindIssue, Resolver: true, + Summary: "Read one issue with its body, labels, and participants", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or an issue URL"}, + {Name: "num", Help: "issue number", Optional: true}, + }, + }, getIssue) + + kit.Handle(app, kit.OpMeta{ + Name: "pr", Group: "read", Single: true, URIType: KindPR, Resolver: true, + Aliases: []string{"pull"}, + Summary: "Read one pull request, merge state and review state included", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a pull request URL"}, + {Name: "num", Help: "pull request number", Optional: true}, + }, + }, getPull) + + kit.Handle(app, kit.OpMeta{ + Name: "discussion", Group: "read", Single: true, URIType: KindDiscussion, Resolver: true, + Summary: "Read one discussion, its category, and its answer", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a discussion URL"}, + {Name: "num", Help: "discussion number", Optional: true}, + }, + }, getDiscussion) + + kit.Handle(app, kit.OpMeta{ + Name: "commit", Group: "read", Single: true, URIType: KindCommit, Resolver: true, + Summary: "Read one commit, with its author, verification, and changed files", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a commit URL"}, + {Name: "sha", Help: "commit sha", Optional: true}, + }, + }, getCommit) + + kit.Handle(app, kit.OpMeta{ + Name: "release", Group: "read", Single: true, URIType: KindRelease, Resolver: true, + Summary: "Read one release by tag, or the latest one", + Long: "The tag defaults to latest, which github.com redirects to whatever that is\n" + + "today. Assets live behind a lazy fragment the page does not load until you\n" + + "scroll, which is why a release page can be 238 KB and show no downloads at\n" + + "all. Reading one release fetches the fragment, because a release without\n" + + "its downloads is not the thing you asked for. The list command makes that\n" + + "a flag, since there the cost is one request per release.", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a release URL"}, + {Name: "tag", Help: "tag name, or latest", Optional: true}, + }, + }, getRelease) + + kit.Handle(app, kit.OpMeta{ + Name: "compare", Group: "read", Single: true, URIType: KindCompare, Resolver: true, + Aliases: []string{"range"}, + Summary: "Compare two refs and list what changed between them", + Long: "Compare has no JSON route: the page answers Rails HTML however you ask.\n" + + "The patch does have everything, so this reads the git-format-patch mailbox\n" + + "and takes the commit list and the per-file changes out of it.", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a compare URL"}, + {Name: "base", Help: "the ref to compare from", Optional: true}, + {Name: "head", Help: "the ref to compare to", Optional: true}, + }, + }, getCompare) + + kit.Handle(app, kit.OpMeta{ + Name: "get", Group: "read", Single: true, + Summary: "Read whatever a reference points at", + Long: "get classifies the reference and dispatches to the right reader, which is\n" + + "what makes `github commits ... -o url | xargs -n1 github get` work across\n" + + "mixed kinds.", + Args: []kit.Arg{{Name: "ref", Help: "any github reference"}}, + }, getAny) +} + +func getRepo(ctx context.Context, in repoIn, emit func(*Repo) error) error { + id, err := ResolveRef(KindRepo, in.Ref) + if err != nil { + return err + } + r, err := in.C.Repo(ctx, id, RepoOptions{Deep: in.Deep || in.C.Deep, Readme: in.Readme}) + if err != nil { + return err + } + return emit(r) +} + +// getUser reads a profile without caring which template answers. Account +// reports the kind it found, so a name that turns out to be an organization +// comes back as one rather than as an error. +func getUser(ctx context.Context, in nameIn, emit func(*Account) error) error { + login, err := ResolveRef(KindUser, in.Name) + if err != nil { + return err + } + a, err := in.C.Account(ctx, login) + if err != nil { + return err + } + return emit(a) +} + +func getOrg(ctx context.Context, in nameIn, emit func(*Org) error) error { + login, err := ResolveRef(KindOrg, in.Name) + if err != nil { + return err + } + o, err := in.C.Org(ctx, login) + if err != nil { + return err + } + return emit(o) +} + +func getIssue(ctx context.Context, in threadIn, emit func(*Issue) error) error { + repo, num, err := resolveThread(KindIssue, in.Ref, in.Num) + if err != nil { + return err + } + i, err := in.C.Issue(ctx, repo, num) + if err != nil { + return err + } + return emit(i) +} + +func getPull(ctx context.Context, in threadIn, emit func(*PullRequest) error) error { + repo, num, err := resolveThread(KindPR, in.Ref, in.Num) + if err != nil { + return err + } + p, err := in.C.PullRequest(ctx, repo, num) + if err != nil { + return err + } + return emit(p) +} + +func getDiscussion(ctx context.Context, in threadIn, emit func(*Discussion) error) error { + repo, num, err := resolveThread(KindDiscussion, in.Ref, in.Num) + if err != nil { + return err + } + d, err := in.C.Discussion(ctx, repo, num) + if err != nil { + return err + } + return emit(d) +} + +// commitIn has no --files flag for the same reason compareIn has none: the +// route ships the whole diff whether it is decoded or not, so the per-file list +// costs parsing rather than a request. +type commitIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a commit URL"` + SHA string `kit:"arg" help:"commit sha, when the reference does not carry one"` + Patch bool `kit:"flag" help:"also fetch the patch, for applying the change rather than describing it"` +} + +func getCommit(ctx context.Context, in commitIn, emit func(*Commit) error) error { + repo, sha, err := resolveRev(KindCommit, in.Ref, in.SHA) + if err != nil { + return err + } + if sha == "" { + return errs.Usage("commit needs a sha, either as a second argument or in the URL") + } + c, err := in.C.CommitInfo(ctx, repo, sha, CommitInfoOptions{Files: true, Patch: in.Patch}) + if err != nil { + return err + } + return emit(c) +} + +type releaseIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a release URL"` + Tag string `kit:"arg" help:"tag name, or latest"` +} + +func getRelease(ctx context.Context, in releaseIn, emit func(*Release) error) error { + repo, tag, err := resolveRev(KindRelease, in.Ref, in.Tag) + if err != nil { + return err + } + if tag == "" { + tag = "latest" + } + r, err := in.C.Release(ctx, repo, tag, ReleaseOptions{Assets: true, Body: true}) + if err != nil { + return err + } + return emit(r) +} + +// compareIn has no --files flag because the per-file list is free: the patch is +// already downloaded and parsing it costs nothing anyone would notice. --patch +// is a flag because keeping the raw stream on the record can be megabytes. +type compareIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a compare URL"` + Base string `kit:"arg" help:"the ref to compare from"` + Head string `kit:"arg" help:"the ref to compare to"` + Patch bool `kit:"flag" help:"keep the raw patch on the record"` +} + +func getCompare(ctx context.Context, in compareIn, emit func(*Compare) error) error { + repo, base, head, err := resolveCompare(in.Ref, in.Base, in.Head) + if err != nil { + return err + } + cmp, err := in.C.CompareRefs(ctx, repo, base, head, CompareOptions{Files: true, Patch: in.Patch}) + if err != nil { + return err + } + return emit(cmp) +} + +// resolveCompare accepts a compare URL, which carries both ends, or a +// repository and two refs. +func resolveCompare(ref, base, head string) (string, string, string, error) { + kind, id, err := Classify(ref) + if err != nil { + return "", "", "", err + } + if kind == KindCompare && base == "" { + repo, rng, ok := cutRev(id) + if !ok { + return "", "", "", errs.Usage("not a range: %q", ref) + } + // Three dots is the merge-base form and two is the direct diff. + // github.com accepts both and means different things by them, so the + // separator is kept as the caller wrote it. + for _, sep := range []string{"...", ".."} { + if a, b, found := strings.Cut(rng, sep); found { + return repo, a, b, nil + } + } + return "", "", "", errs.Usage("no base...head in %q", ref) + } + repo, err := ResolveRepo(ref) + if err != nil { + return "", "", "", err + } + if base == "" || head == "" { + return "", "", "", errs.Usage("compare needs a base and a head, or a compare URL that carries both") + } + return repo, base, head, nil +} + +// getAny dispatches on the kind the reference names. The kinds it declines are +// declined by name, because "not implemented yet" and "no such thing" are +// different answers and a script should be able to tell them apart. +func getAny(ctx context.Context, in bareRefIn, emit func(any) error) error { + kind, id, err := Classify(in.Ref) + if err != nil { + return err + } + rec, err := in.C.fetchOne(ctx, kind, id) + if err != nil { + return err + } + return emit(rec) +} + +// fetchOne reads one record of any kind by id. The graph walk and `github get` +// share it, so a kind that reads correctly in one reads correctly in both. +func (c *Client) fetchOne(ctx context.Context, kind, id string) (any, error) { + switch kind { + case KindRepo: + return c.Repo(ctx, id, RepoOptions{Deep: c.Deep}) + case KindUser: + return c.Account(ctx, id) + case KindOrg: + return c.Org(ctx, id) + case KindIssue, KindPR, KindDiscussion: + repo, n, ok := SplitThreadID(id) + if !ok { + return nil, errs.Usage("not a thread id: %q", id) + } + num, _ := strconv.Atoi(n) + switch kind { + case KindIssue: + return c.Issue(ctx, repo, num) + case KindPR: + return c.PullRequest(ctx, repo, num) + default: + return c.Discussion(ctx, repo, num) + } + case KindCommit: + repo, sha, ok := cutRev(id) + if !ok { + return nil, errs.Usage("not a commit id: %q", id) + } + return c.CommitInfo(ctx, repo, sha, CommitInfoOptions{}) + case KindRelease: + repo, tag, ok := cutRev(id) + if !ok { + return nil, errs.Usage("not a release id: %q", id) + } + return c.Release(ctx, repo, tag, ReleaseOptions{Assets: true, Body: true}) + case KindBranch, KindTag: + repo, name, ok := cutRev(id) + if !ok { + return nil, errs.Usage("not a %s id: %q", kind, id) + } + return c.oneRef(ctx, kind, repo, name) + case KindCompare: + repo, rng, ok := cutRev(id) + if !ok { + return nil, errs.Usage("not a range: %q", id) + } + base, head, found := strings.Cut(rng, "...") + if !found { + base, head, found = strings.Cut(rng, "..") + } + if !found { + return nil, errs.Usage("no base...head in %q", id) + } + return c.CompareRefs(ctx, repo, base, head, CompareOptions{Files: true}) + case KindFile: + repo, ref, path, ok := SplitPathID(id) + if !ok { + return nil, errs.Usage("not a file id: %q", id) + } + return c.Blob(ctx, repo, path, BlobOptions{Ref: ref}) + case KindTopic: + return c.TopicPage(ctx, id) + case KindGist: + return c.Gist(ctx, id, false) + } + return nil, errs.Unsupported("reading a %s is not implemented yet", kind) +} + +// oneRef finds a single branch or tag by name. The git advertisement is one +// request for every ref in the repository, which beats paging the branches page +// looking for one row. +func (c *Client) oneRef(ctx context.Context, kind, repo, name string) (*GitRef, error) { + var found *GitRef + list := c.Branches + if kind == KindTag { + list = c.Tags + } + err := list(ctx, repo, RefOptions{Complete: true}, func(r GitRef) error { + if r.Name == name { + cp := r + found = &cp + } + return nil + }) + if err != nil { + return nil, err + } + if found == nil { + return nil, errs.NotFound("no %s named %s in %s", kind, name, repo) + } + return found, nil +} + +// --- searching --- + +// searchIn is the query surface every search type shares. GitHub's search box +// takes qualifiers inline (repo:, org:, language:, is:open) and this passes the +// query through untouched, so anything that works in the box works here. The +// flags are sugar that appends a qualifier, for the ones people reach for often +// enough that quoting them gets old. +type searchIn struct { + C *Client `kit:"inject"` + Query string `kit:"arg" help:"a search query, with the same qualifiers the search box takes"` + Repo string `kit:"flag" help:"restrict to one repository, owner/name"` + Owner string `kit:"flag" help:"restrict to one user or organization"` + Language string `kit:"flag" help:"restrict to one language"` + Sort string `kit:"flag" help:"a sort qualifier, e.g. stars or updated"` + Limit int `kit:"flag,inherit"` +} + +// query folds the sugar flags into the one string search actually takes. +func (in searchIn) query() string { + q := in.Query + add := func(qualifier, value string) { + if value == "" { + return + } + q = strings.TrimSpace(q + " " + qualifier + ":" + value) + } + add("repo", in.Repo) + add("user", in.Owner) + add("language", in.Language) + add("sort", in.Sort) + return strings.TrimSpace(q) +} + +func registerSearchOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "repos", Group: "search", URIType: KindRepo, List: true, + Summary: "Search repositories", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listRepos) + + kit.Handle(app, kit.OpMeta{ + Name: "issues", Group: "search", URIType: KindIssue, List: true, + Summary: "Search issues", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listIssues) + + kit.Handle(app, kit.OpMeta{ + Name: "prs", Group: "search", URIType: KindPR, List: true, + Aliases: []string{"pulls"}, + Summary: "Search pull requests", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listPulls) + + kit.Handle(app, kit.OpMeta{ + Name: "users", Group: "search", URIType: KindUser, List: true, + Summary: "Search users and organizations", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listUsers) + + kit.Handle(app, kit.OpMeta{ + Name: "topics", Group: "search", URIType: KindTopic, List: true, + Summary: "Search topics", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listTopics) + + kit.Handle(app, kit.OpMeta{ + Name: "packages", Group: "search", URIType: KindPackage, List: true, + Summary: "Search published packages", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listPackages) + + kit.Handle(app, kit.OpMeta{ + Name: "wikis", Group: "search", URIType: KindWiki, List: true, + Summary: "Search wiki pages", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listWikis) + + kit.Handle(app, kit.OpMeta{ + Name: "actions", Group: "search", URIType: KindAction, List: true, + Summary: "Search the marketplace for actions", + Args: []kit.Arg{{Name: "query", Help: "a search query", Optional: true}}, + }, listActions) + + kit.Handle(app, kit.OpMeta{ + Name: "code", Group: "search", + Summary: "Search code, which needs a session and so does not work here", + Long: "Code search is the one search type that requires a signed-in session. It\n" + + "answers 200 with an empty result set to an anonymous request, which looks\n" + + "exactly like a query that found nothing, so this says so instead of\n" + + "returning zero hits and letting you conclude the query was wrong.", + Args: []kit.Arg{{Name: "query", Help: "a search query"}}, + }, listCode) + + kit.Handle(app, kit.OpMeta{ + Name: "search", Group: "search", + Summary: "Search every type at once", + Long: "Nine of GitHub's ten search types answer JSON to an anonymous request.\n" + + "This asks all of them, or the ones named by --type, and streams the\n" + + "records as they arrive.", + Args: []kit.Arg{{Name: "query", Help: "what to look for"}}, + }, searchAll) +} + +func listRepos(ctx context.Context, in searchIn, emit func(*Repo) error) error { + return in.C.SearchRepositories(ctx, in.query(), in.Limit, byValue(emit)) +} + +func listIssues(ctx context.Context, in searchIn, emit func(*Thread) error) error { + return in.C.SearchIssuesAndPulls(ctx, in.query(), SearchIssues, in.Limit, byValue(emit)) +} + +func listPulls(ctx context.Context, in searchIn, emit func(*Thread) error) error { + return in.C.SearchIssuesAndPulls(ctx, in.query(), SearchPulls, in.Limit, byValue(emit)) +} + +func listUsers(ctx context.Context, in searchIn, emit func(*Account) error) error { + return in.C.SearchAccounts(ctx, in.query(), in.Limit, byValue(emit)) +} + +func listTopics(ctx context.Context, in searchIn, emit func(*Topic) error) error { + return in.C.SearchTopicsBy(ctx, in.query(), in.Limit, byValue(emit)) +} + +func listPackages(ctx context.Context, in searchIn, emit func(*Package) error) error { + return in.C.SearchPackagesBy(ctx, in.query(), in.Limit, byValue(emit)) +} + +func listWikis(ctx context.Context, in searchIn, emit func(*WikiPage) error) error { + return in.C.SearchWikisBy(ctx, in.query(), in.Limit, byValue(emit)) +} + +func listActions(ctx context.Context, in searchIn, emit func(*Action) error) error { + return in.C.SearchMarketplace(ctx, in.query(), in.Limit, byValue(emit)) +} + +func listCode(ctx context.Context, in searchIn, emit func(*File) error) error { + return in.C.SearchCodeBy(ctx, in.query(), in.Limit, byValue(emit)) +} + +type searchAllIn struct { + C *Client `kit:"inject"` + Query string `kit:"arg" help:"what to look for"` + Type []string `kit:"flag" help:"restrict to some types, e.g. repositories,issues"` + Limit int `kit:"flag,inherit"` +} + +func searchAll(ctx context.Context, in searchAllIn, emit func(any) error) error { + types := in.Type + if len(types) == 0 { + types = SearchTypes + } + // The per-type limit is the whole limit: a caller asking for ten of + // everything gets ten of each rather than ten split nine ways, which is + // what anyone piping this into a filter wants. + for _, t := range types { + if err := in.C.searchOne(ctx, t, in.Query, in.Limit, emit); err != nil { + return err + } + } + return nil +} + +func (c *Client) searchOne(ctx context.Context, typ, query string, limit int, emit func(any) error) error { + any1 := func(v any) error { return emit(v) } + switch typ { + case SearchRepos: + return c.SearchRepositories(ctx, query, limit, func(r Repo) error { return any1(&r) }) + case SearchIssues, SearchPulls: + return c.SearchIssuesAndPulls(ctx, query, typ, limit, func(t Thread) error { return any1(&t) }) + case SearchUsers: + return c.SearchAccounts(ctx, query, limit, func(a Account) error { return any1(&a) }) + case SearchCommits: + return c.SearchCommitsBy(ctx, query, limit, func(v Commit) error { return any1(&v) }) + case SearchDiscussions: + return c.SearchDiscussionsBy(ctx, query, limit, func(d Discussion) error { return any1(&d) }) + case SearchTopics: + return c.SearchTopicsBy(ctx, query, limit, func(t Topic) error { return any1(&t) }) + case SearchPackages: + return c.SearchPackagesBy(ctx, query, limit, func(p Package) error { return any1(&p) }) + case SearchWikis: + return c.SearchWikisBy(ctx, query, limit, func(w WikiPage) error { return any1(&w) }) + case SearchMarket: + return c.SearchMarketplace(ctx, query, limit, func(a Action) error { return any1(&a) }) + case SearchCode: + return c.SearchCodeBy(ctx, query, limit, func(f File) error { return any1(&f) }) + } + return errs.Usage("not a search type: %q; the types are %s", typ, strings.Join(SearchTypes, ", ")) +} + +// --- contents --- + +type treeIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a tree URL"` + Path string `kit:"arg" help:"a directory inside the repository"` + Rev string `kit:"flag,name=rev" help:"branch, tag, or sha; the default branch when empty"` + Recursive bool `kit:"flag" help:"walk subdirectories, one request each"` + Sizes bool `kit:"flag" help:"fill the byte size of each file, one request each"` + Limit int `kit:"flag,inherit"` +} + +type blobIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a blob URL"` + Path string `kit:"arg" help:"a file inside the repository"` + Rev string `kit:"flag,name=rev" help:"branch, tag, or sha; the default branch when empty"` + Content bool `kit:"flag" help:"include the bytes, fetched from raw"` + Styled bool `kit:"flag" help:"include the rendered lines and the syntax highlighting spans"` +} + +func registerContentOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "tree", Group: "contents", URIType: KindTree, List: true, + Aliases: []string{"ls"}, + Summary: "List a directory, or the whole tree", + Long: "There is no recursive parameter on this route, so --recursive is one\n" + + "request per directory. For a whole large repository, `github archive` is\n" + + "one request instead of hundreds.", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a tree URL"}, + {Name: "path", Help: "a directory inside the repository", Optional: true}, + }, + }, listTree) + + kit.Handle(app, kit.OpMeta{ + Name: "blob", Group: "contents", Single: true, URIType: KindFile, Resolver: true, + Aliases: []string{"file"}, + Summary: "Read one file's metadata, and its bytes on request", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a blob URL"}, + {Name: "path", Help: "a file inside the repository", Optional: true}, + }, + }, getBlob) + + kit.Handle(app, kit.OpMeta{ + Name: "symbols", Group: "contents", + Summary: "List the definitions GitHub extracted from a file", + Long: "GitHub runs a symbol extractor over every blob it renders and used to\n" + + "ship the result in the route payload. There is no unauthenticated REST\n" + + "equivalent anywhere, which is why this command exists.\n\n" + + "As of now it will not return anything. The blob still says symbols are\n" + + "enabled and still renders the button, and the list behind it is empty for\n" + + "a signed-out reader on every file tried. This reports that rather than\n" + + "returning an empty list, and stays here because the field is still in the\n" + + "payload and may fill in again.", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a blob URL"}, + {Name: "path", Help: "a file inside the repository", Optional: true}, + }, + }, listSymbols) +} + +func listTree(ctx context.Context, in treeIn, emit func(*TreeEntry) error) error { + repo, ref, path, err := resolvePath(in.Ref, in.Path, in.Rev) + if err != nil { + return err + } + return in.C.Tree(ctx, repo, path, TreeOptions{ + Ref: ref, + Recursive: in.Recursive, + Sizes: in.Sizes, + Limit: in.Limit, + }, byValue(emit)) +} + +func getBlob(ctx context.Context, in blobIn, emit func(*File) error) error { + repo, ref, path, err := resolvePath(in.Ref, in.Path, in.Rev) + if err != nil { + return err + } + if path == "" { + return errs.Usage("blob needs a path, either as a second argument or in the URL") + } + f, err := in.C.Blob(ctx, repo, path, BlobOptions{Ref: ref, Content: in.Content, Styled: in.Styled}) + if err != nil { + return err + } + return emit(f) +} + +type symbolIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a blob URL"` + Path string `kit:"arg" help:"a file inside the repository"` + Rev string `kit:"flag,name=rev" help:"branch, tag, or sha; the default branch when empty"` +} + +func listSymbols(ctx context.Context, in symbolIn, emit func(*Symbol) error) error { + repo, ref, path, err := resolvePath(in.Ref, in.Path, in.Rev) + if err != nil { + return err + } + if path == "" { + return errs.Usage("symbols needs a path, either as a second argument or in the URL") + } + f, err := in.C.Blob(ctx, repo, path, BlobOptions{Ref: ref}) + if err != nil { + return err + } + // Both messages lead with a plain word. The CLI title-cases whatever an + // error starts with, which turns a path into nonsense and, less obviously, + // turns GitHub into Github. + switch f.SymbolsStatus { + case "not_analyzed": + return errs.Unsupported("no symbols for %s: GitHub does not extract them from the language it is written in", path) + case "unavailable", "timed_out": + // Unsupported rather than a network kind, because asking again does not + // help. GitHub still renders the symbols button and still sets + // symbolsEnabled on the blob, and the list behind it comes back empty + // for a signed-out reader on every file tried. + return errs.Unsupported("no symbol list for %s: GitHub serves none to a signed-out reader, though it still offers the panel", path) + } + return emitEach(f.Symbols, emit) +} + +// --- history --- + +type commitsIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or any URL from the repository"` + Rev string `kit:"flag,name=rev" help:"branch, tag, or sha to walk from"` + Path string `kit:"flag" help:"limit history to one file or directory"` + Author string `kit:"flag" help:"a login, not an email address"` + Since string `kit:"flag" help:"only commits after this date, YYYY-MM-DD"` + Until string `kit:"flag" help:"only commits before this date, YYYY-MM-DD"` + PR int `kit:"flag,name=pr" help:"walk one pull request's commits instead of the branch"` + Limit int `kit:"flag,inherit"` +} + +type refsIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or any URL from the repository"` + Complete bool `kit:"flag" help:"read the git advertisement: every ref in one request, with shas"` + Pulls bool `kit:"flag" help:"include refs/pull/*, which on a busy repository is most of the response"` + Limit int `kit:"flag,inherit"` +} + +type releasesIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or any URL from the repository"` + Assets bool `kit:"flag" help:"fetch each release's lazy asset fragment, one request each"` + Body bool `kit:"flag" help:"keep the rendered release notes"` + Limit int `kit:"flag,inherit"` +} + +func registerHistoryOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "commits", Group: "history", URIType: KindCommit, List: true, + Aliases: []string{"log"}, + Summary: "Walk a repository's history", + Long: "Every filter here is a query parameter the route already understands, so\n" + + "the filtering happens on GitHub's side rather than after a full download.\n" + + "A page is 35 commits, and an unbounded walk of a large repository is\n" + + "thousands of requests, so set --limit unless you mean it.", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listCommits) + + kit.Handle(app, kit.OpMeta{ + Name: "refs", Group: "history", List: true, + Summary: "List every ref in a repository", + Long: "Without --complete this reads the refs fragment, which is names only but\n" + + "is 6 KB where the advertisement is 588 KB. With it, one request returns\n" + + "every branch, tag, and pull head with its sha, and no cap.", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listRefs) + + kit.Handle(app, kit.OpMeta{ + Name: "branches", Group: "history", URIType: KindBranch, List: true, + Summary: "List branches", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listBranches) + + kit.Handle(app, kit.OpMeta{ + Name: "tags", Group: "history", URIType: KindTag, List: true, + Summary: "List tags", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listTags) + + kit.Handle(app, kit.OpMeta{ + Name: "releases", Group: "history", URIType: KindRelease, List: true, + Summary: "List releases", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listReleases) + + kit.Handle(app, kit.OpMeta{ + Name: "timeline", Group: "history", List: true, + Summary: "List everything that happened on one issue or pull request", + Long: "The timeline is the comments and the events in one stream: labels applied,\n" + + "milestones set, commits referenced, reviews requested, the lot.", + Args: []kit.Arg{ + {Name: "ref", Help: "owner/name, or a thread URL"}, + {Name: "num", Help: "the number, when the reference does not carry one", Optional: true}, + }, + }, listTimeline) +} + +func listCommits(ctx context.Context, in commitsIn, emit func(*Commit) error) error { + repo, rev, err := resolveRev(KindCommit, in.Ref, in.Rev) + if err != nil { + return err + } + if in.PR > 0 { + return in.C.PullCommits(ctx, repo, in.PR, in.Limit, byValue(emit)) + } + return in.C.Commits(ctx, repo, CommitOptions{ + Ref: rev, + Path: in.Path, + Author: in.Author, + Since: in.Since, + Until: in.Until, + Limit: in.Limit, + }, byValue(emit)) +} + +func listRefs(ctx context.Context, in refsIn, emit func(*GitRef) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Refs(ctx, repo, in.options(), byValue(emit)) +} + +func listBranches(ctx context.Context, in refsIn, emit func(*GitRef) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Branches(ctx, repo, in.options(), byValue(emit)) +} + +func listTags(ctx context.Context, in refsIn, emit func(*GitRef) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Tags(ctx, repo, in.options(), byValue(emit)) +} + +func (in refsIn) options() RefOptions { + return RefOptions{Complete: in.Complete, Pulls: in.Pulls, Limit: in.Limit} +} + +func listReleases(ctx context.Context, in releasesIn, emit func(*Release) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Releases(ctx, repo, ReleaseOptions{ + Assets: in.Assets, + Body: in.Body, + Limit: in.Limit, + }, byValue(emit)) +} + +type timelineIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or a thread URL"` + Num int `kit:"arg" help:"the number, when the reference does not carry one"` + Limit int `kit:"flag,inherit"` +} + +func listTimeline(ctx context.Context, in timelineIn, emit func(*TimelineItem) error) error { + repo, num, err := resolveThread(KindIssue, in.Ref, in.Num) + if err != nil { + return err + } + return in.C.Timeline(ctx, repo, num, in.Limit, byValue(emit)) +} + +// --- people --- + +type accountListIn struct { + C *Client `kit:"inject"` + Name string `kit:"arg" help:"a login, a profile URL, or a github:// URI"` + Limit int `kit:"flag,inherit"` +} + +type gistIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"a gist id, a gist URL, or a github:// URI"` + Content bool `kit:"flag" help:"fetch each file's raw content, one request per file"` +} + +type contributionsIn struct { + C *Client `kit:"inject"` + Name string `kit:"arg" help:"a login, a profile URL, or a github:// URI"` + Year int `kit:"flag" help:"calendar year, defaulting to the rolling last twelve months"` +} + +type activityIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"a login for a person's stream, or owner/name for a repository's"` + Limit int `kit:"flag,inherit"` +} + +func registerPeopleOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "followers", Group: "people", URIType: KindUser, List: true, + Summary: "List who follows an account", + Long: "The tab is 50 people a page and the pager is a plain next link, so an\n" + + "account with a hundred thousand followers is two thousand requests. Set\n" + + "--limit unless you mean all of them.", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, listFollowers) + + kit.Handle(app, kit.OpMeta{ + Name: "following", Group: "people", URIType: KindUser, List: true, + Summary: "List who an account follows", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, listFollowing) + + kit.Handle(app, kit.OpMeta{ + Name: "members", Group: "people", URIType: KindUser, List: true, + Summary: "List an organization's public members", + Long: "Public members only, which is the organization's own choice per person\n" + + "and not something a token would widen for someone outside the org.", + Args: []kit.Arg{{Name: "name", Help: "organization login, URL, or github:// URI"}}, + }, listMembers) + + kit.Handle(app, kit.OpMeta{ + Name: "stars", Group: "people", URIType: KindRepo, List: true, + Aliases: []string{"starred"}, + Summary: "List what an account has starred", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, listStarred) + + kit.Handle(app, kit.OpMeta{ + Name: "owned", Group: "people", URIType: KindRepo, List: true, + Summary: "List an account's repositories as the profile shows them", + Long: "This reads the profile's repositories tab, which is the only surface that\n" + + "lists forks and archived repositories in the account's own order. It is\n" + + "not called repos because `github repos --owner name` already exists, goes\n" + + "through search, and is the better tool when you want to filter or sort.", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, listAccountRepos) + + kit.Handle(app, kit.OpMeta{ + Name: "gists", Group: "people", URIType: KindGist, List: true, + Summary: "List an account's public gists", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, listGists) + + kit.Handle(app, kit.OpMeta{ + Name: "gist", Group: "people", URIType: KindGist, Single: true, Resolver: true, + Summary: "Read one gist with its files", + Long: "The index gives each file's first few lines only, which is what the page\n" + + "renders. With --content each file is fetched whole from the raw host.", + Args: []kit.Arg{{Name: "ref", Help: "gist id, gist URL, or github:// URI"}}, + }, getGist) + + kit.Handle(app, kit.OpMeta{ + Name: "contributions", Group: "people", URIType: KindContribution, List: true, + Aliases: []string{"calendar"}, + Summary: "Read an account's contribution calendar, one record per day", + Long: "The count is not on the square. Each square points at a tooltip by id and\n" + + "the tooltip holds the sentence with the number in it, so this indexes the\n" + + "tooltips first and reads the squares against that index.", + Args: []kit.Arg{{Name: "name", Help: "login, profile URL, or github:// URI"}}, + }, listContributions) + + kit.Handle(app, kit.OpMeta{ + Name: "activity", Group: "people", URIType: KindEvent, List: true, + Aliases: []string{"events", "feed"}, + Summary: "Read a public activity feed", + Long: "One login gives that person's public events; one owner/name gives that\n" + + "repository's commit feed. Both are Atom, both are public, and neither has\n" + + "a pager, so a feed is however many entries GitHub decided to put in it.", + Args: []kit.Arg{{Name: "ref", Help: "a login, or owner/name"}}, + }, listActivity) +} + +func listFollowers(ctx context.Context, in accountListIn, emit func(*Account) error) error { + login, err := ResolveRef(KindUser, in.Name) + if err != nil { + return err + } + return in.C.Followers(ctx, login, in.Limit, byValue(emit)) +} + +func listFollowing(ctx context.Context, in accountListIn, emit func(*Account) error) error { + login, err := ResolveRef(KindUser, in.Name) + if err != nil { + return err + } + return in.C.Following(ctx, login, in.Limit, byValue(emit)) +} + +func listMembers(ctx context.Context, in accountListIn, emit func(*Account) error) error { + login, err := ResolveRef(KindOrg, in.Name) + if err != nil { + return err + } + return in.C.Members(ctx, login, in.Limit, byValue(emit)) +} + +func listStarred(ctx context.Context, in accountListIn, emit func(*Repo) error) error { + login, err := ResolveRef(KindUser, in.Name) + if err != nil { + return err + } + return in.C.Starred(ctx, login, in.Limit, byValue(emit)) +} + +func listAccountRepos(ctx context.Context, in accountListIn, emit func(*Repo) error) error { + login, err := ResolveRef(KindUser, in.Name) + if err != nil { + return err + } + return in.C.ReposAsShown(ctx, login, in.Limit, byValue(emit)) +} + +func listGists(ctx context.Context, in accountListIn, emit func(*Gist) error) error { + login, err := ResolveRef(KindUser, in.Name) + if err != nil { + return err + } + return in.C.Gists(ctx, login, in.Limit, byValue(emit)) +} + +func getGist(ctx context.Context, in gistIn, emit func(*Gist) error) error { + id, err := ResolveRef(KindGist, in.Ref) + if err != nil { + return err + } + g, err := in.C.Gist(ctx, id, in.Content) + if err != nil { + return err + } + return emit(g) +} + +func listContributions(ctx context.Context, in contributionsIn, emit func(*ContributionDay) error) error { + login, err := ResolveRef(KindUser, in.Name) + if err != nil { + return err + } + return in.C.Contributions(ctx, login, in.Year, byValue(emit)) +} + +// listActivity does not resolve the reference, because the feed takes both a +// login and an owner/name and the reader tells them apart itself. Sending it +// through ResolveRef would force a choice that neither kind wins. +func listActivity(ctx context.Context, in activityIn, emit func(*Event) error) error { + return in.C.Activity(ctx, strings.TrimPrefix(in.Ref, BaseURL+"/"), in.Limit, byValue(emit)) +} + +// --- discovery and statistics --- + +type trendingIn struct { + C *Client `kit:"inject"` + Since string `kit:"flag" help:"daily, weekly, or monthly"` + Language string `kit:"flag" help:"a language slug, as it appears in the trending URL"` + SpokenLanguage string `kit:"flag,name=spoken" help:"a two-letter natural language code"` + Developers bool `kit:"flag" help:"list trending developers instead of repositories"` + Limit int `kit:"flag,inherit"` +} + +type topicIn struct { + C *Client `kit:"inject"` + Name string `kit:"arg" help:"a topic slug, a topic URL, or a github:// URI"` +} + +type repoListIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or any URL from the repository"` + Limit int `kit:"flag,inherit"` +} + +type repoRefIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or any URL from the repository"` +} + +func registerDiscoverOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "trending", Group: "discover", URIType: KindRepo, List: true, + Summary: "List what is trending", + Long: "Trending is the clearest case for reading pages. There is no JSON version\n" + + "of it anywhere, with a token or without, so a page decoder is not a\n" + + "fallback here, it is the only implementation that can exist.", + }, listTrending) + + kit.Handle(app, kit.OpMeta{ + Name: "topic", Group: "discover", URIType: KindTopic, Single: true, Resolver: true, + Summary: "Read one topic page", + Long: "The search result for a topic has a name and a blurb. The page has the long\n" + + "description, the logo, who created the thing, when it was released, the\n" + + "Wikipedia link, and the related topics, which is most of what makes a topic\n" + + "worth a record.", + Args: []kit.Arg{{Name: "name", Help: "topic slug, URL, or github:// URI"}}, + }, getTopic) + + kit.Handle(app, kit.OpMeta{ + Name: "forks", Group: "discover", URIType: KindRepo, List: true, + Summary: "List a repository's public forks", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listForks) + + kit.Handle(app, kit.OpMeta{ + Name: "contributors", Group: "discover", URIType: KindContributor, List: true, + Summary: "List contributors with their commit, addition, and deletion counts", + Long: "This reads the contributor graph's own data route, which answers 202 with\n" + + "an empty body while GitHub computes the numbers. That is normal rather\n" + + "than an error, so the first call on a large repository waits a few seconds\n" + + "and every call after it is instant.", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listContributors) + + kit.Handle(app, kit.OpMeta{ + Name: "languages", Group: "discover", URIType: KindRepo, List: true, + Summary: "Report the language histogram, one record per language", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listLanguages) + + kit.Handle(app, kit.OpMeta{ + Name: "stats", Group: "discover", URIType: KindRepo, Single: true, + Summary: "Report a repository's counts and nothing else", + Long: "Every field here is on the repository record too. The point of having it\n" + + "separately is that a record with eight numbers in it is something you can\n" + + "store once a day and diff, and a record with a readme in it is not.", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, getStats) +} + +func listTrending(ctx context.Context, in trendingIn, emit func(any) error) error { + opts := TrendingOptions{ + Since: in.Since, + Language: in.Language, + SpokenLanguage: in.SpokenLanguage, + Limit: in.Limit, + } + if in.Developers { + return in.C.TrendingDevelopers(ctx, opts, func(a Account) error { return emit(&a) }) + } + return in.C.Trending(ctx, opts, func(t Trending) error { return emit(&t) }) +} + +func getTopic(ctx context.Context, in topicIn, emit func(*Topic) error) error { + slug, err := ResolveRef(KindTopic, in.Name) + if err != nil { + return err + } + t, err := in.C.TopicPage(ctx, slug) + if err != nil { + return err + } + return emit(t) +} + +func listForks(ctx context.Context, in repoListIn, emit func(*Repo) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Forks(ctx, repo, in.Limit, byValue(emit)) +} + +type contributorsIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"owner/name, or any URL from the repository"` + Weeks bool `kit:"flag" help:"keep the per-week breakdown, which is large"` + Limit int `kit:"flag,inherit"` +} + +func listContributors(ctx context.Context, in contributorsIn, emit func(*Contributor) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Contributors(ctx, repo, ContributorOptions{Weeks: in.Weeks, Limit: in.Limit}, byValue(emit)) +} + +func listLanguages(ctx context.Context, in repoRefIn, emit func(*LanguageShare) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Languages(ctx, repo, byValue(emit)) +} + +func getStats(ctx context.Context, in repoRefIn, emit func(*RepoStats) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + s, err := in.C.Stats(ctx, repo) + if err != nil { + return err + } + return emit(s) +} + +// --- meta --- + +// --- the graph plane --- + +func registerGraphOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "graph", Group: "graph", + Summary: "Emit the node, edges, and facts for one entity", + Args: []kit.Arg{{Name: "ref", Help: "any github reference"}}, + }, graph) + + kit.Handle(app, kit.OpMeta{ + Name: "edges", Group: "graph", List: true, + Summary: "Emit only the edges for one entity", + Long: "Every edge carries the rule that produced it. --min-trust id is the\n" + + "interesting case: the edges derived from the id alone need no request at\n" + + "all, so `github edges golang/go#1 --min-trust id` answers offline.", + Args: []kit.Arg{{Name: "ref", Help: "any github reference"}}, + }, edges) + + kit.Handle(app, kit.OpMeta{ + Name: "crawl", Group: "graph", + Summary: "Walk the graph breadth-first from a seed", + Long: "crawl follows only the predicates named by --follow, which defaults to the\n" + + "structural ones: references, stars, follows, and the two dependency\n" + + "predicates fan out without bound and have to be asked for by name. Nodes\n" + + "and edges stream as they are found, so an interrupted walk has still\n" + + "emitted everything it reached.", + Args: []kit.Arg{{Name: "ref", Help: "seed reference"}}, + }, crawl) + + kit.Handle(app, kit.OpMeta{ + Name: "deps", Group: "graph", URIType: KindRepo, List: true, + Summary: "List what a repository depends on", + Long: "The dependency graph is opt-in per repository. A repository with it off\n" + + "answers with a page and no rows, which comes back as an empty list rather\n" + + "than an error, because the page does not say which of the two it is.", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listDeps) + + kit.Handle(app, kit.OpMeta{ + Name: "dependents", Group: "graph", URIType: KindRepo, List: true, + Summary: "List the repositories that depend on this one", + Long: "The list is ordered by stars and it is long, so --limit is the flag that\n" + + "matters: a popular library has tens of thousands of rows at thirty a page.", + Args: []kit.Arg{{Name: "ref", Help: "owner/name, or any URL from the repository"}}, + }, listDependents) +} + +func graph(ctx context.Context, in bareRefIn, emit func(any) error) error { + _, _, g, err := in.C.GraphOfRef(ctx, in.Ref) + if err != nil { + return err + } + for i := range g.Nodes { + if err := emit(&g.Nodes[i]); err != nil { + return err + } + } + for i := range g.Edges { + if err := emit(&g.Edges[i]); err != nil { + return err + } + } + for i := range g.Facts { + if err := emit(&g.Facts[i]); err != nil { + return err + } + } + return nil +} + +type edgesIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"any github reference"` + Predicate string `kit:"flag" help:"keep only this predicate"` + MinTrust string `kit:"flag,name=min-trust" help:"drop edges below this rule: id, payload, feed, html, text" default:"html"` +} + +func edges(ctx context.Context, in edgesIn, emit func(*Edge) error) error { + // The id rules produce their edges without a fetch, so asking for them + // alone is answered from the reference and nothing else. + if in.MinTrust == SrcID { + kind, id, err := Classify(in.Ref) + if err != nil { + return err + } + return emitEach(pickEdges(idEdges(kind, id), in), emit) + } + _, _, g, err := in.C.GraphOfRef(ctx, in.Ref) + if err != nil { + return err + } + return emitEach(pickEdges(g.Edges, in), emit) +} + +// idEdges is what the id alone says. It builds the record shell rather than +// fetching one, which is the whole point: a thread id names its repository and +// a repository id names its owner, and neither fact needs github.com. +func idEdges(kind, id string) []Edge { + var rec any + switch kind { + case KindRepo: + owner, name, _ := SplitRepo(id) + r := &Repo{Owner: owner, Name: name} + r.setIdentity(KindRepo, id) + rec = r + case KindIssue, KindPR, KindDiscussion: + repo, num, ok := SplitThreadID(id) + if !ok { + return nil + } + n, _ := strconv.Atoi(num) + t := &Thread{Repo: repo, Number: n} + t.setIdentity(kind, id) + rec = &Issue{Thread: *t} + default: + return nil + } + _, out, _ := Extract(rec) + return FilterTrust(out, SrcID) +} + +func pickEdges(in []Edge, opts edgesIn) []Edge { + out := FilterTrust(in, opts.MinTrust) + if opts.Predicate == "" { + return out + } + want := strings.TrimPrefix(opts.Predicate, "gh:") + kept := out[:0] + for _, e := range out { + if e.Predicate == want { + kept = append(kept, e) + } + } + return kept +} + +type crawlIn struct { + C *Client `kit:"inject"` + Ref string `kit:"arg" help:"seed reference"` + Depth int `kit:"flag" help:"how many edges out to walk" default:"1"` + Follow []string `kit:"flag" help:"predicates to follow, or all (default: the structural ones)"` + Kinds []string `kit:"flag" help:"expand only these kinds"` + MinTrust string `kit:"flag,name=min-trust" help:"drop edges below this rule" default:"html"` + DryRun bool `kit:"flag,name=dry-run" help:"print the estimate and stop"` + NodesOnly bool `kit:"flag,name=nodes-only" help:"emit nodes only"` + EdgesOnly bool `kit:"flag,name=edges-only" help:"emit edges only"` + Limit int `kit:"flag,inherit"` +} + +func crawl(ctx context.Context, in crawlIn, emit func(any) error) error { + o := CrawlOptions{ + Depth: in.Depth, + Follow: in.Follow, + Kinds: in.Kinds, + MinTrust: in.MinTrust, + Limit: in.Limit, + NodesOnly: in.NodesOnly, + EdgesOnly: in.EdgesOnly, + } + if in.DryRun { + plan, err := in.C.Estimate(ctx, in.Ref, o) + if err != nil { + return err + } + return emit(plan) + } + return in.C.Crawl(ctx, in.Ref, o, CrawlSink{ + Node: func(n *Node) error { return emit(n) }, + Edge: func(e *Edge) error { return emit(e) }, + Fact: func(f *Fact) error { return emit(f) }, + }) +} + +func listDeps(ctx context.Context, in repoListIn, emit func(*Dependency) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Dependencies(ctx, repo, in.Limit, byValue(emit)) +} + +func listDependents(ctx context.Context, in repoListIn, emit func(*Dependent) error) error { + repo, err := ResolveRepo(in.Ref) + if err != nil { + return err + } + return in.C.Dependents(ctx, repo, in.Limit, byValue(emit)) +} + +func registerMetaOps(app *kit.App) { + kit.Handle(app, kit.OpMeta{ + Name: "url", Group: "meta", Single: true, + Aliases: []string{"classify"}, + Summary: "Parse a reference and report what it names", + Long: "url does no network work. It takes anything a person might paste, a\n" + + "github.com URL, a github:// URI, owner/name, owner/name#123, a blob URL\n" + + "with a line anchor, and returns the kind, the canonical id, the URI, and\n" + + "the https location.", + Args: []kit.Arg{{Name: "ref", Help: "any github reference"}}, + }, parseRef) + + kit.Handle(app, kit.OpMeta{ + Name: "routes", Group: "meta", List: true, + Summary: "List which surface answers for which route", + Long: "This is the index the readers work from: for each route, the surface that\n" + + "answers it best and what to fall back to when that surface declines. A\n" + + "route that is not in the table is unsupported rather than guessed at.", + }, listRoutes) + + kit.Handle(app, kit.OpMeta{ + Name: "doctor", Group: "meta", List: true, + Summary: "Check the environment, the site, and the cache", + Long: doctorLong, + }, runDoctor) +} + +type doctorIn struct { + C *Client `kit:"inject"` +} + +func runDoctor(ctx context.Context, in doctorIn, emit func(*Check) error) error { + return in.C.Doctor(ctx, emit) +} + +type parseIn struct { + Ref string `kit:"arg" help:"any github reference"` +} + +func parseRef(_ context.Context, in parseIn, emit func(*Ident) error) error { + id, err := Parse(in.Ref) + if err != nil { + return err + } + return emit(&id) +} + +type noIn struct{} + +func listRoutes(_ context.Context, _ noIn, emit func(*RouteInfo) error) error { + return emitEach(Routes, emit) +} diff --git a/gh/people.go b/gh/people.go new file mode 100644 index 0000000..2723ebe --- /dev/null +++ b/gh/people.go @@ -0,0 +1,738 @@ +package gh + +import ( + "context" + "encoding/xml" + "strconv" + "strings" + "time" + + "golang.org/x/net/html" + + "github.com/tamnd/any-cli/kit/errs" + "github.com/tamnd/github-cli/pkg/page" +) + +// people.go is everything that hangs off an account: who follows whom, what +// they starred, what they published, and what they did. +// +// None of it has a JSON payload. The profile tabs, the organization roster, and +// the gist index are all Rails, and the activity stream is Atom. That is the +// whole reason this file exists as its own unit: the surfaces here share a +// pager and a row shape with each other and with nothing else in the tool. +// +// The pager is the "rails" one from surface.go: fetch a page, decode the rows, +// look for the link that says next. Two templates write that link two ways and +// nextPageHref knows both. + +// Followers lists the accounts following a login, newest first, which is the +// order the page uses and the only order it offers. +func (c *Client) Followers(ctx context.Context, login string, limit int, emit func(Account) error) error { + return c.profileAccounts(ctx, login, "followers", limit, emit) +} + +// Following lists the accounts a login follows. +func (c *Client) Following(ctx context.Context, login string, limit int, emit func(Account) error) error { + return c.profileAccounts(ctx, login, "following", limit, emit) +} + +func (c *Client) profileAccounts(ctx context.Context, login, tab string, limit int, emit func(Account) error) error { + if login == "" || strings.Contains(login, "/") { + return usageBadID("account", login, "a bare login") + } + base := query(accountURL(login), "tab", tab) + fetch := func(ctx context.Context, token string) ([]Account, string, error) { + u := base + if n := pageToken(token); n > 1 { + u = query(u, "page", strconv.Itoa(n)) + } + res, err := c.GetHTML(ctx, u) + if err != nil { + return nil, "", err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return nil, "", structureChanged(login + " " + tab) + } + var out []Account + for _, row := range page.FindAll(doc, page.Sel{Class: "d-table"}) { + a, ok := followRow(row, res.FinalURL) + if ok { + out = append(out, a) + } + } + return out, railsNext(doc, token), nil + } + return paginate(ctx, limit, fetch, emit) +} + +// followRow reads one row of a followers or following list. The row has the +// login twice, once in the avatar link and once in the muted span, and the +// display name in the primary span when the person set one. +func followRow(row *html.Node, source string) (Account, bool) { + link := page.Find(row, page.Sel{Tag: "a", Attr: "data-hovercard-type", AttrValue: "user"}) + if link == nil { + return Account{}, false + } + who := actorFromHref(page.Attr(link, "href")) + if who.Login == "" { + return Account{}, false + } + a := Account{Login: who.Login, Type: "User"} + a.setIdentity(KindUser, who.Login) + if n := page.Find(row, page.Sel{Class: "Link--primary"}); n != nil { + a.Name = page.Text(n) + } + if img := page.Find(row, page.Sel{Tag: "img", Class: "avatar-user"}); img != nil { + a.AvatarURL = page.Attr(img, "src") + } + a.addSource(source) + return a, true +} + +// Members lists an organization's public members. The roster is at +// /orgs/{login}/people rather than on the profile, and the profile's avatar +// strip is a sample of it rather than a short version of it. +func (c *Client) Members(ctx context.Context, login string, limit int, emit func(Account) error) error { + if login == "" || strings.Contains(login, "/") { + return usageBadID("organization", login, "a bare login") + } + base := BaseURL + "/orgs/" + login + "/people" + fetch := func(ctx context.Context, token string) ([]Account, string, error) { + u := base + if n := pageToken(token); n > 1 { + u = query(u, "page", strconv.Itoa(n)) + } + res, err := c.GetHTML(ctx, u) + if err != nil { + return nil, "", err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return nil, "", structureChanged(login + " members") + } + var out []Account + for _, li := range page.FindAll(doc, page.Sel{Class: "member-list-item"}) { + m, ok := memberRow(li, res.FinalURL) + if ok { + out = append(out, m) + } + } + return out, railsNext(doc, token), nil + } + return paginate(ctx, limit, fetch, emit) +} + +// memberRow reads one member. The role is behind a batch-deferred fragment that +// needs a session, so it is absent here rather than wrong. +func memberRow(li *html.Node, source string) (Account, bool) { + name := page.Find(li, page.Sel{Tag: "a", Attr: "id", AttrPrefix: "member-"}) + if name == nil { + return Account{}, false + } + who := actorFromHref(page.Attr(name, "href")) + if who.Login == "" { + return Account{}, false + } + a := Account{Login: who.Login, Type: "User", Name: page.Text(name)} + a.setIdentity(KindUser, who.Login) + // The name anchor falls back to the login when the person set no display + // name, and a Name that repeats the Login says nothing. + if a.Name == a.Login { + a.Name = "" + } + if img := page.Find(li, page.Sel{Tag: "img", Class: "avatar-user"}); img != nil { + a.AvatarURL = page.Attr(img, "src") + if id := avatarUserID(page.Attr(img, "src")); id > 0 { + a.DatabaseID = intp(id) + } + } + a.addSource(source) + return a, true +} + +// avatarUserID pulls the numeric account id out of an avatar URL. It is the +// only place a listing states it, and having it lets a record join to search +// results, which key on the same number. +func avatarUserID(src string) int { + _, rest, ok := strings.Cut(src, "/u/") + if !ok { + return 0 + } + digits, _, _ := strings.Cut(rest, "?") + n, err := strconv.Atoi(digits) + if err != nil { + return 0 + } + return n +} + +// Starred lists the repositories an account has starred. It is a different +// template from the repositories tab, so it gets its own row reader even though +// the two records are the same shape. +func (c *Client) Starred(ctx context.Context, login string, limit int, emit func(Repo) error) error { + return c.profileRepos(ctx, login, "stars", limit, emit) +} + +// ReposAsShown lists an account's repositories in the order and with the +// filters the profile tab itself uses. `github repos --user x` runs a search +// instead, which sorts and pages better; this is what --as-shown selects when +// the exact page order is the point. +func (c *Client) ReposAsShown(ctx context.Context, login string, limit int, emit func(Repo) error) error { + return c.profileRepos(ctx, login, "repositories", limit, emit) +} + +func (c *Client) profileRepos(ctx context.Context, login, tab string, limit int, emit func(Repo) error) error { + if login == "" || strings.Contains(login, "/") { + return usageBadID("account", login, "a bare login") + } + base := query(accountURL(login), "tab", tab) + fetch := func(ctx context.Context, token string) ([]Repo, string, error) { + u := base + if n := pageToken(token); n > 1 { + u = query(u, "page", strconv.Itoa(n)) + } + res, err := c.GetHTML(ctx, u) + if err != nil { + return nil, "", err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return nil, "", structureChanged(login + " " + tab) + } + var out []Repo + for _, h := range repoCardHeadings(doc) { + if r, ok := repoCard(h, res.FinalURL); ok { + out = append(out, r) + } + } + return out, railsNext(doc, token), nil + } + return paginate(ctx, limit, fetch, emit) +} + +// repoCardHeadings finds the heading of every repository card on a listing +// page. The repositories tab marks the name with microdata and the stars tab +// does not, so both hooks are tried and the results are kept in document order +// rather than merged, since no page uses both. +func repoCardHeadings(doc *html.Node) []*html.Node { + if named := page.FindAll(doc, page.Sel{Attr: "itemprop", AttrValue: "name codeRepository"}); len(named) > 0 { + return named + } + var out []*html.Node + for _, h := range page.FindAll(doc, page.Sel{Tag: "h3"}) { + if a := page.Find(h, page.Sel{Tag: "a", Attr: "href"}); a != nil { + if p := hrefPath(page.Attr(a, "href")); strings.Count(p, "/") == 1 { + out = append(out, a) + } + } + } + return out +} + +// repoCard reads a repository out of a listing card, walking up from the name +// link to the row that holds the rest of the fields. +func repoCard(nameLink *html.Node, source string) (Repo, bool) { + id := hrefPath(page.Attr(nameLink, "href")) + if strings.Count(id, "/") != 1 { + return Repo{}, false + } + owner, name, ok := SplitRepo(id) + if !ok { + return Repo{}, false + } + r := Repo{Owner: owner, Name: name} + r.setIdentity(KindRepo, id) + r.addSource(source) + + row := cardRow(nameLink) + if row == nil { + return r, true + } + if d := page.Find(row, page.Sel{Attr: "itemprop", AttrValue: "description"}); d != nil { + r.Description = page.Text(d) + } + if l := page.Find(row, page.Sel{Attr: "itemprop", AttrValue: "programmingLanguage"}); l != nil { + r.Language = page.Text(l) + } + if c := page.Find(row, page.Sel{Class: "repo-language-color"}); c != nil { + r.LanguageColor = styleColor(page.Attr(c, "style")) + } + for _, a := range page.FindAll(row, page.Sel{Tag: "a", Attr: "href"}) { + href := page.Attr(a, "href") + n, _, ok := page.ParseCompactCount(page.Text(a)) + if !ok { + continue + } + switch { + case strings.HasSuffix(href, "/stargazers"): + r.Stars = intp(n) + case strings.HasSuffix(href, "/forks"): + r.Forks = intp(n) + } + } + if t := page.Find(row, page.RelTimeEl); t != nil { + r.PushedAt = parseTime(page.Attr(t, "datetime")) + } + for _, tag := range page.FindAll(row, page.Sel{Class: "topic-tag"}) { + if s := page.Text(tag); s != "" { + r.Topics = append(r.Topics, s) + } + } + return r, true +} + +// cardRow walks up to the element that contains a whole listing card. Four +// levels is what separates the name link from the row on every template that +// uses one, and stopping there keeps a malformed page from handing back the +// document root and with it every field on it. +func cardRow(n *html.Node) *html.Node { + for i := 0; i < 4 && n != nil; i++ { + n = n.Parent + if n == nil { + return nil + } + if n.Type == html.ElementNode && (n.Data == "li" || page.HasClass(n, "col-12") || page.HasClass(n, "Box-row")) { + return n + } + } + return n +} + +// styleColor pulls a colour out of an inline style, which is where GitHub puts +// the language colour on every listing template. +func styleColor(style string) string { + _, rest, ok := strings.Cut(style, "background-color:") + if !ok { + return "" + } + v, _, _ := strings.Cut(rest, ";") + return strings.TrimSpace(v) +} + +// --- gists --- + +// Gists lists an account's public gists. +func (c *Client) Gists(ctx context.Context, login string, limit int, emit func(Gist) error) error { + if login == "" || strings.Contains(login, "/") { + return usageBadID("account", login, "a bare login") + } + base := GistURL + "/" + login + fetch := func(ctx context.Context, token string) ([]Gist, string, error) { + u := base + if n := pageToken(token); n > 1 { + u = query(u, "page", strconv.Itoa(n)) + } + res, err := c.GetHTML(ctx, u) + if err != nil { + return nil, "", err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return nil, "", structureChanged(login + " gists") + } + var out []Gist + for _, snip := range page.FindAll(doc, page.Sel{Class: "gist-snippet"}) { + if g, ok := gistSnippet(snip, res.FinalURL); ok { + out = append(out, g) + } + } + return out, railsNext(doc, token), nil + } + return paginate(ctx, limit, fetch, emit) +} + +// gistSnippet reads one entry of a gist index. The counts are in the text of +// the links beside it: "1 file", "6 forks", "62 stars". +func gistSnippet(snip *html.Node, source string) (Gist, bool) { + var id, owner string + for _, a := range page.FindAll(snip, page.Sel{Tag: "a", Attr: "href"}) { + p := hrefPath(page.Attr(a, "href")) + o, rest, ok := strings.Cut(p, "/") + if !ok { + continue + } + hex, _, _ := strings.Cut(rest, "/") + if isGistID(hex) { + owner, id = o, hex + break + } + } + if id == "" { + return Gist{}, false + } + g := Gist{Owner: owner, IsPublic: true} + g.setIdentity(KindGist, id) + g.addSource(source) + for _, a := range page.FindAll(snip, page.Sel{Tag: "a", Attr: "href"}) { + text := page.Text(a) + n, _, ok := page.CountIn(text) + if !ok { + continue + } + switch { + case strings.HasSuffix(text, "file"), strings.HasSuffix(text, "files"): + g.FileCount = intp(n) + case strings.HasSuffix(text, "fork"), strings.HasSuffix(text, "forks"): + g.Forks = intp(n) + case strings.HasSuffix(text, "star"), strings.HasSuffix(text, "stars"): + g.Stars = intp(n) + } + } + if d := page.Find(snip, page.Sel{Class: "gist-snippet-meta"}); d != nil { + if p := page.Find(d, page.Sel{Tag: "span", Class: "f6"}); p != nil { + g.Description = page.Text(p) + } + } + if t := page.Find(snip, page.RelTimeEl); t != nil { + g.UpdatedAt = parseTime(page.Attr(t, "datetime")) + } + return g, true +} + +// Gist reads one gist and its file list. Contents are a second request per file +// and are opt-in, because a gist can hold a megabyte of log paste. +func (c *Client) Gist(ctx context.Context, id string, withContent bool) (*Gist, error) { + id = strings.TrimSpace(id) + if i := strings.LastIndex(id, "/"); i >= 0 { + id = id[i+1:] + } + if !isGistID(id) { + return nil, usageBadID("gist", id, "a hexadecimal gist id") + } + res, err := c.GetHTML(ctx, GistURL+"/"+id) + if err != nil { + return nil, err + } + p := page.Extract(res.FinalURL, res.Body) + doc := p.Doc() + if doc == nil { + return nil, structureChanged(id) + } + g := &Gist{IsPublic: true} + g.setIdentity(KindGist, id) + g.addSource(res.FinalURL) + g.Owner = hrefOwner(res.FinalURL) + if d := page.Find(doc, page.Sel{Attr: "itemprop", AttrValue: "about"}); d != nil { + g.Description = page.Text(d) + } + if t := page.Find(doc, page.RelTimeEl); t != nil { + g.UpdatedAt = parseTime(page.Attr(t, "datetime")) + } + for _, box := range page.FindAll(doc, page.Sel{Class: "file"}) { + f, ok := gistFile(box) + if !ok { + continue + } + g.Files = append(g.Files, f) + } + if len(g.Files) == 0 { + return nil, structureChanged(id) + } + g.FileCount = intp(len(g.Files)) + if withContent { + for i := range g.Files { + text, err := c.text(ctx, g.Files[i].RawURL) + if err != nil { + return nil, err + } + g.Files[i].Content = text + } + } + return g, nil +} + +// gistFile reads one file block. The raw link is the useful half: it is the +// only address on the page that returns the bytes rather than the rendering. +func gistFile(box *html.Node) (GistFile, bool) { + name := page.Find(box, page.Sel{Class: "gist-blob-name"}) + if name == nil { + return GistFile{}, false + } + f := GistFile{Name: page.Text(name)} + if f.Name == "" { + return GistFile{}, false + } + for _, a := range page.FindAll(box, page.Sel{Tag: "a", Attr: "href", AttrContains: "/raw/"}) { + f.RawURL = absoluteGistURL(page.Attr(a, "href")) + break + } + if f.RawURL == "" { + return GistFile{}, false + } + if i := strings.LastIndex(f.Name, "."); i > 0 { + f.Language = f.Name[i+1:] + } + return f, true +} + +// text fetches a URL and returns it as a string. It is the small sibling of +// Raw, for the addresses that are already absolute. +func (c *Client) text(ctx context.Context, rawURL string) (string, error) { + res, err := c.Get(ctx, rawURL, SurfaceRaw) + if err != nil { + return "", err + } + return string(res.Body), nil +} + +func absoluteGistURL(href string) string { + if strings.Contains(href, "://") { + return href + } + return GistURL + "/" + strings.TrimPrefix(href, "/") +} + +func hrefOwner(rawURL string) string { + p := hrefPath(rawURL) + owner, rest, ok := strings.Cut(p, "/") + if !ok || !isGistID(rest) { + return "" + } + return owner +} + +// isGistID matches the twenty-or-more hexadecimal characters a gist is named +// with. Anything shorter is a login or a route word. +func isGistID(s string) bool { + if len(s) < 20 { + return false + } + for _, r := range s { + switch { + case r >= '0' && r <= '9', r >= 'a' && r <= 'f', r >= 'A' && r <= 'F': + default: + return false + } + } + return true +} + +// --- the contribution calendar --- + +// Contributions reads a year of a profile's contribution graph, one record per +// day. This is the only representation of the numbers that exists without a +// token: the GraphQL field that carries them refuses anonymous callers. +// +// A year is the largest window the fragment serves. Asking for a wider range +// gets the last year, so the range is stated rather than inferred. +func (c *Client) Contributions(ctx context.Context, login string, year int, emit func(ContributionDay) error) error { + if login == "" || strings.Contains(login, "/") { + return usageBadID("account", login, "a bare login") + } + if year == 0 { + year = time.Now().UTC().Year() + } + from := strconv.Itoa(year) + "-01-01" + to := strconv.Itoa(year) + "-12-31" + u := query(BaseURL+"/users/"+login+"/contributions", "from", from, "to", to) + // HTML rather than XHR. This is a fragment the front end swaps into the + // profile, so it serves markup and answers 406 to a request that asks for + // JSON, which the client reports as a response rather than an error and + // would show up here as an empty calendar. + res, err := c.Get(ctx, u, SurfaceHTML) + if err != nil { + return err + } + doc := page.Extract(res.FinalURL, res.Body).Doc() + if doc == nil { + return structureChanged(login + " contributions") + } + + // The count is not on the cell. Each cell points at a tooltip by id and the + // tooltip holds the sentence with the number in it, so the tooltips are + // indexed first and the cells read against that index. + counts := map[string]int{} + for _, tip := range page.FindAll(doc, page.Sel{Tag: "tool-tip"}) { + counts[page.Attr(tip, "for")] = leadingCount(page.Text(tip)) + } + found := false + for _, td := range page.FindAll(doc, page.Sel{Tag: "td", Attr: "data-date"}) { + day := ContributionDay{Login: login} + at := parseTime(page.Attr(td, "data-date")) + if at == nil { + continue + } + found = true + day.Date = *at + day.Level, _ = strconv.Atoi(page.Attr(td, "data-level")) + day.Count = counts[page.Attr(td, "id")] + day.Kind = KindContribution + day.ID = login + "@" + page.Attr(td, "data-date") + day.URI = URI(KindContribution, day.ID) + day.URL = accountURL(login) + day.addSource(res.FinalURL) + if err := emit(day); err != nil { + return err + } + } + if !found { + return structureChanged(login + " contributions") + } + return nil +} + +// leadingCount reads the number off the front of "7 contributions on January +// 4th." and treats "No contributions" as the zero it is. +func leadingCount(s string) int { + field, _, _ := strings.Cut(strings.TrimSpace(s), " ") + n, err := strconv.Atoi(strings.ReplaceAll(field, ",", "")) + if err != nil { + return 0 + } + return n +} + +// --- activity --- + +// Activity reads a public event stream. The same feed shape serves an account +// and a repository, so the argument is either a login or owner/name and the +// URL is the only thing that differs. +// +// This replaces the REST events endpoint outright. The feed is public, cheap, +// and needs no credential, and the event class is encoded in each entry's id, +// so the type comes from a field rather than from matching on prose. +func (c *Client) Activity(ctx context.Context, ref string, limit int, emit func(Event) error) error { + ref = strings.Trim(ref, "/") + if ref == "" { + return usageBadID("account or repository", ref, "a login or owner/name") + } + u := feedURL(ref + ".atom") + if _, _, ok := SplitRepo(ref); ok { + u = repoSubURL(ref, "commits.atom") + } + res, err := c.Get(ctx, u, SurfaceFeed) + if err != nil { + return err + } + var feed atomFeed + if err := xml.Unmarshal(res.Body, &feed); err != nil { + return badPayload(shortURL(u), err) + } + seen := 0 + for _, e := range feed.Entries { + ev := e.event(res.FinalURL) + if err := emit(ev); err != nil { + return err + } + seen++ + if limit > 0 && seen >= limit { + return nil + } + } + if seen == 0 { + return errs.NotFound("empty feed: %s carried no entries", shortURL(u)) + } + return nil +} + +// atomFeed is the shape all five of GitHub's feeds share. Only the id encoding +// differs between them, and that is read per entry rather than per feed. +type atomFeed struct { + Title string `xml:"title"` + Updated string `xml:"updated"` + Entries []atomEntry `xml:"entry"` +} + +type atomEntry struct { + ID string `xml:"id"` + Title string `xml:"title"` + Published string `xml:"published"` + Updated string `xml:"updated"` + Content string `xml:"content"` + Link struct { + Href string `xml:"href,attr"` + } `xml:"link"` + Author struct { + Name string `xml:"name"` + URI string `xml:"uri"` + } `xml:"author"` + Thumbnail struct { + URL string `xml:"url,attr"` + } `xml:"thumbnail"` +} + +// event turns one entry into a record. The id is +// "tag:github.com,2008:push/15757005823", so the segment after the colon is the +// event class and the tool never has to read the localised title to find out +// what happened. +func (e atomEntry) event(source string) Event { + ev := Event{Title: page.FragmentText(e.Title)} + _, tail, _ := strings.Cut(e.ID, "2008:") + class, rest, _ := strings.Cut(tail, "/") + ev.Type = eventType(class) + ev.Kind = KindEvent + ev.ID = e.ID + if ev.Type != "" && rest != "" { + ev.ID = ev.Type + "/" + rest + } + ev.URI = URI(KindEvent, ev.ID) + ev.URL = e.Link.Href + ev.Target = e.Link.Href + ev.At = firstTime(e.Published, e.Updated) + ev.BodyHTML = e.Content + if e.Author.Name != "" { + ev.Actor = actor(e.Author.Name) + ev.Actor.AvatarURL = e.Thumbnail.URL + } + // The alternate link points at whatever the event touched, and the first + // two segments of it are the repository whenever there is one. + if p := hrefPath(e.Link.Href); strings.Count(p, "/") >= 1 { + owner, rest, _ := strings.Cut(p, "/") + name, _, _ := strings.Cut(rest, "/") + if owner != "" && name != "" && !routeWord[name] { + ev.Repo = owner + "/" + name + } + } + ev.addSource(source) + return ev +} + +// eventType turns the class out of the entry id into one word. +// +// A person's feed names the class in lower case, push or fork or watch. A +// repository's commit feed names it after the Ruby object that used to render +// it, Grit::Commit, which is an implementation detail from 2008 and not a thing +// anyone should have to filter on. +func eventType(class string) string { + if _, tail, ok := strings.Cut(class, "::"); ok { + class = tail + } + return strings.ToLower(class) +} + +func firstTime(ss ...string) *time.Time { + for _, s := range ss { + if t := parseTime(s); t != nil { + return t + } + } + return nil +} + +// --- the rails pager --- + +// nextPageToken returns the page number to ask for next, or empty when the page +// says there is no next. +// +// Two templates write the same link two ways: the organization roster marks it +// rel="next" and the profile tabs use a plain anchor whose text is Next. Both +// are checked because both are load-bearing. +func railsNext(doc *html.Node, token string) string { + if !hasNextLink(doc) { + return "" + } + return strconv.Itoa(pageToken(token) + 1) +} + +func hasNextLink(doc *html.Node) bool { + if page.Find(doc, page.NextPage) != nil { + return true + } + for _, a := range page.FindAll(doc, page.Sel{Tag: "a", Attr: "href"}) { + if strings.EqualFold(strings.TrimSpace(page.Text(a)), "next") { + return true + } + } + return false +} diff --git a/gh/rdf.go b/gh/rdf.go new file mode 100644 index 0000000..cda8f99 --- /dev/null +++ b/gh/rdf.go @@ -0,0 +1,617 @@ +package gh + +import ( + "encoding/json" + "fmt" + "io" + "strconv" + "strings" + "time" +) + +// rdf.go serialises the graph. The graph plane is already triples, so this is a +// serialisation and not a transformation. +// +// The one real decision here is the schema.org alignment. A consumer that has +// never heard of gh:forkOf still understands schema:author and schema:isPartOf, +// and using the standard term where one exists is what lets a github export and +// an hf export join in the same triple store. +// +// Subject IRIs are the canonical github.com URLs rather than the github:// URIs. +// A triple whose subject is https://github.com/golang/go is dereferenceable by +// anything on the web; one whose subject is github://repo/golang/go is +// dereferenceable only by this tool. The github:// form survives as a gh:uri +// literal so nothing is lost. + +// The namespaces. +const ( + NSSchema = "https://schema.org/" + NSGH = "https://github.com/ns#" + NSGHR = "https://github.com/" + NSRdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#" + NSRdfs = "http://www.w3.org/2000/01/rdf-schema#" + NSXsd = "http://www.w3.org/2001/XMLSchema#" + NSDoap = "http://usefulinc.com/ns/doap#" + NSFoaf = "http://xmlns.com/foaf/0.1/" +) + +var rdfPrefixes = [][2]string{ + {"schema", NSSchema}, + {"gh", NSGH}, + {"ghr", NSGHR}, + {"rdf", NSRdf}, + {"rdfs", NSRdfs}, + {"xsd", NSXsd}, + {"doap", NSDoap}, + {"foaf", NSFoaf}, +} + +// The datatypes a Fact can carry. They are CURIEs so a Fact reads the same in +// every serialisation. +const ( + TypeInteger = "xsd:integer" + TypeDecimal = "xsd:decimal" + TypeBoolean = "xsd:boolean" + TypeDateTime = "xsd:dateTime" +) + +// rdfTypes is the class mapping. Marking an issue as schema:DiscussionForumPosting +// is not this tool's invention: it is what GitHub's own structured_data block +// says, and following the publisher's vocabulary for its own content is the +// whole point. +var rdfTypes = map[string][]string{ + KindRepo: {"schema:SoftwareSourceCode", "doap:Project"}, + KindUser: {"schema:Person", "foaf:Person"}, + KindOrg: {"schema:Organization"}, + KindIssue: {"gh:Issue", "schema:DiscussionForumPosting"}, + KindPR: {"gh:PullRequest", "schema:DiscussionForumPosting"}, + KindDiscussion: {"schema:DiscussionForumPosting"}, + KindCommit: {"gh:Commit"}, + KindRelease: {"schema:SoftwareApplication"}, + KindTag: {"gh:Ref"}, + KindBranch: {"gh:Ref"}, + KindFile: {"schema:MediaObject"}, + KindTree: {"schema:MediaObject"}, + KindTopic: {"schema:DefinedTerm"}, + KindLabel: {"schema:DefinedTerm"}, + KindPackage: {"schema:SoftwareApplication"}, + KindGist: {"schema:SoftwareSourceCode"}, + KindAction: {"schema:SoftwareApplication"}, + KindWiki: {"schema:Article"}, +} + +// rdfPredicates maps this tool's vocabulary onto RDF terms. A predicate with no +// entry is emitted in the gh: namespace under its own name, which is what makes +// adding a predicate to graph.go a one-line change rather than two. +var rdfPredicates = map[string]string{ + PredOwnedBy: "schema:author", + PredAuthoredBy: "schema:author", + PredPartOf: "schema:isPartOf", + PredMemberOf: "schema:memberOf", + PredHasTopic: "schema:keywords", + PredHasLabel: "schema:keywords", + PredWrittenIn: "schema:programmingLanguage", + PredLicensedUnder: "schema:license", + PredReferences: "schema:citation", + PredFollows: "schema:follows", +} + +// rdfFacts maps the literal predicates. The counts get gh: terms because +// schema.org has no stargazer count, and the dates get the standard ones +// because it does. +var rdfFacts = map[string]string{ + FactName: "schema:name", + FactDescription: "schema:description", + FactHomepage: "schema:url", + FactCreated: "schema:dateCreated", + FactUpdated: "schema:dateModified", + FactStars: "gh:stargazerCount", + FactForks: "gh:forkCount", + FactWatchers: "gh:watcherCount", + FactCommits: "gh:commitCount", + FactURI: "gh:uri", + FactAvatar: "schema:image", +} + +// The output formats. +const ( + FormatNT = "nt" + FormatNQuads = "nq" + FormatTurtle = "ttl" + FormatJSONLD = "jsonld" +) + +// RDFFormats is the accepted set, for help text and validation. +var RDFFormats = []string{FormatNT, FormatNQuads, FormatTurtle, FormatJSONLD} + +// RDFOptions controls a serialisation. +type RDFOptions struct { + Format string + // Graph is the fourth position for N-Quads. Putting the source URL there + // means the provenance survives into the RDF and a quad store can answer + // which page told us this. + Graph string +} + +// WriteRDF serialises a whole graph. N-Triples is the default because it +// streams: nt and nq write a line per triple as it is produced, ttl buffers one +// subject at a time, and jsonld buffers the lot. +func WriteRDF(w io.Writer, g *Graph, o RDFOptions) error { + switch o.Format { + case "", FormatNT: + return writeTriples(w, g, "") + case FormatNQuads: + return writeTriples(w, g, o.Graph) + case FormatTurtle: + return writeTurtle(w, g) + case FormatJSONLD: + return writeJSONLD(w, g) + default: + return fmt.Errorf("unknown rdf format %q, want one of %s", o.Format, strings.Join(RDFFormats, ", ")) + } +} + +// --- streaming --- + +// RDFWriter is the streaming form. A crawl hands it nodes, edges, and facts as +// it finds them and it writes lines, so `github export --depth 3 --format nt` +// over a large organization never holds the graph in memory. The buffered +// formats are handled by collecting into a Graph and calling WriteRDF, and this +// type reports which is which through Streams. +type RDFWriter struct { + w io.Writer + suffix string +} + +// NewRDFWriter returns a streaming writer for nt or nq, and nil for the formats +// that cannot stream. +func NewRDFWriter(w io.Writer, o RDFOptions) *RDFWriter { + switch o.Format { + case "", FormatNT: + return &RDFWriter{w: w, suffix: " .\n"} + case FormatNQuads: + suffix := " .\n" + if o.Graph != "" { + suffix = " <" + o.Graph + "> .\n" + } + return &RDFWriter{w: w, suffix: suffix} + } + return nil +} + +// Streams reports whether a format can be written a triple at a time. +func Streams(format string) bool { + return format == "" || format == FormatNT || format == FormatNQuads +} + +func (r *RDFWriter) Node(n *Node) error { return r.lines(nodeLines(*n)) } + +func (r *RDFWriter) Edge(e *Edge) error { return r.lines(edgeLines(*e)) } + +func (r *RDFWriter) Fact(f *Fact) error { return r.lines(factLines(*f)) } + +func (r *RDFWriter) lines(ls []string) error { + for _, l := range ls { + if _, err := io.WriteString(r.w, l+r.suffix); err != nil { + return err + } + } + return nil +} + +func writeTriples(w io.Writer, g *Graph, graph string) error { + r := NewRDFWriter(w, RDFOptions{Format: FormatNQuads, Graph: graph}) + for i := range g.Nodes { + if err := r.Node(&g.Nodes[i]); err != nil { + return err + } + } + for i := range g.Edges { + if err := r.Edge(&g.Edges[i]); err != nil { + return err + } + } + for i := range g.Facts { + if err := r.Fact(&g.Facts[i]); err != nil { + return err + } + } + return nil +} + +// nodeLines states a node's classes and its label. +func nodeLines(n Node) []string { + subj := iri(n.URI) + var out []string + for _, t := range rdfTypes[n.Kind] { + out = append(out, subj+" <"+NSRdf+"type> "+expand(t)) + } + if n.Label != "" { + out = append(out, subj+" "+expand("rdfs:label")+" "+quote(n.Label)) + } + return out +} + +// edgeLines renders one edge. +// +// A weighted edge is reified: a contributor's commit count is a property of the +// relation and not of either end, and the only honest way to say that in RDF is +// to give the relation a node of its own. +func edgeLines(e Edge) []string { + subj := iri(e.Subject) + obj := objectTerm(e.Predicate, e.Object) + out := []string{subj + " " + expand(rdfPredicate(e.Predicate)) + " " + obj} + if e.Weight != nil { + blank := reifiedNode(e) + out = append(out, + blank+" <"+NSRdf+"subject> "+subj, + blank+" <"+NSRdf+"predicate> "+expand(rdfPredicate(e.Predicate)), + blank+" <"+NSRdf+"object> "+obj, + blank+" "+expand(weightTerm(e.Predicate))+" "+quote(strconv.Itoa(*e.Weight))+"^^"+expand(TypeInteger), + ) + } + if e.At != nil { + out = append(out, reifiedNode(e)+" "+expand("schema:dateCreated")+" "+ + quote(e.At.UTC().Format(time.RFC3339))+"^^"+expand(TypeDateTime)) + } + return out +} + +// weightTerm names what a weight counts. Only contributedTo and reactedWith +// carry one, and calling both of them "count" would throw away the only thing +// that makes the number readable. +func weightTerm(pred string) string { + switch pred { + case PredContributedTo: + return "gh:commitCount" + case PredReactedWith: + return "gh:reactionCount" + default: + return "gh:count" + } +} + +// reifiedNode names the statement itself. The name is derived from the triple, +// so two runs produce the same node and a merge of two exports does not +// duplicate it. +func reifiedNode(e Edge) string { + key := e.Subject + "|" + e.Predicate + "|" + e.Object + return "_:stmt-" + strings.NewReplacer("://", "-", "/", "-", "|", "-", "#", "-", "@", "-", " ", "_").Replace(key) +} + +func factLines(f Fact) []string { + pred, ok := rdfFacts[f.Predicate] + if !ok { + pred = "gh:" + f.Predicate + } + obj := quote(f.Value) + if f.Datatype != "" { + obj += "^^" + expand(f.Datatype) + } + return []string{iri(f.Subject) + " " + expand(pred) + " " + obj} +} + +// rdfPredicate maps a predicate onto its RDF term, defaulting to the gh: +// namespace so a new predicate needs no entry to serialise correctly. +func rdfPredicate(pred string) string { + if p, ok := rdfPredicates[pred]; ok { + return p + } + return "gh:" + pred +} + +// objectTerm renders an edge's object. Most objects are URIs. A language and a +// licence are bare strings on the record plane, and they get synthetic IRIs +// here rather than becoming string literals, because gh:language/go is +// something two exports can join on and "Go" is not. +func objectTerm(pred, object string) string { + if strings.HasPrefix(object, Scheme+"://") { + return iri(object) + } + switch pred { + case PredWrittenIn: + return "<" + NSGH + "language/" + slug(object) + ">" + case PredLicensedUnder: + return "<" + NSGH + "license/" + slug(object) + ">" + case PredReactedWith: + return "<" + NSGH + "reaction/" + slug(object) + ">" + } + return quote(object) +} + +// slug makes a URI path segment out of a rendered name. Spaces and slashes are +// the only characters that actually occur here, in names like "Jupyter Notebook" +// and "BSD 3-Clause", and both have to go. +func slug(s string) string { + s = strings.TrimSpace(s) + var b strings.Builder + for _, r := range s { + switch { + case r == ' ' || r == '/' || r == '\\': + b.WriteByte('-') + case r == '<' || r == '>' || r == '"' || r == '{' || r == '}' || r == '|' || r == '^' || r == '`': + // Characters an IRI may not carry. Dropping them beats escaping + // them, because nobody wants gh:license/BSD%203-Clause. + default: + b.WriteRune(r) + } + } + return b.String() +} + +// iri renders a subject or object. A blank node stays a blank node, and a +// github:// URI becomes the canonical https URL. +func iri(uri string) string { + if strings.HasPrefix(uri, "_:") { + return uri + } + return "<" + IRI(uri) + ">" +} + +// IRI maps a github:// URI to its dereferenceable https form. The github:// form +// stays on the record plane, where it is a stable key rather than a location. +// +// The three derived kinds have no address of their own, so they map into the +// gh: namespace instead of pretending to be a page. +func IRI(uri string) string { + if !strings.HasPrefix(uri, Scheme+"://") { + return uri + } + rest := strings.TrimPrefix(uri, Scheme+"://") + kind, id, ok := strings.Cut(rest, "/") + if !ok { + return uri + } + if u, err := Locate(kind, id); err == nil { + return u + } + return NSGH + kind + "/" + slug(id) +} + +func expand(curie string) string { + prefix, rest, ok := strings.Cut(curie, ":") + if !ok { + return "<" + curie + ">" + } + for _, p := range rdfPrefixes { + if p[0] == prefix { + return "<" + p[1] + rest + ">" + } + } + return "<" + curie + ">" +} + +func quote(s string) string { + var b strings.Builder + b.WriteByte('"') + for _, r := range s { + switch r { + case '"': + b.WriteString(`\"`) + case '\\': + b.WriteString(`\\`) + case '\n': + b.WriteString(`\n`) + case '\r': + b.WriteString(`\r`) + case '\t': + b.WriteString(`\t`) + default: + b.WriteRune(r) + } + } + b.WriteByte('"') + return b.String() +} + +// --- turtle --- + +// writeTurtle groups by subject, which is the whole reason to prefer Turtle: a +// node and everything said about it read as one paragraph. +func writeTurtle(w io.Writer, g *Graph) error { + for _, p := range rdfPrefixes { + if _, err := fmt.Fprintf(w, "@prefix %s: <%s> .\n", p[0], p[1]); err != nil { + return err + } + } + if _, err := io.WriteString(w, "\n"); err != nil { + return err + } + + bySubject := map[string][][2]string{} + var order []string + add := func(subj, pred, obj string) { + if _, seen := bySubject[subj]; !seen { + order = append(order, subj) + } + bySubject[subj] = append(bySubject[subj], [2]string{pred, obj}) + } + // The line renderers already produce N-Triples, and Turtle is the same + // triples with the subject factored out, so this splits each line rather + // than growing a second renderer that could disagree with the first. + collect := func(lines []string) { + for _, l := range lines { + subj, pred, obj, ok := splitTriple(l) + if !ok { + continue + } + add(subj, shorten(pred), shorten(obj)) + } + } + for _, n := range g.Nodes { + collect(nodeLines(n)) + } + for _, e := range g.Edges { + collect(edgeLines(e)) + } + for _, f := range g.Facts { + collect(factLines(f)) + } + + for _, subj := range order { + if _, err := io.WriteString(w, shorten(subj)+"\n"); err != nil { + return err + } + pairs := bySubject[subj] + for i, pair := range pairs { + end := " ;\n" + if i == len(pairs)-1 { + end = " .\n\n" + } + if _, err := io.WriteString(w, " "+pair[0]+" "+pair[1]+end); err != nil { + return err + } + } + } + return nil +} + +// splitTriple pulls a rendered N-Triples line apart. The grammar is regular +// enough for this: the subject and the predicate are always angle-bracketed or +// blank-node terms with no spaces in them, and everything after the second +// space is the object. +func splitTriple(line string) (subj, pred, obj string, ok bool) { + subj, rest, ok := strings.Cut(line, " ") + if !ok { + return "", "", "", false + } + pred, obj, ok = strings.Cut(rest, " ") + if !ok { + return "", "", "", false + } + return subj, pred, obj, true +} + +// shorten turns an expanded IRI back into a CURIE where a prefix covers it, +// which is what makes Turtle readable rather than just grouped. +func shorten(term string) string { + if !strings.HasPrefix(term, "<") { + // A literal, possibly with a datatype that is itself an IRI. + if i := strings.Index(term, "^^<"); i >= 0 { + return term[:i+2] + shorten(term[i+2:]) + } + return term + } + full := strings.TrimSuffix(strings.TrimPrefix(term, "<"), ">") + if full == NSRdf+"type" { + return "a" + } + for _, p := range rdfPrefixes { + // ghr is the whole of github.com, so every subject IRI would collapse + // into it and read as ghr:golang/go, which is not a legal CURIE local + // name once a path has slashes in it. Subjects stay in angle brackets. + if p[0] == "ghr" { + continue + } + if rest, found := strings.CutPrefix(full, p[1]); found && rest != "" && !strings.ContainsAny(rest, "/") { + return p[0] + ":" + rest + } + } + return term +} + +// --- json-ld --- + +// writeJSONLD emits one object per node with its edges and facts folded in, and +// an inline context so the document stands alone. It buffers everything, which +// is why the help text points at it for single records rather than crawls. +func writeJSONLD(w io.Writer, g *Graph) error { + ctx := map[string]any{} + for _, p := range rdfPrefixes { + ctx[p[0]] = p[1] + } + + byURI := map[string]map[string]any{} + var order []string + obj := func(uri string) map[string]any { + o, ok := byURI[uri] + if !ok { + o = map[string]any{"@id": IRI(uri)} + byURI[uri] = o + order = append(order, uri) + } + return o + } + for _, n := range g.Nodes { + o := obj(n.URI) + if types := rdfTypes[n.Kind]; len(types) > 0 { + o["@type"] = types + } + if n.Label != "" { + o["rdfs:label"] = n.Label + } + if n.URL != "" { + o["schema:url"] = n.URL + } + } + for _, e := range g.Edges { + o := obj(e.Subject) + var value any + if strings.HasPrefix(e.Object, Scheme+"://") { + value = map[string]any{"@id": IRI(e.Object)} + } else if term := objectTerm(e.Predicate, e.Object); strings.HasPrefix(term, "<") { + value = map[string]any{"@id": strings.TrimSuffix(strings.TrimPrefix(term, "<"), ">")} + } else { + value = e.Object + } + if e.Weight != nil { + value = map[string]any{"@id": jsonldID(value), weightTerm(e.Predicate): *e.Weight} + } + appendValue(o, rdfPredicate(e.Predicate), value) + } + for _, f := range g.Facts { + pred, ok := rdfFacts[f.Predicate] + if !ok { + pred = "gh:" + f.Predicate + } + appendValue(obj(f.Subject), pred, jsonldLiteral(f)) + } + + graph := make([]map[string]any, 0, len(order)) + for _, uri := range order { + graph = append(graph, byURI[uri]) + } + enc := json.NewEncoder(w) + enc.SetIndent("", " ") + return enc.Encode(map[string]any{"@context": ctx, "@graph": graph}) +} + +func jsonldID(v any) any { + if m, ok := v.(map[string]any); ok { + return m["@id"] + } + return v +} + +// appendValue keeps repeated predicates as a list rather than letting the last +// one win, because a repository with twelve topics has twelve of them. +func appendValue(o map[string]any, pred string, value any) { + switch cur := o[pred].(type) { + case nil: + o[pred] = value + case []any: + o[pred] = append(cur, value) + default: + o[pred] = []any{cur, value} + } +} + +// jsonldLiteral gives a value its type, so a count arrives as a number and a +// timestamp as a typed value rather than as prose. +func jsonldLiteral(f Fact) any { + switch f.Datatype { + case TypeInteger: + if n, err := strconv.ParseInt(f.Value, 10, 64); err == nil { + return n + } + case TypeDecimal: + if v, err := strconv.ParseFloat(f.Value, 64); err == nil { + return v + } + case TypeBoolean: + return f.Value == "true" + case TypeDateTime: + return map[string]any{"@value": f.Value, "@type": TypeDateTime} + } + return f.Value +} diff --git a/gh/repo.go b/gh/repo.go new file mode 100644 index 0000000..51ae7ac --- /dev/null +++ b/gh/repo.go @@ -0,0 +1,604 @@ +package gh + +import ( + "context" + "encoding/json" + "strconv" + "strings" + + "golang.org/x/net/html" + + "github.com/tamnd/github-cli/pkg/page" +) + +// repo.go reads a repository. +// +// It reads HTML rather than the JSON route, and that is not laziness. The JSON +// route returns the file tree and the ids and nothing else; description, +// homepage, topics, stars, watchers, forks, licence, and the release and tag +// counts live in payload.sidebarAbout, which only ever ships inside the HTML +// document. One request gets all of it. +// +// Merge order, later sources overwriting earlier ones field by field and only +// where they actually said something: +// +// 1. codeViewLayoutRoute.repo ids and structure +// 2. sidebarAbout counts and metadata +// 3. codeViewRepoRoute head SHA, tree, commit count, README +// 4. the DOM licence and the language bar +// 5. deferred fragments languages and dependents, --deep only + +// RepoOptions controls how much a repository read costs. +type RepoOptions struct { + // Deep runs the fragment fetches: the language histogram and the dependent + // count. Four to six requests instead of one. + Deep bool + // Readme keeps the rendered README, which is most of the response body on a + // well-documented repository. + Readme bool +} + +// Repo reads one repository. id is owner/name. +func (c *Client) Repo(ctx context.Context, id string, opts RepoOptions) (*Repo, error) { + owner, name, ok := SplitRepo(id) + if !ok { + return nil, usageBadID("repository", id, "owner/name") + } + res, err := c.GetHTML(ctx, repoURL(id)) + if err != nil { + return nil, err + } + p := page.Extract(res.FinalURL, res.Body) + + r := &Repo{Owner: owner, Name: name} + r.setIdentity(KindRepo, id) + r.addSource(res.FinalURL) + if p.Canonical != "" { + // Renames 301, and the payload then reports the new name while the + // caller asked for the old one. The canonical link is the authority. + r.URL = p.Canonical + } + + found := false + if raw, ok := p.Route("codeViewLayoutRoute"); ok { + found = r.readLayoutRoute(raw) || found + } + if raw, ok := p.Payload["sidebarAbout"]; ok { + found = r.readSidebar(raw) || found + } + if raw, ok := p.Route("codeViewRepoRoute"); ok { + found = r.readRepoRoute(raw, opts.Readme) || found + } + if !found { + return nil, structureChanged(id) + } + r.readDOM(p) + + // The language is worth a second request. It is one of the fields people + // most expect on a repository record, the page carries it only in a bar + // that is a loading skeleton on a cold fetch, and search answers it in one + // hop. This runs only when the page did not already say. + if r.Language == "" { + if lang, color, err := c.searchLanguage(ctx, id); err == nil && lang != "" { + r.Language = lang + r.LanguageColor = color + recordVia(&r.Base, "language", "search") + } + } + + if opts.Deep { + if err := c.deepenRepo(ctx, r); err != nil { + return nil, err + } + } + return r, nil +} + +// --- payload blocks --- + +type layoutRepo struct { + ID *int `json:"id"` + Name string `json:"name"` + OwnerLogin string `json:"ownerLogin"` + DefaultBranch string `json:"defaultBranch"` + CreatedAt string `json:"createdAt"` + IsFork bool `json:"isFork"` + IsEmpty bool `json:"isEmpty"` + Private bool `json:"private"` + IsOrgOwned bool `json:"isOrgOwned"` + OwnerAvatar string `json:"ownerAvatar"` +} + +func (r *Repo) readLayoutRoute(raw json.RawMessage) bool { + var v struct { + Repo json.RawMessage `json:"repo"` + } + if err := json.Unmarshal(raw, &v); err != nil || len(v.Repo) == 0 { + return false + } + var lr layoutRepo + if err := json.Unmarshal(v.Repo, &lr); err != nil { + return false + } + r.DatabaseID = lr.ID + if lr.Name != "" { + r.Name = lr.Name + } + if lr.OwnerLogin != "" { + r.Owner = lr.OwnerLogin + } + r.DefaultBranch = lr.DefaultBranch + r.CreatedAt = parseTime(lr.CreatedAt) + r.IsFork = lr.IsFork + r.IsEmpty = lr.IsEmpty + r.IsPrivate = lr.Private + r.IsOrgOwned = lr.IsOrgOwned + r.OwnerAvatarURL = lr.OwnerAvatar + + r.addExtra("codeViewLayoutRoute.repo", decodeExtra(v.Repo, &lr, + // The inverse of private, and both always arrive. + "public", + // Viewer permissions, uniformly false without a session. + "currentUserCanPush", "currentUserCanFork", "currentUserIsOwner", + )) + return true +} + +type sidebarAbout struct { + Description string `json:"description"` + Website string `json:"website"` + StargazerCount *int `json:"stargazerCount"` + WatcherCount *int `json:"watcherCount"` + ForksCount *int `json:"forksCount"` + StargazersPath string `json:"stargazersPath"` + ForkNetworkPath string `json:"forkNetworkPath"` + ActivityPath string `json:"activityPath"` + OwnerLogin string `json:"ownerLogin"` + RepoName string `json:"repoName"` + IsOrg bool `json:"isOrg"` + HasCitation bool `json:"hasCitation"` + + Topics []struct { + Name string `json:"name"` + } `json:"topics"` + + // Sections is deliberately untyped. Most of its members are a plain bool + // meaning "this box is on the page", but a few are objects carrying the + // counts, and which is which changes with what the repository has. A typed + // struct here loses the whole block the moment one member arrives as true + // instead of {}, so each member is decoded on its own below and a member + // that does not fit costs only that member. + Sections map[string]json.RawMessage `json:"sections"` +} + +// releasesSection and usedBySection are the two members that carry numbers. +type releasesSection struct { + ReleaseCount *int `json:"releaseCount"` + TagCount *int `json:"tagCount"` +} + +type usedBySection struct { + DependentsCount *int `json:"dependentsCount"` +} + +// section decodes one member of sidebarAbout.sections, and returns false for +// the bool form rather than treating it as a failure. +func section[T any](m map[string]json.RawMessage, name string) (T, bool) { + var out T + raw, ok := m[name] + if !ok || len(raw) == 0 || raw[0] != '{' { + return out, false + } + if err := json.Unmarshal(raw, &out); err != nil { + return out, false + } + return out, true +} + +func (r *Repo) readSidebar(raw json.RawMessage) bool { + var sa sidebarAbout + if err := json.Unmarshal(raw, &sa); err != nil { + return false + } + r.Description = sa.Description + r.Homepage = sa.Website + r.Stars = sa.StargazerCount + r.Watchers = sa.WatcherCount + r.Forks = sa.ForksCount + r.StargazersPath = sa.StargazersPath + r.ForkNetworkPath = sa.ForkNetworkPath + r.ActivityPath = sa.ActivityPath + r.HasCitation = sa.HasCitation + r.IsOrgOwned = r.IsOrgOwned || sa.IsOrg + if rel, ok := section[releasesSection](sa.Sections, "releases"); ok { + r.ReleaseCount = rel.ReleaseCount + r.TagCount = rel.TagCount + } + if used, ok := section[usedBySection](sa.Sections, "usedBy"); ok && used.DependentsCount != nil { + r.DependentCount = used.DependentsCount + } + for _, t := range sa.Topics { + if t.Name != "" { + r.Topics = append(r.Topics, t.Name) + } + } + if langs := decodeLanguages(sa.Sections["languages"]); len(langs) > 0 { + r.Languages = langs + r.Language = topLanguage(langs) + } + + r.addExtra("sidebarAbout", decodeExtra(raw, &sa, + // The description again with emoji shortcodes expanded. The raw form is + // what the record carries. + "formattedDescription", + // UI state and viewer permissions. + "showInsights", "canEditMetadata", "canEditTopics", + // Routes to pages with nothing public on them. + "reportPath", "customPropertiesPath", "watchersPath", + )) + return true +} + +type repoRoute struct { + Path string `json:"path"` + RefInfo struct { + Name string `json:"name"` + RefType string `json:"refType"` + CurrentOid string `json:"currentOid"` + } `json:"refInfo"` + Tree struct { + Items []treeItem `json:"items"` + TotalCount *int `json:"totalCount"` + } `json:"tree"` + Overview struct { + CommitCount string `json:"commitCount"` + OverviewFiles []struct { + DisplayName string `json:"displayName"` + Path string `json:"path"` + PreferredFileType string `json:"preferredFileType"` + RichText string `json:"richText"` + } `json:"overviewFiles"` + } `json:"overview"` +} + +type treeItem struct { + Name string `json:"name"` + Path string `json:"path"` + ContentType string `json:"contentType"` +} + +func (r *Repo) readRepoRoute(raw json.RawMessage, keepReadme bool) bool { + var rr repoRoute + if err := json.Unmarshal(raw, &rr); err != nil { + return false + } + r.HeadSHA = rr.RefInfo.CurrentOid + if r.DefaultBranch == "" { + r.DefaultBranch = rr.RefInfo.Name + } + r.FileCount = rr.Tree.TotalCount + + ref := firstNonEmpty(rr.RefInfo.Name, r.DefaultBranch) + for _, it := range rr.Tree.Items { + r.Tree = append(r.Tree, newTreeEntry(r.ID, ref, it)) + } + + // "8,112" in one locale and "8.112" in another, for the same number. Both + // forms are kept so that the ambiguity stays visible. + if n, display, ok := page.ParseCompactCount(rr.Overview.CommitCount); ok { + r.CommitCount = intp(n) + r.CommitCountDisplay = display + } + + for _, f := range rr.Overview.OverviewFiles { + if f.PreferredFileType != "readme" { + continue + } + r.ReadmePath = f.Path + if keepReadme { + r.ReadmeHTML = f.RichText + // The payload has no plain-text form, and a README is the one + // field most consumers want as prose rather than as markup. + r.ReadmeText = page.FragmentText(f.RichText) + } + break + } + + r.addExtra("codeViewRepoRoute", decodeExtra(raw, &rr, + // Chrome: which panels the front end opens, which buttons it draws. + "banners", "codeButton", "popovers", "treeExpanded", "symbolsExpanded", + "isOverview", "showBranchInfobar", "userNameDisplayConfiguration", + // Copilot entitlement, which is a property of the viewer, not the repo. + "copilot*", + )) + return true +} + +// newTreeEntry builds one entry. The id form is owner/name@ref/path with a +// slash, not a colon: Locate splits on the first slash after the ref, so a colon +// there lands the whole filename inside the ref and produces a URL nobody can +// follow. +func newTreeEntry(repo, ref string, it treeItem) TreeEntry { + e := TreeEntry{Repo: repo, Ref: ref, Name: it.Name, Path: it.Path, Type: it.ContentType} + kind := KindFile + if it.ContentType == "directory" { + kind = KindTree + } + e.setIdentity(kind, repo+"@"+ref+"/"+it.Path) + return e +} + +// decodeLanguages reads the language histogram out of sections.languages. The +// block is lazily populated and is an empty object on a cold page, which is why +// there is a language-bar fallback below. +func decodeLanguages(raw json.RawMessage) map[string]int64 { + if len(raw) == 0 { + return nil + } + var v struct { + Languages []struct { + Name string `json:"name"` + Size *int64 `json:"size"` + Bytes *int64 `json:"bytes"` + } `json:"languages"` + } + if err := json.Unmarshal(raw, &v); err != nil { + return nil + } + out := map[string]int64{} + for _, l := range v.Languages { + switch { + case l.Size != nil: + out[l.Name] = *l.Size + case l.Bytes != nil: + out[l.Name] = *l.Bytes + } + } + if len(out) == 0 { + return nil + } + return out +} + +func topLanguage(langs map[string]int64) string { + best, bestN := "", int64(-1) + for _, k := range sortedKeys(langs) { + if langs[k] > bestN { + best, bestN = k, langs[k] + } + } + return best +} + +// --- the DOM pass --- + +// readDOM picks up the two fields that exist nowhere in any payload: the +// licence and the language bar. Both are class selectors, both are in the +// bottom tier of the preference ladder, and both degrade to absent rather than +// to wrong. +func (r *Repo) readDOM(p *page.Page) { + doc := p.Doc() + if doc == nil { + return + } + if r.SocialImageURL == "" { + r.SocialImageURL = p.MetaContent("og:image") + } + if lic := licenseFrom(doc); lic != "" { + r.License = lic + } + if len(r.Languages) == 0 { + if langs, pct := languageBar(doc); len(langs) > 0 { + r.Languages = langs + r.Language = topLanguage(langs) + if pct { + // Percentages are not byte counts and the record must not + // pretend otherwise. + recordVia(&r.Base, "languages", "bar") + } + } + } + if r.Description == "" { + r.Description = descriptionFromOG(p.MetaContent("og:title"), r.ID) + } +} + +// licenseFrom reads the About-sidebar licence link. This is the single most +// fragile extraction in the tool: the anchor is found by the icon inside it, +// because the icon outlives the anchor's classes. When it stops matching, the +// field is absent, which is a truthful answer, rather than empty, which would +// be a claim. +func licenseFrom(doc *html.Node) string { + a := page.Find(doc, page.LicenseLink) + if a == nil { + return "" + } + text := page.Text(a) + text = strings.TrimSuffix(text, " license") + text = strings.TrimSuffix(text, " License") + return strings.TrimSpace(text) +} + +// languageBar reads the coloured bar under the About box. The second return +// says the values are percentages times one hundred rather than byte counts, +// which the caller then records so that nobody aggregates the two together. +func languageBar(doc *html.Node) (map[string]int64, bool) { + links := page.FindAll(doc, page.LanguageBarItem) + if len(links) == 0 { + return nil, false + } + out := map[string]int64{} + for _, a := range links { + nameNode := page.Find(a, page.LanguageBarName) + if nameNode == nil { + continue + } + name := page.Text(nameNode) + rest := strings.TrimSpace(strings.TrimPrefix(page.Text(a), name)) + pct, err := strconv.ParseFloat(strings.TrimSuffix(rest, "%"), 64) + if err != nil || name == "" { + continue + } + out[name] = int64(pct * 100) + } + if len(out) == 0 { + return nil, false + } + return out, true +} + +// descriptionFromOG recovers a description from the Open Graph title, which on +// a repository reads "GitHub - owner/name: the description". This looks like a +// hack until the sidebar has not loaded, which happens, and then it is the only +// description on the page. +func descriptionFromOG(title, id string) string { + marker := id + ": " + i := strings.Index(title, marker) + if i < 0 { + return "" + } + return strings.TrimSpace(title[i+len(marker):]) +} + +// --- the deep pass --- + +// deepenRepo runs the two extra fetches --deep opts into: the sidebar +// fragment, which is where the language histogram and the contributor count +// actually live, and the dependency graph for the dependent count. Both +// failures are soft. A dependency graph that is disabled is a fact about the +// repository, not an error in the read. +func (c *Client) deepenRepo(ctx context.Context, r *Repo) error { + if sb, err := c.sidebar(ctx, r.ID); err == nil && sb != nil { + sb.apply(r) + r.addSource(repoSubURL(r.ID, "_sidebar")) + } + if n, err := c.dependents(ctx, r.ID); err == nil && n != nil { + r.DependentCount = n + } + r.addSource(repoSubURL(r.ID, "network/dependents")) + return nil +} + +// sidebarData is the fragment the repository page's own front end fetches to +// fill the About column in. Everything on it is deferred, which is why a cold +// page has a skeleton where the language bar goes. +type sidebarData struct { + Languages *struct { + Languages []sidebarLanguage `json:"languages"` + } `json:"languages"` + Contributors *struct { + ContributorCount *int `json:"contributorCount"` + } `json:"contributors"` + UsedBy *struct { + DependentsCount *int `json:"dependentsCount"` + } `json:"usedBy"` +} + +type sidebarLanguage struct { + Name string `json:"name"` + Percentage float64 `json:"percentage"` + Color string `json:"color"` +} + +// sidebar reads /{owner}/{repo}/_sidebar. +// +// This is the answer to a question the rest of this file used to give up on. +// /{owner}/{repo}/graphs/languages 301s back to the repository page for an +// anonymous client, and none of show_partial, /languages, or +// /graphs/languages-data exist, so the conclusion was that the histogram had no +// keyless source. It has one: the same fragment the page itself waits for, and +// it needs no credential, only the header that says a script is asking. +func (c *Client) sidebar(ctx context.Context, id string) (*sidebarData, error) { + if _, _, ok := SplitRepo(id); !ok { + return nil, usageBadID("repository", id, "owner/name") + } + res, err := c.Get(ctx, repoSubURL(id, "_sidebar"), SurfaceXHR) + if err != nil { + return nil, err + } + var sb sidebarData + if err := json.Unmarshal(res.Body, &sb); err != nil { + return nil, badPayload(id, err) + } + return &sb, nil +} + +// apply folds the fragment into the record. +// +// The percentages are what the fragment states, so they are stored as +// percentages times one hundred and marked as such. A byte count and a +// percentage are not the same number and nothing downstream should be able to +// mistake one for the other. +func (sb *sidebarData) apply(r *Repo) { + if sb.Contributors != nil && sb.Contributors.ContributorCount != nil { + r.ContributorCount = sb.Contributors.ContributorCount + } + if sb.UsedBy != nil && sb.UsedBy.DependentsCount != nil { + r.DependentCount = sb.UsedBy.DependentsCount + } + langs := sb.langs() + if len(langs) == 0 { + return + } + out := map[string]int64{} + for _, l := range langs { + out[l.Name] = int64(l.Percentage * 100) + } + r.Languages = out + r.Language = topLanguage(out) + if r.LanguageColor == "" { + r.LanguageColor = langs[0].Color + } + recordVia(&r.Base, "languages", "sidebar-percent") +} + +func (sb *sidebarData) langs() []sidebarLanguage { + if sb == nil || sb.Languages == nil { + return nil + } + return sb.Languages.Languages +} + +// searchLanguage asks repository search for the repository by name, because the +// search result carries the primary language and its colour and the repository +// page does not. +// +// This is the shallow path. It is one request and gives the primary language +// only; the sidebar fragment gives the whole histogram and is what --deep uses. +func (c *Client) searchLanguage(ctx context.Context, id string) (lang, color string, err error) { + owner, name, ok := SplitRepo(id) + if !ok { + return "", "", usageBadID("repository", id, "owner/name") + } + q := "repo:" + owner + "/" + name + err = c.SearchRepositories(ctx, q, 5, func(r Repo) error { + if r.ID == id && r.Language != "" { + lang, color = r.Language, r.LanguageColor + } + return nil + }) + if err != nil { + return "", "", err + } + return lang, color, nil +} + +// dependents reads the count off the /network/dependents heading. Prose, marked +// fragile, and absent when the graph is off or private. +func (c *Client) dependents(ctx context.Context, id string) (*int, error) { + res, err := c.GetHTML(ctx, repoSubURL(id, "network/dependents")) + if err != nil { + return nil, err + } + p := page.Extract(res.FinalURL, res.Body) + doc := p.Doc() + if doc == nil { + return nil, nil + } + for _, a := range page.FindAll(doc, page.Sel{Tag: "a", Attr: "href", AttrContains: "dependent_type=REPOSITORY"}) { + if n, _, ok := page.CountIn(page.Text(a)); ok { + return intp(n), nil + } + } + return nil, nil +} diff --git a/gh/search.go b/gh/search.go new file mode 100644 index 0000000..198e759 --- /dev/null +++ b/gh/search.go @@ -0,0 +1,812 @@ +package gh + +import ( + "context" + "encoding/json" + "html" + "strconv" + "strings" + + "github.com/tamnd/any-cli/kit/errs" +) + +// search.go is the widest surface this tool has. Ten search types answer JSON +// to an anonymous Accept header, and nine of them return real results. Code +// search is the tenth: it answers 200 with an empty result set, because it +// needs a session, and this file says so out loud rather than returning zero +// hits and letting the caller conclude the query was wrong. +// +// Search results are decoded into the same record types a full read produces. +// A caller doing `github repos --org golang` gets Repo records, thinner but the +// same type, so --fields works the same and the output pipes into `github get` +// for the full read. A record from a listing is a real record with fewer +// fields, never a different type. + +// The site's own names for the search types. They are not guessable (issues but +// pullrequests, registrypackages but wikis) which is exactly why they are +// constants. +const ( + SearchRepos = "repositories" + SearchIssues = "issues" + SearchPulls = "pullrequests" + SearchUsers = "users" + SearchCommits = "commits" + SearchDiscussions = "discussions" + SearchTopics = "topics" + SearchPackages = "registrypackages" + SearchWikis = "wikis" + SearchMarket = "marketplace" + SearchCode = "code" +) + +// SearchTypes is every type in the order the commands present them. +var SearchTypes = []string{ + SearchRepos, SearchIssues, SearchPulls, SearchUsers, SearchCommits, + SearchDiscussions, SearchTopics, SearchPackages, SearchWikis, SearchMarket, +} + +// searchEnvelope is the shape every search type shares. +type searchEnvelope struct { + Payload struct { + Results []json.RawMessage `json:"results"` + Type string `json:"type"` + Page int `json:"page"` + PageCount int `json:"page_count"` + ResultCount int `json:"result_count"` + Errors []string `json:"errors"` + // WarnLimitedResults is set when GitHub capped the result set, which it + // does silently otherwise. + WarnLimitedResults bool `json:"warn_limited_results"` + } `json:"payload"` +} + +// searchFetch turns one search type into a pager. The numbered ?p=N form is the +// only pagination search has, and page_count is authoritative, so the walk stops +// on the count rather than on a short page. +func searchFetch[T any](c *Client, query, typ string, decode func(json.RawMessage) (T, bool)) fetchPage[T] { + return func(ctx context.Context, token string) ([]T, string, error) { + n := pageToken(token) + var env searchEnvelope + res, err := c.GetJSON(ctx, searchURL(query, typ, n), SurfaceSearch, &env) + if err != nil { + return nil, "", err + } + if len(env.Payload.Errors) > 0 { + return nil, "", errs.Usage("search: %s", strings.Join(env.Payload.Errors, "; ")) + } + out := make([]T, 0, len(env.Payload.Results)) + for _, raw := range env.Payload.Results { + rec, ok := decode(raw) + if !ok { + continue + } + if b := baseOf(&rec); b != nil { + b.addSource(res.FinalURL) + } + out = append(out, rec) + } + next := "" + if n < env.Payload.PageCount { + next = strconv.Itoa(n + 1) + } + return out, next, nil + } +} + +// baseOf reaches the embedded Base of a record so the pager can stamp the +// source URL without every decoder repeating it. +func baseOf(v any) *Base { + type based interface{ base() *Base } + if b, ok := v.(based); ok { + return b.base() + } + return nil +} + +func (b *Base) base() *Base { return b } + +// --- the nine working types --- + +// SearchRepositories streams repository records for a query. +func (c *Client) SearchRepositories(ctx context.Context, query string, limit int, emit func(Repo) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchRepos, decodeSearchRepo), emit) +} + +// SearchIssuesAndPulls streams thread records. typ is SearchIssues or +// SearchPulls: the result shape is identical and only the qualifier differs, +// which is why one decoder serves both. +func (c *Client) SearchIssuesAndPulls(ctx context.Context, query, typ string, limit int, emit func(Thread) error) error { + return paginate(ctx, limit, searchFetch(c, query, typ, decodeSearchThread), emit) +} + +// SearchAccounts streams user records. +func (c *Client) SearchAccounts(ctx context.Context, query string, limit int, emit func(Account) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchUsers, decodeSearchAccount), emit) +} + +// SearchCommitsBy streams commit records. Commit search is the only source for +// signature and verification state on a keyless surface. +func (c *Client) SearchCommitsBy(ctx context.Context, query string, limit int, emit func(Commit) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchCommits, decodeSearchCommit), emit) +} + +// SearchDiscussionsBy streams discussion records. +func (c *Client) SearchDiscussionsBy(ctx context.Context, query string, limit int, emit func(Discussion) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchDiscussions, decodeSearchDiscussion), emit) +} + +// SearchTopicsBy streams topic records. +func (c *Client) SearchTopicsBy(ctx context.Context, query string, limit int, emit func(Topic) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchTopics, decodeSearchTopic), emit) +} + +// SearchPackagesBy streams package records. Search is the only source for +// packages, so these records are complete rather than thin. +func (c *Client) SearchPackagesBy(ctx context.Context, query string, limit int, emit func(Package) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchPackages, decodeSearchPackage), emit) +} + +// SearchWikisBy streams wiki page records. +func (c *Client) SearchWikisBy(ctx context.Context, query string, limit int, emit func(WikiPage) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchWikis, decodeSearchWiki), emit) +} + +// SearchMarketplace streams action and app listings. +func (c *Client) SearchMarketplace(ctx context.Context, query string, limit int, emit func(Action) error) error { + return paginate(ctx, limit, searchFetch(c, query, SearchMarket, decodeSearchAction), emit) +} + +// SearchCodeBy is the honest gap. The route answers 200 with zero results +// without a session, which is the worst possible failure mode: it looks like +// the query matched nothing. +func (c *Client) SearchCodeBy(context.Context, string, int, func(File) error) error { + return notPublic("code search", "the route answers 200 with an empty result set to an anonymous client, which is indistinguishable from no matches") +} + +// --- decoders --- + +type searchRepoResult struct { + ID string `json:"id"` + Archived bool `json:"archived"` + Color string `json:"color"` + Followers *int `json:"followers"` + HasFundingFile bool `json:"has_funding_file"` + HLName string `json:"hl_name"` + HLTruncDescription string `json:"hl_trunc_description"` + Language string `json:"language"` + Mirror bool `json:"mirror"` + OwnedByOrganization bool `json:"owned_by_organization"` + Public bool `json:"public"` + Sponsorable bool `json:"sponsorable"` + Topics []string `json:"topics"` + Type string `json:"type"` + HelpWanted *int `json:"help_wanted_issues_count"` + GoodFirstIssue *int `json:"good_first_issue_issues_count"` + Repo repoNest `json:"repo"` +} + +// repoNest is the doubly-wrapped repository reference search uses everywhere: +// {"repo":{"repository":{...}}}. Modelling it once keeps six decoders from each +// spelling it out. +type repoNest struct { + Repository struct { + ID *int `json:"id"` + Name string `json:"name"` + OwnerLogin string `json:"owner_login"` + OwnerID *int `json:"owner_id"` + UpdatedAt string `json:"updated_at"` + HasIssues bool `json:"has_issues"` + } `json:"repository"` +} + +func (r repoNest) id() string { + if r.Repository.OwnerLogin == "" || r.Repository.Name == "" { + return "" + } + return r.Repository.OwnerLogin + "/" + r.Repository.Name +} + +func decodeSearchRepo(raw json.RawMessage) (Repo, bool) { + var s searchRepoResult + if err := json.Unmarshal(raw, &s); err != nil { + return Repo{}, false + } + id := s.Repo.id() + if id == "" { + id = stripHL(s.HLName) + } + owner, name, ok := SplitRepo(id) + if !ok { + return Repo{}, false + } + r := Repo{Owner: owner, Name: name} + r.setIdentity(KindRepo, id) + r.DatabaseID = s.Repo.Repository.ID + r.Description = stripHL(s.HLTruncDescription) + if r.Description != s.HLTruncDescription { + r.DescriptionHighlight = s.HLTruncDescription + } + r.Language = s.Language + r.LanguageColor = s.Color + r.Stars = s.Followers + r.Topics = s.Topics + r.IsArchived = s.Archived + r.IsMirror = s.Mirror + r.IsOrgOwned = s.OwnedByOrganization + r.IsPrivate = !s.Public + r.Visibility = strings.ToLower(s.Type) + r.Sponsorable = s.Sponsorable + r.HasFunding = s.HasFundingFile + r.HelpWantedIssues = s.HelpWanted + r.GoodFirstIssues = s.GoodFirstIssue + r.UpdatedAt = parseTime(s.Repo.Repository.UpdatedAt) + + r.addExtra("search", decodeExtra(raw, &s, + // Viewer state, always false for an anonymous read. + "starred_by_current_user", "followed_by_current_user", "is_current_user", + )) + return r, true +} + +type searchThreadResult struct { + AuthorName string `json:"author_name"` + AuthorAvatarURL string `json:"author_avatar_url"` + ID string `json:"id"` + Repo repoNest `json:"repo"` + Labels []string `json:"labels"` + NumComments *int `json:"num_comments"` + Number int `json:"number"` + State string `json:"state"` + StateReason *string `json:"state_reason"` + HLTitle string `json:"hl_title"` + HLText string `json:"hl_text"` + Created string `json:"created"` + ReviewableState *string `json:"reviewable_state"` + Merged *bool `json:"merged"` + Issue struct { + Issue struct { + PullRequestID *int `json:"pull_request_id"` + } `json:"issue"` + } `json:"issue"` +} + +// decodeSearchThread produces a Thread. The caller decides whether it wanted +// issues or pull requests, and pull_request_id says which one this actually is, +// so a query that mixes them still classifies each result correctly. +func decodeSearchThread(raw json.RawMessage) (Thread, bool) { + var s searchThreadResult + if err := json.Unmarshal(raw, &s); err != nil { + return Thread{}, false + } + repo := s.Repo.id() + if repo == "" || s.Number == 0 { + return Thread{}, false + } + kind := KindIssue + if s.Issue.Issue.PullRequestID != nil { + kind = KindPR + } + t := Thread{Repo: repo, Number: s.Number} + t.setIdentity(kind, repo+"#"+strconv.Itoa(s.Number)) + t.Title = stripHL(s.HLTitle) + if t.Title != s.HLTitle { + t.TitleHighlight = s.HLTitle + } + t.Body = stripHL(s.HLText) + t.State = s.State + if s.StateReason != nil { + t.StateReason = *s.StateReason + } + t.CommentCount = s.NumComments + t.CreatedAt = parseTime(s.Created) + t.Author = actor(s.AuthorName) + t.Author.AvatarURL = s.AuthorAvatarURL + for _, l := range s.Labels { + t.Labels = append(t.Labels, Label{Name: l}) + } + if n, err := strconv.Atoi(s.ID); err == nil { + t.DatabaseID = intp(n) + } + + t.addExtra("search", decodeExtra(raw, &s)) + return t, true +} + +type searchUserResult struct { + AvatarURL string `json:"avatar_url"` + HLLogin string `json:"hl_login"` + HLName string `json:"hl_name"` + HLProfileBio string `json:"hl_profile_bio"` + Followers *int `json:"followers"` + ID string `json:"id"` + Location string `json:"location"` + Login string `json:"login"` + DisplayLogin string `json:"display_login"` + Name string `json:"name"` + ProfileBio string `json:"profile_bio"` + Sponsorable bool `json:"sponsorable"` + Repos *int `json:"repos"` +} + +func decodeSearchAccount(raw json.RawMessage) (Account, bool) { + var s searchUserResult + if err := json.Unmarshal(raw, &s); err != nil { + return Account{}, false + } + login := firstNonEmpty(s.Login, stripHL(s.HLLogin)) + if login == "" { + return Account{}, false + } + a := Account{Login: login, Type: "User"} + a.setIdentity(KindUser, login) + a.Name = s.Name + a.Bio = s.ProfileBio + a.Location = s.Location + a.AvatarURL = s.AvatarURL + a.Followers = s.Followers + a.RepoCount = s.Repos + a.Sponsorable = s.Sponsorable + if n, err := strconv.Atoi(s.ID); err == nil { + a.DatabaseID = intp(n) + } + + a.addExtra("search", decodeExtra(raw, &s, + // Viewer state. + "followed_by_current_user", "is_current_user", + )) + return a, true +} + +type searchCommitResult struct { + ID string `json:"id"` + SHA string `json:"sha"` + AuthorDate string `json:"author_date"` + HLSubject string `json:"hl_subject"` + HLBody string `json:"hl_body"` + Message string `json:"message"` + Repository repoNest `json:"repository"` + VerificationStatus string `json:"verification_status"` + VerificationReason string `json:"signature_verification_reason"` + SignedByGitHub bool `json:"signed_by_github"` + HasSignature bool `json:"has_signature"` + KeyExpired bool `json:"key_expired"` + KeyID string `json:"key_id"` + ChecksState string `json:"checks_header_state"` + ChecksSummary string `json:"checks_status_summary"` + + // The plural is not a typo. Commit search reports co-authors, so this is a + // list where a single author field would lose the trailer. + Authors []struct { + Login string `json:"login"` + DisplayName string `json:"display_name"` + AvatarURL string `json:"avatar_url"` + } `json:"authors"` + + // Committer is present only when it differs from the author, which on + // golang/go means gopherbot on every commit and on most repositories means + // nothing at all. + Committer *struct { + Login string `json:"login"` + DisplayName string `json:"display_name"` + AvatarURL string `json:"avatar_url"` + } `json:"committer"` + + IssueReferences []struct { + ID *int `json:"id"` + Title string `json:"title"` + State string `json:"state"` + IsPullRequest bool `json:"is_pull_request"` + Permalink string `json:"permalink"` + Merged bool `json:"merged"` + } `json:"issue_references"` +} + +func decodeSearchCommit(raw json.RawMessage) (Commit, bool) { + var s searchCommitResult + if err := json.Unmarshal(raw, &s); err != nil { + return Commit{}, false + } + repo := s.Repository.id() + if repo == "" || s.SHA == "" { + return Commit{}, false + } + c := Commit{Repo: repo, SHA: s.SHA} + c.setIdentity(KindCommit, repo+"@"+s.SHA) + subject, body, _ := strings.Cut(s.Message, "\n\n") + c.Subject = strings.TrimSpace(subject) + c.Body = strings.TrimSpace(body) + // hl_subject arrives wrapped in an anchor rather than as plain text, which + // is the one place search returns markup instead of a highlighted string. + if hl := stripTags(s.HLSubject); hl != "" && c.Subject == "" { + c.Subject = hl + } + c.SubjectHighlight = s.HLSubject + c.AuthoredAt = parseTime(s.AuthorDate) + c.Verification = s.VerificationStatus + c.VerificationReason = s.VerificationReason + c.SignedByGitHub = s.SignedByGitHub + c.HasSignature = s.HasSignature + c.KeyID = s.KeyID + c.KeyExpired = s.KeyExpired + c.StatusRollup = s.ChecksState + c.StatusSummary = s.ChecksSummary + for _, a := range s.Authors { + act := actor(a.Login) + act.Name = a.DisplayName + act.AvatarURL = a.AvatarURL + c.Authors = append(c.Authors, act) + } + if s.Committer != nil { + act := actor(s.Committer.Login) + act.Name = s.Committer.DisplayName + act.AvatarURL = s.Committer.AvatarURL + c.Committer = &act + } + for _, r := range s.IssueReferences { + c.IssueRefs = append(c.IssueRefs, ThreadRef{ + DatabaseID: r.ID, + Title: r.Title, + State: r.State, + IsPullRequest: r.IsPullRequest, + Merged: r.Merged, + URL: r.Permalink, + }) + } + + c.addExtra("search", decodeExtra(raw, &s, + // A link to GitHub's own docs about signature verification. + "help_url", + // The whole CI rollup, which is a repository-shaped blob three levels + // deep and belongs to `github checks`, not to a commit listing. + "status_check_rollup", + // Prose assembled from authors and committer_attribution for a + // tooltip, and the flag that generated it. + "commit_author_tooltip", "committer_attribution", + // Viewer state. + "is_viewer", + )) + return c, true +} + +type searchDiscussionResult struct { + Body string `json:"body"` + Created string `json:"created"` + Updated string `json:"updated"` + HLText string `json:"hl_text"` + HLTitle string `json:"hl_title"` + ID string `json:"id"` + NumComments *int `json:"num_comments"` + Number int `json:"number"` + Repo repoNest `json:"repo"` + Title string `json:"title"` + URL string `json:"url"` + UserAvatarURL string `json:"user_avatar_url"` + UserID *int `json:"user_id"` + UserLogin string `json:"user_login"` +} + +func decodeSearchDiscussion(raw json.RawMessage) (Discussion, bool) { + var s searchDiscussionResult + if err := json.Unmarshal(raw, &s); err != nil { + return Discussion{}, false + } + repo := s.Repo.id() + if repo == "" || s.Number == 0 { + return Discussion{}, false + } + var d Discussion + d.Repo = repo + d.Number = s.Number + d.setIdentity(KindDiscussion, repo+"#"+strconv.Itoa(s.Number)) + d.Title = firstNonEmpty(s.Title, stripHL(s.HLTitle)) + d.TitleHighlight = s.HLTitle + d.Body = s.Body + d.CommentCount = s.NumComments + d.CreatedAt = parseTime(s.Created) + d.UpdatedAt = parseTime(s.Updated) + d.Author = actor(s.UserLogin) + d.Author.AvatarURL = s.UserAvatarURL + d.Author.DatabaseID = s.UserID + if n, err := strconv.Atoi(s.ID); err == nil { + d.DatabaseID = intp(n) + } + + d.addExtra("search", decodeExtra(raw, &s)) + return d, true +} + +type searchTopicResult struct { + ID string `json:"id"` + Name string `json:"name"` + ShortDescription string `json:"short_description"` + DisplayName string `json:"display_name"` + Released string `json:"released"` + WikipediaURL string `json:"wikipedia_url"` + URL string `json:"url"` + GitHubURL string `json:"github_url"` + LogoURL string `json:"logo_url"` + Featured bool `json:"featured"` + Curated bool `json:"curated"` + StargazerCount *int `json:"stargazer_count"` + AppliedCount *int `json:"applied_count"` + HLDisplayName string `json:"hl_display_name"` + HLShortDescription string `json:"hl_short_description"` + CreatedBy string `json:"created_by"` + Aliases []string `json:"aliases"` + Related []string `json:"related"` +} + +func decodeSearchTopic(raw json.RawMessage) (Topic, bool) { + var s searchTopicResult + if err := json.Unmarshal(raw, &s); err != nil || s.Name == "" { + return Topic{}, false + } + t := Topic{Name: s.Name} + t.setIdentity(KindTopic, s.Name) + t.DisplayName = s.DisplayName + t.ShortDescription = s.ShortDescription + t.Released = s.Released + t.WikipediaURL = s.WikipediaURL + t.GitHubURL = s.GitHubURL + t.LogoURL = s.LogoURL + t.Featured = s.Featured + t.Curated = s.Curated + t.StargazerCount = s.StargazerCount + t.AppliedCount = s.AppliedCount + t.CreatedBy = s.CreatedBy + t.Aliases = s.Aliases + t.Related = s.Related + + t.addExtra("search", decodeExtra(raw, &s, + // True when logo_url is set, which the record already carries. + "has_logo_url", + // A moderation flag with no public meaning. + "flagged", + // The highlight map repeats hl_display_name and hl_short_description in + // a nested shape, and both are already modelled. + "highlights", + // A ceiling marker on repository_count, kept out because the count it + // qualifies is not one this record claims. + "repository_count", "repository_count_over_max_fetch_limit", + // Viewer state. + "starred_by_current_user", + )) + return t, true +} + +type searchPackageResult struct { + ID string `json:"id"` + Color string `json:"color"` + Downloads *int `json:"downloads"` + Name string `json:"name"` + PackageURL string `json:"package_url"` + PackageType string `json:"package_type"` + Public bool `json:"public"` + Summary string `json:"summary"` + Topics []string `json:"topics"` + UpdatedAt string `json:"updated_at"` + Repo struct { + Name string `json:"name"` + OwnerLogin string `json:"owner_login"` + } `json:"repo"` + Source struct { + PackageType string `json:"package_type"` + } `json:"source"` +} + +func decodeSearchPackage(raw json.RawMessage) (Package, bool) { + var s searchPackageResult + if err := json.Unmarshal(raw, &s); err != nil || s.Name == "" { + return Package{}, false + } + p := Package{Name: s.Name} + if s.Repo.OwnerLogin != "" && s.Repo.Name != "" { + p.Repo = s.Repo.OwnerLogin + "/" + s.Repo.Name + } + p.setIdentity(KindPackage, firstNonEmpty(p.Repo+"/"+s.Name, s.Name)) + if s.PackageURL != "" { + p.URL = BaseURL + s.PackageURL + } + // package_type is null at the top level and populated inside source, which + // looks like an oversight upstream but is consistent enough to rely on. + p.Type = firstNonEmpty(s.PackageType, s.Source.PackageType) + p.Summary = s.Summary + p.Downloads = s.Downloads + p.Topics = s.Topics + p.UpdatedAt = parseTime(s.UpdatedAt) + + p.addExtra("search", decodeExtra(raw, &s, + // The language colour of the source repository, which belongs to the + // repository record and not to the package. + "color", + // The whole source blob: registry internals, ids, and a version list + // that `github package` reads properly. + "source", + )) + return p, true +} + +type searchWikiResult struct { + Body string `json:"body"` + Filename string `json:"filename"` + Format string `json:"format"` + HLBody string `json:"hl_body"` + HLTitle string `json:"hl_title"` + ID string `json:"id"` + Path string `json:"path"` + Public bool `json:"public"` + Repo repoNest `json:"repo"` + RepoID *int `json:"repo_id"` + Title string `json:"title"` + UpdatedAt string `json:"updated_at"` +} + +func decodeSearchWiki(raw json.RawMessage) (WikiPage, bool) { + var s searchWikiResult + if err := json.Unmarshal(raw, &s); err != nil { + return WikiPage{}, false + } + repo := s.Repo.id() + if repo == "" { + return WikiPage{}, false + } + w := WikiPage{Repo: repo, Path: s.Path, Format: s.Format} + w.Title = firstNonEmpty(s.Title, stripHL(s.HLTitle)) + w.setIdentity(KindWiki, repo+"/"+firstNonEmpty(s.Path, s.Filename)) + w.Body = s.Body + w.UpdatedAt = parseTime(s.UpdatedAt) + + w.addExtra("search", decodeExtra(raw, &s)) + return w, true +} + +// searchActionResult covers both listing shapes at once. A repository action +// fills repository_action and a marketplace app fills marketplace_listing, and +// the two never overlap, so decoding both and taking whichever arrived is +// simpler and more honest than branching on type. +type searchActionResult struct { + Type string `json:"type"` + ID string `json:"id"` + Name string `json:"name"` + Free bool `json:"free"` + PrimaryCategory string `json:"primary_category"` + SecondaryCategory string `json:"secondary_category"` + IsVerifiedOwner bool `json:"is_verified_owner"` + Slug string `json:"slug"` + OwnerLogin string `json:"owner_login"` + ResourcePath string `json:"resource_path"` + Description string `json:"description"` + ShortDescription string `json:"short_description"` + FullDescription string `json:"full_description"` + Stars *int `json:"stars"` + DependentsCount *int `json:"dependents_count"` + InstallationCount *int `json:"installation_count"` + ListingLogoURL string `json:"listing_logo_url"` + State string `json:"state"` + Recommended bool `json:"recommended"` + + RepositoryAction struct { + RepositoryAction struct { + ID *int `json:"id"` + Path string `json:"path"` + Name string `json:"name"` + Description string `json:"description"` + IconName string `json:"icon_name"` + Color string `json:"color"` + Featured bool `json:"featured"` + RepositoryID *int `json:"repository_id"` + Slug string `json:"slug"` + } `json:"repository_action"` + } `json:"repository_action"` + + MarketplaceListing struct { + Listing struct { + ID *int `json:"id"` + Name string `json:"name"` + Slug string `json:"slug"` + ShortDescription string `json:"short_description"` + FullDescription string `json:"full_description"` + ExtendedDescription string `json:"extended_description"` + PrivacyPolicyURL string `json:"privacy_policy_url"` + TOSURL string `json:"tos_url"` + CompanyURL string `json:"company_url"` + SupportURL string `json:"support_url"` + DocumentationURL string `json:"documentation_url"` + PricingURL string `json:"pricing_url"` + ByGitHub bool `json:"by_github"` + ListableType string `json:"listable_type"` + ListableID *int `json:"listable_id"` + } `json:"listing"` + } `json:"marketplace_listing"` +} + +func decodeSearchAction(raw json.RawMessage) (Action, bool) { + var s searchActionResult + if err := json.Unmarshal(raw, &s); err != nil { + return Action{}, false + } + ra := s.RepositoryAction.RepositoryAction + ml := s.MarketplaceListing.Listing + + slug := firstNonEmpty(s.Slug, ra.Slug, ml.Slug) + if slug == "" { + return Action{}, false + } + a := Action{Owner: s.OwnerLogin, Slug: slug} + a.Name = firstNonEmpty(s.Name, ra.Name, ml.Name) + a.setIdentity(KindAction, slug) + if s.ResourcePath != "" { + a.URL = BaseURL + s.ResourcePath + } + a.Description = firstNonEmpty(s.Description, ra.Description) + a.ShortDescription = firstNonEmpty(s.ShortDescription, ml.ShortDescription) + a.FullDescription = firstNonEmpty(s.FullDescription, ml.FullDescription) + a.ExtendedDescription = ml.ExtendedDescription + a.Type = s.Type + a.PrimaryCategory = s.PrimaryCategory + a.SecondaryCategory = s.SecondaryCategory + a.IsFree = s.Free + a.IsVerifiedOwner = s.IsVerifiedOwner + a.IsRecommended = s.Recommended + a.State = s.State + a.Stars = s.Stars + a.DependentCount = s.DependentsCount + a.InstallationCount = s.InstallationCount + a.LogoURL = s.ListingLogoURL + + a.Path = ra.Path + a.IconName = ra.IconName + a.IconColor = ra.Color + a.RepositoryID = ra.RepositoryID + a.IsFeatured = ra.Featured + + a.ListingID = ml.ID + a.CompanyURL = ml.CompanyURL + a.DocumentationURL = ml.DocumentationURL + a.SupportURL = ml.SupportURL + a.PrivacyPolicyURL = ml.PrivacyPolicyURL + a.TermsURL = ml.TOSURL + a.PricingURL = ml.PricingURL + a.ByGitHub = ml.ByGitHub + + a.addExtra("search", decodeExtra(raw, &s, + // Repeats name and description with markers, both already modelled. + "highlights", + // The listing's own icon markup, which is a whole inline SVG document + // and is a rendering concern rather than a fact about the action. + "icon_svg", + )) + return a, true +} + +// --- highlight handling --- + +// stripHL turns a hl_ field into plain text. Search wraps matches in and +// entity-escapes the rest, so this is a fixed transform and not an HTML parse: +// nothing else ever appears in those fields. +func stripHL(s string) string { + if s == "" { + return "" + } + s = strings.ReplaceAll(s, "", "") + s = strings.ReplaceAll(s, "", "") + return html.UnescapeString(s) +} + +// stripTags is the wider hammer, needed for hl_subject on commit search, which +// arrives as a whole anchor element rather than as a highlighted string. +func stripTags(s string) string { + var b strings.Builder + depth := 0 + for _, r := range s { + switch { + case r == '<': + depth++ + case r == '>' && depth > 0: + depth-- + case depth == 0: + b.WriteRune(r) + } + } + return strings.TrimSpace(html.UnescapeString(b.String())) +} diff --git a/gh/surface.go b/gh/surface.go new file mode 100644 index 0000000..bbdd552 --- /dev/null +++ b/gh/surface.go @@ -0,0 +1,198 @@ +package gh + +import ( + "context" + "net/url" + "strconv" + "strings" +) + +// surface.go is the route-to-surface table from the spec, written down as data +// rather than scattered through the decoders, plus the four pagers every +// listing in this tool is built from. +// +// The table is not decoration. `github surfaces` prints it, which means the +// answer to "can this tool read X" is a command rather than a reading of the +// source, and a route that is not in the table is Unsupported with exit code 7 +// instead of a guess. + +// RouteInfo is one row: what the route is, which surface answers it best, and +// what to fall back to when that surface declines. +type RouteInfo struct { + Route string `json:"route" table:"route"` + Primary string `json:"primary" table:"primary"` + Fallback string `json:"fallback,omitempty" table:"fallback"` + Note string `json:"note,omitempty" table:"note,truncate"` +} + +// Routes is the whole index. Surface names are the ones in doc 01: html, +// route-json, xhr, search, feed, raw, git, embedded, ld-json. +var Routes = []RouteInfo{ + {"/{owner}/{repo}", "embedded", "route-json", "sidebarAbout lives only in the HTML payload"}, + {"/{owner}/{repo}/tree/{ref}/{path}", "route-json", "embedded", ""}, + {"/{owner}/{repo}/blob/{ref}/{path}", "embedded", "raw", "the route JSON dropped the metadata block, so the page is the read; bytes from raw"}, + {"/{owner}/{repo}/branches", "route-json", "xhr", ""}, + {"/{owner}/{repo}/refs", "xhr", "git", "names only, 6 KB against 588 KB"}, + {"/{owner}/{repo}/commits/{ref}", "route-json", "feed", ""}, + {"/{owner}/{repo}/commit/{sha}", "route-json", "raw", "the diff comes from .patch"}, + {"/{owner}/{repo}/compare/{a}...{b}", "route-json", "raw", ""}, + {"/{owner}/{repo}/issues/{n}", "embedded", "ld-json", "Relay preloaded queries"}, + {"/{owner}/{repo}/pull/{n}", "embedded", "raw", "Relay, plus .diff and .patch"}, + {"/{owner}/{repo}/discussions/{n}", "embedded", "", "Relay"}, + {"/{owner}/{repo}/issues", "search", "", "type=issues with a repo: qualifier"}, + {"/{owner}/{repo}/pulls", "search", "", "type=pullrequests"}, + {"/{owner}/{repo}/releases", "feed", "html", "releases.atom, then a page each for assets"}, + {"/{owner}/{repo}/releases/tag/{tag}", "html", "feed", "download counts exist nowhere else"}, + {"/{owner}/{repo}/tags", "feed", "git", "the feed is recent, git is complete"}, + {"/{owner}/{repo}/graphs/contributors", "xhr", "", "answers 202 while it computes, so it polls"}, + {"/{owner}/{repo}/wiki", "feed", "html", ""}, + {"/{owner}/{repo}.git/info/refs", "git", "", "every ref and its SHA in one request"}, + {"/{login}", "html", "ld-json", "microdata and microformats, no payload at all"}, + {"/{org}", "html", "xhr", "two deferred fragments under --deep"}, + {"/{login}?tab=repositories", "search", "html", "user: qualifier"}, + {"/{login}?tab=stars", "html", "", ""}, + {"/{login}.atom", "feed", "", ""}, + {"/users/{login}/hovercard", "xhr", "html", ""}, + {"/users/{login}/contributions", "xhr", "", "returns an HTML fragment, not JSON"}, + {"/orgs/{org}/people", "html", "", ""}, + {"/search", "search", "", "ten types, code is the one that needs a token"}, + {"/trending", "html", "", "no JSON equivalent exists, tokened or not"}, + {"/topics/{slug}", "html", "search", ""}, + {"gist.github.com/{id}", "html", "raw", ""}, + {"raw.githubusercontent.com/...", "raw", "", ""}, + {"codeload.github.com/...", "raw", "", ""}, +} + +// --- URL builders --- +// +// Every request in this tool goes through one of these. Building URLs in one +// place is what makes the escape hatch (`github url`) tell the truth about what +// the tool would actually fetch. + +func repoURL(repo string) string { return BaseURL + "/" + repo } + +func repoSubURL(repo, sub string) string { return BaseURL + "/" + repo + "/" + sub } + +func treeURL(repo, ref, path string) string { + u := BaseURL + "/" + repo + "/tree/" + ref + if path != "" { + u += "/" + path + } + return u +} + +func blobURL(repo, ref, path string) string { + return BaseURL + "/" + repo + "/blob/" + ref + "/" + path +} + +func rawURL(repo, ref, path string) string { + return RawURL + "/" + repo + "/" + ref + "/" + path +} + +func commitURL(repo, sha string) string { return BaseURL + "/" + repo + "/commit/" + sha } + +func threadURL(repo, segment string, number int) string { + return BaseURL + "/" + repo + "/" + segment + "/" + strconv.Itoa(number) +} + +func feedURL(path string) string { return BaseURL + "/" + strings.TrimPrefix(path, "/") } + +func accountURL(login string) string { return BaseURL + "/" + login } + +// gitRefsURL is the smart-protocol advertisement: every ref and its object id, +// in one request, with no page limit and no login. +func gitRefsURL(repo string) string { + return BaseURL + "/" + repo + ".git/info/refs?service=git-upload-pack" +} + +// searchURL builds a search request. Type is the site's own name for the type: +// repositories, issues, pullrequests, discussions, users, commits, registrypackages, +// wikis, topics, marketplace. +func searchURL(query, typ string, page int) string { + v := url.Values{} + v.Set("q", query) + if typ != "" { + v.Set("type", typ) + } + if page > 1 { + v.Set("p", strconv.Itoa(page)) + } + return BaseURL + "/search?" + v.Encode() +} + +// --- the pagers --- + +// A pager answers one question: given where we are, what is the next batch and +// where does that leave us. Four shapes cover every listing github.com has. +// +// search numbered pages, ?p=N, stops on a short page +// cursor Relay endCursor, stops when hasNextPage is false +// rails a rel="next" anchor in the markup, stops when it is absent +// none one response, and that is the whole list +// +// They share one driver so that --limit, cancellation, and the "stop asking for +// more once the caller has enough" rule are written once. + +// fetchPage returns one batch and the token for the next batch. An empty next +// token ends the walk. +type fetchPage[T any] func(ctx context.Context, token string) (batch []T, next string, err error) + +// paginate walks a listing, handing each record to emit as it arrives. +// +// Records are emitted as they decode, not collected and returned, because a +// listing of every repository in an organization should start printing on the +// first page rather than after the last one. +// +// limit <= 0 means no limit. The walk stops the moment the limit is reached, so +// asking for five records off a thousand-record listing costs one request. +func paginate[T any](ctx context.Context, limit int, fetch fetchPage[T], emit func(T) error) error { + token := "" + seen := 0 + for { + if err := ctx.Err(); err != nil { + return wrapNetwork("", err) + } + batch, next, err := fetch(ctx, token) + if err != nil { + return err + } + for _, rec := range batch { + if err := emit(rec); err != nil { + return err + } + seen++ + if limit > 0 && seen >= limit { + return nil + } + } + // A page that returned nothing ends the walk even when the surface + // still claims a next token. Trusting the token alone is how a paginator + // spins forever against a route that has started answering empty. + if next == "" || next == token || len(batch) == 0 { + return nil + } + token = next + } +} + +// pageToken and cursorToken are the two token encodings. Keeping them as helper +// functions rather than inline strconv calls means the "which page am I on" +// logic reads the same in all nine listings that use it. + +func pageToken(token string) int { + if token == "" { + return 1 + } + n, err := strconv.Atoi(token) + if err != nil || n < 1 { + return 1 + } + return n +} + +func nextPageToken(current int, gotFull bool) string { + if !gotFull { + return "" + } + return strconv.Itoa(current + 1) +} diff --git a/gh/thread.go b/gh/thread.go new file mode 100644 index 0000000..b8d2f9b --- /dev/null +++ b/gh/thread.go @@ -0,0 +1,1186 @@ +package gh + +import ( + "context" + "encoding/base64" + "encoding/json" + "strconv" + "strings" + "time" + + "golang.org/x/net/html" + + "github.com/tamnd/github-cli/pkg/page" +) + +// thread.go reads issues, pull requests, and discussions. +// +// The spec files these three together because they are one concept, and they +// are one concept in GitHub's data model too. They are not one surface. Each of +// the three ships on a different plane, and the differences are large enough +// that pretending otherwise would cost more than it saves: +// +// issue React, Relay. The whole GraphQL issue node is preloaded into +// the page, labels, milestone, reactions, and fifteen timeline +// items included. This is the richest keyless surface on the site. +// pull React, route props. pullRequestsLayoutRoute carries the merge +// metadata and nothing else: no body, no timeline, no labels. The +// body comes from the hovercard fragment and is truncated, and +// that is the best a logged-out client can do. +// discussion Rails. No payload at all. A schema.org QAPage block carries the +// body, the upvotes, and the accepted answer; the category, the +// labels, and the participants come from the markup. +// +// The asymmetry is worth stating in the record rather than hiding, so a pull +// request read this way says where each field came from in Sources and leaves +// the fields it could not reach empty instead of guessing them. + +// --- issues --- + +// Issue reads one issue. repo is owner/name. +// +// A pull request number handed to this function does not 404: GitHub redirects +// /issues/{n} to /pull/{n}, and the read follows that redirect and hands off to +// PullRequest rather than returning a half-decoded record. +func (c *Client) Issue(ctx context.Context, repo string, number int) (*Issue, error) { + if _, _, ok := SplitRepo(repo); !ok { + return nil, usageBadID("repository", repo, "owner/name") + } + if number <= 0 { + return nil, usageBadID("issue number", strconv.Itoa(number), "a positive integer") + } + res, err := c.GetHTML(ctx, threadURL(repo, "issues", number)) + if err != nil { + return nil, err + } + if strings.Contains(res.FinalURL, "/pull/") { + pr, err := c.PullRequest(ctx, repo, number) + if err != nil { + return nil, err + } + // The caller asked for an issue and got a pull request. Returning the + // pull request's thread half is the honest answer: same number, same + // repository, same conversation, and Kind says which it is. + return &Issue{Thread: pr.Thread}, nil + } + + p := page.Extract(res.FinalURL, res.Body) + raw, ok := issueNode(p) + if !ok { + return nil, structureChanged(repo + "#" + strconv.Itoa(number)) + } + + iss, err := decodeRelayIssue(repo, raw) + if err != nil { + return nil, err + } + iss.addSource(res.FinalURL) + if p.Canonical != "" { + iss.URL = p.Canonical + } + return iss, nil +} + +// issueNode digs the issue out of the preloaded Relay result. The query is +// named IssueViewerViewQuery today and the name is generated, so the lookup is +// by suffix with the first query as the fallback: on a thread page there is +// only ever one. +func issueNode(p *page.Page) (json.RawMessage, bool) { + data, ok := p.Query("IssueViewerViewQuery") + if !ok { + _, data, ok = p.FirstQuery() + } + if !ok { + return nil, false + } + var q struct { + Repository struct { + Issue json.RawMessage `json:"issue"` + } `json:"repository"` + } + if err := json.Unmarshal(data, &q); err != nil || len(q.Repository.Issue) == 0 { + return nil, false + } + return q.Repository.Issue, true +} + +// relayIssue is the GraphQL issue node, modelled field for field. Everything +// not named here lands in Extra, which is how the day GitHub adds a field +// becomes a failing test rather than a year of silently dropping it. +type relayIssue struct { + ID string `json:"id"` + Number int `json:"number"` + DatabaseID *int `json:"databaseId"` + Title string `json:"title"` + TitleHTML string `json:"titleHTML"` + URL string `json:"url"` + State string `json:"state"` + StateReason string `json:"stateReason"` + Locked bool `json:"locked"` + IsPinned bool `json:"isPinned"` + CreatedAt string `json:"createdAt"` + UpdatedAt string `json:"updatedAt"` + Body string `json:"body"` + BodyHTML string `json:"bodyHTML"` + + Repository struct { + NameWithOwner string `json:"nameWithOwner"` + DatabaseID *int `json:"databaseId"` + } `json:"repository"` + + Author relayActor `json:"author"` + IssueType *struct { + Name string `json:"name"` + } `json:"issueType"` + DuplicateOf *struct { + URL string `json:"url"` + } `json:"duplicateOf"` + + Labels struct { + Edges []struct { + Node relayLabel `json:"node"` + } `json:"edges"` + } `json:"labels"` + + Milestone *relayMilestone `json:"milestone"` + + AssignedActors struct { + Nodes []relayActor `json:"nodes"` + } `json:"assignedActors"` + + ReactionGroups []relayReactionGroup `json:"reactionGroups"` + + SubIssuesSummary *struct { + Total *int `json:"total"` + Completed *int `json:"completed"` + } `json:"subIssuesSummary"` + + LinkedPullRequests struct { + Nodes []struct { + URL string `json:"url"` + } `json:"nodes"` + } `json:"linkedPullRequests"` + ClosedByPullRequestsReferences struct { + Nodes []struct { + URL string `json:"url"` + } `json:"nodes"` + } `json:"closedByPullRequestsReferences"` + + ProjectItems struct { + Edges []struct { + Node struct { + Project struct { + Title string `json:"title"` + URL string `json:"url"` + } `json:"project"` + } `json:"node"` + } `json:"edges"` + } `json:"projectItems"` + + FrontTimelineItems relayTimeline `json:"frontTimelineItems"` + BackTimelineItems relayTimeline `json:"backTimelineItems"` +} + +type relayActor struct { + Typename string `json:"__typename"` + Login string `json:"login"` + Name string `json:"name"` + ID string `json:"id"` + AvatarURL string `json:"avatarUrl"` + ProfileURL string `json:"profileUrl"` +} + +func (a relayActor) actor() Actor { + if a.Login == "" { + return Actor{} + } + out := actor(a.Login) + out.Name = a.Name + out.Type = a.Typename + out.NodeID = a.ID + out.AvatarURL = a.AvatarURL + if a.ProfileURL != "" { + out.URL = a.ProfileURL + } + return out +} + +type relayLabel struct { + ID string `json:"id"` + Name string `json:"name"` + Color string `json:"color"` + Description string `json:"description"` + URL string `json:"url"` +} + +func (l relayLabel) label() Label { + return Label{ + Name: l.Name, + Color: l.Color, + Description: l.Description, + URL: l.URL, + NodeID: l.ID, + } +} + +type relayMilestone struct { + ID string `json:"id"` + Title string `json:"title"` + Number *int `json:"number"` + Closed bool `json:"closed"` + DueOn string `json:"dueOn"` + ClosedAt string `json:"closedAt"` + Progress *float64 `json:"progressPercentage"` + URL string `json:"url"` +} + +func (m *relayMilestone) milestone() *Milestone { + if m == nil || m.Title == "" { + return nil + } + out := &Milestone{ + Title: m.Title, + Number: m.Number, + Closed: m.Closed, + DueOn: parseTime(m.DueOn), + ClosedAt: parseTime(m.ClosedAt), + Progress: m.Progress, + URL: m.URL, + } + // The milestone number is not in the node, but it is the last segment of + // the URL, and a milestone without a number is awkward to join on. + if out.Number == nil { + if i := strings.LastIndex(m.URL, "/"); i >= 0 { + if n, err := strconv.Atoi(m.URL[i+1:]); err == nil { + out.Number = intp(n) + } + } + } + return out +} + +type relayReactionGroup struct { + Content string `json:"content"` + Reactors struct { + TotalCount int `json:"totalCount"` + } `json:"reactors"` +} + +// reactions drops the zero groups. All eight always arrive, most of them empty, +// and eight zeroes on every record is noise that would bury the real ones. +func reactions(groups []relayReactionGroup) []Reaction { + var out []Reaction + for _, g := range groups { + if g.Reactors.TotalCount == 0 { + continue + } + out = append(out, Reaction{Content: g.Content, Count: g.Reactors.TotalCount}) + } + return out +} + +type relayTimeline struct { + TotalCount *int `json:"totalCount"` + PageInfo struct { + HasNextPage bool `json:"hasNextPage"` + HasPreviousPage bool `json:"hasPreviousPage"` + EndCursor string `json:"endCursor"` + StartCursor string `json:"startCursor"` + } `json:"pageInfo"` + Edges []struct { + Cursor string `json:"cursor"` + Node json.RawMessage `json:"node"` + } `json:"edges"` +} + +func decodeRelayIssue(repo string, raw json.RawMessage) (*Issue, error) { + var v relayIssue + if err := json.Unmarshal(raw, &v); err != nil { + return nil, badPayload(repo, err) + } + if v.Number == 0 { + return nil, structureChanged(repo) + } + if v.Repository.NameWithOwner != "" { + // A transferred issue reports its new home here, and the new home is + // the truth. The caller's repo argument is where the redirect started. + repo = v.Repository.NameWithOwner + } + + iss := &Issue{} + t := &iss.Thread + t.Repo = repo + t.Number = v.Number + t.setIdentity(KindIssue, repo+"#"+strconv.Itoa(v.Number)) + if v.URL != "" { + t.URL = v.URL + } + t.Title = v.Title + if v.TitleHTML != v.Title { + t.TitleHTML = v.TitleHTML + } + t.State = enum(v.State) + t.StateReason = enum(v.StateReason) + t.Body = v.Body + t.BodyHTML = v.BodyHTML + t.Author = v.Author.actor() + t.Locked = v.Locked + t.IsPinned = v.IsPinned + t.CreatedAt = parseTime(v.CreatedAt) + t.UpdatedAt = parseTime(v.UpdatedAt) + t.NodeID = v.ID + t.DatabaseID = v.DatabaseID + for _, e := range v.Labels.Edges { + t.Labels = append(t.Labels, e.Node.label()) + } + t.Milestone = v.Milestone.milestone() + for _, a := range v.AssignedActors.Nodes { + t.Assignees = append(t.Assignees, a.actor()) + } + t.Reactions = reactions(v.ReactionGroups) + // CommentCount stays nil deliberately. The only count on this payload is + // the timeline total, and a timeline counts labellings and assignments as + // well as comments, so reporting it as the comment count would be wrong by + // however much housekeeping the thread has had. Search states the real one. + // + // ClosedAt is not on the issue node at all. It is a ClosedEvent on the + // timeline, so it gets filled from there when the timeline carries one. + t.ClosedAt = closedAtFrom(v.FrontTimelineItems, v.BackTimelineItems) + + if v.IssueType != nil { + iss.IssueType = v.IssueType.Name + } + if v.DuplicateOf != nil { + iss.DuplicateOf = v.DuplicateOf.URL + } + if s := v.SubIssuesSummary; s != nil { + iss.SubIssueTotal = s.Total + iss.SubIssueDone = s.Completed + } + for _, n := range v.LinkedPullRequests.Nodes { + iss.LinkedPRs = append(iss.LinkedPRs, n.URL) + } + for _, n := range v.ClosedByPullRequestsReferences.Nodes { + iss.ClosedByPRs = append(iss.ClosedByPRs, n.URL) + } + for _, e := range v.ProjectItems.Edges { + if e.Node.Project.Title != "" { + iss.ProjectItems = append(iss.ProjectItems, e.Node.Project.Title) + } + } + + t.addExtra("issue", decodeExtra(raw, &v, issueSkips...)) + return iss, nil +} + +// issueSkips are the keys the issue node carries that a logged-out reader has +// no use for. Every one of them is a permission or a draft-state flag scoped to +// the viewer, and this tool has no viewer. +var issueSkips = []string{ + // Session state. All of these are false for us by definition. + "viewer*", + // Relay type discriminators. They exist so the client can pick a fragment, + // and __typename is already read where it decides something. + "__is*", "__typename", + // Editing scaffolding: the body hash the editor posts back, the suggestion + // drafts, the agent handoff, and the pinned-comment slot. None of them are + // facts about the issue. + "bodyVersion", "pendingSuggestions", "agentAssignments", "pinnedIssueComment", + // Custom issue fields, which are private-repository projects plumbing and + // arrive empty on every public issue seen so far. + "issueFieldValues", +} + +// closedAtFrom digs the close date out of the timeline, because the issue node +// does not carry one. The back half is searched first: a close is usually the +// last thing that happened, and the back half is the end of the thread. +func closedAtFrom(front, back relayTimeline) *time.Time { + for _, tl := range []relayTimeline{back, front} { + for i := len(tl.Edges) - 1; i >= 0; i-- { + var n struct { + Typename string `json:"__typename"` + CreatedAt string `json:"createdAt"` + } + if err := json.Unmarshal(tl.Edges[i].Node, &n); err != nil { + continue + } + if n.Typename == "ClosedEvent" { + return parseTime(n.CreatedAt) + } + } + } + return nil +} + +// --- the timeline --- + +// Timeline emits the events the issue page carries. +// +// It is deliberately not a pager. The page preloads the first fifteen events +// and, on a long thread, the last few; walking past those needs the GraphQL +// endpoint, which needs a session. So this returns what the page had and +// Truncated says whether there is more, rather than pretending to a +// completeness it cannot deliver. +func (c *Client) Timeline(ctx context.Context, repo string, number int, limit int, emit func(TimelineItem) error) error { + if _, _, ok := SplitRepo(repo); !ok { + return usageBadID("repository", repo, "owner/name") + } + res, err := c.GetHTML(ctx, threadURL(repo, "issues", number)) + if err != nil { + return err + } + p := page.Extract(res.FinalURL, res.Body) + raw, ok := issueNode(p) + if !ok { + return structureChanged(repo + "#" + strconv.Itoa(number)) + } + var v relayIssue + if err := json.Unmarshal(raw, &v); err != nil { + return badPayload(repo, err) + } + + thread := repo + "#" + strconv.Itoa(number) + seen := 0 + for _, tl := range []relayTimeline{v.FrontTimelineItems, v.BackTimelineItems} { + for _, e := range tl.Edges { + item, ok := decodeTimelineItem(thread, e.Cursor, e.Node) + if !ok { + continue + } + item.addSource(res.FinalURL) + if err := emit(item); err != nil { + return err + } + seen++ + if limit > 0 && seen >= limit { + return nil + } + } + } + return nil +} + +// timelineNode is the union of every field the timeline typenames use. Relay +// sends a different shape per __typename, and the fields do not collide, so one +// struct decodes all of them and the typename decides which half is filled. +type timelineNode struct { + Typename string `json:"__typename"` + DatabaseID *int `json:"databaseId"` + ID string `json:"id"` + CreatedAt string `json:"createdAt"` + Actor relayActor `json:"actor"` + Author relayActor `json:"author"` + Body string `json:"body"` + BodyHTML string `json:"bodyHTML"` + URL string `json:"url"` + Association string `json:"authorAssociation"` + IsHidden bool `json:"isHidden"` + MinimizedReason string `json:"minimizedReason"` + CreatedViaEmail bool `json:"createdViaEmail"` + LastEditedAt string `json:"lastEditedAt"` + + Label *relayLabel `json:"label"` + Milestone *relayMilestone `json:"milestone"` + Assignee *relayActor `json:"assignee"` + Reactions []relayReactionGroup `json:"reactionGroups"` + + PreviousTitle string `json:"previousTitle"` + CurrentTitle string `json:"currentTitle"` + + Commit *struct { + OID string `json:"oid"` + URL string `json:"url"` + } `json:"commit"` + Source *struct { + URL string `json:"url"` + } `json:"source"` + Subject *struct { + URL string `json:"url"` + } `json:"subject"` +} + +func decodeTimelineItem(thread, cursor string, raw json.RawMessage) (TimelineItem, bool) { + var n timelineNode + if err := json.Unmarshal(raw, &n); err != nil { + return TimelineItem{}, false + } + if n.Typename == "" { + return TimelineItem{}, false + } + + item := TimelineItem{Thread: thread, Type: snakeCase(n.Typename), Cursor: cursor} + id := thread + ":" + item.Type + if n.DatabaseID != nil { + id = thread + ":" + strconv.Itoa(*n.DatabaseID) + } + item.setIdentity(KindIssue, id) + // setIdentity built a URL from the id, which is wrong for an event: it is + // not an issue. The node's own URL is right when it has one, and no URL is + // better than one that resolves somewhere else. + item.URL = n.URL + + item.CreatedAt = parseTime(n.CreatedAt) + if a := n.Actor.actor(); a.Login != "" { + item.Actor = &a + } else if a := n.Author.actor(); a.Login != "" { + item.Actor = &a + } + item.Body = n.Body + item.BodyHTML = n.BodyHTML + if n.Label != nil { + l := n.Label.label() + item.Label = &l + } + item.Milestone = n.Milestone.milestone() + if n.Assignee != nil { + a := n.Assignee.actor() + item.Assignee = &a + } + item.FromTitle = n.PreviousTitle + item.ToTitle = n.CurrentTitle + if n.Commit != nil { + item.Commit = n.Commit.OID + } + switch { + case n.Source != nil && n.Source.URL != "": + item.Source = n.Source.URL + case n.Subject != nil: + item.Source = n.Subject.URL + } + item.Reactions = reactions(n.Reactions) + item.Minimized = n.IsHidden + item.MinimizedReason = n.MinimizedReason + item.CreatedViaEmail = n.CreatedViaEmail + item.LastEditedAt = parseTime(n.LastEditedAt) + + item.addExtra("timeline", decodeExtra(raw, &n, timelineSkips...)) + return item, true +} + +// timelineSkips is the same reasoning as issueSkips, plus the back-references +// the node carries to its own thread and repository, which the caller already +// has because it asked for them. +var timelineSkips = []string{ + "viewer*", "__is*", + "bodyVersion", "issue", "pullRequest", "repository", + // Sponsorship badges, spam flags, and the app that posted the comment. + // They are display decisions, not events. + "authorToRepoOwnerSponsorship", "showSpammyBadge", "viaApp", + "lastUserContentEdit", "pinnedBy", "intent", "willCloseTarget", + "willCloseSubject", "innerSource", "referencedAt", +} + +// snakeCase turns a GraphQL typename into the wire form the records use: +// IssueComment becomes issue_comment, CrossReferencedEvent becomes +// cross_referenced_event. +func snakeCase(s string) string { + var b strings.Builder + for i, r := range s { + if r >= 'A' && r <= 'Z' { + if i > 0 { + b.WriteByte('_') + } + b.WriteRune(r - 'A' + 'a') + continue + } + b.WriteRune(r) + } + return b.String() +} + +// --- pull requests --- + +// PullRequest reads one pull request. +// +// This one takes the HTML page rather than the route JSON, and the reason is +// worth writing down. Asking a React route for JSON returns the props for that +// route only, as a delta against the layout that is already mounted in a real +// browser. /pull/{n} with Accept: application/json is a kilobyte of websocket +// channel tokens: no title, no author, no merge state. The layout route that +// holds those lives in the page's embedded payload and nowhere else, so the +// page is what we fetch. It is about 165 KB on the wire and that is the price. +// +// One fetch then covers everything a logged-out client can see: +// +// pullRequestsLayoutRoute title, author, refs, head sha, merge metadata +// pullRequestsConversationsRoute node id and the lock flag +// the sidebar markup labels +// og:description the body, truncated to 200 characters +// +// The body really is a snippet. GitHub renders the conversation client-side and +// serves none of it to a logged-out client, so 200 characters of open graph +// text is the whole of what is reachable. _via records body: og for it, and the +// hovercard fragment is the fallback when the page has no open graph body. +func (c *Client) PullRequest(ctx context.Context, repo string, number int) (*PullRequest, error) { + if _, _, ok := SplitRepo(repo); !ok { + return nil, usageBadID("repository", repo, "owner/name") + } + if number <= 0 { + return nil, usageBadID("pull request number", strconv.Itoa(number), "a positive integer") + } + + url := threadURL(repo, "pull", number) + res, err := c.GetHTML(ctx, url) + if err != nil { + return nil, err + } + p := page.Extract(res.FinalURL, res.Body) + raw, ok := p.Route("pullRequestsLayoutRoute") + if !ok { + return nil, structureChanged(repo + "#" + strconv.Itoa(number)) + } + + pr, err := decodePullRoute(repo, number, raw) + if err != nil { + return nil, err + } + pr.addSource(res.FinalURL) + if p.Canonical != "" { + pr.URL = p.Canonical + } + readPullPage(p, pr) + + // The hovercard is the fallback for the body and for the labels, and it is + // only worth a second request when the page gave neither. A failure here + // keeps the record: a pull request with no body text is still useful. + if pr.Body == "" && len(pr.Labels) == 0 { + if hc, err := c.hovercard(ctx, url); err == nil { + hc.applyTo(pr) + pr.addSource(url + "/hovercard") + recordVia(&pr.Base, "body", "xhr") + } + } + return pr, nil +} + +// readPullPage takes the three facts the layout route does not carry off the +// rest of the page: the node id and the lock flag from the conversation route, +// the labels from the server-rendered sidebar, and the body from open graph. +func readPullPage(p *page.Page, pr *PullRequest) { + if raw, ok := p.Route("pullRequestsConversationsRoute"); ok { + var conv struct { + ID string `json:"id"` + Locked bool `json:"locked"` + } + if json.Unmarshal(raw, &conv) == nil { + if pr.NodeID == "" { + pr.NodeID = conv.ID + } + pr.Locked = conv.Locked + } + } + + if doc := p.Doc(); doc != nil { + seen := map[string]bool{} + for _, n := range page.FindAll(doc, page.HovercardLabel) { + // The sidebar renders each label twice, once for the wide layout + // and once for the narrow one, so the same name comes back twice. + name := page.Attr(n, "data-name") + if name == "" || seen[name] { + continue + } + seen[name] = true + pr.Labels = append(pr.Labels, Label{Name: name}) + } + } + + // og:description is the body on a thread page and the repository blurb on a + // page with no body, and meta description is always the repository blurb. + // Comparing the two is how you tell an empty body from a real one. + if body := p.MetaContent("og:description"); body != "" && body != p.MetaContent("description") { + pr.Body = body + recordVia(&pr.Base, "body", "og") + } +} + +type pullRoute struct { + PullRequest struct { + ID *int `json:"id"` + RelayID string `json:"relayId"` + Number int `json:"number"` + Title string `json:"title"` + TitleHTML string `json:"titleHtml"` + State string `json:"state"` + BaseBranch string `json:"baseBranch"` + HeadBranch string `json:"headBranch"` + HeadSha string `json:"headSha"` + HeadRepo string `json:"headRepositoryName"` + HeadOwner string `json:"headRepositoryOwnerLogin"` + CommitCount *int `json:"commitsCount"` + CreatedTime string `json:"createdTime"` + ClosedTime string `json:"closedTime"` + MergedTime string `json:"mergedTime"` + MergedBy string `json:"mergedBy"` + MergedName string `json:"mergedByName"` + MergedAvat string `json:"mergedByAvatarUrl"` + Author struct { + Login string `json:"login"` + DisplayName string `json:"displayName"` + AvatarURL string `json:"avatarUrl"` + } `json:"author"` + } `json:"pullRequest"` + Repository struct { + ID *int `json:"id"` + Name string `json:"name"` + OwnerLogin string `json:"ownerLogin"` + DefaultBranch string `json:"defaultBranch"` + } `json:"repository"` +} + +func decodePullRoute(repo string, number int, raw json.RawMessage) (*PullRequest, error) { + var v pullRoute + if err := json.Unmarshal(raw, &v); err != nil { + return nil, badPayload(repo, err) + } + p := v.PullRequest + if p.Number == 0 { + return nil, structureChanged(repo + "#" + strconv.Itoa(number)) + } + if v.Repository.OwnerLogin != "" && v.Repository.Name != "" { + repo = v.Repository.OwnerLogin + "/" + v.Repository.Name + } + + pr := &PullRequest{} + t := &pr.Thread + t.Repo = repo + t.Number = p.Number + t.setIdentity(KindPR, repo+"#"+strconv.Itoa(p.Number)) + t.Title = p.Title + if p.TitleHTML != p.Title { + t.TitleHTML = p.TitleHTML + } + t.State = enum(p.State) + t.CreatedAt = parseTime(p.CreatedTime) + t.ClosedAt = parseTime(p.ClosedTime) + t.NodeID = p.RelayID + t.DatabaseID = p.ID + t.Author = actor(p.Author.Login) + t.Author.Name = p.Author.DisplayName + t.Author.AvatarURL = p.Author.AvatarURL + + pr.BaseRef = p.BaseBranch + pr.HeadRef = p.HeadBranch + if p.HeadOwner != "" && p.HeadOwner+"/"+p.HeadRepo != repo { + // A fork's head is only unambiguous with the owner on it, and the + // owner is the whole point of the field on a cross-repository PR. + pr.HeadRef = p.HeadOwner + ":" + p.HeadBranch + } + pr.HeadOID = p.HeadSha + pr.CommitCount = p.CommitCount + pr.MergedAt = parseTime(p.MergedTime) + pr.Merged = pr.MergedAt != nil || enum(p.State) == "merged" + if p.MergedBy != "" { + a := actor(p.MergedBy) + a.Name = p.MergedName + a.AvatarURL = p.MergedAvat + pr.MergedBy = &a + } + pr.IsDraft = enum(p.State) == "draft" + + t.addExtra("pullRequestsLayoutRoute", decodeExtra(raw, &v, pullSkips...)) + return pr, nil +} + +// pullSkips: the pull request route is mostly front-end plumbing. What is left +// after these is the twenty-odd fields decoded above. +var pullSkips = []string{ + // Live-update websocket tokens. They expire in minutes. + "aliveChannel", "aliveChannels", "mergeboxChannels", "markAsReadChannel", + // Feature flags, banners, and view state. + "featureStates", "bannersData", "viewSettings", "viewerPendingReview", + "mergeStatusButtonData", "user", "stack", "helpUrl", + // urls and pageTitle restate what the record already computes. + "urls", "pageTitle", +} + +// --- hovercards --- + +// hovercardData is the handful of facts the hovercard fragment carries that the +// pull request route does not. +type hovercardData struct { + Body string + State string + Labels []Label + Base string + Head string +} + +// hovercard fetches the XHR fragment behind a link's popover. Any thread URL +// plus /hovercard serves it, and it is the only keyless source for a pull +// request's body text. +func (c *Client) hovercard(ctx context.Context, threadURL string) (*hovercardData, error) { + res, err := c.Get(ctx, threadURL+"/hovercard", SurfaceXHR) + if err != nil { + return nil, err + } + doc, err := html.Parse(strings.NewReader(string(res.Body))) + if err != nil { + return nil, wrapNetwork(threadURL, err) + } + + hc := &hovercardData{} + if n := page.Find(doc, page.HovercardBody); n != nil { + hc.Body = page.Text(n) + } + if n := page.Find(doc, page.HovercardState); n != nil { + // The title reads "Status: Merged". The word after the colon is the + // state, and the element text is the same word, so either works and + // the text is the one that survives a title rewording. + hc.State = enum(page.Text(n)) + } + for _, n := range page.FindAll(doc, page.HovercardLabel) { + if name := page.Attr(n, "data-name"); name != "" { + hc.Labels = append(hc.Labels, Label{Name: name}) + } + } + refs := page.FindAll(doc, page.HovercardRef) + if len(refs) == 2 { + // Rendered as "base <- head", in that order, and there is nothing else + // on the card with this class. + hc.Base = page.Text(refs[0]) + hc.Head = page.Text(refs[1]) + } + return hc, nil +} + +func (hc *hovercardData) applyTo(pr *PullRequest) { + if pr.Body == "" { + pr.Body = hc.Body + } + if pr.State == "" { + pr.State = hc.State + } + if len(pr.Labels) == 0 { + pr.Labels = hc.Labels + } + if pr.BaseRef == "" { + pr.BaseRef = hc.Base + } + if pr.HeadRef == "" { + pr.HeadRef = hc.Head + } +} + +// --- pull request commits --- + +// PullCommits emits the commits on a pull request. The route serves them all in +// one response grouped by push, which is why there is no pager here. +func (c *Client) PullCommits(ctx context.Context, repo string, number int, limit int, emit func(Commit) error) error { + if _, _, ok := SplitRepo(repo); !ok { + return usageBadID("repository", repo, "owner/name") + } + var env struct { + Payload struct { + Route struct { + CommitGroups []struct { + Commits []routeCommit `json:"commits"` + } `json:"commitGroups"` + } `json:"pullRequestsCommitsRoute"` + } `json:"payload"` + } + url := threadURL(repo, "pull", number) + "/commits" + res, err := c.GetJSON(ctx, url, SurfaceRouteJSON, &env) + if err != nil { + return err + } + + seen := 0 + for _, g := range env.Payload.Route.CommitGroups { + for _, rc := range g.Commits { + cm := rc.commit(repo) + cm.addSource(res.FinalURL) + if err := emit(cm); err != nil { + return err + } + seen++ + if limit > 0 && seen >= limit { + return nil + } + } + } + return nil +} + +// routeCommit is the commit shape the pull request commits route uses. It is +// not the same shape as the repository commits route, which is why it is here +// and not in commit.go. +type routeCommit struct { + OID string `json:"oid"` + ShortMessage string `json:"shortMessage"` + BodyHTML string `json:"bodyMessageHtml"` + AuthoredDate string `json:"authoredDate"` + CommittedDate string `json:"committedDate"` + Authors []routeActor `json:"authors"` + Committer *routeActor `json:"committer"` +} + +type routeActor struct { + Login string `json:"login"` + DisplayName string `json:"displayName"` + AvatarURL string `json:"avatarUrl"` + Path string `json:"path"` + IsGitHub bool `json:"isGitHub"` +} + +func (a routeActor) actor() Actor { + if a.Login == "" { + // Commits from an address with no GitHub account have a display name + // and nothing else. Dropping them would lose the authorship entirely. + return Actor{Name: a.DisplayName, AvatarURL: a.AvatarURL} + } + out := actor(a.Login) + out.Name = a.DisplayName + out.AvatarURL = a.AvatarURL + return out +} + +func (rc routeCommit) commit(repo string) Commit { + cm := Commit{Repo: repo, SHA: rc.OID} + cm.setIdentity(KindCommit, repo+"@"+rc.OID) + cm.Subject = rc.ShortMessage + cm.BodyHTML = rc.BodyHTML + cm.Body = stripTags(rc.BodyHTML) + cm.AuthoredAt = parseTime(rc.AuthoredDate) + cm.CommittedAt = parseTime(rc.CommittedDate) + for _, a := range rc.Authors { + cm.Authors = append(cm.Authors, a.actor()) + } + if rc.Committer != nil { + c := rc.Committer.actor() + cm.Committer = &c + } + return cm +} + +// --- discussions --- + +// Discussion reads one discussion. +// +// Discussions never migrated to React, so there is no payload to decode. What +// there is instead is a schema.org QAPage block, which carries the body, the +// upvote count, and the accepted answer, and is the most reliable thing on the +// page because GitHub publishes it for search engines and therefore keeps it +// working. Everything the block does not have comes from the markup. +// +// repo may be owner/name or an organization login: organization-level +// discussions live at /orgs/{login}/discussions/{n} and the read follows the +// redirect either way, then takes the true repository off the sidebar. +func (c *Client) Discussion(ctx context.Context, repo string, number int) (*Discussion, error) { + if number <= 0 { + return nil, usageBadID("discussion number", strconv.Itoa(number), "a positive integer") + } + url := threadURL(repo, "discussions", number) + if !strings.Contains(repo, "/") { + url = BaseURL + "/orgs/" + repo + "/discussions/" + strconv.Itoa(number) + } + res, err := c.GetHTML(ctx, url) + if err != nil { + return nil, err + } + p := page.Extract(res.FinalURL, res.Body) + doc := p.Doc() + if doc == nil { + return nil, structureChanged(url) + } + + sidebar := page.Find(doc, page.DiscussionSidebar) + if sidebar == nil { + return nil, structureChanged(url) + } + + d := &Discussion{} + t := &d.Thread + t.Repo = repoFromSidebar(sidebar, repo) + t.Number = number + t.setIdentity(KindDiscussion, t.Repo+"#"+strconv.Itoa(number)) + t.NodeID = page.Attr(sidebar, "data-gid") + t.addSource(res.FinalURL) + if p.Canonical != "" { + t.URL = p.Canonical + } + + readDiscussionHeader(d, doc) + readDiscussionSidebar(d, sidebar) + readDiscussionBody(d, doc) + readQAPage(d, p) + return d, nil +} + +// repoFromSidebar reads the true repository off the sidebar's deferred-load +// URL, which is /{owner}/{repo}/discussions/{n}/sidebar even when the page was +// served from the /orgs/ path. +func repoFromSidebar(sidebar *html.Node, fallback string) string { + u := page.Attr(sidebar, "data-url") + parts := strings.Split(strings.Trim(u, "/"), "/") + if len(parts) >= 2 && parts[0] != "" && parts[1] != "" { + return parts[0] + "/" + parts[1] + } + return fallback +} + +func readDiscussionHeader(d *Discussion, doc *html.Node) { + if n := page.Find(doc, page.DiscussionTitle); n != nil { + d.Title = page.Text(n) + } + for _, n := range page.FindAll(doc, page.DiscussionState) { + title := page.Attr(n, "title") + switch { + case strings.HasPrefix(title, "Status: "): + // "Status: Closed as resolved" is more than the state: it is the + // state and the reason, and both are worth keeping. + rest := strings.TrimPrefix(title, "Status: ") + state, reason, found := strings.Cut(rest, " as ") + d.State = enum(state) + if found { + d.StateReason = enum(reason) + } + case title == "Answered": + d.IsAnswered = true + } + } + if n := page.Find(doc, page.DiscussionAuthor); n != nil { + d.Author = actorFromHref(page.Attr(n, "href")) + } + if n := page.Find(doc, page.RelTimeEl); n != nil { + d.CreatedAt = parseTime(page.Attr(n, "datetime")) + } + if n := page.Find(doc, page.DiscussionUpvote); n != nil { + // aria-label is "Upvote: 12". The number in the button text is the + // same, but the label survives the button being replaced by a form. + if _, after, ok := strings.Cut(page.Attr(n, "aria-label"), ":"); ok { + if v, err := strconv.Atoi(strings.TrimSpace(after)); err == nil { + d.Upvotes = intp(v) + } + } + } + if n := page.Find(doc, page.DiscussionAnswerLink); n != nil { + d.IsAnswered = true + // The answer author is the bold link right after the "by" that follows + // the answer link, and the answer link's parent holds both. + if parent := n.Parent; parent != nil { + for _, a := range page.FindAll(parent, page.ProfileAnyLink) { + if who := actorFromHref(page.Attr(a, "href")); who.Login != "" { + d.AnswerAuthor = &who + break + } + } + } + } +} + +func readDiscussionSidebar(d *Discussion, sidebar *html.Node) { + if n := page.Find(sidebar, page.DiscussionCategory); n != nil { + d.Category = page.Text(n) + } + for _, n := range page.FindAll(sidebar, page.DiscussionLabel) { + d.Labels = append(d.Labels, Label{ + Name: page.Attr(n, "data-name"), + URL: absolute(page.Attr(n, "href")), + }) + } +} + +func readDiscussionBody(d *Discussion, doc *html.Node) { + // The first comment container is the discussion body itself; the ones after + // it are replies. The container carries the node id, which is how the body + // is told apart from a reply that happens to come first in the markup. + for _, n := range page.FindAll(doc, page.DiscussionComment) { + gid := page.Attr(n, "data-gid") + if !strings.Contains(decodeGID(gid), "Discussion") || + strings.Contains(decodeGID(gid), "DiscussionComment") { + continue + } + if body := page.Find(n, page.DiscussionBody); body != nil { + d.BodyHTML = page.OuterHTML(body) + d.Body = page.BlockText(body) + } + return + } +} + +// readQAPage takes what the schema.org block says and lets it win where the two +// sources disagree, because it is the one GitHub maintains for search engines +// and the markup is the one that gets restyled. +func readQAPage(d *Discussion, p *page.Page) { + for _, raw := range p.LinkedData { + var v struct { + Type string `json:"@type"` + MainEntity struct { + Type string `json:"@type"` + Name string `json:"name"` + Text string `json:"text"` + UpvoteCount *int `json:"upvoteCount"` + AnswerCount *int `json:"answerCount"` + AcceptedAnswer *struct { + Text string `json:"text"` + } `json:"acceptedAnswer"` + } `json:"mainEntity"` + } + if err := json.Unmarshal(raw, &v); err != nil || v.Type != "QAPage" { + continue + } + e := v.MainEntity + if e.Name != "" { + d.Title = e.Name + } + if e.Text != "" { + // Both halves move together. Letting the markup win here while the + // prose still came off the DOM would leave a record whose two body + // fields describe different text. + d.BodyHTML = e.Text + d.Body = page.FragmentText(e.Text) + } + if e.UpvoteCount != nil { + d.Upvotes = e.UpvoteCount + } + if e.AnswerCount != nil { + d.CommentCount = e.AnswerCount + } + if e.AcceptedAnswer != nil { + d.IsAnswered = true + } + recordVia(&d.Base, "body_html", "ld-json") + return + } +} + +// --- small shared helpers --- + +// enum lowercases a GraphQL enum so that a state means the same string whatever +// surface it arrived on: the Relay node says CLOSED, the route says MERGED, and +// search says open. +func enum(s string) string { return strings.ToLower(strings.TrimSpace(s)) } + +// decodeGID unpacks a base64 global node id far enough to read the type off the +// front of it. "MDEwOkRpc2N1c3Npb24yNDQyOA==" decodes to "010:Discussion24428", +// which is how a discussion body is told apart from a reply without relying on +// document order. A gid that does not decode returns empty, and the caller +// treats that as "not the type I wanted", which is the safe direction. +func decodeGID(gid string) string { + if gid == "" { + return "" + } + b, err := base64.StdEncoding.DecodeString(gid) + if err != nil { + b, err = base64.RawURLEncoding.DecodeString(gid) + if err != nil { + return "" + } + } + return string(b) +} + +// absolute turns a site-relative href into a full URL and leaves an already +// absolute one alone. +func absolute(href string) string { + switch { + case href == "": + return "" + case strings.HasPrefix(href, "http"): + return href + case strings.HasPrefix(href, "/"): + return BaseURL + href + default: + return BaseURL + "/" + href + } +} diff --git a/gh/types.go b/gh/types.go new file mode 100644 index 0000000..c921b7f --- /dev/null +++ b/gh/types.go @@ -0,0 +1,901 @@ +package gh + +import "time" + +// types.go is the record model: twenty-five types that between them describe +// everything this tool can read off github.com without a token. +// +// Three conventions run through all of them. +// +// Counts are *int. nil means the surface did not carry it, 0 means the surface +// said zero. Collapsing those two is how aggregates end up quietly wrong. +// +// Timestamps are *time.Time and come from a datetime attribute or an ISO +// string. Rendered relative text ("3 days ago") is never parsed. +// +// Where a surface offers only a rendered count, the record carries both the +// parsed integer and the original string, because "8,112" and "8.112" are the +// same number in different locales and throwing away the original hides that. + +// --- repository --- + +// Repo is the centre of the model. A read from a page fills most of it, a read +// from a search result fills a thinner but honest subset, and Sources says +// which happened. +type Repo struct { + Base + + Owner string `json:"owner" table:"owner"` + Name string `json:"name" table:"name"` + + Description string `json:"description,omitempty" table:"description,truncate"` + DescriptionHighlight string `json:"description_highlight,omitempty" table:"-"` + Homepage string `json:"homepage,omitempty" table:"-"` + Topics []string `json:"topics,omitempty" table:"topics"` + + DatabaseID *int `json:"database_id,omitempty" table:"-"` + NodeID string `json:"node_id,omitempty" table:"-"` + + DefaultBranch string `json:"default_branch,omitempty" table:"branch"` + HeadSHA string `json:"head_sha,omitempty" table:"-"` + + Language string `json:"language,omitempty" table:"language"` + LanguageColor string `json:"language_color,omitempty" table:"-"` + Languages map[string]int64 `json:"languages,omitempty" table:"-"` + + Stars *int `json:"stars,omitempty" table:"stars"` + StarsDisplay string `json:"stars_display,omitempty" table:"-"` + Forks *int `json:"forks,omitempty" table:"forks"` + Watchers *int `json:"watchers,omitempty" table:"watchers"` + + OpenIssues *int `json:"open_issues,omitempty" table:"issues"` + GoodFirstIssues *int `json:"good_first_issues,omitempty" table:"-"` + HelpWantedIssues *int `json:"help_wanted_issues,omitempty" table:"-"` + + CommitCount *int `json:"commit_count,omitempty" table:"-"` + CommitCountDisplay string `json:"commit_count_display,omitempty" table:"-"` + ReleaseCount *int `json:"release_count,omitempty" table:"-"` + TagCount *int `json:"tag_count,omitempty" table:"-"` + FileCount *int `json:"file_count,omitempty" table:"-"` + DependentCount *int `json:"dependent_count,omitempty" table:"-"` + ContributorCount *int `json:"contributor_count,omitempty" table:"-"` + + // License comes from one sidebar anchor and from nowhere else on any + // keyless surface. See page.LicenseLink. + License string `json:"license,omitempty" table:"license"` + + IsFork bool `json:"is_fork" table:"-"` + ForkOf string `json:"fork_of,omitempty" table:"-"` + IsArchived bool `json:"is_archived" table:"-"` + IsMirror bool `json:"is_mirror" table:"-"` + IsTemplate bool `json:"is_template" table:"-"` + IsEmpty bool `json:"is_empty" table:"-"` + IsPrivate bool `json:"is_private" table:"-"` + IsOrgOwned bool `json:"is_org_owned" table:"-"` + Visibility string `json:"visibility,omitempty" table:"-"` + + Sponsorable bool `json:"sponsorable" table:"-"` + HasFunding bool `json:"has_funding" table:"-"` + HasCitation bool `json:"has_citation" table:"-"` + HasDiscussions bool `json:"has_discussions" table:"-"` + HasWiki bool `json:"has_wiki" table:"-"` + HasPages bool `json:"has_pages" table:"-"` + + CreatedAt *time.Time `json:"created_at,omitempty" table:"created,time"` + PushedAt *time.Time `json:"pushed_at,omitempty" table:"pushed,time"` + UpdatedAt *time.Time `json:"updated_at,omitempty" table:"updated,time"` + + OwnerAvatarURL string `json:"owner_avatar_url,omitempty" table:"-"` + SocialImageURL string `json:"social_image_url,omitempty" table:"-"` + + ReadmePath string `json:"readme_path,omitempty" table:"-"` + ReadmeHTML string `json:"readme_html,omitempty" table:"-"` + ReadmeText string `json:"readme_text,omitempty" table:"-"` + + Tree []TreeEntry `json:"tree,omitempty" table:"-"` + + StargazersPath string `json:"stargazers_path,omitempty" table:"-"` + ForkNetworkPath string `json:"fork_network_path,omitempty" table:"-"` + ActivityPath string `json:"activity_path,omitempty" table:"-"` +} + +// --- trees and files --- + +// TreeEntry is one row of a directory listing. Size and SHA are absent from the +// tree route and cost one request each, which is what `--sizes` opts into. +type TreeEntry struct { + Base + + Repo string `json:"repo" table:"-"` + Ref string `json:"ref" table:"-"` + + Name string `json:"name" table:"name"` + Path string `json:"path" table:"path"` + // Type is contentType verbatim: file, directory, symlink_file, + // symlink_directory, submodule. + Type string `json:"type" table:"type"` + + Size *int64 `json:"size,omitempty" table:"size"` + SHA string `json:"sha,omitempty" table:"-"` +} + +// File is a blob. The interesting part is Symbols: GitHub runs a symbol +// extractor over every blob it renders and ships the result in the route +// payload, and there is no unauthenticated REST equivalent anywhere. +type File struct { + Base + + Repo string `json:"repo" table:"-"` + Ref string `json:"ref" table:"-"` + Path string `json:"path" table:"path"` + + Size *int64 `json:"size,omitempty" table:"size"` + // SizeDisplay is what the page shows, "13.3 KB". The page has no byte + // count anywhere, so an exact Size costs a request to raw and is filled + // only when the bytes were fetched anyway. + SizeDisplay string `json:"size_display,omitempty" table:"-"` + Lines *int `json:"lines,omitempty" table:"lines"` + Language string `json:"language,omitempty" table:"language"` + IsBinary bool `json:"is_binary" table:"-"` + IsLFS bool `json:"is_lfs" table:"-"` + IsGenerated bool `json:"is_generated" table:"-"` + IsTruncated bool `json:"is_truncated" table:"-"` + + RawURL string `json:"raw_url" table:"-"` + + Content string `json:"content,omitempty" table:"-"` + RawLines []string `json:"raw_lines,omitempty" table:"-"` + RichText string `json:"rich_text,omitempty" table:"-"` + + TOC []Heading `json:"toc,omitempty" table:"-"` + Symbols []Symbol `json:"symbols,omitempty" table:"-"` + // SymbolsStatus is ok, timed_out, not_analyzed, or unavailable. An empty + // symbol list with not_analyzed means the language is unsupported, which is + // a different fact from a file that genuinely has no symbols, and + // unavailable means GitHub's analyser had not finished when we asked. The + // caller should not have to guess which one it got. + SymbolsStatus string `json:"symbols_status,omitempty" table:"-"` +} + +// Heading is one entry of a rendered markdown table of contents. +type Heading struct { + Level int `json:"level" table:"level"` + Text string `json:"text" table:"text"` + Anchor string `json:"anchor" table:"anchor"` +} + +// Symbol is one extracted definition, with byte offsets into the blob. +type Symbol struct { + Name string `json:"name" table:"name"` + Kind string `json:"kind" table:"kind"` + FullyQualifiedName string `json:"fqn,omitempty" table:"-"` + IdentStart int `json:"ident_start" table:"-"` + IdentEnd int `json:"ident_end" table:"-"` + ExtentStart int `json:"extent_start" table:"-"` + ExtentEnd int `json:"extent_end" table:"-"` +} + +// --- accounts --- + +// Account is a user or an organization. Profiles carry no JSON payload at all, +// so every field here comes from microdata, a microformat class, a stable data +// attribute, or a counted link. That makes accounts the most selector-dependent +// records in the tool, and the reason every field has a golden pinning it. +type Account struct { + Base + + Login string `json:"login" table:"login"` + Name string `json:"name,omitempty" table:"name"` + // Type is User or Organization, decided by which blocks the page carries + // rather than guessed from the login. + Type string `json:"type" table:"type"` + + Bio string `json:"bio,omitempty" table:"bio,truncate"` + Company string `json:"company,omitempty" table:"company"` + Location string `json:"location,omitempty" table:"location"` + Website string `json:"website,omitempty" table:"-"` + Email string `json:"email,omitempty" table:"-"` + Pronouns string `json:"pronouns,omitempty" table:"-"` + SocialLinks []string `json:"social_links,omitempty" table:"-"` + + DatabaseID *int `json:"database_id,omitempty" table:"-"` + NodeID string `json:"node_id,omitempty" table:"-"` + AvatarURL string `json:"avatar_url,omitempty" table:"-"` + + Followers *int `json:"followers,omitempty" table:"followers"` + FollowersDisplay string `json:"followers_display,omitempty" table:"-"` + Following *int `json:"following,omitempty" table:"following"` + Starred *int `json:"starred,omitempty" table:"-"` + RepoCount *int `json:"repo_count,omitempty" table:"repos"` + // These three come from the navigation tabs, which is the only place + // either template states them. A tab with nothing in it carries no counter + // at all, so zero and absent are the same thing here and the field stays + // nil rather than claiming a zero it did not read. + PackageCount *int `json:"package_count,omitempty" table:"-"` + ProjectCount *int `json:"project_count,omitempty" table:"-"` + SponsoringCount *int `json:"sponsoring_count,omitempty" table:"-"` + + CreatedAt *time.Time `json:"created_at,omitempty" table:"joined,time"` + + Sponsorable bool `json:"sponsorable" table:"-"` + IsVerified bool `json:"is_verified" table:"-"` + IsHireable bool `json:"is_hireable" table:"-"` + + ReadmeHTML string `json:"readme_html,omitempty" table:"-"` + ReadmeText string `json:"readme_text,omitempty" table:"-"` + + PinnedRepos []string `json:"pinned_repos,omitempty" table:"-"` + Organizations []string `json:"organizations,omitempty" table:"-"` + Achievements []string `json:"achievements,omitempty" table:"-"` + + SocialImageURL string `json:"social_image_url,omitempty" table:"-"` +} + +// Org is an Account plus the five things only an organization page has. +// TopLanguages and TopTopics are deferred fragments and arrive only with +// --deep. +type Org struct { + Account + + VerifiedDomains []string `json:"verified_domains,omitempty" table:"-"` + MemberCount *int `json:"member_count,omitempty" table:"members"` + // Members is the avatar strip on the front page, which is a sample and not + // the roster. It never sets MemberCount for that reason: `github members` + // walks /orgs/{login}/people for the real list. + Members []string `json:"members,omitempty" table:"-"` + TopLanguages map[string]int `json:"top_languages,omitempty" table:"-"` + TopTopics []string `json:"top_topics,omitempty" table:"-"` + IsEnterprise bool `json:"is_enterprise" table:"-"` +} + +// --- threads --- + +// Thread is what issues, pull requests, and discussions have in common, which +// is most of it: their pages share a Relay payload shape. +type Thread struct { + Base + + Repo string `json:"repo" table:"repo"` + Number int `json:"number" table:"number"` + + Title string `json:"title" table:"title,truncate"` + TitleHighlight string `json:"title_highlight,omitempty" table:"-"` + TitleHTML string `json:"title_html,omitempty" table:"-"` + + State string `json:"state" table:"state"` + StateReason string `json:"state_reason,omitempty" table:"-"` + + Body string `json:"body,omitempty" table:"-"` + BodyHTML string `json:"body_html,omitempty" table:"-"` + + Author Actor `json:"author" table:"author"` + + Labels []Label `json:"labels,omitempty" table:"labels"` + Milestone *Milestone `json:"milestone,omitempty" table:"-"` + Assignees []Actor `json:"assignees,omitempty" table:"-"` + Reactions []Reaction `json:"reactions,omitempty" table:"-"` + + CommentCount *int `json:"comment_count,omitempty" table:"comments"` + + Locked bool `json:"locked" table:"-"` + IsPinned bool `json:"is_pinned" table:"-"` + + CreatedAt *time.Time `json:"created_at,omitempty" table:"created,time"` + UpdatedAt *time.Time `json:"updated_at,omitempty" table:"updated,time"` + ClosedAt *time.Time `json:"closed_at,omitempty" table:"-"` + + AuthorAssociation string `json:"author_association,omitempty" table:"-"` + + NodeID string `json:"node_id,omitempty" table:"-"` + DatabaseID *int `json:"database_id,omitempty" table:"-"` +} + +// Issue adds the tracking relationships GitHub keeps between issues and the +// work that closes them. +type Issue struct { + Thread + + IssueType string `json:"issue_type,omitempty" table:"-"` + SubIssueTotal *int `json:"sub_issue_total,omitempty" table:"-"` + SubIssueDone *int `json:"sub_issue_done,omitempty" table:"-"` + DuplicateOf string `json:"duplicate_of,omitempty" table:"-"` + LinkedPRs []string `json:"linked_prs,omitempty" table:"-"` + ClosedByPRs []string `json:"closed_by_prs,omitempty" table:"-"` + ProjectItems []string `json:"project_items,omitempty" table:"-"` +} + +// PullRequest adds the diff and the merge state. +type PullRequest struct { + Thread + + BaseRef string `json:"base_ref,omitempty" table:"base"` + HeadRef string `json:"head_ref,omitempty" table:"head"` + BaseOID string `json:"base_oid,omitempty" table:"-"` + HeadOID string `json:"head_oid,omitempty" table:"-"` + + Merged bool `json:"merged" table:"-"` + MergedAt *time.Time `json:"merged_at,omitempty" table:"merged,time"` + MergedBy *Actor `json:"merged_by,omitempty" table:"-"` + Mergeable string `json:"mergeable,omitempty" table:"-"` + IsDraft bool `json:"is_draft" table:"-"` + + Additions *int `json:"additions,omitempty" table:"+"` + // The render tag grammar uses "-" to mean "skip this column", and "-" is + // also the natural header for deletions. The grammar wins: deletions is + // hidden by default and shown with --fields deletions. This is deliberate, + // please do not "fix" it. + Deletions *int `json:"deletions,omitempty" table:"-"` + ChangedFiles *int `json:"changed_files,omitempty" table:"files"` + CommitCount *int `json:"commit_count,omitempty" table:"-"` + + ReviewDecision string `json:"review_decision,omitempty" table:"review"` + ReviewRequests []Actor `json:"review_requests,omitempty" table:"-"` + ClosesIssues []string `json:"closes_issues,omitempty" table:"-"` +} + +// Discussion adds the answer, which is the thing discussions have that issues +// do not. +type Discussion struct { + Thread + + Category string `json:"category,omitempty" table:"category"` + IsAnswered bool `json:"is_answered" table:"answered"` + AnswerChosenAt *time.Time `json:"answer_chosen_at,omitempty" table:"-"` + AnswerAuthor *Actor `json:"answer_author,omitempty" table:"-"` + Upvotes *int `json:"upvotes,omitempty" table:"upvotes"` +} + +// Label is a thread label. +type Label struct { + Name string `json:"name" table:"name"` + Color string `json:"color,omitempty" table:"color"` + Description string `json:"description,omitempty" table:"description,truncate"` + URL string `json:"url,omitempty" table:"url,url"` + NodeID string `json:"node_id,omitempty" table:"-"` +} + +// Milestone is a thread milestone. +type Milestone struct { + Title string `json:"title" table:"title"` + Number *int `json:"number,omitempty" table:"number"` + Closed bool `json:"closed" table:"closed"` + DueOn *time.Time `json:"due_on,omitempty" table:"due,time"` + ClosedAt *time.Time `json:"closed_at,omitempty" table:"-"` + Progress *float64 `json:"progress,omitempty" table:"progress"` + URL string `json:"url,omitempty" table:"url,url"` +} + +// Reaction is one emoji group. Content is the GraphQL enum: THUMBS_UP, +// THUMBS_DOWN, LAUGH, HOORAY, CONFUSED, HEART, ROCKET, EYES. All eight always +// arrive, most with a zero count, and the decoder drops the zeroes so an +// unreacted thread has an empty list rather than eight noisy nothings. +type Reaction struct { + Content string `json:"content" table:"content"` + Count int `json:"count" table:"count"` +} + +// TimelineItem is one event on a thread. Type is the GraphQL __typename, +// lower-snake-cased. +// +// An unrecognised typename does not get dropped: Type is set, the common fields +// are filled, the whole node goes into Extra, and the fixture suite fails and +// names it. That is the entire strategy for union drift. +type TimelineItem struct { + Base + + Thread string `json:"thread" table:"-"` + Type string `json:"type" table:"type"` + Cursor string `json:"cursor,omitempty" table:"-"` + + Actor *Actor `json:"actor,omitempty" table:"actor"` + CreatedAt *time.Time `json:"created_at,omitempty" table:"created,time"` + + Body string `json:"body,omitempty" table:"body,truncate"` + BodyHTML string `json:"body_html,omitempty" table:"-"` + + Label *Label `json:"label,omitempty" table:"-"` + Milestone *Milestone `json:"milestone,omitempty" table:"-"` + Assignee *Actor `json:"assignee,omitempty" table:"-"` + FromTitle string `json:"from_title,omitempty" table:"-"` + ToTitle string `json:"to_title,omitempty" table:"-"` + Commit string `json:"commit,omitempty" table:"-"` + Source string `json:"source,omitempty" table:"-"` + Reactions []Reaction `json:"reactions,omitempty" table:"-"` + + Minimized bool `json:"minimized" table:"-"` + MinimizedReason string `json:"minimized_reason,omitempty" table:"-"` + CreatedViaEmail bool `json:"created_via_email" table:"-"` + LastEditedAt *time.Time `json:"last_edited_at,omitempty" table:"-"` +} + +// --- commits --- + +// Commit is one commit. Authors is a list because co-authored commits are +// common and the payload already ships an array; Committer is separate and only +// differs from the author when the surface says it does. +type Commit struct { + Base + + Repo string `json:"repo" table:"-"` + SHA string `json:"sha" table:"sha"` + NodeID string `json:"node_id,omitempty" table:"-"` + + Subject string `json:"subject" table:"subject,truncate"` + SubjectHighlight string `json:"subject_highlight,omitempty" table:"-"` + Body string `json:"body,omitempty" table:"-"` + BodyHTML string `json:"body_html,omitempty" table:"-"` + + Authors []Actor `json:"authors,omitempty" table:"authors"` + Committer *Actor `json:"committer,omitempty" table:"-"` + Pusher *Actor `json:"pusher,omitempty" table:"-"` + + AuthoredAt *time.Time `json:"authored_at,omitempty" table:"authored,time"` + CommittedAt *time.Time `json:"committed_at,omitempty" table:"-"` + PushedAt *time.Time `json:"pushed_at,omitempty" table:"-"` + + // DateGroup is the calendar-day heading the commit list grouped this commit + // under. It is kept because it is the only place the surface tells you what + // timezone it grouped in. + DateGroup string `json:"date_group,omitempty" table:"-"` + + Verification string `json:"verification,omitempty" table:"-"` + // VerificationReason is the why behind Verification: "unsigned", + // "valid", "expired_key", and so on. Verification alone says a commit is + // unverified without saying whether that is because nobody signed it or + // because the signature failed, which are very different facts. + VerificationReason string `json:"verification_reason,omitempty" table:"-"` + SignedByGitHub bool `json:"signed_by_github" table:"-"` + HasSignature bool `json:"has_signature" table:"-"` + KeyID string `json:"key_id,omitempty" table:"-"` + KeyExpired bool `json:"key_expired" table:"-"` + + StatusRollup string `json:"status_rollup,omitempty" table:"status"` + StatusSummary string `json:"status_summary,omitempty" table:"-"` + CommentCount *int `json:"comment_count,omitempty" table:"-"` + + // IssueRefs are the issues and pull requests this commit's message closes + // or mentions, already resolved by GitHub. This is the commit-to-thread + // edge of the graph, handed over for free, and it is the reason commit + // search is worth reading even when you already have the commit. + IssueRefs []ThreadRef `json:"issue_refs,omitempty" table:"-"` + + Parents []string `json:"parents,omitempty" table:"-"` + Additions *int `json:"additions,omitempty" table:"-"` + Deletions *int `json:"deletions,omitempty" table:"-"` + Files []FileChange `json:"files,omitempty" table:"-"` +} + +// ThreadRef is a pointer to an issue or a pull request from somewhere else. It +// is not a Thread: it carries only what the referring surface knew, and the +// caller resolves it with `github get` when it wants the rest. +type ThreadRef struct { + DatabaseID *int `json:"database_id,omitempty" table:"id"` + Title string `json:"title,omitempty" table:"title,truncate"` + State string `json:"state,omitempty" table:"state"` + IsPullRequest bool `json:"is_pull_request" table:"-"` + Merged bool `json:"merged" table:"-"` + URL string `json:"url,omitempty" table:"url"` +} + +// FileChange is one file in a commit or a diff. +type FileChange struct { + Path string `json:"path" table:"path"` + PrevPath string `json:"prev_path,omitempty" table:"-"` + Status string `json:"status" table:"status"` + Additions *int `json:"additions,omitempty" table:"+"` + // Hidden by the same tag-grammar collision as PullRequest.Deletions. + Deletions *int `json:"deletions,omitempty" table:"-"` + IsBinary bool `json:"is_binary" table:"-"` +} + +// --- refs and releases --- + +// GitRef is a branch or a tag. It is not called Ref because Ident already owns +// the word "reference" in this package, and a git ref and a parsed URI are very +// different things to confuse in a stack trace. +// +// Three surfaces carry refs and each is incomplete differently: the branches +// page has authors and dates but a truncated list, the refs XHR has every name +// and nothing else, and the git protocol has every name with its SHA. The +// commands pick per question, which is why `github refs --names-only` is 6 KB +// where `github refs` is 588 KB. +type GitRef struct { + Base + + Repo string `json:"repo" table:"-"` + Name string `json:"name" table:"name"` + Type string `json:"type" table:"type"` + SHA string `json:"sha,omitempty" table:"sha"` + + // PeeledSHA is set for annotated tags, from the ^{} entry in the git + // protocol advertisement. + PeeledSHA string `json:"peeled_sha,omitempty" table:"-"` + IsDefault bool `json:"is_default" table:"default"` + Protected bool `json:"protected" table:"protected"` + + Author *Actor `json:"author,omitempty" table:"author"` + AuthoredAt *time.Time `json:"authored_at,omitempty" table:"authored,time"` +} + +// Release is one published release. Assets and download counts exist only on +// the per-release HTML page, so the feed-driven listing leaves Assets nil and +// `--assets` opts into one request per release. +type Release struct { + Base + + Repo string `json:"repo" table:"-"` + Tag string `json:"tag" table:"tag"` + + Title string `json:"title,omitempty" table:"title,truncate"` + Body string `json:"body,omitempty" table:"-"` + BodyHTML string `json:"body_html,omitempty" table:"-"` + + Author *Actor `json:"author,omitempty" table:"author"` + + PublishedAt *time.Time `json:"published_at,omitempty" table:"published,time"` + UpdatedAt *time.Time `json:"updated_at,omitempty" table:"-"` + + IsPrerelease bool `json:"is_prerelease" table:"pre"` + IsLatest bool `json:"is_latest" table:"latest"` + IsDraft bool `json:"is_draft" table:"-"` + + CommitSHA string `json:"commit_sha,omitempty" table:"-"` + + Assets []Asset `json:"assets,omitempty" table:"-"` + TarballURL string `json:"tarball_url,omitempty" table:"-"` + ZipballURL string `json:"zipball_url,omitempty" table:"-"` + + // RepoDatabaseID comes free from the Atom , which is + // tag:github.com,2008:Repository/11180687/v0.164.0. That is how a release + // read from a feed joins to a repository record without a second fetch. + RepoDatabaseID *int `json:"repo_database_id,omitempty" table:"-"` +} + +// Asset is one release download. +type Asset struct { + Name string `json:"name" table:"name"` + // Label is the text GitHub prints in place of the filename, "GitHub CLI + // 2.63.2 checksums" for gh_2.63.2_checksums.txt. It is set per asset at + // upload time and is usually the only human-readable thing in the row. + Label string `json:"label,omitempty" table:"-"` + Size *int64 `json:"size,omitempty" table:"-"` + SizeDisplay string `json:"size_display,omitempty" table:"size"` + // Digest is the sha256 the assets fragment publishes, prefixed "sha256:". + // It is new: GitHub added it around the time it stopped showing download + // counts to logged-out clients, so this record trades a popularity number + // for something you can actually verify a download against. + Digest string `json:"digest,omitempty" table:"-"` + // DownloadCount is left absent on a keyless read. The release page used to + // print it next to each asset and no longer does, and no other public + // surface carries it. The field stays because the shape of the record + // should not change when GitHub changes its mind again. + DownloadCount *int `json:"download_count,omitempty" table:"-"` + URL string `json:"url" table:"url,url"` + UpdatedAt *time.Time `json:"updated_at,omitempty" table:"-"` + ContentType string `json:"content_type,omitempty" table:"-"` +} + +// Compare is a range between two commits: what `github compare` returns. +// +// It is built from the plain-text patch mailbox rather than the compare page. +// The page is 227 KB of HTML with no JSON payload at all, and the .patch suffix +// on the same range is a git-format-patch stream that carries every commit's +// author, date, subject, and diff with no markup to guess at. Parsing a format +// GitHub cannot restyle is the whole point. +type Compare struct { + Base + + Repo string `json:"repo" table:"-"` + BaseRef string `json:"base_ref" table:"base"` + HeadRef string `json:"head_ref" table:"head"` + PatchURL string `json:"patch_url" table:"-"` + DiffURL string `json:"diff_url" table:"-"` + + Commits []Commit `json:"commits,omitempty" table:"-"` + // CommitCount is len(Commits) and is here so a table row says something + // useful without the caller reaching into the slice. + CommitCount int `json:"commit_count" table:"commits"` + + Files []FileChange `json:"files,omitempty" table:"-"` + FileCount int `json:"file_count" table:"files"` + Additions int `json:"additions" table:"+"` + Deletions int `json:"deletions" table:"-"` + + // Patch is the raw stream, kept only when the caller asked for it. It is + // megabytes on a wide range. + Patch string `json:"patch,omitempty" table:"-"` +} + +// --- the long tail --- + +// Topic is a curated or uncurated topic. The search result carries most of it; +// the long description, the logo, the creator, the release year, the Wikipedia +// link, and the aliases need the topic page. +type Topic struct { + Base + + Name string `json:"name" table:"name"` + DisplayName string `json:"display_name,omitempty" table:"display"` + + ShortDescription string `json:"short_description,omitempty" table:"description,truncate"` + Description string `json:"description,omitempty" table:"-"` + DescriptionHTML string `json:"description_html,omitempty" table:"-"` + + LogoURL string `json:"logo_url,omitempty" table:"-"` + WikipediaURL string `json:"wikipedia_url,omitempty" table:"-"` + GitHubURL string `json:"github_url,omitempty" table:"-"` + CreatedBy string `json:"created_by,omitempty" table:"-"` + Released string `json:"released,omitempty" table:"released"` + Aliases []string `json:"aliases,omitempty" table:"-"` + Related []string `json:"related,omitempty" table:"-"` + + StargazerCount *int `json:"stargazer_count,omitempty" table:"stars"` + AppliedCount *int `json:"applied_count,omitempty" table:"repos"` + + Featured bool `json:"featured" table:"-"` + Curated bool `json:"curated" table:"-"` +} + +// Package is a published package. Search is the only source, which means the +// record is complete the moment it is read. +type Package struct { + Base + + Repo string `json:"repo,omitempty" table:"repo"` + Name string `json:"name" table:"name"` + Type string `json:"type" table:"type"` + + Summary string `json:"summary,omitempty" table:"summary,truncate"` + Downloads *int `json:"downloads,omitempty" table:"downloads"` + Topics []string `json:"topics,omitempty" table:"-"` + Source string `json:"source,omitempty" table:"-"` + + UpdatedAt *time.Time `json:"updated_at,omitempty" table:"updated,time"` +} + +// WikiPage is one page of a repository wiki. +type WikiPage struct { + Base + + Repo string `json:"repo" table:"repo"` + Title string `json:"title" table:"title"` + Path string `json:"path" table:"path"` + Format string `json:"format,omitempty" table:"-"` + + Body string `json:"body,omitempty" table:"-"` + BodyHTML string `json:"body_html,omitempty" table:"-"` + + UpdatedAt *time.Time `json:"updated_at,omitempty" table:"updated,time"` + Author *Actor `json:"author,omitempty" table:"author"` +} + +// Gist is a gist and its files. +type Gist struct { + Base + + Owner string `json:"owner,omitempty" table:"owner"` + Description string `json:"description,omitempty" table:"description,truncate"` + + IsPublic bool `json:"is_public" table:"public"` + FileCount *int `json:"file_count,omitempty" table:"files"` + Forks *int `json:"forks,omitempty" table:"forks"` + Stars *int `json:"stars,omitempty" table:"stars"` + Revisions *int `json:"revisions,omitempty" table:"-"` + + Files []GistFile `json:"files,omitempty" table:"-"` + + CreatedAt *time.Time `json:"created_at,omitempty" table:"created,time"` + UpdatedAt *time.Time `json:"updated_at,omitempty" table:"-"` +} + +// GistFile is one file in a gist. +type GistFile struct { + Name string `json:"name" table:"name"` + Language string `json:"language,omitempty" table:"language"` + Size *int64 `json:"size,omitempty" table:"size"` + RawURL string `json:"raw_url" table:"-"` + Content string `json:"content,omitempty" table:"-"` +} + +// Action is a marketplace listing. The type covers both actions and apps, and +// Type says which. +type Action struct { + Base + + Name string `json:"name" table:"name"` + Slug string `json:"slug" table:"-"` + Owner string `json:"owner,omitempty" table:"owner"` + Description string `json:"description,omitempty" table:"description,truncate"` + + // ShortDescription is the one-line blurb on the listing card, which is a + // different string from Description on an app and the same one on a + // repository action. Both are kept rather than picked between. + ShortDescription string `json:"short_description,omitempty" table:"-"` + FullDescription string `json:"full_description,omitempty" table:"-"` + ExtendedDescription string `json:"extended_description,omitempty" table:"-"` + + Type string `json:"type,omitempty" table:"type"` + PrimaryCategory string `json:"primary_category,omitempty" table:"category"` + SecondaryCategory string `json:"secondary_category,omitempty" table:"-"` + Highlights []string `json:"highlights,omitempty" table:"-"` + + // A repository action and a marketplace app are both listings and the + // search results are interleaved, but only one of these two groups is ever + // populated for a given record. Which group is filled in is itself the + // answer to "what kind of thing is this". + Path string `json:"path,omitempty" table:"-"` + RepositoryID *int `json:"repository_id,omitempty" table:"-"` + IconName string `json:"icon_name,omitempty" table:"-"` + IconColor string `json:"icon_color,omitempty" table:"-"` + + ListingID *int `json:"listing_id,omitempty" table:"-"` + LogoURL string `json:"logo_url,omitempty" table:"-"` + InstallationCount *int `json:"installation_count,omitempty" table:"installs"` + State string `json:"state,omitempty" table:"state"` + CompanyURL string `json:"company_url,omitempty" table:"-"` + DocumentationURL string `json:"documentation_url,omitempty" table:"-"` + SupportURL string `json:"support_url,omitempty" table:"-"` + PrivacyPolicyURL string `json:"privacy_policy_url,omitempty" table:"-"` + TermsURL string `json:"terms_url,omitempty" table:"-"` + PricingURL string `json:"pricing_url,omitempty" table:"-"` + + Stars *int `json:"stars,omitempty" table:"stars"` + DependentCount *int `json:"dependent_count,omitempty" table:"used_by"` + + IsFree bool `json:"is_free" table:"-"` + IsVerifiedOwner bool `json:"is_verified_owner" table:"verified"` + IsFeatured bool `json:"is_featured" table:"-"` + IsRecommended bool `json:"is_recommended" table:"-"` + ByGitHub bool `json:"by_github" table:"-"` +} + +// Trending embeds Repo because a trending entry is a repository with three +// extra facts. Embedding is what makes `github trending -o url | xargs -n1 +// github get` work with no special case anywhere. +type Trending struct { + Repo + + StarsInPeriod *int `json:"stars_in_period,omitempty" table:"period_stars"` + Period string `json:"period" table:"period"` + BuiltBy []Actor `json:"built_by,omitempty" table:"-"` + Rank int `json:"rank" table:"rank"` +} + +// --- contributions --- + +// Contributor is one person's contribution statistics for a repository. Weeks +// arrives with the response but is dropped unless asked for, because the route +// sends every week since the repository began for every contributor and that is +// megabytes of mostly zeroes. It is never a table column either way, because a +// hundred weeks is not a column. +type Contributor struct { + Base + + Repo string `json:"repo" table:"repo"` + Login string `json:"login" table:"login"` + + Commits *int `json:"commits,omitempty" table:"commits"` + Additions *int `json:"additions,omitempty" table:"+"` + // Hidden by the tag-grammar collision, as everywhere else. + Deletions *int `json:"deletions,omitempty" table:"-"` + + // FirstWeek and LastWeek are derived by trimming the leading and trailing + // zero weeks, which turns a six-hundred-element array into two dates a + // table can show. + FirstWeek *time.Time `json:"first_week,omitempty" table:"first,time"` + LastWeek *time.Time `json:"last_week,omitempty" table:"last,time"` + + Weeks []ContributorWeek `json:"weeks,omitempty" table:"-"` + + AvatarURL string `json:"avatar_url,omitempty" table:"-"` + DatabaseID *int `json:"database_id,omitempty" table:"-"` +} + +// ContributorWeek is one week of one contributor's statistics. +type ContributorWeek struct { + Week time.Time `json:"week" table:"week,time"` + Additions int `json:"additions" table:"+"` + Deletions int `json:"deletions" table:"-"` + Commits int `json:"commits" table:"commits"` +} + +// ContributionDay is one square of a profile contribution graph. +type ContributionDay struct { + Base + + Login string `json:"login" table:"login"` + Date time.Time `json:"date" table:"date,time"` + Count int `json:"count" table:"count"` + Level int `json:"level" table:"level"` +} + +// Event is one entry of an activity feed. Type is derived from the entry id, +// which encodes the event class, rather than from the title text, which is +// prose and is localised. +type Event struct { + Base + + Actor Actor `json:"actor" table:"actor"` + Type string `json:"type" table:"type"` + Repo string `json:"repo,omitempty" table:"repo"` + + Title string `json:"title,omitempty" table:"title,truncate"` + BodyHTML string `json:"body_html,omitempty" table:"-"` + Target string `json:"target,omitempty" table:"-"` + At *time.Time `json:"at,omitempty" table:"at,time"` +} + +// --- projections --- + +// LanguageShare is one language of one repository. The repository record +// carries the same numbers as a map, which is the right shape to keep and the +// wrong shape to print, so this is the row form of it. +type LanguageShare struct { + Base + + Repo string `json:"repo" table:"repo"` + Language string `json:"language" table:"language"` + Percent float64 `json:"percent" table:"percent"` + Color string `json:"color,omitempty" table:"-"` +} + +// RepoStats is the counts and nothing else. +// +// Every field is already on Repo. The reason to have it separately is that a +// record with eight numbers in it is something you can store once a day and +// diff; a record with a readme in it is not. +type RepoStats struct { + Base + + Repo string `json:"repo" table:"repo"` + + Stars *int `json:"stars,omitempty" table:"stars"` + Forks *int `json:"forks,omitempty" table:"forks"` + Watchers *int `json:"watchers,omitempty" table:"watching"` + OpenIssues *int `json:"open_issues,omitempty" table:"issues"` + Commits *int `json:"commits,omitempty" table:"commits"` + Releases *int `json:"releases,omitempty" table:"releases"` + Tags *int `json:"tags,omitempty" table:"tags"` + Contributors *int `json:"contributors,omitempty" table:"people"` + Dependents *int `json:"dependents,omitempty" table:"used_by"` + + PushedAt *time.Time `json:"pushed_at,omitempty" table:"pushed,time"` +} + +// Dependency is one row of /network/dependencies: a package this repository +// declares in one of its manifests. +// +// The identity is the repository the package resolves to, because that is the +// only thing on the row with an address on github.com. A package GitHub cannot +// resolve to a repository has no Kind and no ID, and its name is still on the +// record, because a dependency list with the unresolvable rows silently dropped +// is a lie about what the manifest contains. +type Dependency struct { + Base + + Repo string `json:"repo" table:"repo"` + Package string `json:"package" table:"package"` + + SourceRepo string `json:"source_repo,omitempty" table:"source"` + Version string `json:"version,omitempty" table:"version"` + Relationship string `json:"relationship,omitempty" table:"rel"` + Ecosystem string `json:"ecosystem,omitempty" table:"ecosystem"` + Manifest string `json:"manifest,omitempty" table:"manifest"` + License string `json:"license,omitempty" table:"-"` +} + +// Dependent is one row of /network/dependents: a repository that depends on +// this one. The two counts are on the row, so a caller sorting the dependents +// of a popular library by stars does not need a fetch per row. +type Dependent struct { + Base + + Repo string `json:"repo" table:"repo"` + Dependent string `json:"dependent" table:"dependent"` + Owner string `json:"owner" table:"-"` + + Stars *int `json:"stars,omitempty" table:"stars"` + Forks *int `json:"forks,omitempty" table:"forks"` + + AvatarURL string `json:"avatar_url,omitempty" table:"-"` +} diff --git a/gh/uri.go b/gh/uri.go new file mode 100644 index 0000000..3ad1153 --- /dev/null +++ b/gh/uri.go @@ -0,0 +1,646 @@ +package gh + +import ( + "net/url" + "strings" + + "github.com/tamnd/any-cli/kit/errs" +) + +// uri.go is the whole identity scheme. Everything downstream, the graph, the +// RDF subjects, the cache keys, the `github get` dispatch, resolves through +// here, so an id that parses wrong is a bug that shows up everywhere at once. +// +// The grammar has three separators and each one means exactly one thing: +// +// / separates a namespace from a name, and a repository from a path +// # introduces a thread number +// @ introduces a git revision +// +// A path may contain slashes but never a `#` or an `@` in a position that +// matters, because the revision always comes immediately after the repository +// and the number is always last. So parsing splits on `#`, then on the first +// `@` after the second `/`, and the rest is a path. That is the entire trick. + +// The kinds. Twenty are addressable as github:// URIs; compare is a recognised +// route that names a range rather than a thing, and is here because people +// paste compare URLs. +const ( + KindRepo = "repo" + KindUser = "user" + KindOrg = "org" + KindIssue = "issue" + KindPR = "pr" + KindDiscussion = "discussion" + KindCommit = "commit" + KindBranch = "branch" + KindTag = "tag" + KindRelease = "release" + KindFile = "file" + KindTree = "tree" + KindLabel = "label" + KindMilestone = "milestone" + KindTopic = "topic" + KindGist = "gist" + KindPackage = "package" + KindAction = "action" + KindWiki = "wiki" + KindAdvisory = "advisory" + KindCompare = "compare" + + // These three name records GitHub derives rather than serves. There is no + // page whose address is one contributor's statistics or one day of a + // calendar, so they get a URI and no canonical URL, and Locate points at + // the page they were read from instead of inventing one. + KindContributor = "contributor" + KindContribution = "contribution" + KindEvent = "event" +) + +// Kinds is the whole set, in the order above, for help text and for the error a +// bad kind produces. Listing them is the difference between an error a reader +// can act on and one that sends them to the source. +var Kinds = []string{ + KindRepo, KindUser, KindOrg, KindIssue, KindPR, KindDiscussion, + KindCommit, KindBranch, KindTag, KindRelease, KindFile, KindTree, + KindLabel, KindMilestone, KindTopic, KindGist, KindPackage, KindAction, + KindWiki, KindAdvisory, KindCompare, + KindContributor, KindContribution, KindEvent, +} + +// Scheme is the URI scheme this package mints and dereferences. +const Scheme = "github" + +// Ident is a parsed reference: what kind of thing, its canonical id, and the +// fragment the URL carried. The fragment never changes the kind. A link to +// #issuecomment-66046293 is still a link to the issue, and a link to #L10-L20 +// is still a link to the file, so the anchor is recorded and set aside. +type Ident struct { + Kind string `json:"kind" table:"kind"` + ID string `json:"id" table:"id"` + Anchor string `json:"anchor,omitempty" table:"anchor"` + URI string `json:"uri" table:"uri"` + URL string `json:"url" table:"url,url"` +} + +// reserved lists the top-level github.com paths that are site routes rather +// than accounts. Without it, `github get https://github.com/topics/go` would +// classify topics as a user, which is the kind of wrong answer that only shows +// up in someone else's script. +var reserved = map[string]bool{ + "about": true, "advisories": true, "apps": true, "collections": true, + "contact": true, "customer-stories": true, "dashboard": true, "enterprise": true, + "events": true, "explore": true, "features": true, "issues": true, "join": true, + "login": true, "logout": true, "marketplace": true, "new": true, "notifications": true, + "orgs": true, "pricing": true, "pulls": true, "search": true, "security": true, + "settings": true, "site": true, "sponsors": true, "stars": true, "topics": true, + "trending": true, "readme": true, "codespaces": true, "sessions": true, +} + +// Classify turns anything a person might paste into a kind and an id. It does +// no I/O, and it never fails on a well-formed github.com URL. +// +// Two of its answers are guesses and both are documented as such. A bare word +// is a user, because a pure function cannot tell a user from an organization +// without asking. A bare owner/name is a repository. `github get` reads the +// page and returns a record whose Kind is the truth; classification is a +// routing hint, not an answer. +func Classify(input string) (kind, id string, err error) { + r, err := Parse(input) + if err != nil { + return "", "", err + } + return r.Kind, r.ID, nil +} + +// Parse is Classify with the fragment and the derived forms kept. +func Parse(input string) (Ident, error) { + s := strings.TrimSpace(input) + if s == "" { + return Ident{}, errs.Usage("empty reference") + } + var anchor string + switch { + case strings.HasPrefix(s, Scheme+"://"): + kind, id, a, err := parseURI(s) + if err != nil { + return Ident{}, err + } + return finish(kind, id, a) + case strings.Contains(s, "://"): + kind, id, a, err := parseURL(s) + if err != nil { + return Ident{}, err + } + return finish(kind, id, a) + } + // A bare reference. Strip a fragment the same way a URL would, so + // golang/go#1#issuecomment-1 and a pasted anchor both behave. + if i := strings.Index(s, "#"); i >= 0 { + if j := strings.Index(s[i+1:], "#"); j >= 0 { + anchor = s[i+1+j+1:] + s = s[:i+1+j] + } + } + s = strings.TrimPrefix(s, "/") + s = strings.TrimSuffix(s, "/") + kind, id, err := classifyBare(s) + if err != nil { + return Ident{}, err + } + return finish(kind, id, anchor) +} + +func finish(kind, id, anchor string) (Ident, error) { + u, err := Locate(kind, id) + if err != nil { + return Ident{}, err + } + return Ident{Kind: kind, ID: id, Anchor: anchor, URI: URI(kind, id), URL: u}, nil +} + +// URI renders the github:// form. It is a string join and not a url.URL, +// because ids contain `#` and `@` on purpose and url.URL would escape them. +func URI(kind, id string) string { return Scheme + "://" + kind + "/" + id } + +func parseURI(s string) (kind, id, anchor string, err error) { + rest := strings.TrimPrefix(s, Scheme+"://") + if i := strings.LastIndex(rest, "#issuecomment-"); i >= 0 { + anchor, rest = rest[i+1:], rest[:i] + } + kind, id, ok := strings.Cut(rest, "/") + if !ok || kind == "" || id == "" { + return "", "", "", errs.Usage("not a %s:// URI: %q", Scheme, s) + } + if !knownKind(kind) { + return "", "", "", errs.Usage("unknown kind %q; the kinds are %s", kind, strings.Join(Kinds, ", ")) + } + return kind, strings.TrimSuffix(id, "/"), anchor, nil +} + +// knownKind reads the same list the error message prints, so a kind cannot be +// accepted here and left out of the list a reader is shown. +func knownKind(k string) bool { + for _, want := range Kinds { + if k == want { + return true + } + } + return false +} + +// parseURL handles every github.com host that serves content, plus the two +// static hosts. Query strings are dropped: ?tab=repositories names a tab on a +// profile, not a different profile. +func parseURL(raw string) (kind, id, anchor string, err error) { + u, perr := url.Parse(raw) + if perr != nil { + return "", "", "", errs.Usage("not a URL: %q, %v", raw, perr) + } + host := strings.ToLower(u.Host) + host = strings.TrimPrefix(host, "www.") + path := strings.Trim(u.Path, "/") + anchor = u.Fragment + + switch host { + case "raw.githubusercontent.com": + // /{owner}/{repo}/{ref}/{path...} + p := strings.Split(path, "/") + if len(p) < 4 { + return "", "", "", errs.Usage("not a raw file URL: %q", raw) + } + return KindFile, p[0] + "/" + p[1] + "@" + p[2] + "/" + strings.Join(p[3:], "/"), anchor, nil + case "gist.github.com", "gist.githubusercontent.com": + p := strings.Split(path, "/") + if len(p) == 0 || p[0] == "" { + return "", "", "", errs.Usage("no gist named in %q", raw) + } + // A gist URL is either /{id} or /{owner}/{id}. The id is the last + // segment that looks like one. + return KindGist, p[len(p)-1], anchor, nil + case "github.com", "codeload.github.com": + return classifyPath(path, anchor, raw) + default: + return "", "", "", errs.Usage("not a github.com URL: %q", raw) + } +} + +func classifyPath(path, anchor, raw string) (kind, id, a string, err error) { + if path == "" { + return "", "", "", errs.Usage("no resource named in %q", raw) + } + p := strings.Split(path, "/") + + // Site routes first, so a repository named "topics" cannot shadow one. + switch p[0] { + case "topics": + if len(p) >= 2 { + return KindTopic, p[1], anchor, nil + } + case "marketplace": + if len(p) >= 3 && p[1] == "actions" { + return KindAction, p[2], anchor, nil + } + case "advisories": + if len(p) >= 2 { + return KindAdvisory, p[1], anchor, nil + } + case "orgs": + if len(p) >= 2 { + return KindOrg, p[1], anchor, nil + } + } + if reserved[p[0]] { + return "", "", "", errs.Usage("no resource behind %q; it is a github.com page, not a thing this tool reads", raw) + } + if len(p) == 1 { + return KindUser, p[0], anchor, nil + } + owner, name := p[0], strings.TrimSuffix(p[1], ".git") + repo := owner + "/" + name + if len(p) == 2 { + return KindRepo, repo, anchor, nil + } + + rest := p[2:] + switch rest[0] { + case "issues": + if len(rest) >= 2 && isNumber(rest[1]) { + return KindIssue, repo + "#" + rest[1], anchor, nil + } + return KindRepo, repo, anchor, nil + case "pull", "pulls": + if len(rest) >= 2 && isNumber(rest[1]) { + return KindPR, repo + "#" + rest[1], anchor, nil + } + return KindRepo, repo, anchor, nil + case "discussions": + if len(rest) >= 2 && isNumber(rest[1]) { + return KindDiscussion, repo + "#" + rest[1], anchor, nil + } + return KindRepo, repo, anchor, nil + case "commit": + if len(rest) >= 2 { + return KindCommit, repo + "@" + rest[1], anchor, nil + } + case "commits": + if len(rest) >= 2 { + return KindBranch, repo + "@" + rest[1], anchor, nil + } + return KindRepo, repo, anchor, nil + case "tree": + if len(rest) == 2 { + return KindBranch, repo + "@" + rest[1], anchor, nil + } + if len(rest) > 2 { + return KindTree, repo + "@" + rest[1] + "/" + strings.Join(rest[2:], "/"), anchor, nil + } + case "blob", "raw", "blame": + if len(rest) >= 3 { + return KindFile, repo + "@" + rest[1] + "/" + strings.Join(rest[2:], "/"), anchor, nil + } + case "releases": + if len(rest) >= 3 && rest[1] == "tag" { + return KindRelease, repo + "@" + strings.Join(rest[2:], "/"), anchor, nil + } + if len(rest) >= 3 && rest[1] == "download" { + return KindRelease, repo + "@" + rest[2], anchor, nil + } + return KindRepo, repo, anchor, nil + case "labels": + if len(rest) >= 2 { + name, _ := url.PathUnescape(strings.Join(rest[1:], "/")) + return KindLabel, repo + "/" + name, anchor, nil + } + case "milestone": + if len(rest) >= 2 { + return KindMilestone, repo + "/" + rest[1], anchor, nil + } + case "wiki": + if len(rest) >= 2 { + return KindWiki, repo + "/" + strings.Join(rest[1:], "/"), anchor, nil + } + return KindWiki, repo + "/Home", anchor, nil + case "pkgs": + // /{owner}/{repo}/pkgs/{type}/{name}, where the name is usually the + // repository and the thing inside it and so carries a %2F. Parsing + // decoded that back into a slash before the split, so the name is + // everything from the type onwards rather than the last segment. + if len(rest) >= 3 { + return KindPackage, repo + "/" + strings.Join(rest[2:], "/"), anchor, nil + } + case "compare": + if len(rest) >= 2 { + return KindCompare, repo + "@" + strings.Join(rest[1:], "/"), anchor, nil + } + case "archive": + if len(rest) >= 2 { + ref := strings.TrimSuffix(strings.TrimSuffix(rest[len(rest)-1], ".zip"), ".tar.gz") + return KindBranch, repo + "@" + ref, anchor, nil + } + } + // Every other repository tab (actions, settings, network, graphs, stargazers) + // is a view of the repository, so that is what it resolves to. + return KindRepo, repo, anchor, nil +} + +// classifyBare reads the compact forms people type: owner/name, owner/name#12, +// owner/name@sha, owner/name@ref/path, and a bare login. +func classifyBare(s string) (kind, id string, err error) { + if s == "" { + return "", "", errs.Usage("empty reference") + } + if strings.HasPrefix(strings.ToUpper(s), "GHSA-") { + return KindAdvisory, s, nil + } + if base, num, ok := strings.Cut(s, "#"); ok { + if !isNumber(num) { + return "", "", errs.Usage("the part after # must be a number, in %q", s) + } + if strings.Count(base, "/") != 1 { + return "", "", errs.Usage("not a thread reference: %q, which should look like owner/name#123", s) + } + // Bare owner/name#N is an issue, which is the same guess github.com + // makes: /issues/N redirects to /pull/N when N is a pull request. + return KindIssue, base + "#" + num, nil + } + if i := strings.Index(s, "@"); i >= 0 && strings.Count(s[:i], "/") == 1 { + repo, rev := s[:i], s[i+1:] + if rev == "" { + return "", "", errs.Usage("nothing after the @ in %q", s) + } + if r, path, ok := strings.Cut(rev, "/"); ok { + return KindFile, repo + "@" + r + "/" + path, nil + } + if isSHA(rev) { + return KindCommit, repo + "@" + rev, nil + } + // A short ref with no path is a branch by default. `github tag` and + // `github release` name their own kind and override this. + return KindBranch, repo + "@" + rev, nil + } + switch strings.Count(s, "/") { + case 0: + if reserved[s] { + return "", "", errs.Usage("no account behind %q; it is a github.com page, not a profile", s) + } + return KindUser, s, nil + case 1: + return KindRepo, s, nil + default: + // owner/name/something. When "something" is one of github.com's own + // route words this is a URL with the host left off, and the URL parser + // already knows exactly what it means, so hand it over rather than + // guess. That is what makes `github url cli/cli/blob/trunk/go.mod` and + // the full URL agree, which they did not when this guessed first. + p := strings.SplitN(s, "/", 3) + if word, _, _ := strings.Cut(p[2], "/"); routeWord[word] { + kind, id, _, err := classifyPath(s, "", s) + return kind, id, err + } + // Anything else is ambiguous between wiki, label, milestone, and + // package, so it goes to the one whose ids are numeric when it is + // numeric and to a wiki page otherwise. + if isNumber(p[2]) { + return KindMilestone, s, nil + } + return KindWiki, s, nil + } +} + +// routeWord is the set of third segments that make a bare reference a route +// rather than a name. It is exactly the case list of the switch in +// classifyPath, and the two have to stay in step: a word here that the switch +// does not handle resolves to the repository instead of to the thing named. +var routeWord = map[string]bool{ + "issues": true, "pull": true, "pulls": true, "discussions": true, + "commit": true, "commits": true, "tree": true, "blob": true, "raw": true, + "blame": true, "releases": true, "labels": true, "milestone": true, + "wiki": true, "pkgs": true, "compare": true, "archive": true, +} + +// Locate turns a kind and id back into the canonical github.com URL. +// Locate(Classify(u)) is the canonical form of u, which is what makes -o url +// safe to pipe back into the tool. +func Locate(kind, id string) (string, error) { + if id == "" { + return "", errs.Usage("no id given for a %s", kind) + } + switch kind { + case KindRepo: + return BaseURL + "/" + id, nil + case KindUser, KindOrg: + return BaseURL + "/" + id, nil + case KindIssue, KindPR, KindDiscussion: + repo, num, ok := strings.Cut(id, "#") + if !ok { + return "", errs.Usage("missing number: the %s id %q needs one", kind, id) + } + seg := map[string]string{KindIssue: "issues", KindPR: "pull", KindDiscussion: "discussions"}[kind] + return BaseURL + "/" + repo + "/" + seg + "/" + num, nil + case KindCommit: + repo, sha, ok := cutRev(id) + if !ok { + return "", errs.Usage("commit id %q is missing its sha", id) + } + return BaseURL + "/" + repo + "/commit/" + sha, nil + case KindBranch: + repo, ref, ok := cutRev(id) + if !ok { + return "", errs.Usage("branch id %q is missing its ref", id) + } + return BaseURL + "/" + repo + "/tree/" + ref, nil + case KindTag, KindRelease: + repo, tag, ok := cutRev(id) + if !ok { + return "", errs.Usage("missing tag: the %s id %q needs one", kind, id) + } + return BaseURL + "/" + repo + "/releases/tag/" + tag, nil + case KindFile, KindTree: + repo, ref, path, ok := SplitPathID(id) + if !ok { + return "", errs.Usage("wrong shape: the %s id %q is not owner/name@ref/path", kind, id) + } + seg := "blob" + if kind == KindTree { + seg = "tree" + } + if path == "" { + return BaseURL + "/" + repo + "/tree/" + ref, nil + } + return BaseURL + "/" + repo + "/" + seg + "/" + ref + "/" + path, nil + case KindLabel: + repo, name, ok := cutRepoRest(id) + if !ok { + return "", errs.Usage("label id %q is not owner/name/label", id) + } + return BaseURL + "/" + repo + "/labels/" + url.PathEscape(name), nil + case KindMilestone: + repo, num, ok := cutRepoRest(id) + if !ok { + return "", errs.Usage("milestone id %q is not owner/name/number", id) + } + return BaseURL + "/" + repo + "/milestone/" + num, nil + case KindWiki: + repo, page, ok := cutRepoRest(id) + if !ok { + return "", errs.Usage("wiki id %q is not owner/name/page", id) + } + return BaseURL + "/" + repo + "/wiki/" + page, nil + case KindPackage: + repo, name, ok := cutRepoRest(id) + if !ok { + return "", errs.Usage("package id %q is not owner/name/package", id) + } + // The name is escaped because a container package is usually called + // after the repository and the thing inside it, so it has a slash in + // it. GitHub wants that slash as %2F: the unescaped form 404s and the + // escaped one is the page. + return BaseURL + "/" + repo + "/pkgs/container/" + url.PathEscape(name), nil + case KindTopic: + return BaseURL + "/topics/" + id, nil + case KindAction: + return BaseURL + "/marketplace/actions/" + id, nil + case KindAdvisory: + return BaseURL + "/advisories/" + id, nil + case KindGist: + return GistURL + "/" + id, nil + case KindContributor: + // The id is owner/name@login and the page that states it is the graph, + // which is the whole roster rather than the one row. That is the + // closest true address, so it is the one given. + repo, _, ok := cutRev(id) + if !ok { + return "", errs.Usage("contributor id %q is not owner/name@login", id) + } + return BaseURL + "/" + repo + "/graphs/contributors", nil + case KindContribution: + login, _, ok := cutRev(id) + if !ok { + return "", errs.Usage("contribution id %q is not login@date", id) + } + return BaseURL + "/" + login, nil + case KindEvent: + // An event's address is the thing it happened to, which the feed states + // per entry and no rule can reconstruct from the id. + return "", errs.Usage("an event has no address of its own; read its url field") + case KindCompare: + repo, rng, ok := cutRev(id) + if !ok { + return "", errs.Usage("compare id %q is missing its range", id) + } + return BaseURL + "/" + repo + "/compare/" + rng, nil + } + return "", errs.Usage("unknown kind %q; the kinds are %s", kind, strings.Join(Kinds, ", ")) +} + +// cutRev splits owner/name@rev. It looks for the `@` after the second slash so +// that an owner with an `@` in it, which github.com does not allow but a +// hand-written id might contain, cannot confuse it. +func cutRev(id string) (repo, rev string, ok bool) { + i := strings.Index(id, "@") + if i <= 0 || i == len(id)-1 { + return "", "", false + } + return id[:i], id[i+1:], true +} + +// SplitPathID splits owner/name@ref/path/to/file into its three parts. The ref +// runs to the next slash, which means a branch with a slash in its name +// (feature/x) parses as ref "feature" and path "x/...". That is a real +// ambiguity in GitHub's own URLs and nothing here can resolve it; a caller who +// knows better passes --ref. +func SplitPathID(id string) (repo, ref, path string, ok bool) { + repo, rest, ok := cutRev(id) + if !ok { + return "", "", "", false + } + ref, path, _ = strings.Cut(rest, "/") + if ref == "" { + return "", "", "", false + } + return repo, ref, path, true +} + +// cutRepoRest splits owner/name/rest, keeping any slashes in rest. +func cutRepoRest(id string) (repo, rest string, ok bool) { + p := strings.SplitN(id, "/", 3) + if len(p) != 3 || p[0] == "" || p[1] == "" || p[2] == "" { + return "", "", false + } + return p[0] + "/" + p[1], p[2], true +} + +// SplitThreadID splits owner/name#123. +func SplitThreadID(id string) (repo string, num string, ok bool) { + repo, num, ok = strings.Cut(id, "#") + if !ok || repo == "" || !isNumber(num) { + return "", "", false + } + return repo, num, true +} + +// SplitRepo splits owner/name. +func SplitRepo(id string) (owner, name string, ok bool) { + owner, name, ok = strings.Cut(id, "/") + if !ok || owner == "" || name == "" || strings.Contains(name, "/") { + return "", "", false + } + return owner, name, true +} + +// RepoOf returns the repository an id belongs to, for the kinds whose id +// carries one. This is what makes `github tree ` work. +func RepoOf(kind, id string) (string, bool) { + switch kind { + case KindRepo: + if _, _, ok := SplitRepo(id); ok { + return id, true + } + case KindIssue, KindPR, KindDiscussion: + if repo, _, ok := SplitThreadID(id); ok { + return repo, true + } + case KindCommit, KindBranch, KindTag, KindRelease, KindCompare: + if repo, _, ok := cutRev(id); ok { + return repo, true + } + case KindFile, KindTree: + if repo, _, _, ok := SplitPathID(id); ok { + return repo, true + } + case KindLabel, KindMilestone, KindWiki, KindPackage: + if repo, _, ok := cutRepoRest(id); ok { + return repo, true + } + } + return "", false +} + +func isNumber(s string) bool { + if s == "" { + return false + } + for _, r := range s { + if r < '0' || r > '9' { + return false + } + } + return true +} + +// isSHA reports whether a revision looks like an object name rather than a +// branch. Seven is git's own abbreviation floor, and a seven-character branch +// name made only of hex digits (`decade`, `facade` are six) is rare enough that +// this is the right default and --ref is the override. +func isSHA(s string) bool { + if len(s) < 7 || len(s) > 40 { + return false + } + for _, r := range s { + if (r < '0' || r > '9') && (r < 'a' || r > 'f') && (r < 'A' || r > 'F') { + return false + } + } + return true +} diff --git a/github/github.go b/github/github.go deleted file mode 100644 index 8fd347b..0000000 --- a/github/github.go +++ /dev/null @@ -1,349 +0,0 @@ -// Package github is the library behind the ghb command: the HTTP client, -// request shaping, and the typed data models for GitHub. -// -// The GitHub REST API v3 at https://api.github.com is open for public data -// with no authentication required. Unauthenticated access is rate-limited to -// 60 requests per hour per IP address. The client paces to 1 request per second -// to stay safely within that budget. -package github - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "net/url" - "strconv" - "sync" - "time" -) - -// DefaultUserAgent identifies the client to the GitHub API. -const DefaultUserAgent = "ghb/0.1.0 (+https://github.com/tamnd/github-cli)" - -// ErrNotFound is returned when the API responds with HTTP 404. -var ErrNotFound = errors.New("not found") - -// ErrRateLimit is returned when the unauthenticated rate limit (60 req/hr) is exhausted. -var ErrRateLimit = errors.New("GitHub rate limit reached (60 req/hr for unauthenticated); retry after a minute") - -// Config holds constructor parameters for Client. -type Config struct { - BaseURL string // default: "https://api.github.com" - UserAgent string // default: DefaultUserAgent - Rate time.Duration // default: 1s - Retries int // default: 2 - Timeout time.Duration // default: 30s -} - -// DefaultConfig returns sensible defaults. -func DefaultConfig() Config { - return Config{ - BaseURL: "https://api.github.com", - UserAgent: DefaultUserAgent, - Rate: 1 * time.Second, - Retries: 2, - Timeout: 30 * time.Second, - } -} - -// Client talks to the GitHub REST API v3. -type Client struct { - httpClient *http.Client - baseURL string - userAgent string - rate time.Duration - retries int - mu sync.Mutex - last time.Time -} - -// NewClient returns a Client with the given config. -func NewClient(cfg Config) *Client { - return &Client{ - httpClient: &http.Client{Timeout: cfg.Timeout}, - baseURL: cfg.BaseURL, - userAgent: cfg.UserAgent, - rate: cfg.Rate, - retries: cfg.Retries, - } -} - -// get fetches a URL with pacing and retries. -func (c *Client) get(ctx context.Context, rawURL string) ([]byte, error) { - var lastErr error - for attempt := 0; attempt <= c.retries; attempt++ { - if attempt > 0 { - select { - case <-ctx.Done(): - return nil, ctx.Err() - case <-time.After(backoff(attempt)): - } - } - body, retry, err := c.do(ctx, rawURL) - if err == nil { - return body, nil - } - lastErr = err - if !retry { - return nil, err - } - } - return nil, fmt.Errorf("get %s: %w", rawURL, lastErr) -} - -func (c *Client) do(ctx context.Context, rawURL string) ([]byte, bool, error) { - c.pace() - req, err := http.NewRequestWithContext(ctx, http.MethodGet, rawURL, nil) - if err != nil { - return nil, false, err - } - req.Header.Set("User-Agent", c.userAgent) - req.Header.Set("Accept", "application/vnd.github.v3+json") - - resp, err := c.httpClient.Do(req) - if err != nil { - return nil, true, err - } - defer func() { _ = resp.Body.Close() }() - - if resp.StatusCode == http.StatusForbidden { - if resp.Header.Get("X-RateLimit-Remaining") == "0" { - return nil, false, ErrRateLimit - } - return nil, false, fmt.Errorf("http 403") - } - if resp.StatusCode == http.StatusNotFound { - return nil, false, ErrNotFound - } - if resp.StatusCode == http.StatusTooManyRequests || resp.StatusCode >= 500 { - return nil, true, fmt.Errorf("http %d", resp.StatusCode) - } - if resp.StatusCode != http.StatusOK { - return nil, false, fmt.Errorf("http %d", resp.StatusCode) - } - b, err := io.ReadAll(io.LimitReader(resp.Body, 8<<20)) - if err != nil { - return nil, true, err - } - return b, false, nil -} - -func (c *Client) pace() { - c.mu.Lock() - defer c.mu.Unlock() - if c.rate <= 0 { - return - } - if wait := c.rate - time.Since(c.last); wait > 0 { - time.Sleep(wait) - } - c.last = time.Now() -} - -func backoff(attempt int) time.Duration { - d := time.Duration(attempt) * 500 * time.Millisecond - if d > 5*time.Second { - d = 5 * time.Second - } - return d -} - -func (c *Client) getJSON(ctx context.Context, rawURL string, v any) error { - body, err := c.get(ctx, rawURL) - if err != nil { - return err - } - if err := json.Unmarshal(body, v); err != nil { - return fmt.Errorf("decode %s: %w", rawURL, err) - } - return nil -} - -// ─── API methods ────────────────────────────────────────────────────────────── - -// SearchRepoOptions controls repository search. -type SearchRepoOptions struct { - Query string // raw keyword query - Language string // language filter (empty = all) - Sort string // stars|forks|updated|help-wanted-issues - Limit int // max records to return -} - -// SearchRepos searches repositories. -func (c *Client) SearchRepos(ctx context.Context, opts SearchRepoOptions) ([]Repo, error) { - q := opts.Query - if opts.Language != "" { - q += " language:" + opts.Language - } - sort := opts.Sort - if sort == "" { - sort = "stars" - } - limit := opts.Limit - if limit <= 0 { - limit = 20 - } - perPage := limit - if perPage > 100 { - perPage = 100 - } - - params := url.Values{} - params.Set("q", q) - params.Set("sort", sort) - params.Set("order", "desc") - params.Set("per_page", strconv.Itoa(perPage)) - params.Set("page", "1") - - rawURL := c.baseURL + "/search/repositories?" + params.Encode() - var resp searchReposResp - if err := c.getJSON(ctx, rawURL, &resp); err != nil { - return nil, err - } - - out := make([]Repo, 0, len(resp.Items)) - for i, w := range resp.Items { - if i >= limit { - break - } - out = append(out, wireRepoToRepo(w, i+1)) - } - return out, nil -} - -// GetRepo fetches a single repository. -func (c *Client) GetRepo(ctx context.Context, owner, repo string) (Repo, error) { - rawURL := fmt.Sprintf("%s/repos/%s/%s", c.baseURL, - url.PathEscape(owner), url.PathEscape(repo)) - var w wireRepo - if err := c.getJSON(ctx, rawURL, &w); err != nil { - return Repo{}, err - } - return wireRepoToRepo(w, 1), nil -} - -// TrendingOptions controls the trending proxy query. -type TrendingOptions struct { - Language string // language filter (empty = all) - Days int // 7|30|365 - Limit int // max records -} - -// Trending returns repos created after a cutoff date sorted by stars. -func (c *Client) Trending(ctx context.Context, opts TrendingOptions) ([]Repo, error) { - days := opts.Days - if days <= 0 { - days = 7 - } - cutoff := time.Now().AddDate(0, 0, -days).Format("2006-01-02") - q := "stars:>10 created:>" + cutoff - if opts.Language != "" { - q += " language:" + opts.Language - } - limit := opts.Limit - if limit <= 0 { - limit = 25 - } - perPage := limit - if perPage > 100 { - perPage = 100 - } - - params := url.Values{} - params.Set("q", q) - params.Set("sort", "stars") - params.Set("order", "desc") - params.Set("per_page", strconv.Itoa(perPage)) - params.Set("page", "1") - - rawURL := c.baseURL + "/search/repositories?" + params.Encode() - var resp searchReposResp - if err := c.getJSON(ctx, rawURL, &resp); err != nil { - return nil, err - } - - out := make([]Repo, 0, len(resp.Items)) - for i, w := range resp.Items { - if i >= limit { - break - } - out = append(out, wireRepoToRepo(w, i+1)) - } - return out, nil -} - -// GetUser fetches a single user profile. -func (c *Client) GetUser(ctx context.Context, username string) (User, error) { - rawURL := fmt.Sprintf("%s/users/%s", c.baseURL, url.PathEscape(username)) - var w wireUser - if err := c.getJSON(ctx, rawURL, &w); err != nil { - return User{}, err - } - return wireUserToUser(w), nil -} - -// UserRepos returns a user's public repos sorted by most recently pushed. -func (c *Client) UserRepos(ctx context.Context, username string, limit int) ([]Repo, error) { - if limit <= 0 { - limit = 10 - } - perPage := limit - if perPage > 100 { - perPage = 100 - } - - params := url.Values{} - params.Set("sort", "pushed") - params.Set("per_page", strconv.Itoa(perPage)) - - rawURL := fmt.Sprintf("%s/users/%s/repos?%s", - c.baseURL, url.PathEscape(username), params.Encode()) - - var items []wireRepo - if err := c.getJSON(ctx, rawURL, &items); err != nil { - return nil, err - } - - out := make([]Repo, 0, len(items)) - for i, w := range items { - if i >= limit { - break - } - out = append(out, wireRepoToRepo(w, i+1)) - } - return out, nil -} - -// Releases returns releases for a repo, most recent first. -func (c *Client) Releases(ctx context.Context, owner, repo string, limit int) ([]Release, error) { - if limit <= 0 { - limit = 20 - } - perPage := limit - if perPage > 100 { - perPage = 100 - } - - params := url.Values{} - params.Set("per_page", strconv.Itoa(perPage)) - - rawURL := fmt.Sprintf("%s/repos/%s/%s/releases?%s", - c.baseURL, url.PathEscape(owner), url.PathEscape(repo), params.Encode()) - - var items []wireRelease - if err := c.getJSON(ctx, rawURL, &items); err != nil { - return nil, err - } - - out := make([]Release, 0, len(items)) - for i, w := range items { - if i >= limit { - break - } - out = append(out, wireReleaseToRelease(w, i+1)) - } - return out, nil -} diff --git a/github/github_test.go b/github/github_test.go deleted file mode 100644 index 760acfa..0000000 --- a/github/github_test.go +++ /dev/null @@ -1,311 +0,0 @@ -package github - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "strings" - "testing" - "time" -) - -func testClient(t *testing.T, handler http.HandlerFunc) (*Client, *httptest.Server) { - t.Helper() - srv := httptest.NewServer(handler) - t.Cleanup(srv.Close) - cfg := DefaultConfig() - cfg.BaseURL = srv.URL - cfg.Rate = 0 - return NewClient(cfg), srv -} - -func TestGetSendsHeaders(t *testing.T) { - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.Header.Get("User-Agent") == "" { - t.Error("request carried no User-Agent") - } - if r.Header.Get("Accept") != "application/vnd.github.v3+json" { - t.Errorf("Accept header = %q, want application/vnd.github.v3+json", r.Header.Get("Accept")) - } - _, _ = w.Write([]byte(`"hello"`)) - }) - - body, err := c.get(context.Background(), c.baseURL) - if err != nil { - t.Fatal(err) - } - if string(body) != `"hello"` { - t.Errorf("body = %q", body) - } -} - -func TestGetRetriesOn503(t *testing.T) { - var hits int - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - hits++ - if hits < 3 { - w.WriteHeader(http.StatusServiceUnavailable) - return - } - _, _ = w.Write([]byte(`"recovered"`)) - }) - c.retries = 5 - - start := time.Now() - body, err := c.get(context.Background(), c.baseURL) - if err != nil { - t.Fatal(err) - } - if string(body) != `"recovered"` { - t.Errorf("body = %q after retries", body) - } - if hits != 3 { - t.Errorf("server saw %d hits, want 3", hits) - } - if time.Since(start) < 500*time.Millisecond { - t.Error("retries did not back off") - } -} - -func TestSearchRepos(t *testing.T) { - desc := "A great project" - lang := "Go" - license := "MIT" - resp := searchReposResp{ - TotalCount: 1, - Items: []wireRepo{ - { - ID: 1, - FullName: "owner/repo", - Description: &desc, - HTMLURL: "https://github.com/owner/repo", - Stars: 5000, - Forks: 1200, - Language: &lang, - License: &struct { - SPDXID string `json:"spdx_id"` - }{SPDXID: license}, - PushedAt: "2024-06-01T12:00:00Z", - }, - }, - } - - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - if !strings.HasPrefix(r.URL.Path, "/search/repositories") { - t.Errorf("unexpected path: %s", r.URL.Path) - } - _ = json.NewEncoder(w).Encode(resp) - }) - - repos, err := c.SearchRepos(context.Background(), SearchRepoOptions{ - Query: "great project", - Limit: 5, - }) - if err != nil { - t.Fatal(err) - } - if len(repos) != 1 { - t.Fatalf("got %d repos, want 1", len(repos)) - } - r := repos[0] - if r.FullName != "owner/repo" { - t.Errorf("FullName = %q", r.FullName) - } - if r.Stars != 5000 { - t.Errorf("Stars = %d, want 5000", r.Stars) - } - if r.License != "MIT" { - t.Errorf("License = %q, want MIT", r.License) - } - if r.URL != "https://github.com/owner/repo" { - t.Errorf("URL = %q", r.URL) - } - if r.Rank != 1 { - t.Errorf("Rank = %d, want 1", r.Rank) - } -} - -func TestGetRepo(t *testing.T) { - desc := "Linux kernel source tree" - lang := "C" - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/repos/torvalds/linux" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - _ = json.NewEncoder(w).Encode(wireRepo{ - ID: 1234, - FullName: "torvalds/linux", - Description: &desc, - HTMLURL: "https://github.com/torvalds/linux", - Stars: 220000, - Forks: 64000, - Language: &lang, - PushedAt: "2024-06-01T00:00:00Z", - }) - }) - - repo, err := c.GetRepo(context.Background(), "torvalds", "linux") - if err != nil { - t.Fatal(err) - } - if repo.FullName != "torvalds/linux" { - t.Errorf("FullName = %q", repo.FullName) - } - if repo.Stars != 220000 { - t.Errorf("Stars = %d, want 220000", repo.Stars) - } - if repo.Language != "C" { - t.Errorf("Language = %q, want C", repo.Language) - } - if repo.License != "" { - t.Errorf("License = %q, want empty", repo.License) - } -} - -func TestReleases(t *testing.T) { - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/repos/golang/go/releases" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - _ = json.NewEncoder(w).Encode([]wireRelease{ - { - TagName: "go1.22.0", - Name: "Go 1.22", - Prerelease: false, - Draft: false, - CreatedAt: "2024-02-06T00:00:00Z", - HTMLURL: "https://github.com/golang/go/releases/tag/go1.22.0", - }, - { - TagName: "go1.22rc1", - Name: "Go 1.22 RC1", - Prerelease: true, - Draft: false, - CreatedAt: "2024-01-23T00:00:00Z", - HTMLURL: "https://github.com/golang/go/releases/tag/go1.22rc1", - }, - }) - }) - - releases, err := c.Releases(context.Background(), "golang", "go", 5) - if err != nil { - t.Fatal(err) - } - if len(releases) != 2 { - t.Fatalf("got %d releases, want 2", len(releases)) - } - if releases[0].TagName != "go1.22.0" { - t.Errorf("TagName = %q", releases[0].TagName) - } - if releases[0].Rank != 1 { - t.Errorf("Rank = %d, want 1", releases[0].Rank) - } - if releases[1].Prerelease != true { - t.Errorf("Prerelease = %v, want true", releases[1].Prerelease) - } - if releases[0].URL != "https://github.com/golang/go/releases/tag/go1.22.0" { - t.Errorf("URL = %q", releases[0].URL) - } -} - -func TestGetUser(t *testing.T) { - name := "Linus Torvalds" - company := "Linux Foundation" - location := "Portland, OR" - bio := "Creator of Linux" - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - if r.URL.Path != "/users/torvalds" { - t.Errorf("unexpected path: %s", r.URL.Path) - } - _ = json.NewEncoder(w).Encode(wireUser{ - Login: "torvalds", - Name: &name, - Company: &company, - Location: &location, - Bio: &bio, - PublicRepos: 10, - Followers: 200000, - HTMLURL: "https://github.com/torvalds", - }) - }) - - user, err := c.GetUser(context.Background(), "torvalds") - if err != nil { - t.Fatal(err) - } - if user.Login != "torvalds" { - t.Errorf("Login = %q", user.Login) - } - if user.Name != "Linus Torvalds" { - t.Errorf("Name = %q", user.Name) - } - if user.Followers != 200000 { - t.Errorf("Followers = %d, want 200000", user.Followers) - } - if user.URL != "https://github.com/torvalds" { - t.Errorf("URL = %q, want https://github.com/torvalds", user.URL) - } -} - -func TestGetUserNullableName(t *testing.T) { - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - _ = json.NewEncoder(w).Encode(wireUser{ - Login: "nobody", - Name: nil, - PublicRepos: 0, - Followers: 0, - HTMLURL: "https://github.com/nobody", - }) - }) - - user, err := c.GetUser(context.Background(), "nobody") - if err != nil { - t.Fatal(err) - } - if user.Name != "" { - t.Errorf("Name = %q, want empty for nil", user.Name) - } - if user.URL != "https://github.com/nobody" { - t.Errorf("URL = %q", user.URL) - } -} - -func TestTrendingQueryContainsCutoff(t *testing.T) { - var gotQuery string - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - gotQuery = r.URL.Query().Get("q") - sort := r.URL.Query().Get("sort") - if sort != "stars" { - t.Errorf("sort = %q, want stars", sort) - } - _ = json.NewEncoder(w).Encode(searchReposResp{}) - }) - - _, err := c.Trending(context.Background(), TrendingOptions{Days: 7, Limit: 5}) - if err != nil { - t.Fatal(err) - } - if !strings.Contains(gotQuery, "created:>") { - t.Errorf("query %q missing created:>", gotQuery) - } - if !strings.Contains(gotQuery, "stars:>10") { - t.Errorf("query %q missing stars:>10", gotQuery) - } -} - -func TestRateLimitError(t *testing.T) { - c, _ := testClient(t, func(w http.ResponseWriter, r *http.Request) { - w.Header().Set("X-RateLimit-Remaining", "0") - w.WriteHeader(http.StatusForbidden) - _, _ = w.Write([]byte(`{"message":"API rate limit exceeded"}`)) - }) - - _, err := c.get(context.Background(), c.baseURL) - if err == nil { - t.Fatal("expected error, got nil") - } - if err != ErrRateLimit { - t.Errorf("err = %v, want ErrRateLimit", err) - } -} diff --git a/github/types.go b/github/types.go deleted file mode 100644 index f45ce98..0000000 --- a/github/types.go +++ /dev/null @@ -1,130 +0,0 @@ -package github - -import "fmt" - -// Repo is the record emitted for repository commands. -type Repo struct { - Rank int `json:"rank"` - FullName string `json:"full_name"` - Description string `json:"description"` - Language string `json:"language"` - Stars int `json:"stars"` - Forks int `json:"forks"` - License string `json:"license"` - PushedAt string `json:"pushed_at"` - URL string `json:"url"` -} - -// User is the record emitted for user commands. -type User struct { - Login string `json:"login"` - Name string `json:"name"` - Company string `json:"company"` - Location string `json:"location"` - Followers int `json:"followers"` - Repos int `json:"repos"` - Bio string `json:"bio"` - URL string `json:"url"` -} - -// Release is the record emitted for the releases command. -type Release struct { - Rank int `json:"rank"` - TagName string `json:"tag_name"` - Name string `json:"name"` - Prerelease bool `json:"prerelease"` - CreatedAt string `json:"created_at"` - URL string `json:"url"` -} - -// ─── wire types from GitHub REST API ───────────────────────────────────────── - -type wireRepo struct { - ID int `json:"id"` - FullName string `json:"full_name"` - Description *string `json:"description"` - HTMLURL string `json:"html_url"` - Stars int `json:"stargazers_count"` - Forks int `json:"forks_count"` - Language *string `json:"language"` - License *struct { - SPDXID string `json:"spdx_id"` - } `json:"license"` - PushedAt string `json:"pushed_at"` -} - -type wireUser struct { - Login string `json:"login"` - Name *string `json:"name"` - Company *string `json:"company"` - Location *string `json:"location"` - Bio *string `json:"bio"` - PublicRepos int `json:"public_repos"` - Followers int `json:"followers"` - HTMLURL string `json:"html_url"` -} - -type wireRelease struct { - TagName string `json:"tag_name"` - Name string `json:"name"` - Prerelease bool `json:"prerelease"` - Draft bool `json:"draft"` - CreatedAt string `json:"created_at"` - HTMLURL string `json:"html_url"` -} - -type searchReposResp struct { - TotalCount int `json:"total_count"` - Items []wireRepo `json:"items"` -} - -// ─── converters ────────────────────────────────────────────────────────────── - -func deref(s *string) string { - if s == nil { - return "" - } - return *s -} - -func wireRepoToRepo(w wireRepo, rank int) Repo { - lic := "" - if w.License != nil { - lic = w.License.SPDXID - } - return Repo{ - Rank: rank, - FullName: w.FullName, - Description: deref(w.Description), - Language: deref(w.Language), - Stars: w.Stars, - Forks: w.Forks, - License: lic, - PushedAt: w.PushedAt, - URL: w.HTMLURL, - } -} - -func wireUserToUser(w wireUser) User { - return User{ - Login: w.Login, - Name: deref(w.Name), - Company: deref(w.Company), - Location: deref(w.Location), - Followers: w.Followers, - Repos: w.PublicRepos, - Bio: deref(w.Bio), - URL: fmt.Sprintf("https://github.com/%s", w.Login), - } -} - -func wireReleaseToRelease(w wireRelease, rank int) Release { - return Release{ - Rank: rank, - TagName: w.TagName, - Name: w.Name, - Prerelease: w.Prerelease, - CreatedAt: w.CreatedAt, - URL: w.HTMLURL, - } -} diff --git a/go.mod b/go.mod index 667b111..762fbfe 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,8 @@ go 1.26 require ( github.com/charmbracelet/fang v1.0.0 - github.com/mattn/go-isatty v0.0.22 github.com/spf13/cobra v1.10.2 + github.com/tamnd/any-cli v0.4.7 ) require ( @@ -20,18 +20,28 @@ require ( github.com/clipperhouse/displaywidth v0.4.1 // indirect github.com/clipperhouse/stringish v0.1.1 // indirect github.com/clipperhouse/uax29/v2 v2.3.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/lucasb-eyer/go-colorful v1.3.0 // indirect + github.com/mattn/go-isatty v0.0.22 // indirect github.com/mattn/go-runewidth v0.0.19 // indirect github.com/muesli/cancelreader v0.2.2 // indirect github.com/muesli/mango v0.1.0 // indirect github.com/muesli/mango-cobra v1.2.0 // indirect github.com/muesli/mango-pflag v0.1.0 // indirect github.com/muesli/roff v0.1.0 // indirect + github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect github.com/spf13/pflag v1.0.9 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/sync v0.17.0 // indirect - golang.org/x/sys v0.37.0 // indirect - golang.org/x/text v0.24.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + modernc.org/libc v1.72.3 // indirect + modernc.org/mathutil v1.7.1 // indirect + modernc.org/memory v1.11.0 // indirect + modernc.org/sqlite v1.52.0 // indirect ) diff --git a/go.sum b/go.sum index e7d4564..5c1b552 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,14 @@ github.com/clipperhouse/uax29/v2 v2.3.0/go.mod h1:Wn1g7MK6OoeDT0vL+Q0SQLDz/KpfsV github.com/cpuguy83/go-md2man/v2 v2.0.6/go.mod h1:oOW0eioCTA6cOiMLiUPZOpcVxMig6NIQQ7OS05n1F4g= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e h1:ijClszYn+mADRFY17kjQEVQ1XRhq2/JR1M3sGqeJoxs= +github.com/google/pprof v0.0.0-20250317173921-a4b03ec1a45e/go.mod h1:boTsfXsheKC2y+lKOCMpSfarhxDeIzfZG1jqGcPl3cA= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/lucasb-eyer/go-colorful v1.3.0 h1:2/yBRLdWBZKrf7gB40FoiKfAWYQ0lqNcbuQwVHXptag= @@ -47,8 +55,12 @@ github.com/muesli/mango-pflag v0.1.0 h1:UADqbYgpUyRoBja3g6LUL+3LErjpsOwaC9ywvBWe github.com/muesli/mango-pflag v0.1.0/go.mod h1:YEQomTxaCUp8PrbhFh10UfbhbQrM/xJ4i2PB8VTLLW0= github.com/muesli/roff v0.1.0 h1:YD0lalCotmYuF5HhZliKWlIx7IEhiXeSfq7hNjFqGF8= github.com/muesli/roff v0.1.0/go.mod h1:pjAHQM9hdUUwm/krAfrLGgJkXJ+YuhtsfZ42kieB2Ig= +github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= +github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= +github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -58,17 +70,69 @@ github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY= github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/tamnd/any-cli v0.4.0 h1:ngyRJBvjZ2X1iBlwlmDLvY2S9aQWlDjVE7CiOwxtt5Y= +github.com/tamnd/any-cli v0.4.0/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk= +github.com/tamnd/any-cli v0.4.4 h1:mOo3JJ7M3ZWQtOocYVtMEN7Zhfc3ogVWcVrbarlt9jE= +github.com/tamnd/any-cli v0.4.4/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk= +github.com/tamnd/any-cli v0.4.5 h1:dEeniLDoneCxK4A9SixIyND2xlS2A/RxUtRdqTWXlQw= +github.com/tamnd/any-cli v0.4.5/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk= +github.com/tamnd/any-cli v0.4.6 h1:5GHwOsr8Z9oRYCtFt49Q9bwUKf9NBqkhXUqfOB5So+4= +github.com/tamnd/any-cli v0.4.6/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk= +github.com/tamnd/any-cli v0.4.7 h1:aHjifufpIy0M4HQQo+Ex90M4xaHyYoAMNtNPqVWw8ug= +github.com/tamnd/any-cli v0.4.7/go.mod h1:lns3VfQVrC9hMy7YKBzIQoYpobnfSDIzJ8c27H2ILmk= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= -golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= -golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI= -golang.org/x/sys v0.37.0 h1:fdNQudmxPjkdUTPnLn5mdQv7Zwvbvpaxqs831goi9kQ= -golang.org/x/sys v0.37.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks= +golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8= +golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= +golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.42.0 h1:omrd2nAlyT5ESRdCLYdm3+fMfNFE/+Rf4bDIQImRJeo= +golang.org/x/sys v0.42.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= golang.org/x/text v0.24.0 h1:dd5Bzh4yt5KYA8f9CJHCP4FB4D51c2c6JvN37xJJkJ0= golang.org/x/text v0.24.0/go.mod h1:L8rBsPeo2pSS+xqN0d5u2ikmjtmoJbDBT1b7nHvFCdU= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.42.0 h1:uNgphsn75Tdz5Ji2q36v/nsFSfR/9BRFvqhGBaJGd5k= +golang.org/x/tools v0.42.0/go.mod h1:Ma6lCIwGZvHK6XtgbswSoWroEkhugApmsXyrUmBhfr0= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +modernc.org/cc/v4 v4.28.2 h1:3tQ0lf2ADtoby2EtSP+J7IE2SHwEJdP8ioR59wx7XpY= +modernc.org/cc/v4 v4.28.2/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI= +modernc.org/ccgo/v4 v4.34.0 h1:yRLPFZieg532OT4rp4JFNIVcquwalMX26G95WQDqwCQ= +modernc.org/ccgo/v4 v4.34.0/go.mod h1:AS5WYMyBakQ+fhsHhtP8mWB82KTGPkNNJDGfGQCe0/A= +modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM= +modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU= +modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI= +modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito= +modernc.org/gc/v3 v3.1.2 h1:ZtDCnhonXSZexk/AYsegNRV1lJGgaNZJuKjJSWKyEqo= +modernc.org/gc/v3 v3.1.2/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY= +modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks= +modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI= +modernc.org/libc v1.72.3 h1:ZnDF4tXn4NBXFutMMQC4vtbTFSXhhKzR73fv0beZEAU= +modernc.org/libc v1.72.3/go.mod h1:dn0dZNnnn1clLyvRxLxYExxiKRZIRENOfqQ8XEeg4Qs= +modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU= +modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg= +modernc.org/memory v1.11.0 h1:o4QC8aMQzmcwCK3t3Ux/ZHmwFPzE6hf2Y5LbkRs+hbI= +modernc.org/memory v1.11.0/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw= +modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg= +modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns= +modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w= +modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE= +modernc.org/sqlite v1.52.0 h1:p4dhYh2tXZCiyaqHwRVJDjIGKWyXayiQpThxgDzJaxo= +modernc.org/sqlite v1.52.0/go.mod h1:tcNzv5p84E0skkmJn038y+hWJbLQXQqEnQfeh5r2JLM= +modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0= +modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A= +modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y= +modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM= diff --git a/pkg/gitproto/gitproto.go b/pkg/gitproto/gitproto.go new file mode 100644 index 0000000..6caff88 --- /dev/null +++ b/pkg/gitproto/gitproto.go @@ -0,0 +1,161 @@ +// Package gitproto reads the git smart HTTP advertisement. +// +// This is the one surface on github.com that answers "what refs does this +// repository have" completely, in a single request, with no login, no page +// limit, and no truncation. The branches page caps its list and says so, the +// refs fragment gives names and nothing else, and the tags feed gives ten. The +// advertisement gives every branch, every tag, every pull request head, and the +// object each one points at. +// +// The format is pkt-line, which is four hex digits of length followed by that +// many bytes including the four. 0000 is a flush packet. The first line after +// the service header carries the capability list after a NUL byte, and one of +// those capabilities is symref=HEAD:refs/heads/main, which is where the default +// branch comes from for free. +package gitproto + +import ( + "errors" + "strconv" + "strings" +) + +// Ref is one advertised ref. +type Ref struct { + Name string + SHA string + // Peeled is set on the ^{} entry of an annotated tag: the tag object's own + // SHA is in SHA and the commit it points at is here. A lightweight tag has + // no peeled entry, which is how you tell the two apart. + Peeled string +} + +// Advertisement is a parsed info/refs response. +type Advertisement struct { + Refs []Ref + // Head is the SHA the HEAD line advertised. + Head string + // DefaultBranch is the target of symref=HEAD:..., short form, so "main" + // rather than "refs/heads/main". Empty when the server did not advertise it. + DefaultBranch string + Capabilities []string +} + +// ErrNotGit is returned when the body is not an advertisement. It usually means +// github.com answered with an HTML page, which is what a private or missing +// repository does. +var ErrNotGit = errors.New("not a git upload-pack advertisement") + +// Parse reads an info/refs?service=git-upload-pack body. +// +// Peeled entries are folded into the ref they belong to rather than kept as +// separate refs, because "refs/tags/v1.0.0^{}" is not a ref anybody can check +// out and a caller that has to know about the fold is a caller doing the +// parser's job. +func Parse(body []byte) (*Advertisement, error) { + lines, err := pktLines(body) + if err != nil { + return nil, err + } + ad := &Advertisement{} + byName := map[string]int{} + for _, line := range lines { + line = strings.TrimRight(line, "\n") + if line == "" || strings.HasPrefix(line, "# service=") { + continue + } + // The first ref line carries the capabilities after a NUL. + if i := strings.IndexByte(line, 0); i >= 0 { + ad.Capabilities = strings.Fields(line[i+1:]) + for _, c := range ad.Capabilities { + if v, ok := strings.CutPrefix(c, "symref=HEAD:"); ok { + ad.DefaultBranch = shortName(v) + } + } + line = line[:i] + } + sha, name, ok := strings.Cut(line, " ") + if !ok || len(sha) != 40 { + continue + } + if name == "HEAD" { + ad.Head = sha + continue + } + if base, ok := strings.CutSuffix(name, "^{}"); ok { + if i, seen := byName[base]; seen { + ad.Refs[i].Peeled = sha + } + continue + } + byName[name] = len(ad.Refs) + ad.Refs = append(ad.Refs, Ref{Name: name, SHA: sha}) + } + if len(ad.Refs) == 0 && ad.Head == "" { + return nil, ErrNotGit + } + return ad, nil +} + +// pktLines splits a pkt-line stream into its payloads. +// +// A malformed length is a hard error rather than a skipped line. Half-reading a +// binary protocol and carrying on gives a ref list that looks fine and is +// missing entries, which is worse than not answering. +func pktLines(body []byte) ([]string, error) { + var out []string + for len(body) > 0 { + if len(body) < 4 { + return nil, ErrNotGit + } + n, err := strconv.ParseUint(string(body[:4]), 16, 32) + if err != nil { + return nil, ErrNotGit + } + if n == 0 { + // Flush packet. The advertisement carries one after the service + // header and one at the end, and neither ends the stream for us. + body = body[4:] + continue + } + if n < 4 || int(n) > len(body) { + return nil, ErrNotGit + } + out = append(out, string(body[4:n])) + body = body[n:] + } + return out, nil +} + +// Branches returns the refs under refs/heads, with the prefix stripped. +func (a *Advertisement) Branches() []Ref { return a.under("refs/heads/") } + +// Tags returns the refs under refs/tags, with the prefix stripped. An annotated +// tag keeps both SHAs: SHA is the tag object and Peeled is the commit. +func (a *Advertisement) Tags() []Ref { return a.under("refs/tags/") } + +// PullHeads returns the refs under refs/pull, which github.com advertises for +// every pull request ever opened against the repository. The name keeps its +// shape, "1234/head" or "1234/merge", because those two are different objects +// and flattening them would lose that. +func (a *Advertisement) PullHeads() []Ref { return a.under("refs/pull/") } + +func (a *Advertisement) under(prefix string) []Ref { + var out []Ref + for _, r := range a.Refs { + if name, ok := strings.CutPrefix(r.Name, prefix); ok { + r.Name = name + out = append(out, r) + } + } + return out +} + +func shortName(full string) string { + for _, p := range []string{"refs/heads/", "refs/tags/", "refs/remotes/"} { + if s, ok := strings.CutPrefix(full, p); ok { + return s + } + } + return full +} diff --git a/pkg/gitproto/gitproto_test.go b/pkg/gitproto/gitproto_test.go new file mode 100644 index 0000000..2527e66 --- /dev/null +++ b/pkg/gitproto/gitproto_test.go @@ -0,0 +1,84 @@ +package gitproto + +import "testing" + +// pkt builds a pkt-line stream from payloads. A "" payload is a flush packet. +func pkt(payloads ...string) []byte { + var b []byte + for _, p := range payloads { + if p == "" { + b = append(b, "0000"...) + continue + } + n := len(p) + 4 + const hex = "0123456789abcdef" + b = append(b, hex[n>>12&0xf], hex[n>>8&0xf], hex[n>>4&0xf], hex[n&0xf]) + b = append(b, p...) + } + return b +} + +const ( + shaHead = "1111111111111111111111111111111111111111" + shaMain = "2222222222222222222222222222222222222222" + shaTag = "3333333333333333333333333333333333333333" + shaPeel = "4444444444444444444444444444444444444444" + shaPull = "5555555555555555555555555555555555555555" +) + +func TestParse(t *testing.T) { + body := pkt( + "# service=git-upload-pack\n", + "", + shaHead+" HEAD\x00multi_ack symref=HEAD:refs/heads/trunk object-format=sha1\n", + shaMain+" refs/heads/trunk\n", + shaTag+" refs/tags/v1.0.0\n", + shaPeel+" refs/tags/v1.0.0^{}\n", + shaPull+" refs/pull/42/head\n", + "", + ) + ad, err := Parse(body) + if err != nil { + t.Fatal(err) + } + if ad.Head != shaHead { + t.Errorf("head %q", ad.Head) + } + if ad.DefaultBranch != "trunk" { + t.Errorf("default branch %q", ad.DefaultBranch) + } + if len(ad.Refs) != 3 { + t.Fatalf("refs %d, the peeled entry should have folded into its tag", len(ad.Refs)) + } + branches := ad.Branches() + if len(branches) != 1 || branches[0].Name != "trunk" || branches[0].SHA != shaMain { + t.Errorf("branches %+v", branches) + } + tags := ad.Tags() + if len(tags) != 1 || tags[0].Name != "v1.0.0" { + t.Fatalf("tags %+v", tags) + } + // An annotated tag keeps both: the tag object and the commit it points at. + if tags[0].SHA != shaTag || tags[0].Peeled != shaPeel { + t.Errorf("tag sha %q peeled %q", tags[0].SHA, tags[0].Peeled) + } + pulls := ad.PullHeads() + if len(pulls) != 1 || pulls[0].Name != "42/head" { + t.Errorf("pull heads %+v", pulls) + } +} + +func TestParseRejectsHTML(t *testing.T) { + // A private or missing repository answers with a page, and a parser that + // shrugs at that hands back an empty ref list that looks like a real answer. + for _, body := range []string{ + "\nGitHub", + "", + "00x4bad length", + "0005", + } { + if _, err := Parse([]byte(body)); err == nil { + t.Errorf("accepted %q", body[:min(len(body), 20)]) + } + } +} diff --git a/pkg/page/compact.go b/pkg/page/compact.go new file mode 100644 index 0000000..bc3ff0e --- /dev/null +++ b/pkg/page/compact.go @@ -0,0 +1,68 @@ +package page + +import ( + "strconv" + "strings" +) + +// compact.go parses the numbers GitHub renders for people rather than for +// programs: 313k followers, 1.2k stars, 8,112 commits. +// +// The rule elsewhere in this tool is never to parse a rendered number when a +// raw one exists, and it holds: sidebarAbout.stargazerCount is an integer and +// is always preferred. These functions are for the pages where the rendered +// form is the only form there is. + +// ParseCompactCount reads a rendered count and returns the integer plus the +// original string. Both are kept because the grouping separator depends on the +// locale GitHub infers, so "8,112" and "8.112" are the same number, and +// throwing away the original would make that ambiguity invisible. +// +// Returns ok=false rather than zero when the string is not a number at all. +// Absent is not zero: a parser that returns 0 on failure is a bug that ships +// quietly and is discovered a year later in someone's aggregate. +func ParseCompactCount(s string) (n int, display string, ok bool) { + display = strings.TrimSpace(s) + t := strings.ToLower(display) + t = strings.TrimSpace(strings.NewReplacer(" ", " ", "+", "").Replace(t)) + if t == "" { + return 0, display, false + } + // A suffix multiplier turns the rest into a float: 1.2k is 1200. + mult := 1 + switch { + case strings.HasSuffix(t, "k"): + mult, t = 1_000, strings.TrimSuffix(t, "k") + case strings.HasSuffix(t, "m"): + mult, t = 1_000_000, strings.TrimSuffix(t, "m") + case strings.HasSuffix(t, "b"): + mult, t = 1_000_000_000, strings.TrimSuffix(t, "b") + } + t = strings.TrimSpace(t) + if mult > 1 { + f, err := strconv.ParseFloat(strings.Replace(t, ",", ".", 1), 64) + if err != nil { + return 0, display, false + } + return int(f * float64(mult)), display, true + } + // No suffix, so any separator is a thousands separator. GitHub uses a + // comma, a period, or a thin space depending on the inferred locale, and + // all three mean the same thing here. + t = strings.NewReplacer(",", "", ".", "", " ", "", " ", "", "'", "").Replace(t) + v, err := strconv.Atoi(t) + if err != nil { + return 0, display, false + } + return v, display, true +} + +// CountIn parses the first token of a rendered label, which is how the profile +// counters read: "313k followers", "1.2k following". +func CountIn(s string) (int, string, bool) { + fields := strings.Fields(strings.TrimSpace(s)) + if len(fields) == 0 { + return 0, "", false + } + return ParseCompactCount(fields[0]) +} diff --git a/pkg/page/dom.go b/pkg/page/dom.go new file mode 100644 index 0000000..051c26e --- /dev/null +++ b/pkg/page/dom.go @@ -0,0 +1,439 @@ +package page + +import ( + "bytes" + "strings" + + "golang.org/x/net/html" +) + +// dom.go is the parsed-document half of the extractor, and the matcher it +// needs. The matcher handles tag name, id, class membership, attribute +// presence, attribute value, prefix and suffix on an attribute, and one level +// of descendant. That is every selector in selectors.go and nothing more, +// which is the reason there is no CSS selector dependency here. + +// Doc parses the page once and caches the tree. Most reads never call it: the +// React pages carry their data as JSON and the scanner alone is enough. +func (p *Page) Doc() *html.Node { + if p.doc != nil || len(p.HTML) == 0 { + return p.doc + } + n, err := html.Parse(bytes.NewReader(p.HTML)) + if err != nil { + return nil + } + p.doc = n + return p.doc +} + +// Sel is one selector. A zero field is "do not care", so a selector that only +// sets Class matches on class alone. +type Sel struct { + Tag string + ID string + Class string // one class, matched against the whitespace-separated list + Attr string // attribute that must be present + AttrValue string // and, if set, must equal this + AttrPrefix string + AttrSuffix string + AttrContains string + // HasDescendantClass requires a descendant element carrying this class. + // It exists for the licence link, which is identified by the icon inside + // it because the icon changes less often than the anchor's own classes. + HasDescendantClass string +} + +// Match reports whether n satisfies every field the selector set. +func (s Sel) Match(n *html.Node) bool { + if n.Type != html.ElementNode { + return false + } + if s.Tag != "" && n.Data != s.Tag { + return false + } + if s.ID != "" && Attr(n, "id") != s.ID { + return false + } + if s.Class != "" && !HasClass(n, s.Class) { + return false + } + if s.Attr != "" { + v, ok := lookupAttr(n, s.Attr) + if !ok { + return false + } + if s.AttrValue != "" && v != s.AttrValue { + return false + } + if s.AttrPrefix != "" && !strings.HasPrefix(v, s.AttrPrefix) { + return false + } + if s.AttrSuffix != "" && !strings.HasSuffix(v, s.AttrSuffix) { + return false + } + if s.AttrContains != "" && !strings.Contains(v, s.AttrContains) { + return false + } + } + if s.HasDescendantClass != "" && Find(n, Sel{Class: s.HasDescendantClass}) == nil { + return false + } + return true +} + +// Attr reads one attribute, empty when it is absent. +func Attr(n *html.Node, name string) string { + v, _ := lookupAttr(n, name) + return v +} + +func lookupAttr(n *html.Node, name string) (string, bool) { + for _, a := range n.Attr { + if a.Key == name { + return a.Val, true + } + } + return "", false +} + +// HasClass matches one class in the whitespace-separated list, never a +// substring. `Box-row` must not match `Box-row-hover`. +func HasClass(n *html.Node, want string) bool { + v, ok := lookupAttr(n, "class") + if !ok { + return false + } + for _, c := range strings.Fields(v) { + if c == want { + return true + } + } + return false +} + +// Find returns the first matching element, or nil. +func Find(root *html.Node, s Sel) *html.Node { + var found *html.Node + Walk(root, func(n *html.Node) bool { + // Walk keeps visiting siblings after a subtree says stop, so the + // answer has to be latched. Without this the last match wins instead + // of the first, which on a page where a dialog repeats the same shape + // as the content silently returns the dialog. + if found != nil { + return false + } + if s.Match(n) { + found = n + return false + } + return true + }) + return found +} + +// FindAll returns every matching element in document order. +func FindAll(root *html.Node, s Sel) []*html.Node { + var out []*html.Node + Walk(root, func(n *html.Node) bool { + if s.Match(n) { + out = append(out, n) + } + return true + }) + return out +} + +// Walk visits every node depth-first. Returning false from fn stops the +// traversal of that subtree and, once found is set, the search as a whole. +func Walk(n *html.Node, fn func(*html.Node) bool) { + if n == nil { + return + } + if !fn(n) { + return + } + for c := n.FirstChild; c != nil; c = c.NextSibling { + Walk(c, fn) + } +} + +// Text returns the concatenated, whitespace-collapsed text of a subtree. It is +// the last-resort extractor and every caller of it is marked as such. +func Text(n *html.Node) string { + if n == nil { + return "" + } + var b strings.Builder + Walk(n, func(x *html.Node) bool { + if x.Type == html.TextNode { + b.WriteString(x.Data) + } + return true + }) + return collapse(b.String()) +} + +func collapse(s string) string { return strings.Join(strings.Fields(s), " ") } + +// blockTag is the set of elements that end a line of prose. It does not need to +// be the full HTML block list, only the tags GitHub's renderer actually emits +// into a README, a release note, or a comment body. +var blockTag = map[string]bool{ + "address": true, "article": true, "aside": true, "blockquote": true, + "br": true, "dd": true, "details": true, "div": true, "dl": true, + "dt": true, "figcaption": true, "figure": true, "footer": true, + "h1": true, "h2": true, "h3": true, "h4": true, "h5": true, "h6": true, + "header": true, "hr": true, "li": true, "main": true, "nav": true, + "ol": true, "p": true, "pre": true, "section": true, "summary": true, + "table": true, "tbody": true, "td": true, "th": true, "thead": true, + "tr": true, "ul": true, +} + +// BlockText returns the prose of a subtree with the line structure the markup +// implies, which is what Text deliberately throws away. +// +// Text collapses a whole subtree onto one line, which is right for a label and +// wrong for a document: a twenty-kilobyte README as a single line is not a +// readable rendering of anything. This keeps one line per block element, one +// blank line between paragraphs, and the interior whitespace of a 
 exactly
+// as it was, since indentation is the meaning of a code block rather than
+// decoration on it.
+func BlockText(n *html.Node) string {
+	if n == nil {
+		return ""
+	}
+	var t textLines
+	t.walk(n)
+	return t.done()
+}
+
+// FragmentText is BlockText over an HTML fragment that arrived as a string.
+// Several of GitHub's payloads carry rendered markup as a JSON value rather
+// than as part of the document, so there is no node to walk until this parses
+// one.
+func FragmentText(s string) string {
+	if strings.TrimSpace(s) == "" {
+		return ""
+	}
+	doc, err := html.Parse(strings.NewReader(s))
+	if err != nil {
+		return ""
+	}
+	return BlockText(doc)
+}
+
+// textLines accumulates prose one line at a time. It exists because whether a
+// line keeps its whitespace depends on where the line started, which a single
+// pass over a string builder cannot know after the fact.
+type textLines struct {
+	out []string
+	cur strings.Builder
+	pre int  // depth inside 
+	raw bool // the line being built started inside a 
+}
+
+func (t *textLines) walk(n *html.Node) {
+	switch n.Type {
+	case html.TextNode:
+		t.text(n.Data)
+		return
+	case html.ElementNode:
+		switch n.Data {
+		case "script", "style", "template":
+			return
+		case "pre":
+			t.pre++
+			defer func() { t.pre-- }()
+		}
+		if blockTag[n.Data] {
+			t.brk()
+		}
+	}
+	for c := n.FirstChild; c != nil; c = c.NextSibling {
+		t.walk(c)
+	}
+	if n.Type == html.ElementNode && blockTag[n.Data] {
+		t.brk()
+	}
+}
+
+func (t *textLines) text(s string) {
+	if t.pre == 0 {
+		t.cur.WriteString(s)
+		return
+	}
+	t.raw = true
+	for i, part := range strings.Split(s, "\n") {
+		if i > 0 {
+			t.brk()
+			t.raw = true
+		}
+		t.cur.WriteString(part)
+	}
+}
+
+func (t *textLines) brk() {
+	line := t.cur.String()
+	t.cur.Reset()
+	if t.raw {
+		line = strings.TrimRight(line, " \t\r")
+	} else {
+		line = collapse(line)
+	}
+	t.raw = false
+	t.out = append(t.out, line)
+}
+
+// done joins the lines, dropping runs of blank ones. A rendered document is
+// full of wrapper divs, and one blank line between paragraphs is the intent
+// while six is an artifact of the markup.
+func (t *textLines) done() string {
+	t.brk()
+	var b strings.Builder
+	blank := false
+	for _, line := range t.out {
+		if line == "" {
+			blank = true
+			continue
+		}
+		if blank && b.Len() > 0 {
+			b.WriteString("\n")
+		}
+		blank = false
+		if b.Len() > 0 {
+			b.WriteString("\n")
+		}
+		b.WriteString(line)
+	}
+	return b.String()
+}
+
+// RelTime returns the datetime attribute of the first 
+// descendant. The element's own text is never read: it is localised and
+// relative, and parsing it would be a whole class of bug for no gain.
+func RelTime(n *html.Node) string {
+	rt := Find(n, Sel{Tag: "relative-time"})
+	if rt == nil {
+		rt = Find(n, Sel{Tag: "time-ago"})
+	}
+	if rt == nil {
+		return ""
+	}
+	return Attr(rt, "datetime")
+}
+
+// readDOM is steps 5 to 8 of the extraction: meta tags, canonical, microdata,
+// and the deferred fragments. It runs on every page because all four are cheap
+// once the tree exists and all four are useful on both planes.
+func (p *Page) readDOM() {
+	doc := p.Doc()
+	if doc == nil {
+		return
+	}
+	p.Meta = map[string]string{}
+	p.Microdata = map[string][]string{}
+
+	Walk(doc, func(n *html.Node) bool {
+		if n.Type != html.ElementNode {
+			return true
+		}
+		switch n.Data {
+		case "meta":
+			key := firstAttr(n, "property", "name")
+			content := Attr(n, "content")
+			if content == "" {
+				return true
+			}
+			if strings.HasPrefix(key, "og:") || strings.HasPrefix(key, "twitter:") ||
+				key == "description" || key == "octolytics-dimension-user_login" ||
+				key == "octolytics-dimension-repository_id" || key == "route-pattern" {
+				p.Meta[key] = content
+			}
+		case "link":
+			if Attr(n, "rel") == "canonical" {
+				p.Canonical = Attr(n, "href")
+			}
+		case "include-fragment", "turbo-frame":
+			src := Attr(n, "src")
+			// In-product messaging is growth tooling, not content, and it is on
+			// nearly every page.
+			if src != "" && !strings.Contains(src, "/in-product-messaging") {
+				p.Fragments = appendUnique(p.Fragments, src)
+			}
+		case "title":
+			if p.Title == "" {
+				p.Title = Text(n)
+			}
+		}
+		if v, ok := lookupAttr(n, "itemprop"); ok {
+			// itemprop can name several properties at once, as in
+			// itemprop="name codeRepository" on the repositories tab.
+			for _, name := range strings.Fields(v) {
+				p.Microdata[name] = append(p.Microdata[name], itemValue(n))
+			}
+		}
+		return true
+	})
+	if len(p.Meta) == 0 {
+		p.Meta = nil
+	}
+	if len(p.Microdata) == 0 {
+		p.Microdata = nil
+	}
+}
+
+// itemValue reads a microdata property the way the specification says to: the
+// attribute that carries the machine-readable form when there is one, and the
+// element text otherwise.
+func itemValue(n *html.Node) string {
+	switch n.Data {
+	case "meta":
+		return Attr(n, "content")
+	case "a", "area", "link":
+		return Attr(n, "href")
+	case "img", "audio", "embed", "iframe", "source", "video":
+		return Attr(n, "src")
+	case "time":
+		if v := Attr(n, "datetime"); v != "" {
+			return v
+		}
+	case "data":
+		if v := Attr(n, "value"); v != "" {
+			return v
+		}
+	}
+	return Text(n)
+}
+
+func firstAttr(n *html.Node, names ...string) string {
+	for _, name := range names {
+		if v, ok := lookupAttr(n, name); ok {
+			return v
+		}
+	}
+	return ""
+}
+
+func appendUnique(ss []string, s string) []string {
+	for _, x := range ss {
+		if x == s {
+			return ss
+		}
+	}
+	return append(ss, s)
+}
+
+// OuterHTML re-renders a subtree. It is how a README makes it into a record
+// with its markup intact: the alternative is refetching the fragment, and the
+// fragment is already here.
+func OuterHTML(n *html.Node) string {
+	if n == nil {
+		return ""
+	}
+	var b strings.Builder
+	if err := html.Render(&b, n); err != nil {
+		return ""
+	}
+	return b.String()
+}
diff --git a/pkg/page/dom_test.go b/pkg/page/dom_test.go
new file mode 100644
index 0000000..64f868b
--- /dev/null
+++ b/pkg/page/dom_test.go
@@ -0,0 +1,174 @@
+package page
+
+import (
+	"strings"
+	"testing"
+
+	"golang.org/x/net/html"
+)
+
+func parse(t *testing.T, s string) *html.Node {
+	t.Helper()
+	doc, err := html.Parse(strings.NewReader(s))
+	if err != nil {
+		t.Fatal(err)
+	}
+	return doc
+}
+
+// The shape here is GitHub's, not an invention: a page renders its content and
+// then renders a dialog that repeats the same tags for the picker overlay. A
+// Find that answers with the last match reads the dialog and looks like it
+// worked.
+const twoOfEverything = `
+

+  

+    
GitHub CLI 2.63.2

+    a while ago
+    
+      
Choose a tag to compare

+      then
+    
+  

+

+`
+
+func TestFindReturnsTheFirstMatch(t *testing.T) {
+	doc := parse(t, twoOfEverything)
+	h := Find(doc, Sel{Tag: "h1"})
+	if h == nil {
+		t.Fatal("no h1")
+	}
+	if got := Text(h); got != "GitHub CLI 2.63.2" {
+		t.Errorf("h1 is %q, the dialog's copy won", got)
+	}
+	rt := Find(doc, RelTimeEl)
+	if rt == nil {
+		t.Fatal("no relative-time")
+	}
+	if got := Attr(rt, "datetime"); got != "2024-12-05T18:15:12Z" {
+		t.Errorf("datetime is %q, the dialog's copy won", got)
+	}
+}
+
+func TestFindAllIsInDocumentOrder(t *testing.T) {
+	doc := parse(t, twoOfEverything)
+	all := FindAll(doc, Sel{Tag: "h1"})
+	if len(all) != 2 {
+		t.Fatalf("found %d h1s", len(all))
+	}
+	if Text(all[0]) != "GitHub CLI 2.63.2" || Text(all[1]) != "Choose a tag to compare" {
+		t.Errorf("out of order: %q then %q", Text(all[0]), Text(all[1]))
+	}
+}
+
+func TestFindNested(t *testing.T) {
+	// A match inside a match is still a match, and the outer one is first.
+	doc := parse(t, `
inner
`)
+	n := Find(doc, Sel{Class: "Box"})
+	if n == nil {
+		t.Fatal("no Box")
+	}
+	if inner := Find(n.FirstChild, Sel{Class: "Box"}); inner == nil {
+		t.Error("the outer Box has no inner Box, so Find picked the inner one")
+	}
+}
+
+func TestSelMatch(t *testing.T) {
+	doc := parse(t, `
+tag
+sha256:abc
+`)
+	cases := []struct {
+		name string
+		sel  Sel
+		want bool
+	}{
+		{"tag", Sel{Tag: "a"}, true},
+		{"id", Sel{ID: "x"}, true},
+		{"wrong id", Sel{ID: "y"}, false},
+		{"one class of several", Sel{Class: "Link--muted"}, true},
+		{"class is not a substring", Sel{Class: "Link--mut"}, false},
+		{"bare attribute", Sel{Tag: "a", Attr: "data-hpc"}, true},
+		{"attr contains", Sel{Attr: "href", AttrContains: "/releases/tag/"}, true},
+		{"attr prefix", Sel{Attr: "href", AttrPrefix: "/cli/"}, true},
+		{"attr suffix", Sel{Attr: "href", AttrSuffix: "v1.0.0"}, true},
+		{"attr value must be exact", Sel{Attr: "id", AttrValue: "x"}, true},
+		{"attr value mismatch", Sel{Attr: "id", AttrValue: "xx"}, false},
+	}
+	for _, c := range cases {
+		if got := Find(doc, c.sel) != nil; got != c.want {
+			t.Errorf("%s: matched %v, want %v", c.name, got, c.want)
+		}
+	}
+}
+
+// The markup here is the shape GitHub's markdown renderer emits into a README:
+// a heading, a paragraph broken across source lines, a list, and a fenced code
+// block that came through as 
.
+func TestBlockText(t *testing.T) {
+	doc := parse(t, `

+
gh

+
GitHub on
+the command line.

+
It brings pull requests to the terminal.

+
  • one
  • two

+
func main() {
+	println("hi")
+}
+

+
Done.

+
`)
+
+	want := strings.Join([]string{
+		"gh",
+		"",
+		"GitHub on the command line.",
+		"",
+		"It brings pull requests to the terminal.",
+		"",
+		"one",
+		"",
+		"two",
+		"",
+		"func main() {",
+		"\tprintln(\"hi\")",
+		"}",
+		"",
+		"Done.",
+	}, "\n")
+
+	got := BlockText(Find(doc, Sel{Class: "markdown-body"}))
+	if got != want {
+		t.Errorf("BlockText:\n%q\nwant:\n%q", got, want)
+	}
+}
+
+func TestBlockTextKeepsCodeIndentation(t *testing.T) {
+	// A code block's leading whitespace is its meaning, so it survives even
+	// though every other line gets collapsed.
+	got := FragmentText("
  indented\n    more\n
")
+	if got != "  indented\n    more" {
+		t.Errorf("FragmentText(pre) = %q", got)
+	}
+}
+
+func TestBlockTextDropsChrome(t *testing.T) {
+	// Wrapper divs are the bulk of GitHub's markup and none of its prose, so a
+	// stack of them must not turn into a stack of blank lines.
+	got := FragmentText(`
a

+
+
b
`)
+	if got != "a\n\nb" {
+		t.Errorf("FragmentText = %q, want %q", got, "a\n\nb")
+	}
+}
+
+func TestBlockTextEmpty(t *testing.T) {
+	if got := BlockText(nil); got != "" {
+		t.Errorf("BlockText(nil) = %q", got)
+	}
+	if got := FragmentText("   "); got != "" {
+		t.Errorf("FragmentText(blank) = %q", got)
+	}
+}
diff --git a/pkg/page/page.go b/pkg/page/page.go
new file mode 100644
index 0000000..193c981
--- /dev/null
+++ b/pkg/page/page.go
@@ -0,0 +1,252 @@
+// Package page turns a github.com HTML document into structured data.
+//
+// github.com is two applications sharing a domain. The React half ships its
+// route props as JSON inside a script tag, and the Rails half ships schema.org
+// microdata. Both are structured, which means most of what looks like scraping
+// is really JSON decoding with an HTML document as the envelope.
+//
+// Pulling text out of rendered markup is the third choice here, not the first.
+// Where it is unavoidable, the selector lives in selectors.go with the date it
+// was last checked against a live page, so a break is a one-file diff.
+package page
+
+import (
+	"encoding/json"
+	"strings"
+
+	"golang.org/x/net/html"
+)
+
+// Plane says which of the two applications rendered a page. It is worth
+// knowing because it decides where the data is: React pages carry an app
+// payload and Rails pages carry microdata.
+type Plane string
+
+const (
+	PlaneReact Plane = "react"
+	PlaneRails Plane = "rails"
+)
+
+// Page is the whole HTML plane of one document in one struct. Every
+// page-derived record is built from a Page and never from a raw string, which
+// is what makes `github page` possible: it prints this, and every record is a
+// projection of it.
+type Page struct {
+	URL       string `json:"url"`
+	Canonical string `json:"canonical,omitempty"`
+	Title     string `json:"title,omitempty"`
+	Plane     Plane  `json:"plane"`
+
+	// Payload is the React route props, the union of every route object the
+	// server sent for this page.
+	Payload map[string]json.RawMessage `json:"payload,omitempty"`
+
+	// Queries holds the Relay results the server preloaded, keyed by query
+	// name. This is a GraphQL response without a GraphQL token, and on issue
+	// and pull request pages it is where everything lives.
+	Queries map[string]json.RawMessage `json:"queries,omitempty"`
+
+	// StructuredData is payload.structured_data, GitHub's own schema.org view
+	// of the page. Its url field is wrong, see Canonical.
+	StructuredData json.RawMessage `json:"structured_data,omitempty"`
+
+	// LinkedData is every ` is a byte scan, and the content between them is JSON that never
+// contains `` because GitHub escapes it before it goes out.
+//
+// This matters for cost. A repository page is 300 KB and an issue page is 340
+// KB, and the app payload is the only part of either that most reads need. The
+// scanner allocates once per block and never builds a node tree.
+
+type jsonBlock struct {
+	dataTarget string
+	id         string
+	body       []byte
+}
+
+var (
+	scriptOpen  = []byte("")
+	typeJSON    = []byte(`type="application/json"`)
+	typeLDJSON  = []byte(`type="application/ld+json"`)
+)
+
+// scanJSONScripts returns every