diff --git a/README.md b/README.md index 7e3a4ef..b1f72a3 100644 --- a/README.md +++ b/README.md @@ -18,6 +18,7 @@ This exporter is configured via environment variables. All variables are optiona | `GITHUB_APP_INSTALLATION_ID` | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`. | | | `GITHUB_APP_KEY_PATH` | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`. | | | `GITHUB_RATE_LIMIT_ENABLED` | Whether to fetch GitHub API rate limit metrics (`true` or `false`). | `true` | +| `GITHUB_RATE_LIMIT` | Core API quota threshold for proactive GitHub App token refresh. When the remaining `core` requests drop below this value, a new installation token is requested. `0` disables this behaviour. | `0` | | `GITHUB_RESULTS_PER_PAGE` | Number of results to request per page from the GitHub API (max 100). | `100` | | `FETCH_REPO_RELEASES_ENABLED` | Whether to fetch repository release metrics (`true` or `false`). | `true` | | `FETCH_ORGS_CONCURRENCY` | Number of concurrent requests to make when fetching organization data. | `1` | diff --git a/config/config.go b/config/config.go index b3b296e..b218910 100644 --- a/config/config.go +++ b/config/config.go @@ -28,6 +28,7 @@ type Config struct { GithubTokenFile string `envconfig:"GITHUB_TOKEN_FILE" required:"false"` GitHubApp bool `envconfig:"GITHUB_APP" required:"false" default:"false"` GitHubRateLimitEnabled bool `envconfig:"GITHUB_RATE_LIMIT_ENABLED" required:"false" default:"true"` + GitHubRateLimit float64 `envconfig:"GITHUB_RATE_LIMIT" required:"false" default:"0"` FetchRepoReleasesEnabled bool `envconfig:"FETCH_REPO_RELEASES_ENABLED" required:"false" default:"true"` FetchOrgsConcurrency int `envconfig:"FETCH_ORGS_CONCURRENCY" required:"false" default:"1"` FetchOrgReposConcurrency int `envconfig:"FETCH_ORG_REPOS_CONCURRENCY" required:"false" default:"1"` @@ -88,7 +89,7 @@ func (c *Config) GetClient() (*github.Client, error) { // Add custom transport for GitHub App authentication if enabled if c.GitHubApp { - itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath) + itr, err := ghinstallation.NewKeyFromFile(transport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath) if err != nil { return nil, fmt.Errorf("creating GitHub App installation transport: %v", err) } diff --git a/config/config_test.go b/config/config_test.go index 166937e..20a2aae 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -34,6 +34,7 @@ func TestConfig(t *testing.T) { GitHubApp: false, GitHubAppConfig: nil, GitHubRateLimitEnabled: true, + GitHubRateLimit: 0, FetchRepoReleasesEnabled: true, FetchOrgsConcurrency: 1, FetchOrgReposConcurrency: 1, @@ -87,6 +88,7 @@ func TestConfig(t *testing.T) { GitHubApp: false, GitHubAppConfig: nil, GitHubRateLimitEnabled: false, + GitHubRateLimit: 0, FetchRepoReleasesEnabled: false, FetchOrgsConcurrency: 2, FetchOrgReposConcurrency: 3, @@ -95,6 +97,37 @@ func TestConfig(t *testing.T) { }, expectedErr: nil, }, + { + name: "github rate limit threshold config", + envVars: map[string]string{ + "GITHUB_RATE_LIMIT": "15000", + }, + expectedCfg: &Config{ + MetricsPath: "/metrics", + ListenPort: "9171", + LogLevel: "INFO", + ApiUrl: &url.URL{ + Scheme: "https", + Host: "api.github.com", + }, + Repositories: []string{}, + Organisations: []string{}, + Users: []string{}, + GitHubResultsPerPage: 100, + GithubToken: "", + GithubTokenFile: "", + GitHubApp: false, + GitHubAppConfig: nil, + GitHubRateLimitEnabled: true, + GitHubRateLimit: 15000, + FetchRepoReleasesEnabled: true, + FetchOrgsConcurrency: 1, + FetchOrgReposConcurrency: 1, + FetchReposConcurrency: 1, + FetchUsersConcurrency: 1, + }, + expectedErr: nil, + }, { name: "invalid url", expectedCfg: nil, diff --git a/exporter/prometheus.go b/exporter/prometheus.go index d0229c4..b322a1a 100644 --- a/exporter/prometheus.go +++ b/exporter/prometheus.go @@ -57,6 +57,9 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { return } + // Refresh client if rate limit threshold enabled + e.rateLimitRefresh(r) + // Set prometheus gauge metrics using the data gathered err = e.processMetrics(data, r, ch) if err != nil { @@ -107,6 +110,32 @@ func (e *Exporter) getRateLimits(ctx context.Context) (*[]RateLimit, error) { return &rls, nil } +func (e *Exporter) rateLimitRefresh(rates *[]RateLimit) { + if e.GitHubRateLimit <= 0 || rates == nil { + return + } + + for _, rl := range *rates { + if rl.Resource == "core" && rl.Remaining <= e.GitHubRateLimit { + log.Infof("GitHub API core rate limit is low (%.0f remaining, threshold: %.0f)", rl.Remaining, e.GitHubRateLimit) + if !e.GitHubApp { + log.Warn("GITHUB_RATE_LIMIT threshold breached but GitHub App auth is not configured; cannot refresh token automatically") + return + } + + log.Info("Refreshing GitHub App installation token due to low rate limit") + newClient, err := e.GetClient() + if err != nil { + log.Errorf("refreshing GitHub App client after low rate limit: %v", err) + return + } + + e.Client = newClient + return + } + } +} + // getRepoMetrics fetches metrics for the configured repositories func (e *Exporter) getRepoMetrics(ctx context.Context) ([]*Datum, error) { var data []*Datum diff --git a/exporter/prometheus_test.go b/exporter/prometheus_test.go new file mode 100644 index 0000000..dc00b03 --- /dev/null +++ b/exporter/prometheus_test.go @@ -0,0 +1,193 @@ +package exporter + +import ( + "crypto/rand" + "crypto/rsa" + "crypto/x509" + "encoding/pem" + "os" + "testing" + + "github.com/githubexporter/github-exporter/config" + "github.com/google/go-github/v76/github" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// writeTempRSAKey generates a 2048-bit RSA private key and writes it as a +// PKCS#1 PEM file to a temp path. The file is removed automatically when the +// test finishes. +func writeTempRSAKey(t *testing.T) string { + t.Helper() + key, err := rsa.GenerateKey(rand.Reader, 2048) + require.NoError(t, err) + + f, err := os.CreateTemp("", "test-github-app-key-*.pem") + require.NoError(t, err) + defer f.Close() + + err = pem.Encode(f, &pem.Block{ + Type: "RSA PRIVATE KEY", + Bytes: x509.MarshalPKCS1PrivateKey(key), + }) + require.NoError(t, err) + + t.Cleanup(func() { os.Remove(f.Name()) }) + + return f.Name() +} + +func newTestExporter(cfg config.Config) *Exporter { + return &Exporter{ + Client: github.NewClient(nil), + Config: cfg, + } +} + +func TestRateLimitRefresh(t *testing.T) { + // Shared rate-limit slices used across sub-tests. + aboveThreshold := []RateLimit{{Resource: "core", Remaining: 200, Limit: 5000}} + atThreshold := []RateLimit{{Resource: "core", Remaining: 100, Limit: 5000}} + belowThreshold := []RateLimit{{Resource: "core", Remaining: 50, Limit: 5000}} + noCoreEntry := []RateLimit{{Resource: "search", Remaining: 5, Limit: 30}} + + baseCfg := config.Config{ + GitHubResultsPerPage: 100, + FetchReposConcurrency: 1, + FetchOrgsConcurrency: 1, + FetchOrgReposConcurrency: 1, + FetchUsersConcurrency: 1, + } + + t.Run("no-op when GitHubRateLimit is zero", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = 0 + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&atThreshold) + + assert.Same(t, original, e.Client) + }) + + t.Run("no-op when GitHubRateLimit is negative", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = -1 + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&atThreshold) + + assert.Same(t, original, e.Client) + }) + + t.Run("no-op when rates is nil", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = 100 + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(nil) + + assert.Same(t, original, e.Client) + }) + + t.Run("no-op when core remaining is above threshold", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = 100 // remaining=200 > threshold=100 + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&aboveThreshold) + + assert.Same(t, original, e.Client) + }) + + t.Run("no-op when rates has no core entry", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = 100 + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&noCoreEntry) + + assert.Same(t, original, e.Client) + }) + + t.Run("no-op when GitHubApp is false and core is at threshold", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = 100 // remaining=100 <= threshold=100 + cfg.GitHubApp = false + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&atThreshold) + + assert.Same(t, original, e.Client) + }) + + t.Run("no-op when GitHubApp is false and core is below threshold", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = 100 // remaining=50 < threshold=100 + cfg.GitHubApp = false + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&belowThreshold) + + assert.Same(t, original, e.Client) + }) + + t.Run("no-op when GitHubApp is true but GetClient fails", func(t *testing.T) { + cfg := baseCfg + cfg.GitHubRateLimit = 100 + cfg.GitHubApp = true + cfg.GitHubAppConfig = &config.GitHubAppConfig{ + GitHubAppKeyPath: "/nonexistent/key.pem", + GitHubAppId: 1, + GitHubAppInstallationId: 1, + } + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&atThreshold) + + assert.Same(t, original, e.Client) + }) + + t.Run("replaces client when GitHubApp is true and core is at threshold", func(t *testing.T) { + keyPath := writeTempRSAKey(t) + cfg := baseCfg + cfg.GitHubRateLimit = 100 // remaining=100 <= threshold=100 + cfg.GitHubApp = true + cfg.GitHubAppConfig = &config.GitHubAppConfig{ + GitHubAppKeyPath: keyPath, + GitHubAppId: 1, + GitHubAppInstallationId: 1, + } + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&atThreshold) + + assert.NotSame(t, original, e.Client, "expected client to be replaced after rate limit refresh") + }) + + t.Run("replaces client when GitHubApp is true and core is below threshold", func(t *testing.T) { + keyPath := writeTempRSAKey(t) + cfg := baseCfg + cfg.GitHubRateLimit = 100 // remaining=50 < threshold=100 + cfg.GitHubApp = true + cfg.GitHubAppConfig = &config.GitHubAppConfig{ + GitHubAppKeyPath: keyPath, + GitHubAppId: 1, + GitHubAppInstallationId: 1, + } + e := newTestExporter(cfg) + original := e.Client + + e.rateLimitRefresh(&belowThreshold) + + assert.NotSame(t, original, e.Client, "expected client to be replaced after rate limit refresh") + }) +}