Reinstate GITHUB_RATE_LIMIT for quota-based token refreshes

README.md

@@ -18,6 +18,7 @@ This exporter is configured via environment variables. All variables are optiona
 | `GITHUB_APP_INSTALLATION_ID`  | The Installation ID of the GitHub App. Required if `GITHUB_APP` is `true`.         |                          |
 | `GITHUB_APP_KEY_PATH`         | Path to the GitHub App private key file. Required if `GITHUB_APP` is `true`.       |                          |
 | `GITHUB_RATE_LIMIT_ENABLED`   | Whether to fetch GitHub API rate limit metrics (`true` or `false`).                | `true`                   |
+| `GITHUB_RATE_LIMIT`           | Core API quota threshold for proactive GitHub App token refresh. When the remaining `core` requests drop below this value, a new installation token is requested. `0` disables this behaviour. | `0` |
 | `GITHUB_RESULTS_PER_PAGE`     | Number of results to request per page from the GitHub API (max 100).               | `100`                    |
 | `FETCH_REPO_RELEASES_ENABLED` | Whether to fetch repository release metrics (`true` or `false`).                   | `true`                   |
 | `FETCH_ORGS_CONCURRENCY`      | Number of concurrent requests to make when fetching organization data.             | `1`                      |

config/config.go

@@ -28,6 +28,7 @@ type Config struct {
 	GithubTokenFile          string   `envconfig:"GITHUB_TOKEN_FILE" required:"false"`
 	GitHubApp                bool     `envconfig:"GITHUB_APP" required:"false" default:"false"`
 	GitHubRateLimitEnabled   bool     `envconfig:"GITHUB_RATE_LIMIT_ENABLED" required:"false" default:"true"`
+	GitHubRateLimit          float64  `envconfig:"GITHUB_RATE_LIMIT" required:"false" default:"0"`
 	FetchRepoReleasesEnabled bool     `envconfig:"FETCH_REPO_RELEASES_ENABLED" required:"false" default:"true"`
 	FetchOrgsConcurrency     int      `envconfig:"FETCH_ORGS_CONCURRENCY" required:"false" default:"1"`
 	FetchOrgReposConcurrency int      `envconfig:"FETCH_ORG_REPOS_CONCURRENCY" required:"false" default:"1"`
@@ -88,7 +89,7 @@ func (c *Config) GetClient() (*github.Client, error) {
 
 	// Add custom transport for GitHub App authentication if enabled
 	if c.GitHubApp {
-		itr, err := ghinstallation.NewKeyFromFile(http.DefaultTransport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath)
+		itr, err := ghinstallation.NewKeyFromFile(transport, c.GitHubAppId, c.GitHubAppInstallationId, c.GitHubAppKeyPath)
 		if err != nil {
 			return nil, fmt.Errorf("creating GitHub App installation transport: %v", err)
 		}

config/config_test.go

@@ -34,6 +34,7 @@ func TestConfig(t *testing.T) {
 				GitHubApp:                false,
 				GitHubAppConfig:          nil,
 				GitHubRateLimitEnabled:   true,
+				GitHubRateLimit:          0,
 				FetchRepoReleasesEnabled: true,
 				FetchOrgsConcurrency:     1,
 				FetchOrgReposConcurrency: 1,
@@ -87,6 +88,7 @@ func TestConfig(t *testing.T) {
 				GitHubApp:                false,
 				GitHubAppConfig:          nil,
 				GitHubRateLimitEnabled:   false,
+				GitHubRateLimit:          0,
 				FetchRepoReleasesEnabled: false,
 				FetchOrgsConcurrency:     2,
 				FetchOrgReposConcurrency: 3,
@@ -95,6 +97,37 @@ func TestConfig(t *testing.T) {
 			},
 			expectedErr: nil,
 		},
+		{
+			name: "github rate limit threshold config",
+			envVars: map[string]string{
+				"GITHUB_RATE_LIMIT": "15000",
+			},
+			expectedCfg: &Config{
+				MetricsPath: "/metrics",
+				ListenPort:  "9171",
+				LogLevel:    "INFO",
+				ApiUrl: &url.URL{
+					Scheme: "https",
+					Host:   "api.github.com",
+				},
+				Repositories:             []string{},
+				Organisations:            []string{},
+				Users:                    []string{},
+				GitHubResultsPerPage:     100,
+				GithubToken:              "",
+				GithubTokenFile:          "",
+				GitHubApp:                false,
+				GitHubAppConfig:          nil,
+				GitHubRateLimitEnabled:   true,
+				GitHubRateLimit:          15000,
+				FetchRepoReleasesEnabled: true,
+				FetchOrgsConcurrency:     1,
+				FetchOrgReposConcurrency: 1,
+				FetchReposConcurrency:    1,
+				FetchUsersConcurrency:    1,
+			},
+			expectedErr: nil,
+		},
 		{
 			name:        "invalid url",
 			expectedCfg: nil,

exporter/prometheus.go

@@ -57,6 +57,9 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 		return
 	}
 
+	// Refresh client if rate limit threshold enabled
+	e.rateLimitRefresh(r)
+
 	// Set prometheus gauge metrics using the data gathered
 	err = e.processMetrics(data, r, ch)
 	if err != nil {
@@ -107,6 +110,32 @@ func (e *Exporter) getRateLimits(ctx context.Context) (*[]RateLimit, error) {
 	return &rls, nil
 }
 
+func (e *Exporter) rateLimitRefresh(rates *[]RateLimit) {
+	if e.GitHubRateLimit <= 0 || rates == nil {
+		return
+	}
+
+	for _, rl := range *rates {
+		if rl.Resource == "core" && rl.Remaining <= e.GitHubRateLimit {
+			log.Infof("GitHub API core rate limit is low (%.0f remaining, threshold: %.0f)", rl.Remaining, e.GitHubRateLimit)
+			if !e.GitHubApp {
+				log.Warn("GITHUB_RATE_LIMIT threshold breached but GitHub App auth is not configured; cannot refresh token automatically")
+				return
+			}
+
+			log.Info("Refreshing GitHub App installation token due to low rate limit")
+			newClient, err := e.GetClient()
+			if err != nil {
+				log.Errorf("refreshing GitHub App client after low rate limit: %v", err)
+				return
+			}
+
+			e.Client = newClient
+			return
+		}
+	}
+}
+
 // getRepoMetrics fetches metrics for the configured repositories
 func (e *Exporter) getRepoMetrics(ctx context.Context) ([]*Datum, error) {
 	var data []*Datum

exporter/prometheus_test.go

@@ -0,0 +1,193 @@
+package exporter
+
+import (
+	"crypto/rand"
+	"crypto/rsa"
+	"crypto/x509"
+	"encoding/pem"
+	"os"
+	"testing"
+
+	"github.com/githubexporter/github-exporter/config"
+	"github.com/google/go-github/v76/github"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// writeTempRSAKey generates a 2048-bit RSA private key and writes it as a
+// PKCS#1 PEM file to a temp path. The file is removed automatically when the
+// test finishes.
+func writeTempRSAKey(t *testing.T) string {
+	t.Helper()
+	key, err := rsa.GenerateKey(rand.Reader, 2048)
+	require.NoError(t, err)
+
+	f, err := os.CreateTemp("", "test-github-app-key-*.pem")
+	require.NoError(t, err)
+	defer f.Close()
+
+	err = pem.Encode(f, &pem.Block{
+		Type:  "RSA PRIVATE KEY",
+		Bytes: x509.MarshalPKCS1PrivateKey(key),
+	})
+	require.NoError(t, err)
+
+	t.Cleanup(func() { os.Remove(f.Name()) })
+
+	return f.Name()
+}
+
+func newTestExporter(cfg config.Config) *Exporter {
+	return &Exporter{
+		Client: github.NewClient(nil),
+		Config: cfg,
+	}
+}
+
+func TestRateLimitRefresh(t *testing.T) {
+	// Shared rate-limit slices used across sub-tests.
+	aboveThreshold := []RateLimit{{Resource: "core", Remaining: 200, Limit: 5000}}
+	atThreshold := []RateLimit{{Resource: "core", Remaining: 100, Limit: 5000}}
+	belowThreshold := []RateLimit{{Resource: "core", Remaining: 50, Limit: 5000}}
+	noCoreEntry := []RateLimit{{Resource: "search", Remaining: 5, Limit: 30}}
+
+	baseCfg := config.Config{
+		GitHubResultsPerPage:     100,
+		FetchReposConcurrency:    1,
+		FetchOrgsConcurrency:     1,
+		FetchOrgReposConcurrency: 1,
+		FetchUsersConcurrency:    1,
+	}
+
+	t.Run("no-op when GitHubRateLimit is zero", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 0
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&atThreshold)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("no-op when GitHubRateLimit is negative", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = -1
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&atThreshold)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("no-op when rates is nil", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(nil)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("no-op when core remaining is above threshold", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100 // remaining=200 > threshold=100
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&aboveThreshold)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("no-op when rates has no core entry", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&noCoreEntry)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("no-op when GitHubApp is false and core is at threshold", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100 // remaining=100 <= threshold=100
+		cfg.GitHubApp = false
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&atThreshold)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("no-op when GitHubApp is false and core is below threshold", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100 // remaining=50 < threshold=100
+		cfg.GitHubApp = false
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&belowThreshold)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("no-op when GitHubApp is true but GetClient fails", func(t *testing.T) {
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100
+		cfg.GitHubApp = true
+		cfg.GitHubAppConfig = &config.GitHubAppConfig{
+			GitHubAppKeyPath:        "/nonexistent/key.pem",
+			GitHubAppId:             1,
+			GitHubAppInstallationId: 1,
+		}
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&atThreshold)
+
+		assert.Same(t, original, e.Client)
+	})
+
+	t.Run("replaces client when GitHubApp is true and core is at threshold", func(t *testing.T) {
+		keyPath := writeTempRSAKey(t)
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100 // remaining=100 <= threshold=100
+		cfg.GitHubApp = true
+		cfg.GitHubAppConfig = &config.GitHubAppConfig{
+			GitHubAppKeyPath:        keyPath,
+			GitHubAppId:             1,
+			GitHubAppInstallationId: 1,
+		}
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&atThreshold)
+
+		assert.NotSame(t, original, e.Client, "expected client to be replaced after rate limit refresh")
+	})
+
+	t.Run("replaces client when GitHubApp is true and core is below threshold", func(t *testing.T) {
+		keyPath := writeTempRSAKey(t)
+		cfg := baseCfg
+		cfg.GitHubRateLimit = 100 // remaining=50 < threshold=100
+		cfg.GitHubApp = true
+		cfg.GitHubAppConfig = &config.GitHubAppConfig{
+			GitHubAppKeyPath:        keyPath,
+			GitHubAppId:             1,
+			GitHubAppInstallationId: 1,
+		}
+		e := newTestExporter(cfg)
+		original := e.Client
+
+		e.rateLimitRefresh(&belowThreshold)
+
+		assert.NotSame(t, original, e.Client, "expected client to be replaced after rate limit refresh")
+	})
+}