import axios from 'axios'; import { Repository } from './config'; interface CacheEntry { data: T; timestamp: number; ttl: number; } interface RateLimitInfo { limit: number; remaining: number; resetTime: number; used: number; } class RequestQueue { private activeRequests = 0; private requestTimes: number[] = []; private maxConcurrent = 10; // Maximum concurrent requests private maxRequestsPerSecond = 10; // GitHub allows up to 5000/hour, so 10/second is very conservative async add(request: () => Promise): Promise { // Wait if we're at max concurrent requests while (this.activeRequests >= this.maxConcurrent) { console.log(`⏳ Waiting for concurrent request slot (${this.activeRequests}/${this.maxConcurrent})`); await new Promise(resolve => setTimeout(resolve, 50)); } // Clean up old request times (older than 1 second) const now = Date.now(); this.requestTimes = this.requestTimes.filter(time => now - time < 1000); // Wait if we're hitting rate limit if (this.requestTimes.length >= this.maxRequestsPerSecond) { const waitTime = 1000 - (now - this.requestTimes[0]); console.log(`⏳ Rate limiting: waiting ${waitTime}ms (${this.requestTimes.length}/${this.maxRequestsPerSecond} requests/sec)`); await new Promise(resolve => setTimeout(resolve, waitTime)); } this.activeRequests++; this.requestTimes.push(now); console.log(`🔄 Starting request (${this.activeRequests}/${this.maxConcurrent} active, ${this.requestTimes.length}/sec)`); try { const result = await request(); console.log(`✅ Request completed (${this.activeRequests - 1}/${this.maxConcurrent} remaining)`); return result; } catch (error) { console.error('Queue request failed:', error); throw error; } finally { this.activeRequests--; } } } export interface WorkflowRun { id: number; name: string; display_title: string; status: 'queued' | 'in_progress' | 'completed'; conclusion: 'success' | 'failure' | 'neutral' | 'cancelled' | 'skipped' | 'timed_out' | 'action_required' | null; workflow_id: number; head_branch: string; head_sha: string; run_number: number; event: string; created_at: string; updated_at: string; html_url: string; repository: { id: number; name: string; full_name: string; owner: { login: string; avatar_url: string; }; }; head_commit: { id: string; message: string; author: { name: string; email: string; }; }; actor: { login: string; avatar_url: string; }; } const GITHUB_API_BASE = 'https://api.github.com'; export class GitHubService { private token: string; private cache = new Map>(); private rateLimitInfo: RateLimitInfo = { limit: 5000, remaining: 5000, resetTime: Date.now() + (60 * 60 * 1000), // 1 hour from now used: 0 }; private requestQueue = new RequestQueue(); private readonly DEFAULT_TTL: number; private readonly WORKFLOW_RUNS_TTL: number; constructor(token: string, cacheTimeoutSeconds: number = 300) { this.token = token; this.DEFAULT_TTL = cacheTimeoutSeconds * 1000; this.WORKFLOW_RUNS_TTL = cacheTimeoutSeconds * 1000; console.log(`🚀 GitHubService initialized with caching (${cacheTimeoutSeconds}s) and rate limiting`); } private getHeaders() { return { 'Authorization': `token ${this.token}`, 'Accept': 'application/vnd.github.v3+json', 'X-GitHub-Api-Version': '2022-11-28' }; } private getCacheKey(endpoint: string, params?: any): string { // Extract repository info from endpoint for better cache key readability const repoMatch = endpoint.match(/\/repos\/([^/]+\/[^/]+)\//); const repoInfo = repoMatch ? repoMatch[1] : 'unknown'; // Create a sorted, clean parameter string const cleanParams = params ? Object.keys(params).sort().reduce((obj: any, key) => { obj[key] = params[key]; return obj; }, {}) : {}; const cacheKey = `${repoInfo}:${endpoint.split('/').pop()}:${JSON.stringify(cleanParams)}`; console.log(`🔑 Generated cache key: ${cacheKey} for ${endpoint}`); return cacheKey; } private isCacheValid(entry: CacheEntry): boolean { return Date.now() - entry.timestamp < entry.ttl; } private setCache(key: string, data: T, ttl: number = this.DEFAULT_TTL): void { this.cache.set(key, { data, timestamp: Date.now(), ttl }); console.log(`💾 SET Cache: ${key} (TTL: ${ttl}ms, Size: ${this.cache.size})`); } private getCache(key: string): T | null { const entry = this.cache.get(key); if (!entry) { console.log(`💾 GET Cache: ${key} - NOT FOUND`); return null; } if (!this.isCacheValid(entry)) { console.log(`💾 GET Cache: ${key} - EXPIRED (age: ${Date.now() - entry.timestamp}ms, ttl: ${entry.ttl}ms)`); this.cache.delete(key); return null; } console.log(`💾 GET Cache: ${key} - HIT (age: ${Date.now() - entry.timestamp}ms, ttl: ${entry.ttl}ms)`); return entry.data; } private updateRateLimitInfo(headers: any): void { if (headers['x-ratelimit-limit']) { this.rateLimitInfo.limit = parseInt(headers['x-ratelimit-limit']); } if (headers['x-ratelimit-remaining']) { this.rateLimitInfo.remaining = parseInt(headers['x-ratelimit-remaining']); } if (headers['x-ratelimit-reset']) { this.rateLimitInfo.resetTime = parseInt(headers['x-ratelimit-reset']) * 1000; } if (headers['x-ratelimit-used']) { this.rateLimitInfo.used = parseInt(headers['x-ratelimit-used']); } } private async shouldWaitForRateLimit(): Promise { if (this.rateLimitInfo.remaining <= 10) { const waitTime = Math.max(0, this.rateLimitInfo.resetTime - Date.now()); if (waitTime > 0) { console.log(`⏳ Rate limit nearly exceeded. Waiting ${Math.ceil(waitTime / 1000)} seconds...`); await new Promise(resolve => setTimeout(resolve, waitTime)); } } } private async makeRequest(url: string, params?: any, ttl: number = this.DEFAULT_TTL): Promise { const cacheKey = this.getCacheKey(url, params); // Check cache first const cached = this.getCache(cacheKey); if (cached) { const repoMatch = url.match(/\/repos\/([^/]+\/[^/]+)\//); const repoInfo = repoMatch ? repoMatch[1] : 'unknown'; console.log(`💾 Cache HIT: ${repoInfo} - ${url.split('/').pop()}`); return cached; } const repoMatch = url.match(/\/repos\/([^/]+\/[^/]+)\//); const repoInfo = repoMatch ? repoMatch[1] : 'unknown'; console.log(`🌐 Cache MISS: ${repoInfo} - ${url.split('/').pop()} - Making API request`); // Check rate limit await this.shouldWaitForRateLimit(); // Make the request through the queue return this.requestQueue.add(async () => { try { const response = await axios.get(url, { headers: this.getHeaders(), params }); // Update rate limit info this.updateRateLimitInfo(response.headers); console.log(`📊 API Rate Limit: ${this.rateLimitInfo.remaining}/${this.rateLimitInfo.limit} remaining`); // Cache the response this.setCache(cacheKey, response.data, ttl); console.log(`💾 Cached response for ${Math.round(ttl/1000)}s`); return response.data; } catch (error: any) { if (error.response?.status === 403 && error.response?.headers['x-ratelimit-remaining'] === '0') { console.error('🚫 GitHub API rate limit exceeded'); throw new Error('GitHub API rate limit exceeded. Please wait before making more requests.'); } throw error; } }); } async getWorkflowRuns(repository: Repository, per_page = 1, branch?: string): Promise { console.log(`📊 getWorkflowRuns called for ${repository.owner}/${repository.name} (per_page: ${per_page}, branch: ${branch})`); try { const params: any = { per_page, page: 1 }; if (branch) { params.branch = branch; } const url = `${GITHUB_API_BASE}/repos/${repository.owner}/${repository.name}/actions/runs`; const response = await this.makeRequest(url, params, this.WORKFLOW_RUNS_TTL); console.log(`✅ getWorkflowRuns completed for ${repository.owner}/${repository.name} - ${response.workflow_runs.length} runs`); return response.workflow_runs; } catch (error) { console.error(`Error fetching workflow runs for ${repository.owner}/${repository.name}:`, error); return []; } } async getLatestMainBranchRuns(repositories: Repository[]): Promise { console.log(`🚀 getLatestMainBranchRuns called for ${repositories.length} repositories`); // Create a cache key for the aggregate request const repoListHash = repositories.map(r => `${r.owner}/${r.name}`).sort().join(','); const aggregateCacheKey = `aggregate:latest-main-runs:${repoListHash}`; console.log(`🔑 Aggregate cache key: ${aggregateCacheKey}`); // Check if we have a cached result for this exact set of repositories const cached = this.getCache(aggregateCacheKey); if (cached) { console.log(`💾 Cache HIT: Aggregate latest runs for ${repositories.length} repositories`); return cached; } console.log(`🌐 Cache MISS: Aggregate latest runs for ${repositories.length} repositories - Fetching individual repos in parallel`); // Process repositories in parallel - the individual cache and request queue will handle rate limiting const promises = repositories.map(async (repo, index) => { console.log(`📊 Starting repo ${index + 1}/${repositories.length}: ${repo.owner}/${repo.name}`); const runs = await this.getWorkflowRuns(repo, 1, 'main'); if (runs.length > 0) { console.log(`✅ Completed run for ${repo.owner}/${repo.name}`); return runs[0]; } else { console.log(`⚠️ No runs found for ${repo.owner}/${repo.name}`); return null; } }); // Wait for all parallel requests to complete const results = await Promise.all(promises); const filteredResults = results.filter((run): run is WorkflowRun => run !== null); // Cache the aggregate result this.setCache(aggregateCacheKey, filteredResults, this.WORKFLOW_RUNS_TTL); console.log(`💾 Cached aggregate result for ${repositories.length} repositories (${Math.round(this.WORKFLOW_RUNS_TTL/1000)}s)`); return filteredResults; } async getRepositoryWorkflowRuns(repository: Repository, limit = 10): Promise { return this.getWorkflowRuns(repository, limit); } // Add method to get current rate limit status getRateLimitInfo(): RateLimitInfo { return { ...this.rateLimitInfo }; } // Add method to clear cache if needed clearCache(): void { this.cache.clear(); console.log('🗑️ Cache cleared'); } // Add method to get cache stats getCacheStats(): { size: number; entries: string[] } { return { size: this.cache.size, entries: Array.from(this.cache.keys()) }; } // Add method to update token without losing cache updateToken(newToken: string): void { this.token = newToken; console.log('🔧 GitHub token updated, cache preserved'); } // Add method to get current cache timeout getCacheTimeout(): number { return this.DEFAULT_TTL / 1000; // Convert back to seconds } }