Huge speed improvements to commit fetching

src/components/charts/GithubContributionChart.js

@@ -1,9 +1,11 @@
 import React, { useState, useEffect, useCallback } from 'react';
-import { getContributions } from '../../lib/github';
+import { getAllCommits } from '@/lib/github';
 
 const GithubContributionChart = ({ username }) => {
     const [contributionData, setContributionData] = useState({});
     const [loading, setLoading] = useState(true);
+    const [timeTaken, setTimeTaken] = useState(null);
+    const [totalCommits, setTotalCommits] = useState(0);
 
     const getColor = useCallback((count) => {
         if (count === 0) return 'bg-gray-100';
@@ -30,15 +32,27 @@ const GithubContributionChart = ({ username }) => {
     useEffect(() => {
         const fetchData = async () => {
             setLoading(true);
-            try {
-                await getContributions(username, (data) => {
-                    setContributionData(data);
-                    setLoading(false);
+            const startTime = Date.now();
+            const sixMonthsAgo = new Date();
+            sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
+            
+            let commitCount = 0;
+            
+            for await (const commit of getAllCommits(username, sixMonthsAgo)) {
+                const date = commit.commit.author.date.split('T')[0];
+                setContributionData(prev => {
+                    const newData = { ...prev };
+                    newData[date] = (newData[date] || 0) + 1;
+                    return newData;
                 });
-            } catch (error) {
-                console.error("Error fetching contribution data:", error);
-                setLoading(false);
+                commitCount++;
+                setTotalCommits(commitCount);
             }
+
+            const endTime = Date.now();
+            setTimeTaken(((endTime - startTime) / 1000).toFixed(2));
+            setLoading(false);
+            console.log(`Total commits fetched: ${commitCount}`);
         };
 
         fetchData();
@@ -49,7 +63,9 @@ const GithubContributionChart = ({ username }) => {
     return (
         <div className="p-4">
             <h2 className="text-xl font-bold mb-4">Github Contributions for {username}</h2>
-            {loading && <p>Loading contribution data...</p>}
+            {loading && <p>Loading contribution data... ({totalCommits} commits fetched)</p>}
+            {timeTaken && <p className="mb-2">Time taken to fetch data: {timeTaken} seconds</p>}
+            {!loading && <p className="mb-2">Total commits: {totalCommits}</p>}
             <div className="flex flex-wrap gap-1">
                 {calendar.map((day, index) => (
                     <div

src/lib/github.js

@@ -1,11 +1,10 @@
-const { Octokit } = require("@octokit/rest");
-const { throttling } = require("@octokit/plugin-throttling");
+import { Octokit } from "@octokit/rest";
+import { throttling } from "@octokit/plugin-throttling";
+
 const ThrottledOctokit = Octokit.plugin(throttling);
 
-const ACCESS_TOKEN = process.env.NEXT_PUBLIC_GITHUB_ACCESS_KEY;
-
 const octokit = new ThrottledOctokit({
-  auth: ACCESS_TOKEN,
+  auth: process.env.NEXT_PUBLIC_GITHUB_ACCESS_KEY,
   throttle: {
     onRateLimit: (retryAfter, options, octokit, retryCount) => {
       octokit.log.warn(`Request quota exhausted for request ${options.method} ${options.url}`);
@@ -21,55 +20,45 @@ const octokit = new ThrottledOctokit({
   },
 });
 
-async function getContributions(username, updateCallback) {
-  const sixMonthsAgo = new Date();
-  sixMonthsAgo.setMonth(sixMonthsAgo.getMonth() - 6);
-  const sinceDate = sixMonthsAgo.toISOString();
+export async function* getAllCommits(username, since) {
+    let page = 1;
 
-  const contributionData = {};
+    while (true) {
+        try {
+            const { data: repos } = await octokit.repos.listForUser({
+                username,
+                per_page: 100,
+                page,
+            });
 
-  try {
-    const repos = await octokit.paginate(octokit.repos.listForUser, {
-      username,
-      per_page: 100,
-    });
+            if (repos.length === 0) break;
 
-    console.log(`Fetched ${repos.length} repositories for ${username}`);
+            const repoPromises = repos.map(repo => 
+                octokit.repos.listCommits({
+                    owner: username,
+                    repo: repo.name,
+                    since: since.toISOString(),
+                    per_page: 100,
+                })
+            );
 
-    // Call updateCallback immediately after fetching repos
-    updateCallback({});
+            const repoResults = await Promise.allSettled(repoPromises);
 
-    for (const repo of repos) {
-      console.log(`Fetching commits for ${repo.name}`);
-      try {
-        const commits = await octokit.paginate(octokit.repos.listCommits, {
-          owner: repo.owner.login,
-          repo: repo.name,
-          author: username,
-          since: sinceDate,
-          per_page: 100,
-        });
+            for (const result of repoResults) {
+                if (result.status === 'fulfilled') {
+                    for (const commit of result.value.data) {
+                        yield commit;
+                    }
+                } else {
+                    console.warn(`Error fetching commits: ${result.reason}`);
+                }
+            }
 
-        console.log(`Fetched ${commits.length} commits for ${repo.name}`);
-
-        commits.forEach(commit => {
-          const date = commit.commit.author.date.split('T')[0];
-          contributionData[date] = (contributionData[date] || 0) + 1;
-          // Call the update callback after processing each commit
-          updateCallback({...contributionData});
-        });
-      } catch (repoError) {
-        console.error(`Error fetching commits for ${repo.name}:`, repoError.message);
-      }
+            page++;
+        } catch (error) {
+            console.error("Error fetching repositories:", error.message);
+            break;
+        }
     }
-
-    console.log('Final contribution data:', contributionData);
-    return contributionData;
-  } catch (error) {
-    console.error("Error fetching contribution data:", error);
-    throw error;
-  }
 }
 
-export { getContributions };
-