feat(pushes)

This commit is contained in:
Othmane Ataallah 2026-05-04 10:29:44 +01:00
parent b5e5d9867a
commit 658307f4be
3 changed files with 269 additions and 23 deletions

View File

@ -101,6 +101,18 @@ The adapter creates and manages the following sheets (tabs) with canonical heade
- `ProcessedAt`, `IssueKey`, `IssueNumber`, `Assignee`, `UserID`, `TotalMinutes`
- The adapter converts the time values returned from Gitea (seconds) into minutes, rounded. It upserts rows by `IssueNumber`+`UserID` to avoid duplicates for the same issue/user.
- `Pushes` — organization-wide push events. Header columns:
- `ProcessedAt`, `EventHash`, `RepoFullName`, `Ref`, `Branch`, `Pusher`, `PusherEmail`, `CommitCount`, `HeadCommitId`, `HeadCommitMessage`, `CompareURL`, `RepoURL`, `CommitsSummary`
- Push rows are idempotent by `eventHash` (if the same push webhook is received twice the existing row is updated instead of duplicated).
- Push rows are idempotent by `eventHash` (if the same push webhook is received twice the existing row is updated instead of duplicated).
- `CommitsSummary` is truncated to a configurable per-commit snippet length and a configurable number of commits; a `CommitsTruncated` column indicates truncation.
- Per-commit rows can be enabled using `PUSH_COMMITS_ENABLED=true`. When enabled a `PushCommits` sheet is created with one row per commit (upserts on EventHash+CommitId).
Configuration for push commits:
- `PUSH_COMMITS_ENABLED` — set to `true` to write per-commit rows to `PushCommits`.
- `PUSH_COMMIT_SNIPPET_LENGTH` — maximum characters per commit message snippet (default 80).
- `PUSH_COMMITS_SUMMARY_LIMIT` — maximum number of commits included in the `CommitsSummary` (default 10).
- `DeadLetters` — failed events for manual inspection. Header columns:
- `ProcessedAt`, `EventHash`, `IssueKey`, `Error`, `PayloadBase64`, `Notes`

View File

@ -40,6 +40,14 @@ const DEADLETTERS_HEADER = ["ProcessedAt", "EventHash", "IssueKey", "Error", "Pa
const COMMENTS_HEADER = process.env.GOOGLE_COMMENTS_HEADER ? JSON.parse(process.env.GOOGLE_COMMENTS_HEADER) : ["ProcessedAt", "IssueNumber", "Actor", "CommentURL"];
const TIME_ENTRIES_SHEET = process.env.GOOGLE_TIME_ENTRIES_SHEET || "TimeEntries";
const TIME_ENTRIES_HEADER = ["ProcessedAt", "IssueKey", "IssueNumber", "Assignee", "UserID", "TotalMinutes"];
const PUSHES_SHEET = process.env.GOOGLE_PUSHES_SHEET || "Pushes";
// Add a boolean indicator if the commits summary was truncated
const PUSHES_HEADER = ["ProcessedAt", "EventHash", "RepoFullName", "Ref", "Branch", "Pusher", "PusherEmail", "CommitCount", "HeadCommitId", "HeadCommitMessage", "CompareURL", "RepoURL", "CommitsSummary", "CommitsTruncated"];
const PUSH_COMMITS_SHEET = process.env.GOOGLE_PUSH_COMMITS_SHEET || "PushCommits";
const PUSH_COMMITS_ENABLED = String(process.env.PUSH_COMMITS_ENABLED || "").toLowerCase() === "true";
const PUSH_COMMITS_HEADER = ["ProcessedAt", "EventHash", "RepoFullName", "Ref", "Branch", "CommitId", "CommitMessage", "CommitAuthor", "CommitURL", "CommitTimestamp"];
const PUSH_COMMIT_SNIPPET_LENGTH = Number(process.env.PUSH_COMMIT_SNIPPET_LENGTH || 80);
const PUSH_COMMITS_SUMMARY_LIMIT = Number(process.env.PUSH_COMMITS_SUMMARY_LIMIT || 10);
// Default number of rows to keep on a freshly-created sheet. Can be overridden with env var
const DEFAULT_SHEET_ROWS = Number(process.env.GOOGLE_SHEET_DEFAULT_ROWS || 100);
@ -468,10 +476,45 @@ export async function appendEvent(evt = {}) {
const processedAt = formatDate(new Date().toISOString());
const eventHash = evt.eventHash || (evt.meta && evt.meta.eventHash) || "";
const eventType = evt.type || (evt.event && evt.event.action) || "";
const issueKey = evt.issueKey || (evt.meta && evt.meta.issueKey) || (evt.event && evt.event.repository && evt.event.issue ? `${evt.event.repository.full_name}#${evt.event.issue.number}` : "");
const actor = (evt.actor || (evt.event && evt.event.sender && evt.event.sender.login) || (evt.meta && evt.meta.actor)) || "";
const summary = evt.summary || (evt.event && evt.event.action) || "";
const eventObj = evt.event || evt;
let eventType = evt.type || (eventObj && eventObj.action) || "";
const issueKey = evt.issueKey || (evt.meta && evt.meta.issueKey) || (eventObj && eventObj.repository && eventObj.issue ? `${eventObj.repository.full_name}#${eventObj.issue.number}` : "");
const actor = (evt.actor || (eventObj && eventObj.sender && eventObj.sender.login) || (evt.meta && evt.meta.actor)) || "";
let summary = evt.summary || (eventObj && eventObj.action) || "";
// If push event, provide a helpful summary
try {
if (eventObj && !eventType) {
// detect push-like events and special cases
const ref = eventObj.ref || "";
const branch = String(ref).replace(/^refs\/heads\//, "");
const commitCount = Array.isArray(eventObj.commits) ? eventObj.commits.length : 0;
const isDeleted = eventObj.deleted === true || (typeof eventObj.after === 'string' && /^0+$/.test(eventObj.after));
if ((eventObj.ref || eventObj.commits)) {
if (isDeleted) {
eventType = 'push-delete';
summary = summary || `deleted ${branch}`;
} else if (commitCount === 0 && !eventObj.head_commit) {
eventType = 'push-empty';
summary = summary || 'no head commit';
} else {
eventType = 'push';
if (!summary) {
if (eventObj.ref) summary = `push to ${branch}`;
else if (commitCount > 0) summary = `${commitCount} commit(s)`;
}
}
}
} else if (!summary && eventObj) {
if (eventObj.ref) {
const branch = String(eventObj.ref).replace(/^refs\/heads\//, "");
summary = `push to ${branch}`;
} else if (Array.isArray(eventObj.commits) && eventObj.commits.length > 0) {
summary = `${eventObj.commits.length} commit(s)`;
}
}
} catch (e) {
// ignore
}
const status = evt.status || "pending";
// If eventHash exists in Events, update only the Status (and ProcessedAt)
@ -483,11 +526,34 @@ export async function appendEvent(evt = {}) {
if (String(values[i][0]) === String(eventHash)) {
const rowIndex = i + 1;
const statusCol = EVENTS_HEADER.indexOf('Status') + 1;
const processedAtCol = 1; // ProcessedAt
const statusRange = `${EVENTS_SHEET}!${colNumberToLetter(statusCol)}${rowIndex}`;
const processedAtRange = `${EVENTS_SHEET}!A${rowIndex}`;
const statusRange = `${EVENTS_SHEET}!${colNumberToLetter(statusCol)}${rowIndex}`;
// update status and processedAt
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: statusRange, valueInputOption: 'RAW', requestBody: { values: [[status]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: processedAtRange, valueInputOption: 'RAW', requestBody: { values: [[processedAt]] } }));
// update EventType, Summary, and IssueKey if present
try {
const eventTypeCol = EVENTS_HEADER.indexOf('EventType') + 1;
const summaryCol = EVENTS_HEADER.indexOf('Summary') + 1;
const issueKeyCol = EVENTS_HEADER.indexOf('IssueKey') + 1;
if (eventType) {
const range = `${EVENTS_SHEET}!${colNumberToLetter(eventTypeCol)}${rowIndex}`;
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range, valueInputOption: 'RAW', requestBody: { values: [[eventType]] } }));
}
if (summary) {
const range = `${EVENTS_SHEET}!${colNumberToLetter(summaryCol)}${rowIndex}`;
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range, valueInputOption: 'RAW', requestBody: { values: [[summary]] } }));
}
if (issueKey) {
const range = `${EVENTS_SHEET}!${colNumberToLetter(issueKeyCol)}${rowIndex}`;
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range, valueInputOption: 'RAW', requestBody: { values: [[issueKey]] } }));
}
} catch (e2) {
logger.warn('Failed updating event extra fields', { err: e2 && e2.message ? e2.message : e2 });
}
logger.info('sheetsAdapter updated event status', { issueKey, eventHash, status });
return;
}
@ -614,6 +680,151 @@ export async function appendTimeEntries(issueKey, issueNumber, times = []) {
return perUserMinutes;
}
export async function appendPush(evt = {}) {
const sheets = await sheetsClient();
const spreadsheetId = process.env.GOOGLE_SHEET_ID || config.googleSheetId;
await ensureSheet(sheets, spreadsheetId, PUSHES_SHEET, PUSHES_HEADER);
const processedAt = formatDate(new Date().toISOString());
const event = evt.event || evt;
const eventHash = evt.eventHash || (evt.meta && evt.meta.eventHash) || "";
const repoFullName = (event.repository && (event.repository.full_name || (event.repository.owner && event.repository.owner.login && event.repository.name && `${event.repository.owner.login}/${event.repository.name}`))) || "";
const ref = event.ref || "";
const branch = String(ref).replace(/^refs\/heads\//, "");
const pusher = (event.pusher && (event.pusher.name || event.pusher.username)) || (event.sender && event.sender.login) || (evt.actor) || "";
const pusherEmail = (event.pusher && event.pusher.email) || "";
const commitCount = Array.isArray(event.commits) ? event.commits.length : 0;
const headCommitId = event.after || (event.head_commit && (event.head_commit.id || event.head_commit.sha)) || (commitCount > 0 && event.commits[commitCount - 1] && (event.commits[commitCount - 1].id || event.commits[commitCount - 1].sha)) || "";
const headCommitMessage = (event.head_commit && (event.head_commit.message || event.head_commit.msg)) || (commitCount > 0 && event.commits[commitCount - 1] && (event.commits[commitCount - 1].message || event.commits[commitCount - 1].msg)) || "";
const compareUrl = event.compare_url || event.compareUrl || "";
const repoUrl = (event.repository && (event.repository.html_url || event.repository.url)) || "";
// Build commits summary with truncation awareness
let commitsTruncated = false;
let commitsSummary = '';
if (Array.isArray(event.commits) && event.commits.length > 0) {
const parts = [];
const limit = Math.max(1, PUSH_COMMITS_SUMMARY_LIMIT || 10);
for (let i = 0; i < Math.min(limit, event.commits.length); i++) {
const c = event.commits[i] || {};
let msg = (c.message || c.msg || '').replace(/\s+/g, ' ').trim();
if (msg.length > PUSH_COMMIT_SNIPPET_LENGTH) {
msg = msg.substring(0, PUSH_COMMIT_SNIPPET_LENGTH);
commitsTruncated = true;
}
const id = (c.id || c.sha || '').substring(0, 7);
parts.push(`${id}:${msg}`);
}
if (event.commits.length > limit) commitsTruncated = true;
commitsSummary = parts.join(' | ');
}
// If eventHash exists in Pushes, update some fields instead of appending duplicate
if (eventHash) {
try {
const res = await requestWithRetry(() => sheets.spreadsheets.values.get({ spreadsheetId, range: `${PUSHES_SHEET}!B:B` }));
const values = (res && res.data && res.data.values) || [];
for (let i = 0; i < values.length; i++) {
if (String(values[i][0]) === String(eventHash)) {
const rowIndex = i + 1;
const processedAtRange = `${PUSHES_SHEET}!A${rowIndex}`;
const commitCountCol = PUSHES_HEADER.indexOf('CommitCount') + 1;
const headIdCol = PUSHES_HEADER.indexOf('HeadCommitId') + 1;
const headMsgCol = PUSHES_HEADER.indexOf('HeadCommitMessage') + 1;
const commitsSummaryCol = PUSHES_HEADER.indexOf('CommitsSummary') + 1;
const commitsTruncatedCol = PUSHES_HEADER.indexOf('CommitsTruncated') + 1;
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: processedAtRange, valueInputOption: 'RAW', requestBody: { values: [[processedAt]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSHES_SHEET}!${colNumberToLetter(commitCountCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[commitCount]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSHES_SHEET}!${colNumberToLetter(headIdCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[headCommitId]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSHES_SHEET}!${colNumberToLetter(headMsgCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[headCommitMessage]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSHES_SHEET}!${colNumberToLetter(commitsSummaryCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[commitsSummary]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSHES_SHEET}!${colNumberToLetter(commitsTruncatedCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[commitsTruncated ? 'true' : 'false']] } }));
logger.info('sheetsAdapter updated push row', { repoFullName, eventHash });
if (PUSH_COMMITS_ENABLED) await appendPushCommits(evt);
return;
}
}
} catch (e) {
logger.warn('Error checking for existing push eventHash, will append', { err: e && e.message ? e.message : e });
}
}
const row = [processedAt, eventHash, repoFullName, ref, branch, pusher, pusherEmail, commitCount, headCommitId, headCommitMessage, compareUrl, repoUrl, commitsSummary, commitsTruncated ? 'true' : 'false'];
const range = `${PUSHES_SHEET}!A:${colNumberToLetter(PUSHES_HEADER.length)}`;
await requestWithRetry(() => sheets.spreadsheets.values.append({ spreadsheetId, range, valueInputOption: 'RAW', requestBody: { values: [row] } }));
logger.info('sheetsAdapter appended push', { repoFullName, branch, commitCount, eventHash });
if (PUSH_COMMITS_ENABLED) await appendPushCommits(evt);
}
export async function appendPushCommits(evt = {}) {
const sheets = await sheetsClient();
const spreadsheetId = process.env.GOOGLE_SHEET_ID || config.googleSheetId;
await ensureSheet(sheets, spreadsheetId, PUSH_COMMITS_SHEET, PUSH_COMMITS_HEADER);
const processedAt = formatDate(new Date().toISOString());
const event = evt.event || evt;
const eventHash = evt.eventHash || (evt.meta && evt.meta.eventHash) || "";
const repoFullName = (event.repository && (event.repository.full_name || (event.repository.owner && event.repository.owner.login && event.repository.name && `${event.repository.owner.login}/${event.repository.name}`))) || "";
const ref = event.ref || "";
const branch = String(ref).replace(/^refs\/heads\//, "");
const commits = Array.isArray(event.commits) ? event.commits : [];
if (commits.length === 0) return;
const existingRes = await requestWithRetry(() => sheets.spreadsheets.values.get({ spreadsheetId, range: `${PUSH_COMMITS_SHEET}!A2:${colNumberToLetter(PUSH_COMMITS_HEADER.length)}` })).catch(() => ({ data: { values: [] } }));
const existing = (existingRes && existingRes.data && existingRes.data.values) || [];
const toAppend = [];
for (let i = 0; i < commits.length; i++) {
const c = commits[i] || {};
const commitId = (c.id || c.sha || '') + '';
let msg = (c.message || c.msg || '').replace(/\s+/g, ' ').trim();
let truncated = false;
if (msg.length > PUSH_COMMIT_SNIPPET_LENGTH) {
msg = msg.substring(0, PUSH_COMMIT_SNIPPET_LENGTH);
truncated = true;
}
const author = (c.author && (c.author.name || c.author.username)) || (c.commit && c.commit.author && c.commit.author.name) || '';
const url = c.url || c.html_url || '';
const ts = (c.timestamp || c.date || (c.commit && c.commit.author && c.commit.author.date)) || '';
// find existing row by EventHash + CommitId
let found = false;
for (let j = 0; j < existing.length; j++) {
const row = existing[j] || [];
const rowEventHash = row[1] !== undefined ? String(row[1]) : '';
const rowCommitId = row[5] !== undefined ? String(row[5]) : '';
if (String(rowEventHash) === String(eventHash) && String(rowCommitId) === String(commitId)) {
const rowIndex = 2 + j; // because we started at A2
// update CommitMessage, CommitAuthor, CommitURL, CommitTimestamp, ProcessedAt
const commitMsgCol = PUSH_COMMITS_HEADER.indexOf('CommitMessage') + 1;
const commitAuthorCol = PUSH_COMMITS_HEADER.indexOf('CommitAuthor') + 1;
const commitUrlCol = PUSH_COMMITS_HEADER.indexOf('CommitURL') + 1;
const commitTsCol = PUSH_COMMITS_HEADER.indexOf('CommitTimestamp') + 1;
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSH_COMMITS_SHEET}!${colNumberToLetter(commitMsgCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[msg]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSH_COMMITS_SHEET}!${colNumberToLetter(commitAuthorCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[author]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSH_COMMITS_SHEET}!${colNumberToLetter(commitUrlCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[url]] } }));
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSH_COMMITS_SHEET}!${colNumberToLetter(commitTsCol)}${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[ts]] } }));
// update processedAt
await requestWithRetry(() => sheets.spreadsheets.values.update({ spreadsheetId, range: `${PUSH_COMMITS_SHEET}!A${rowIndex}`, valueInputOption: 'RAW', requestBody: { values: [[processedAt]] } }));
found = true;
break;
}
}
if (!found) {
toAppend.push([processedAt, eventHash, repoFullName, ref, branch, commitId, msg, author, url, ts]);
}
}
if (toAppend.length > 0) {
const range = `${PUSH_COMMITS_SHEET}!A:${colNumberToLetter(PUSH_COMMITS_HEADER.length)}`;
await requestWithRetry(() => sheets.spreadsheets.values.append({ spreadsheetId, range, valueInputOption: 'RAW', requestBody: { values: toAppend } }));
}
logger.info('sheetsAdapter appended/updated push commits', { eventHash, appended: toAppend.length });
}
export async function updateIssueFields(issueKey, updates = {}) {
const sheets = await sheetsClient();
const spreadsheetId = process.env.GOOGLE_SHEET_ID || config.googleSheetId;

View File

@ -14,35 +14,35 @@ async function handler(payload) {
const headers = payload.headers || {};
const issue = event && event.issue;
if (!issue) {
logger.info("Event has no issue field; skipping processing");
return;
}
const giteaEvent = (headers && (headers['x-gitea-event'] || headers['x-gitea-event'.toLowerCase()])) || null;
const isPush = giteaEvent === 'push' || Boolean(event && (event.ref || event.commits));
const issueKey = meta.issueKey || (event.repository && event.repository.full_name ? `${event.repository.full_name}#${issue.number}` : null);
const issueKey = meta.issueKey || (issue && event.repository && event.repository.full_name ? `${event.repository.full_name}#${issue.number}` : null);
// dedupe: if eventHash already exists in Events or matches issue.LastEventHash, skip
// dedupe: if eventHash already exists in Events, skip. For issues also compare Issue.LastEventHash
try {
const seen = await sheetsAdapter.isEventKnown(meta.eventHash);
if (seen) {
logger.info("Duplicate event detected in Events sheet; skipping", { issueKey, eventHash: meta.eventHash });
logger.info("Duplicate event detected in Events sheet; skipping", { issueKey, eventHash: meta.eventHash, isPush });
try {
await sheetsAdapter.appendEvent({ event, eventHash: meta.eventHash, issueKey, actor: meta.actor, summary: event.action || "", status: "duplicate", meta });
await sheetsAdapter.appendEvent({ event, eventHash: meta.eventHash, issueKey, actor: meta.actor, summary: event.action || (isPush ? 'push' : ''), status: "duplicate", meta });
} catch (e) {
logger.warn("Failed to append duplicate event record", { err: e && e.message ? e.message : e });
}
return;
}
const existing = await sheetsAdapter.getIssueByIssueKey(issueKey);
if (existing && existing.LastEventHash && String(existing.LastEventHash) === String(meta.eventHash)) {
logger.info("Duplicate event detected by Issue.LastEventHash; skipping", { issueKey, eventHash: meta.eventHash });
try {
await sheetsAdapter.appendEvent({ event, eventHash: meta.eventHash, issueKey, actor: meta.actor, summary: event.action || "", status: "duplicate", meta });
} catch (e) {
logger.warn("Failed to append duplicate event record", { err: e && e.message ? e.message : e });
if (issue) {
const existing = await sheetsAdapter.getIssueByIssueKey(issueKey);
if (existing && existing.LastEventHash && String(existing.LastEventHash) === String(meta.eventHash)) {
logger.info("Duplicate event detected by Issue.LastEventHash; skipping", { issueKey, eventHash: meta.eventHash });
try {
await sheetsAdapter.appendEvent({ event, eventHash: meta.eventHash, issueKey, actor: meta.actor, summary: event.action || "", status: "duplicate", meta });
} catch (e) {
logger.warn("Failed to append duplicate event record", { err: e && e.message ? e.message : e });
}
return;
}
return;
}
} catch (e) {
logger.warn("Error during duplicate check; continuing processing", { err: e && e.message ? e.message : e });
@ -55,8 +55,31 @@ async function handler(payload) {
logger.warn("Failed to append processing event", { err: e && e.message ? e.message : e });
}
// If this is a push event (no issue attached), handle and persist push data
if (isPush && !issue) {
try {
await sheetsAdapter.appendPush({ event, eventHash: meta.eventHash, actor: meta.actor });
await sheetsAdapter.appendEvent({ event, eventHash: meta.eventHash, issueKey: null, actor: meta.actor, summary: 'push', status: 'success', meta });
logger.info('Worker processed push event', { repo: (event.repository && event.repository.full_name) || (event.repository && event.repository.name) || '', ref: event.ref, commits: (event.commits && event.commits.length) || 0 });
return;
} catch (err) {
logger.error('Worker error processing push', { err: err && err.message ? err.message : err });
try {
await sheetsAdapter.appendEvent({ event, eventHash: meta.eventHash, issueKey: null, actor: meta.actor, summary: 'push', status: 'failed', meta });
} catch (e2) {
logger.warn('Failed to append failed push event', { err: e2 && e2.message ? e2.message : e2 });
}
try {
await sheetsAdapter.appendDeadLetter({ eventHash: meta.eventHash, issueKey: null, error: err, payload: event, notes: 'push-processing-failed' });
} catch (e3) {
logger.warn('Failed to append dead letter for push', { err: e3 && e3.message ? e3.message : e3 });
}
return;
}
}
// If this is a comment event, append comment row and update issue
const giteaEvent = (headers && (headers['x-gitea-event'] || headers['x-gitea-event'.toLowerCase()])) || null;
const isComment = Boolean(event && event.comment) || giteaEvent === 'issue_comment';
try {