//! Importing a GitHub pull request as a jacquard repo.
//!
//! The point of the exercise is the part GitHub cannot tell you. It records
//! *who pushed*, which is not the same as whose hands made the work, and it
//! records a PR description, which is a summary rather than a rationale
//! anybody put their name to. Jacquard's whole position is that those are
//! different things, so the import refuses to blur them:
//!
//! - Provenance is **inferred**, and every inference carries the rule that produced it. A
//! bot author is an agent because GitHub says so; a human author is a human *by
//! assumption*, because nothing in the API knows whether a person typed the diff or
//! pasted it out of a model.
//! - The PR body becomes a **proposed, unsettled** decision, authored by an importer
//! agent. It is never attested. It cannot be: the engine's attestation constructor
//! takes a `HumanIdentity` and the compiler proves an agent cannot reach it.
//!
//! What the import produces is therefore a *synthetic* history: honest about
//! its shape, explicit about its guesses, and carrying a list of exactly what
//! a person still has to answer. Jackie's interview is what turns those gaps
//! into attested decisions.
//!
//! Only the changed files are imported, at their base and head states. A
//! milestone-1 commit carries its whole tree, and reconstructing every tree of
//! a real repository would be thousands of requests for content nobody is
//! going to read.
use axum::Json;
use axum::extract::State;
use axum::http::StatusCode;
use jac_core::{AgentId, Author, HumanId, Provenance};
use jac_decision::{Decision, DecisionScope, RationaleFamily};
use jac_telemetry::Envelope;
use reqwest::StatusCode as GhStatus;
use serde::{Deserialize, Serialize};
use serde_json::{Value, json};
use crate::dto;
use crate::error::ApiError;
use crate::routes::{repos, with_repo_mut};
use crate::state::SharedState;
/// Beyond this the import stops being a review surface and starts being a
/// mirror of somebody's repository.
const MAX_FILES: usize = 60;
/// Files larger than this are almost never read line by line in review.
const MAX_BYTES: usize = 400_000;
const UA: &str = "jacquard-lovelace-importer";
#[derive(Debug, Deserialize)]
pub(crate) struct ImportRequest {
pub owner: String,
pub repo: String,
/// The pull request to import. This is the unit of review.
pub pr: u64,
/// Optional token. Raises the rate limit and reaches private repos; it is
/// used for this request and never stored.
#[serde(default)]
pub token: Option<String>,
}
/// How one commit's provenance was decided, and on what evidence.
#[derive(Debug, Serialize)]
pub(crate) struct Inference {
pub subject: String,
pub provenance: &'static str,
/// The rule that fired, in words a reader can argue with.
pub because: String,
/// True when the label is a working assumption rather than evidence.
pub assumed: bool,
}
fn client() -> Result<reqwest::Client, ApiError> {
reqwest::Client::builder()
.user_agent(UA)
.build()
.map_err(|e| ApiError::internal(format!("could not build an http client: {e}")))
}
async fn get_json(
client: &reqwest::Client,
url: &str,
token: Option<&str>,
) -> Result<Value, ApiError> {
let mut request = client
.get(url)
.header("Accept", "application/vnd.github+json");
if let Some(token) = token {
request = request.bearer_auth(token);
}
let response = request
.send()
.await
.map_err(|e| ApiError::Unavailable(format!("could not reach GitHub: {e}")))?;
let status = response.status();
if status == GhStatus::NOT_FOUND {
return Err(ApiError::not_found(format!("GitHub has no {url}")));
}
if status == GhStatus::FORBIDDEN {
return Err(ApiError::Unavailable(
"GitHub refused: rate limited, or this repo needs a token".to_owned(),
));
}
if !status.is_success() {
return Err(ApiError::Unavailable(format!("GitHub answered {status}")));
}
response
.json()
.await
.map_err(|e| ApiError::internal(format!("GitHub sent something unreadable: {e}")))
}
/// Raw file bytes at a ref, or `None` when the path does not exist there.
async fn contents_at(
client: &reqwest::Client,
owner: &str,
repo: &str,
path: &str,
at: &str,
token: Option<&str>,
) -> Option<String> {
let url = format!("https://api.github.com/repos/{owner}/{repo}/contents/{path}?ref={at}");
let mut request = client
.get(&url)
// The raw media type hands back the file itself rather than base64.
.header("Accept", "application/vnd.github.raw");
if let Some(token) = token {
request = request.bearer_auth(token);
}
let response = request.send().await.ok()?;
if !response.status().is_success() {
return None;
}
let text = response.text().await.ok()?;
if text.len() > MAX_BYTES {
return None;
}
Some(text)
}
/// The inference rules, in one place so they can be read and disputed.
fn infer(login: Option<&str>, kind: Option<&str>, message: &str) -> (Provenance, String, bool) {
let bot_login = login.is_some_and(|l| l.ends_with("[bot]"));
let bot_type = kind == Some("Bot");
if bot_login || bot_type {
return (
Provenance::Agent,
format!(
"GitHub reports the author `{}` as a bot",
login.unwrap_or("unknown")
),
false,
);
}
// A co-author line naming a bot is the one place GitHub records that a
// machine had a hand in work a person pushed.
let lower = message.to_lowercase();
if lower.contains("co-authored-by:")
&& (lower.contains("[bot]")
|| lower.contains("copilot")
|| lower.contains("claude")
|| lower.contains("gpt"))
{
return (
Provenance::Mixed,
"the commit message co-authors a machine".to_owned(),
false,
);
}
(
Provenance::Human,
"assumed: GitHub records who pushed, not whose hands made the work".to_owned(),
true,
)
}
/// `POST /api/import/github`.
#[expect(
clippy::too_many_lines,
reason = "the import is one sequence; splitting it would scatter what it assumes"
)]
pub(crate) async fn github(
State(state): State<SharedState>,
Json(req): Json<ImportRequest>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let owner = req.owner.trim().to_owned();
let repo = req.repo.trim().to_owned();
if owner.is_empty() || repo.is_empty() {
return Err(ApiError::invalid("owner and repo are required"));
}
let token = req
.token
.as_deref()
.map(str::trim)
.filter(|t| !t.is_empty());
let client = client()?;
let pr_num = req.pr;
let pr = get_json(
&client,
&format!("https://api.github.com/repos/{owner}/{repo}/pulls/{pr_num}"),
token,
)
.await?;
let title = pr["title"].as_str().unwrap_or("imported pull request");
let body = pr["body"].as_str().unwrap_or("");
let base_sha = pr["base"]["sha"]
.as_str()
.ok_or_else(|| ApiError::internal("the PR has no base sha"))?
.to_owned();
let head_sha = pr["head"]["sha"]
.as_str()
.ok_or_else(|| ApiError::internal("the PR has no head sha"))?
.to_owned();
let pr_author = pr["user"]["login"].as_str().unwrap_or("unknown").to_owned();
let pr_author_type = pr["user"]["type"].as_str().map(str::to_owned);
// Changed paths only. See the module docs for why not the whole tree.
let files = get_json(
&client,
&format!("https://api.github.com/repos/{owner}/{repo}/pulls/{pr_num}/files?per_page=100"),
token,
)
.await?;
let listed = files.as_array().cloned().unwrap_or_default();
let total_changed = listed.len();
let paths: Vec<String> = listed
.iter()
.filter_map(|f| f["filename"].as_str().map(str::to_owned))
.take(MAX_FILES)
.collect();
if paths.is_empty() {
return Err(ApiError::invalid("that pull request changes no files"));
}
// Both sides of every path, so the diff is real rather than a patch blob.
let mut base_tree: Vec<(String, String)> = Vec::new();
let mut head_tree: Vec<(String, String)> = Vec::new();
let mut skipped: Vec<String> = Vec::new();
for path in &paths {
let (before, after) = tokio::join!(
contents_at(&client, &owner, &repo, path, &base_sha, token),
contents_at(&client, &owner, &repo, path, &head_sha, token),
);
// Binary or oversized on both sides means there is nothing to read.
if before.is_none() && after.is_none() {
skipped.push(path.clone());
continue;
}
if let Some(text) = before {
base_tree.push((path.clone(), text));
}
if let Some(text) = after {
head_tree.push((path.clone(), text));
}
}
if head_tree.is_empty() {
return Err(ApiError::invalid(
"none of the changed files could be read as text",
));
}
let (provenance, because, assumed) = infer(Some(&pr_author), pr_author_type.as_deref(), body);
// Mint the repo through the normal init path, so an imported repo is the
// same kind of object as one warped by hand.
let created = repos::create_repo(
&state,
repos::InitRepoRequest {
name: format!("{owner}/{repo} #{pr_num}"),
founder: repos::FounderReq {
// Named for the PR author, but declared — not authenticated,
// and emphatically not a claim that this person attested.
display_name: pr_author.clone(),
},
default_ref: Some("main".to_owned()),
agents: vec![repos::AgentReq {
model: "github-importer".to_owned(),
}],
initial_commit: None,
founding_decision: None,
},
)?;
let slug = created.slug;
let report = with_repo_mut(&state, &slug, |entry| {
let founder: HumanId = *entry
.humans
.keys()
.next()
.ok_or_else(|| ApiError::internal("import lost its founder"))?;
let importer: AgentId = *entry
.agents
.keys()
.next()
.ok_or_else(|| ApiError::internal("import lost its importer"))?;
let main = entry.default_ref.clone();
// The base state, so the head has something to be a change *from*.
if !base_tree.is_empty() {
let refs: Vec<(&str, &[u8])> = base_tree
.iter()
.map(|(p, c)| (p.as_str(), c.as_bytes()))
.collect();
entry.repo.commit(
&main,
&refs,
Author::Agent(importer),
Provenance::Agent,
&format!(
"import: {owner}/{repo} at {}",
&base_sha[..7.min(base_sha.len())]
),
)?;
}
let refs: Vec<(&str, &[u8])> = head_tree
.iter()
.map(|(p, c)| (p.as_str(), c.as_bytes()))
.collect();
entry
.repo
.commit(&main, &refs, Author::Human(founder), provenance, title)?;
// The PR body as a *proposal*. Unsettled, and it stays that way: an
// agent cannot attest, and the importer will not pretend the author
// did.
let at = entry.repo.now();
let scope = scope_of(&paths);
let path_prefixes = scope
.iter()
.map(|p| dto::parse_repo_path(p))
.collect::<Result<Vec<_>, _>>()?;
let rationale = if body.trim().is_empty() {
format!(
"Imported from {owner}/{repo}#{pr_num}. The pull request carried no \
description, so there is no stated reason for this change — only \
the change itself."
)
} else {
format!(
"Imported from {owner}/{repo}#{pr_num}. What the pull request said:\n\n{}",
body.trim()
)
};
let decision = entry.repo.propose_decision(Decision {
title: title.to_owned(),
rationale,
families: vec![RationaleFamily::DeferredWork],
proposed_by: Author::Agent(importer),
scope: DecisionScope { path_prefixes },
at,
});
Ok(json!({
"slug": slug,
"source": {
"owner": owner,
"repo": repo,
"pull_request": pr_num,
"base": base_sha,
"head": head_sha,
"author": pr_author,
},
"imported": {
"files": head_tree.len(),
"changed_on_github": total_changed,
"truncated": total_changed > MAX_FILES,
"skipped": skipped,
},
"decision": decision.digest().to_hex().as_str().to_owned(),
"inference": Inference {
subject: title.to_owned(),
provenance: match provenance {
Provenance::Agent => "agent",
Provenance::Mixed => "mixed",
// Fail toward the weaker claim: an unrecognised variant is
// not evidence that a person made this.
_ => "human",
},
because,
assumed,
},
// The whole reason this endpoint exists: naming what was guessed.
"gaps": gaps(assumed, body, total_changed > MAX_FILES),
"honesty":
"provenance here is inferred, not attested. The decision is proposed and \
unsettled — an agent imported it and an agent cannot attest. Only a person \
can settle it, in their own words.",
}))
})?;
state.telemetry.emit(
Envelope::event("jac.import.github", state.now_ms())
.attr("repo", &slug)
.attr_bool("assumed_provenance", assumed)
.measurement(
"files",
f64::from(u32::try_from(head_tree.len()).unwrap_or(u32::MAX)),
),
);
Ok((StatusCode::CREATED, Json(report)))
}
/// What a person still has to answer, in the order Jackie should ask.
fn gaps(assumed: bool, body: &str, truncated: bool) -> Vec<Value> {
let mut out = Vec::new();
if assumed {
out.push(json!({
"kind": "provenance",
"ask": "Whose hands actually made this — yours, a tool's, or both?",
"why": "GitHub records who pushed. It cannot tell the difference between \
work you wrote and work you accepted.",
}));
}
if body.trim().is_empty() {
out.push(json!({
"kind": "rationale",
"ask": "What were you trying to make true, that wasn't true before?",
"why": "The pull request carried no description, so nothing here records why \
the change was made.",
}));
} else {
out.push(json!({
"kind": "rationale",
"ask": "The description says what changed. What did you rule out, and why?",
"why": "A summary is not a rationale. Nobody has put their name to a reason.",
}));
}
if truncated {
out.push(json!({
"kind": "scope",
"ask": "Which of these files carry the decision, and which just came along?",
"why": "The pull request changed more files than were imported.",
}));
}
out
}
/// Longest common directory prefix of the changed paths.
fn scope_of(paths: &[String]) -> Vec<String> {
let Some(first) = paths.first() else {
return Vec::new();
};
let split: Vec<Vec<&str>> = paths.iter().map(|p| p.split('/').collect()).collect();
let head: Vec<&str> = first.split('/').collect();
let mut prefix: Vec<&str> = Vec::new();
// The last segment is a filename, never part of a directory prefix.
for (i, seg) in head.iter().take(head.len().saturating_sub(1)).enumerate() {
if split.iter().all(|p| p.get(i) == Some(seg)) {
prefix.push(seg);
} else {
break;
}
}
if prefix.is_empty() {
Vec::new()
} else {
vec![prefix.join("/")]
}
}
// ---------------------------------------------------------------------------
// Local git
// ---------------------------------------------------------------------------
#[derive(Debug, Deserialize)]
pub(crate) struct GitImportRequest {
/// Absolute path to a git working copy on this machine.
pub path: String,
/// How far back to read. Defaults to a dozen.
#[serde(default)]
pub commits: Option<usize>,
/// Ceiling on distinct paths carried across the window.
#[serde(default)]
pub paths: Option<usize>,
}
/// `POST /api/import/git` — import a local repository into the running host.
///
/// The same inference as the batch importer, but the result stays in memory
/// where the rest of the surface can browse it. Nothing is written to disk and
/// nothing in the source repository is touched: the import only ever runs
/// reading commands.
#[expect(
clippy::too_many_lines,
reason = "one import is one sequence; splitting it would scatter what it assumes"
)]
pub(crate) async fn git(
State(state): State<SharedState>,
Json(req): Json<GitImportRequest>,
) -> Result<(StatusCode, Json<Value>), ApiError> {
let path = std::path::PathBuf::from(shellexpand(&req.path));
if !path.join(".git").exists() {
return Err(ApiError::not_found(format!(
"{} is not a git working copy",
path.display()
)));
}
let bounds = crate::gitimport::Bounds {
commits: req.commits.unwrap_or(12).clamp(1, 200),
paths: req.paths.unwrap_or(120).clamp(1, 2000),
};
// git shells out and blocks; keep it off the async runtime's threads.
let read = {
let path = path.clone();
tokio::task::spawn_blocking(move || crate::gitimport::import(&path, &bounds))
.await
.map_err(|e| ApiError::internal(format!("import panicked: {e}")))?
.map_err(|e| ApiError::invalid(e.to_string()))?
};
let name = path
.file_name()
.map_or_else(|| "repo".to_owned(), |n| n.to_string_lossy().into_owned());
let created = repos::create_repo(
&state,
repos::InitRepoRequest {
name,
founder: repos::FounderReq {
display_name: read
.last()
.map_or_else(|| "imported".to_owned(), |c| c.commit.author_name.clone()),
},
default_ref: Some("main".to_owned()),
agents: vec![repos::AgentReq {
model: "git-importer".to_owned(),
}],
initial_commit: None,
founding_decision: None,
},
)?;
let slug = created.slug;
let report = with_repo_mut(&state, &slug, |entry| {
let founder: HumanId = *entry
.humans
.keys()
.next()
.ok_or_else(|| ApiError::internal("import lost its founder"))?;
let main = entry.default_ref.clone();
let mut rows = Vec::new();
let (mut human, mut agent, mut mixed, mut assumed) = (0usize, 0, 0, 0);
for c in &read {
let files: Vec<(&str, &[u8])> = c
.files
.iter()
.map(|f| (f.path.as_str(), f.content.as_slice()))
.collect();
let id = entry.repo.commit(
&main,
&files,
Author::Human(founder),
c.inferred.provenance,
&c.commit.subject,
)?;
let label = match c.inferred.provenance {
Provenance::Agent => {
agent += 1;
"agent"
}
Provenance::Mixed => {
mixed += 1;
"mixed"
}
_ => {
human += 1;
"human"
}
};
if c.inferred.assumed {
assumed += 1;
}
rows.push(json!({
"id": id.digest().to_hex().as_str().to_owned(),
"git_sha": c.commit.sha,
"message": c.commit.subject,
"provenance": label,
"because": c.inferred.because,
"assumed": c.inferred.assumed,
"files": c.files.len(),
}));
}
Ok(json!({
"slug": slug,
"source": path.to_string_lossy(),
"snapshots": rows,
"provenance": {
"human": human, "agent": agent, "mixed": mixed, "assumed": assumed,
},
"bounds": { "commits": bounds.commits, "paths": bounds.paths },
"honesty":
"a bounded slice of recent history, not a mirror. Provenance is inferred: a \
co-author trailer naming a model, or a bot author, is evidence; everything \
else is a person by assumption. Nothing here is attested.",
}))
})?;
state
.telemetry
.emit(Envelope::event("jac.import.git", state.now_ms()).attr("repo", &slug));
Ok((StatusCode::CREATED, Json(report)))
}
/// A leading `~` is the shell's job, and an API caller has no shell.
fn shellexpand(path: &str) -> String {
match path.strip_prefix("~/") {
Some(rest) => {
std::env::var("HOME").map_or_else(|_| path.to_owned(), |home| format!("{home}/{rest}"))
}
None => path.to_owned(),
}
}
// ---------------------------------------------------------------------------
// The shelf
// ---------------------------------------------------------------------------
#[derive(Debug, Deserialize)]
pub(crate) struct ShelfQuery {
/// Directory a batch import wrote to.
pub path: String,
}
/// `GET /api/shelf?path=` — read the index a batch import left behind.
///
/// The exports are files on disk rather than state in this process, so this
/// only reads them back. It is the one place the surface admits to persistence
/// existing at all, and it does so by pointing at a directory the importer
/// wrote, not by pretending the engine gained a store.
pub(crate) async fn shelf(
axum::extract::Query(query): axum::extract::Query<ShelfQuery>,
) -> Result<Json<Value>, ApiError> {
let root = std::path::PathBuf::from(shellexpand(&query.path));
let index = root.join("index.json");
let bytes = tokio::fs::read(&index).await.map_err(|_| {
ApiError::not_found(format!(
"no shelf at {} — run `jac-serve --import-dir <dir> --out {}`",
index.display(),
root.display()
))
})?;
let parsed: Value = serde_json::from_slice(&bytes)
.map_err(|e| ApiError::internal(format!("that shelf index is unreadable: {e}")))?;
Ok(Json(parsed))
}