//! Importing a whole shelf of git repositories at once.
//!
//! One repo is a demonstration; a directory of them is the case that actually
//! tells you something. Running this over a real working directory produces a
//! provenance tally across everything you have written — how much of it a
//! machine had a hand in, according to what people actually wrote down.
//!
//! Nothing here is destructive. Source repositories are read through `git`
//! with no writing commands, and the output goes to a directory this tool
//! creates.
use std::path::Path;
use jac_core::{Author, HumanId, HumanIdentity, OrgId};
use jac_decision::MemoryLedger;
use jac_object::MemoryObjectStore;
use jac_repo::{MemoryRefStore, RefName, Repo};
use serde_json::{Value, json};
use crate::export::{self, Manifest};
use crate::gitimport::{self, Bounds};
/// Turns a directory name into a URL-safe slug.
fn slugify(name: &str) -> String {
let mut out = String::new();
let mut dash = false;
for ch in name.chars() {
if ch.is_ascii_alphanumeric() {
out.push(ch.to_ascii_lowercase());
dash = false;
} else if !dash && !out.is_empty() {
out.push('-');
dash = true;
}
}
let trimmed = out.trim_end_matches('-').to_owned();
if trimmed.is_empty() {
"repo".to_owned()
} else {
trimmed
}
}
/// Imports one repository and writes its export. Returns its manifest.
///
/// # Errors
/// When git cannot read the repository, when the window holds nothing
/// importable, or when the export cannot be written.
pub fn one(source: &Path, out: &Path, bounds: &Bounds) -> anyhow::Result<Value> {
let name = source
.file_name()
.map_or_else(|| "repo".to_owned(), |n| n.to_string_lossy().into_owned());
let slug = slugify(&name);
let imported = gitimport::import(source, bounds).map_err(|e| anyhow::anyhow!("{name}: {e}"))?;
// A fresh engine per repo: these are separate histories, and mixing them
// would put unrelated work under one address space.
//
// Committed at each commit's *own* time rather than at import time. A
// snapshot's timestamp is hashed into its address, so stamping "now" would
// mean the same history imported twice lands on two different sets of ids,
// and the ids written to disk could never be reproduced by reading them
// back. Git's time is also simply the truer answer to "when did this
// happen".
let mut repo = Repo::new(
MemoryObjectStore::new(),
MemoryRefStore::new(),
MemoryLedger::new(),
crate::hostclock::HostClock::replaying(imported.iter().map(|c| c.commit.at)),
);
let main = RefName::parse("main")?;
// Declared, not authenticated — the same rule the whole surface follows.
let founder = HumanIdentity {
id: HumanId::from_non_zero(core::num::NonZeroU64::MIN),
display_name: "imported".to_owned(),
};
let _org = OrgId::from_non_zero(core::num::NonZeroU64::MIN);
let mut ids = Vec::new();
for commit in &imported {
let files: Vec<(&str, &[u8])> = commit
.files
.iter()
.map(|f| (f.path.as_str(), f.content.as_slice()))
.collect();
let id = repo.commit(
&main,
&files,
Author::Human(founder.id),
commit.inferred.provenance,
&commit.commit.subject,
)?;
ids.push(id);
}
let manifest = export::write(
out,
&Manifest {
slug: &slug,
source: &source.to_string_lossy(),
commits_requested: bounds.commits,
paths_cap: bounds.paths,
},
&repo,
&imported,
&ids,
)?;
Ok(manifest)
}
/// Imports every git repository directly under `root`.
///
/// A repository that cannot be read does not stop the run — it is reported and
/// the shelf continues. Half a shelf imported with the failures named is worth
/// more than an abort on the first oddity.
///
/// # Errors
/// When `root` holds no git repositories at all, or the output directory
/// cannot be created or written.
pub fn shelf(root: &Path, out: &Path, bounds: &Bounds) -> anyhow::Result<Value> {
// Pointing at one repository should import that repository, not complain
// that it contains none.
let repos = if root.join(".git").exists() {
vec![root.to_path_buf()]
} else {
gitimport::discover(root)
};
if repos.is_empty() {
anyhow::bail!(
"{} is not a git repository and contains none",
root.display()
);
}
std::fs::create_dir_all(out)?;
let mut manifests = Vec::new();
let mut failures = Vec::new();
let (mut human, mut agent, mut mixed, mut assumed) = (0u64, 0u64, 0u64, 0u64);
for (i, source) in repos.iter().enumerate() {
let name = source
.file_name()
.map_or_else(String::new, |n| n.to_string_lossy().into_owned());
eprint!("\r[{}/{}] {name:<40}", i + 1, repos.len());
match one(source, out, bounds) {
Ok(m) => {
human += m["provenance"]["human"].as_u64().unwrap_or(0);
agent += m["provenance"]["agent"].as_u64().unwrap_or(0);
mixed += m["provenance"]["mixed"].as_u64().unwrap_or(0);
assumed += m["provenance"]["assumed"].as_u64().unwrap_or(0);
manifests.push(m);
}
Err(e) => failures.push(json!({ "repo": name, "why": e.to_string() })),
}
}
eprintln!("\r{:<60}", "");
let index = json!({
"kind": "jacquard-shelf/1",
"note": "each entry is a jacquard repo; host them with `jac-serve --host-dir <root>`",
"root": root.to_string_lossy(),
"imported": manifests.len(),
"failed": failures.len(),
"failures": failures,
"provenance": {
"human": human,
"agent": agent,
"mixed": mixed,
"assumed": assumed,
"note": "`assumed` counts snapshots labelled human with no evidence either \
way. `mixed` and `agent` are evidence: a co-author trailer naming a \
model, or a bot author.",
},
"repos": manifests
.iter()
.map(|m| json!({
"slug": m["slug"],
"source": m["source"],
"snapshots": m["snapshots"],
"provenance": m["provenance"],
}))
.collect::<Vec<_>>(),
});
std::fs::write(
out.join("index.json"),
format!("{}\n", serde_json::to_string_pretty(&index)?),
)?;
Ok(index)
}