Import Quarto posts and expand blog sync bridge
Some checks failed
Build / build (push) Has been cancelled

This commit is contained in:
Neeldhara Misra 2026-06-20 04:01:13 +01:00
parent 4ddaba7c7d
commit e7227844c7
406 changed files with 7943 additions and 7637 deletions

View file

@ -0,0 +1,634 @@
#!/usr/bin/env node
import fs from "node:fs/promises";
import nodeFs from "node:fs";
import path from "node:path";
const DEFAULT_SOURCE = "/private/tmp/quartosite-shallow/blog";
const DEFAULT_OUTPUT = "/private/tmp/quarto-blog-import";
const sourceRoot = path.resolve(process.argv[2] || DEFAULT_SOURCE);
const outputRoot = path.resolve(process.argv[3] || DEFAULT_OUTPUT);
const blogs = [
{ key: "research", vaultFolder: "01-research", title: "Research" },
{ key: "vibes", vaultFolder: "02-vibes", title: "Vibes" },
{ key: "puzzles", vaultFolder: "03-puzzles", title: "Puzzles" },
{ key: "reflections", vaultFolder: "04-reflections", title: "Reflections" },
{ key: "poetry", vaultFolder: "05-poetry", title: "Poetry" },
{ key: "reviews", vaultFolder: "06-reviews", title: "Reviews" },
{ key: "art", vaultFolder: "07-art", title: "Art" },
];
const blogByKey = new Map(blogs.map((blog) => [blog.key, blog]));
const explicitDestinations = new Map([
["13sheep/index.qmd", "vibes"],
["15-puzzle/index.qmd", "puzzles"],
["2023-exportober-workflows/index.qmd", "reviews"],
["building-interactives/index.qmd", "vibes"],
["building-websites-with-notion/index.qmd", "reviews"],
["comms/index.qmd", "art"],
["communication-complexity-equality/index.qmd", "art"],
["course-plan/2024-fall/index.qmd", "vibes"],
["course-plan/2024-spring/index.qmd", "vibes"],
["course-plan/index.qmd", "vibes"],
["cp/sam-i-am/index.qmd", "research"],
["crypto-intro/index.qmd", "research"],
["django-app/index.qmd", "reviews"],
["dogs-bunny-puzzle/index.qmd", "puzzles"],
["eight-self-sabotaging-behaviors/index.qmd", "reflections"],
["envelope-budgeting-notion/index.qmd", "reviews"],
["exportober/2021-tracker/index.qmd", "reflections"],
["exportober/2021/index.qmd", "reflections"],
["exportober/2022-tracker/index.qmd", "reflections"],
["exportober/about/index.qmd", "reflections"],
["homework-help/index.qmd", "research"],
["iit-rankings/index.qmd", "reflections"],
["indiamini-crosswords/index.qmd", "art"],
["kidney-exchanges/index.qmd", "research"],
["massren/index.qmd", "reviews"],
["moving-blocks-ctis/index.qmd", "puzzles"],
["new-mac/index.qmd", "reviews"],
["note-taking-resources/index.qmd", "reviews"],
["notion-powered-websites/index.qmd", "reviews"],
["on-career-choices/index.qmd", "reflections"],
["on-teaching/index.qmd", "reflections"],
["pandoc-letters/index.qmd", "reviews"],
["skj/index.qmd", "reflections"],
["solo-chess/index.qmd", "puzzles"],
["women-in-mathematics/index.qmd", "reviews"],
]);
const skippedDrafts = [];
const importedPosts = [];
await fs.rm(outputRoot, { recursive: true, force: true });
await fs.mkdir(outputRoot, { recursive: true });
for (const blog of blogs) {
await fs.mkdir(path.join(outputRoot, "obsidian", "blogs", blog.vaultFolder), {
recursive: true,
});
await fs.mkdir(
path.join(
outputRoot,
"astro",
"sites",
blog.key,
"src",
"content",
blog.key,
),
{ recursive: true },
);
}
const sourceFiles = (await walk(sourceRoot))
.filter((file) => /\.(qmd|md|markdown)$/i.test(file))
.sort();
for (const sourceFile of sourceFiles) {
const relativePath = toPosix(path.relative(sourceRoot, sourceFile));
const sourceText = await fs.readFile(sourceFile, "utf8");
const { frontmatter, body } = parseFrontmatter(sourceText);
if (isDraft(frontmatter)) {
skippedDrafts.push(relativePath);
continue;
}
const blogKey = classifyPost(relativePath, frontmatter);
const blog = blogByKey.get(blogKey);
if (!blog) {
throw new Error(`Unknown blog key "${blogKey}" for ${relativePath}`);
}
const markdownRelativePath = relativePath.replace(
/\.(qmd|markdown)$/i,
".md",
);
const convertedBody = convertQuartoMarkdown(body).trim();
const title = frontmatter.title || titleFromPath(relativePath);
const pubDate = normalizeDate(frontmatter.date);
const description =
frontmatter.description || summarize(convertedBody, title);
const categories = parseList(frontmatter.categories);
const markdown = [
"---",
`title: ${yamlString(title)}`,
`description: ${yamlString(description)}`,
`pubDate: ${yamlString(pubDate)}`,
'authorName: "Neeldhara"',
categories.length
? `sourceCategories: [${categories.map(yamlString).join(", ")}]`
: "",
`sourcePath: ${yamlString(relativePath)}`,
"---",
"",
convertedBody,
"",
]
.filter((line, index, lines) => line || lines[index - 1] !== "")
.join("\n");
const obsidianTarget = path.join(
outputRoot,
"obsidian",
"blogs",
blog.vaultFolder,
fromPosix(markdownRelativePath),
);
const astroTarget = path.join(
outputRoot,
"astro",
"sites",
blog.key,
"src",
"content",
blog.key,
fromPosix(markdownRelativePath),
);
await writeFileEnsured(obsidianTarget, markdown);
await writeFileEnsured(astroTarget, markdown);
await copyPostAssets(path.dirname(sourceFile), path.dirname(obsidianTarget));
await copyPostAssets(path.dirname(sourceFile), path.dirname(astroTarget));
importedPosts.push({
source: relativePath,
destination: `${blog.vaultFolder}/${markdownRelativePath}`,
blog: blog.key,
title,
date: pubDate,
categories,
});
}
await writeImportReport();
await writeVaultReadme();
await writeAstroManifest();
console.log(`Imported ${importedPosts.length} posts into ${outputRoot}`);
console.log(`Skipped ${skippedDrafts.length} draft posts.`);
async function writeImportReport() {
const rows = [
"# Quarto Blog Import Report",
"",
`Source: ${sourceRoot}`,
`Generated: ${new Date().toISOString()}`,
"",
"## Imported Posts",
"",
"| Blog | Date | Title | Source | Destination |",
"| --- | --- | --- | --- | --- |",
];
for (const post of importedPosts.sort(
(left, right) =>
left.blog.localeCompare(right.blog) ||
right.date.localeCompare(left.date),
)) {
rows.push(
`| ${post.blog} | ${post.date} | ${escapeTable(post.title)} | \`${post.source}\` | \`${post.destination}\` |`,
);
}
rows.push("", "## Skipped Drafts", "");
if (skippedDrafts.length) {
for (const draft of skippedDrafts) {
rows.push(`- \`${draft}\``);
}
} else {
rows.push("- None");
}
await writeFileEnsured(
path.join(outputRoot, "IMPORT_REPORT.md"),
`${rows.join("\n")}\n`,
);
}
async function writeVaultReadme() {
const readme = `# Blog Content Bridge
This folder is the Obsidian source of truth for the public blog family.
## Folder Map
| Vault folder | Astro site | Public domain |
| --- | --- | --- |
| \`blogs/01-research\` | \`allblogs/sites/research/src/content/research\` | \`https://research.neeldhara.blog\` |
| \`blogs/02-vibes\` | \`allblogs/sites/vibes/src/content/vibes\` | \`https://vibes.neeldhara.blog\` |
| \`blogs/03-puzzles\` | \`allblogs/sites/puzzles/src/content/puzzles\` | \`https://puzzles.neeldhara.blog\` |
| \`blogs/04-reflections\` | \`allblogs/sites/reflections/src/content/reflections\` | \`https://reflections.neeldhara.blog\` |
| \`blogs/05-poetry\` | \`allblogs/sites/poetry/src/content/poetry\` | \`https://poetry.neeldhara.blog\` |
| \`blogs/06-reviews\` | \`allblogs/sites/reviews/src/content/reviews\` | \`https://reviews.neeldhara.blog\` |
| \`blogs/07-art\` | \`allblogs/sites/art/src/content/art\` | \`https://art.neeldhara.blog\` |
| \`blogs/micro\` | \`micro/src/content/blog\` | \`https://micro.neeldhara.blog\` |
The installed Obsidian plugin is still stored at:
\`\`\`text
.obsidian/plugins/microblog-sync
\`\`\`
The plugin name in Obsidian is **Blog Family Sync**. It watches all folders above, syncs edited files into the matching Astro repo, can start a local preview server for the current note, and can publish by running build, commit, and push.
## Post Layout
Use one folder per post:
\`\`\`text
blogs/01-research/example-post/index.md
blogs/01-research/example-post/diagram.png
\`\`\`
The folder path becomes the public URL:
\`\`\`text
blogs/01-research/example-post/index.md
https://research.neeldhara.blog/example-post/
\`\`\`
Nested folders are allowed. For example:
\`\`\`text
blogs/04-reflections/exportober/2021/index.md
https://reflections.neeldhara.blog/exportober/2021/
\`\`\`
## Frontmatter
Every public Markdown post should start with:
\`\`\`yaml
---
title: "Post title"
description: "One or two sentence summary for cards and RSS."
pubDate: "2024-05-10"
authorName: "Neeldhara"
---
\`\`\`
\`title\`, \`description\`, and \`pubDate\` are required by the Astro schemas.
\`updatedDate\`, \`image\`, and \`authorImage\` are optional. Prefer colocated images in the body over remote stock images.
## Markdown Conventions
- Standard Markdown headings, lists, links, tables, code fences, footnotes, and raw HTML are preserved.
- Use colocated relative assets: \`![Caption](image.png)\`.
- Quarto image attributes such as \`{width=70%}\` are removed during import. Use normal Markdown now; layout should be handled by Astro CSS.
- Quarto callouts are converted to simple blockquotes:
\`\`\`markdown
> **Note**
>
> Text of the callout.
\`\`\`
- For new notes, use the same blockquote convention for portable callouts:
\`\`\`markdown
> **Warning**
>
> This point needs attention.
\`\`\`
- Quarto margin notes such as \`[text]{.aside}\` are converted to:
\`\`\`markdown
> **Aside:** text
\`\`\`
## Sync Rules
- Obsidian is the source of truth.
- Auto-sync copies changed files from the matching \`blogs/...\` folder into the matching local Astro content folder.
- The plugin includes Markdown plus common web assets: images, PDFs, JSON/YAML, JS, CSS, HTML, and text files.
- One-way sync does not delete target files unless **Delete orphaned target files** is enabled.
- The initial Quarto import used replace-style sync to clear placeholder Astro posts. Day-to-day editing should use the plugin.
## Publish
Use **Blog Family Sync: Publish current note** or **Blog Family Sync: Publish all changes for current blog** from Obsidian. Publish does three things:
1. Sync vault content into the local repo.
2. Run the configured Astro build command.
3. Commit and push to Forgejo, where Dokploy picks up the change.
`;
await writeFileEnsured(
path.join(outputRoot, "obsidian", "blogs", "README.md"),
readme,
);
}
async function writeAstroManifest() {
const manifest = {
generatedAt: new Date().toISOString(),
sourceRoot,
importedPosts,
skippedDrafts,
};
await writeFileEnsured(
path.join(outputRoot, "manifest.json"),
`${JSON.stringify(manifest, null, 2)}\n`,
);
}
function classifyPost(relativePath, frontmatter) {
if (explicitDestinations.has(relativePath)) {
return explicitDestinations.get(relativePath);
}
const categories = parseList(frontmatter.categories).map((category) =>
category.toLowerCase(),
);
const haystack =
`${relativePath} ${frontmatter.title || ""} ${categories.join(" ")}`.toLowerCase();
if (relativePath.startsWith("poems/") || categories.includes("poem")) {
return "poetry";
}
if (categories.some((category) => ["puzzles", "games"].includes(category))) {
return "puzzles";
}
if (
categories.some((category) =>
[
"books",
"apps",
"workflows",
"tutorial",
"websites",
"notion",
"pandoc",
"lists",
].includes(category),
)
) {
return "reviews";
}
if (categories.some((category) => ["sketchnotes"].includes(category))) {
return "art";
}
if (haystack.includes("chatgpt") || haystack.includes("interactive")) {
return "vibes";
}
if (
categories.some((category) =>
[
"lecturenotes",
"talk",
"exposition",
"parameterized-algorithms",
].includes(category),
)
) {
return "research";
}
return "reflections";
}
function parseFrontmatter(text) {
if (!text.startsWith("---")) {
return { frontmatter: {}, body: text };
}
const end = text.indexOf("\n---", 3);
if (end < 0) {
return { frontmatter: {}, body: text };
}
const rawFrontmatter = text.slice(4, end).trim();
const body = text.slice(end + 4);
const frontmatter = {};
for (const line of rawFrontmatter.split(/\r?\n/)) {
const match = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
if (!match) {
continue;
}
const [, key, rawValue] = match;
frontmatter[key] = unquote(rawValue.trim());
}
return { frontmatter, body };
}
function convertQuartoMarkdown(body) {
const lines = body.replace(/\r\n/g, "\n").split("\n");
const converted = [];
const calloutStack = [];
let inCodeFence = false;
for (const rawLine of lines) {
const line = rawLine.replace(/\s+$/, "");
if (/^```/.test(line.trim())) {
inCodeFence = !inCodeFence;
converted.push(prefixCalloutLine(line, calloutStack.length));
continue;
}
if (!inCodeFence) {
const calloutOpen = line.match(
/^\s*:::\s*\{?\.?callout-([A-Za-z0-9_-]+)[^}]*\}?\s*$/,
);
if (calloutOpen) {
calloutStack.push(calloutOpen[1]);
converted.push(
prefixCalloutLine(
`**${titleCase(calloutOpen[1])}**`,
calloutStack.length,
),
);
converted.push(prefixCalloutLine("", calloutStack.length));
continue;
}
if (/^\s*:::\s*$/.test(line) && calloutStack.length) {
calloutStack.pop();
converted.push("");
continue;
}
if (/^\s*:{3,4}\s*\{/.test(line) || /^\s*:{3,4}\s*$/.test(line)) {
continue;
}
}
let cleaned = line;
if (!inCodeFence) {
cleaned = cleaned
.replace(
/(<script\b[^>]*><\/script>|<script\b[^>]*>.*?<\/script>)/gi,
"",
)
.replace(/!\[([^\]]*)\]\(([^)]+)\)\{[^}]+\}/g, "![$1]($2)")
.replace(/^\s*\[([^\]]+)\]\{\.aside\}\s*$/, "> **Aside:** $1")
.replace(/\[([^\]]+)\]\{\.smallcaps\}/g, "$1")
.replace(/\[([^\]]+)\]\{[^}]+\}/g, "$1");
}
converted.push(prefixCalloutLine(cleaned, calloutStack.length));
}
return converted.join("\n").replace(/\n{4,}/g, "\n\n\n");
}
function prefixCalloutLine(line, depth) {
if (!depth) {
return line;
}
return `${"> ".repeat(depth)}${line}`;
}
async function copyPostAssets(sourceDir, targetDir) {
const files = await fs.readdir(sourceDir, { withFileTypes: true });
for (const entry of files) {
if (
entry.name.startsWith(".") ||
/\.(qmd|md|markdown)$/i.test(entry.name)
) {
continue;
}
const source = path.join(sourceDir, entry.name);
const target = path.join(targetDir, entry.name);
if (entry.isDirectory()) {
await fs.mkdir(target, { recursive: true });
await copyPostAssets(source, target);
continue;
}
if (entry.isFile()) {
await fs.mkdir(path.dirname(target), { recursive: true });
await fs.copyFile(source, target);
}
}
}
async function walk(root) {
const entries = await fs.readdir(root, { withFileTypes: true });
const files = [];
for (const entry of entries) {
if (entry.name.startsWith(".")) {
continue;
}
const absolutePath = path.join(root, entry.name);
if (entry.isDirectory()) {
files.push(...(await walk(absolutePath)));
} else if (entry.isFile()) {
files.push(absolutePath);
}
}
return files;
}
async function writeFileEnsured(target, content) {
await fs.mkdir(path.dirname(target), { recursive: true });
await fs.writeFile(target, content);
}
function parseList(value) {
if (!value) {
return [];
}
const trimmed = String(value).trim();
if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
return trimmed
.slice(1, -1)
.split(",")
.map((item) => unquote(item.trim()))
.filter(Boolean);
}
return [unquote(trimmed)];
}
function isDraft(frontmatter) {
return String(frontmatter.draft || "").toLowerCase() === "true";
}
function normalizeDate(value) {
if (!value) {
return "2000-01-01";
}
const cleaned = String(value).replace(/^["']|["']$/g, "");
const match = cleaned.match(/^(\d{4})-(\d{2})-(\d{2})/);
if (match) {
return `${match[1]}-${match[2]}-${match[3]}`;
}
const date = new Date(cleaned);
if (Number.isNaN(date.valueOf())) {
return "2000-01-01";
}
return date.toISOString().slice(0, 10);
}
function summarize(markdown, title) {
const withoutBlocks = markdown
.replace(/```[\s\S]*?```/g, " ")
.replace(/<[^>]+>/g, " ")
.replace(/!\[[^\]]*\]\([^)]+\)/g, " ")
.replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
.replace(/[#>*_`~$]/g, " ")
.replace(/\s+/g, " ")
.trim();
const summary = withoutBlocks || title;
return summary.length > 180 ? `${summary.slice(0, 177).trim()}...` : summary;
}
function titleFromPath(relativePath) {
const folder = relativePath.replace(/\/index\.(qmd|md|markdown)$/i, "");
const leaf = folder.split("/").pop() || "Untitled";
return leaf
.split(/[-_]+/)
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
.join(" ");
}
function titleCase(value) {
return String(value)
.split(/[-_]+/)
.map((part) => part.charAt(0).toUpperCase() + part.slice(1))
.join(" ");
}
function yamlString(value) {
return JSON.stringify(String(value || ""));
}
function unquote(value) {
if (
(value.startsWith('"') && value.endsWith('"')) ||
(value.startsWith("'") && value.endsWith("'"))
) {
return value.slice(1, -1);
}
return value;
}
function escapeTable(value) {
return String(value).replace(/\|/g, "\\|");
}
function toPosix(value) {
return value.split(path.sep).join("/");
}
function fromPosix(value) {
return value.split("/").join(path.sep);
}