blogs/scripts/import-quarto-blog.mjs

#!/usr/bin/env node
import fs from "node:fs/promises";
import nodeFs from "node:fs";
import path from "node:path";

const DEFAULT_SOURCE = "/private/tmp/quartosite-shallow/blog";
const DEFAULT_OUTPUT = "/private/tmp/quarto-blog-import";

const sourceRoot = path.resolve(process.argv[2] || DEFAULT_SOURCE);
const outputRoot = path.resolve(process.argv[3] || DEFAULT_OUTPUT);

const blogs = [
  { key: "research", vaultFolder: "01-research", title: "Research" },
  { key: "vibes", vaultFolder: "02-vibes", title: "Vibes" },
  { key: "puzzles", vaultFolder: "03-puzzles", title: "Puzzles" },
  { key: "reflections", vaultFolder: "04-reflections", title: "Reflections" },
  { key: "poetry", vaultFolder: "05-poetry", title: "Pieces" },
  { key: "reviews", vaultFolder: "06-reviews", title: "Reviews" },
  { key: "art", vaultFolder: "07-art", title: "Art" },
];

const blogByKey = new Map(blogs.map((blog) => [blog.key, blog]));

const explicitDestinations = new Map([
  ["13sheep/index.qmd", "vibes"],
  ["15-puzzle/index.qmd", "puzzles"],
  ["2023-exportober-workflows/index.qmd", "reviews"],
  ["building-interactives/index.qmd", "vibes"],
  ["building-websites-with-notion/index.qmd", "reviews"],
  ["comms/index.qmd", "art"],
  ["communication-complexity-equality/index.qmd", "art"],
  ["course-plan/2024-fall/index.qmd", "vibes"],
  ["course-plan/2024-spring/index.qmd", "vibes"],
  ["course-plan/index.qmd", "vibes"],
  ["cp/sam-i-am/index.qmd", "research"],
  ["crypto-intro/index.qmd", "research"],
  ["django-app/index.qmd", "reviews"],
  ["dogs-bunny-puzzle/index.qmd", "puzzles"],
  ["eight-self-sabotaging-behaviors/index.qmd", "reflections"],
  ["envelope-budgeting-notion/index.qmd", "reviews"],
  ["exportober/2021-tracker/index.qmd", "reflections"],
  ["exportober/2021/index.qmd", "reflections"],
  ["exportober/2022-tracker/index.qmd", "reflections"],
  ["exportober/about/index.qmd", "reflections"],
  ["homework-help/index.qmd", "research"],
  ["iit-rankings/index.qmd", "reflections"],
  ["indiamini-crosswords/index.qmd", "art"],
  ["kidney-exchanges/index.qmd", "research"],
  ["massren/index.qmd", "reviews"],
  ["moving-blocks-ctis/index.qmd", "puzzles"],
  ["new-mac/index.qmd", "reviews"],
  ["note-taking-resources/index.qmd", "reviews"],
  ["notion-powered-websites/index.qmd", "reviews"],
  ["on-career-choices/index.qmd", "reflections"],
  ["on-teaching/index.qmd", "reflections"],
  ["pandoc-letters/index.qmd", "reviews"],
  ["skj/index.qmd", "reflections"],
  ["solo-chess/index.qmd", "puzzles"],
  ["women-in-mathematics/index.qmd", "reviews"],
]);

const skippedDrafts = [];
const importedPosts = [];

await fs.rm(outputRoot, { recursive: true, force: true });
await fs.mkdir(outputRoot, { recursive: true });

for (const blog of blogs) {
  await fs.mkdir(path.join(outputRoot, "obsidian", "blogs", blog.vaultFolder), {
    recursive: true,
  });
  await fs.mkdir(
    path.join(
      outputRoot,
      "astro",
      "sites",
      blog.key,
      "src",
      "content",
      blog.key,
    ),
    { recursive: true },
  );
}

const sourceFiles = (await walk(sourceRoot))
  .filter((file) => /\.(qmd|md|markdown)$/i.test(file))
  .sort();

for (const sourceFile of sourceFiles) {
  const relativePath = toPosix(path.relative(sourceRoot, sourceFile));
  const sourceText = await fs.readFile(sourceFile, "utf8");
  const { frontmatter, body } = parseFrontmatter(sourceText);

  if (isDraft(frontmatter)) {
    skippedDrafts.push(relativePath);
    continue;
  }

  const blogKey = classifyPost(relativePath, frontmatter);
  const blog = blogByKey.get(blogKey);
  if (!blog) {
    throw new Error(`Unknown blog key "${blogKey}" for ${relativePath}`);
  }

  const markdownRelativePath = relativePath.replace(
    /\.(qmd|markdown)$/i,
    ".md",
  );
  const convertedBody = convertQuartoMarkdown(body).trim();
  const title = frontmatter.title || titleFromPath(relativePath);
  const pubDate = normalizeDate(frontmatter.date);
  const description =
    frontmatter.description || summarize(convertedBody, title);
  const categories = parseList(frontmatter.categories);
  const markdown = [
    "---",
    `title: ${yamlString(title)}`,
    `description: ${yamlString(description)}`,
    `pubDate: ${yamlString(pubDate)}`,
    'authorName: "Neeldhara"',
    categories.length
      ? `sourceCategories: [${categories.map(yamlString).join(", ")}]`
      : "",
    `sourcePath: ${yamlString(relativePath)}`,
    "---",
    "",
    convertedBody,
    "",
  ]
    .filter((line, index, lines) => line || lines[index - 1] !== "")
    .join("\n");

  const obsidianTarget = path.join(
    outputRoot,
    "obsidian",
    "blogs",
    blog.vaultFolder,
    fromPosix(markdownRelativePath),
  );
  const astroTarget = path.join(
    outputRoot,
    "astro",
    "sites",
    blog.key,
    "src",
    "content",
    blog.key,
    fromPosix(markdownRelativePath),
  );

  await writeFileEnsured(obsidianTarget, markdown);
  await writeFileEnsured(astroTarget, markdown);
  await copyPostAssets(path.dirname(sourceFile), path.dirname(obsidianTarget));
  await copyPostAssets(path.dirname(sourceFile), path.dirname(astroTarget));

  importedPosts.push({
    source: relativePath,
    destination: `${blog.vaultFolder}/${markdownRelativePath}`,
    blog: blog.key,
    title,
    date: pubDate,
    categories,
  });
}

await writeImportReport();
await writeVaultReadme();
await writeAstroManifest();

console.log(`Imported ${importedPosts.length} posts into ${outputRoot}`);
console.log(`Skipped ${skippedDrafts.length} draft posts.`);

async function writeImportReport() {
  const rows = [
    "# Quarto Blog Import Report",
    "",
    `Source: ${sourceRoot}`,
    `Generated: ${new Date().toISOString()}`,
    "",
    "## Imported Posts",
    "",
    "| Blog | Date | Title | Source | Destination |",
    "| --- | --- | --- | --- | --- |",
  ];

  for (const post of importedPosts.sort(
    (left, right) =>
      left.blog.localeCompare(right.blog) ||
      right.date.localeCompare(left.date),
  )) {
    rows.push(
      `| ${post.blog} | ${post.date} | ${escapeTable(post.title)} | \`${post.source}\` | \`${post.destination}\` |`,
    );
  }

  rows.push("", "## Skipped Drafts", "");
  if (skippedDrafts.length) {
    for (const draft of skippedDrafts) {
      rows.push(`- \`${draft}\``);
    }
  } else {
    rows.push("- None");
  }

  await writeFileEnsured(
    path.join(outputRoot, "IMPORT_REPORT.md"),
    `${rows.join("\n")}\n`,
  );
}

async function writeVaultReadme() {
  const readme = `# Blog Content Bridge

This folder is the Obsidian source of truth for the public blog family.

## Folder Map

| Vault folder | Astro site | Public domain |
| --- | --- | --- |
| \`blogs/01-research\` | \`allblogs/sites/research/src/content/research\` | \`https://research.neeldhara.blog\` |
| \`blogs/02-vibes\` | \`allblogs/sites/vibes/src/content/vibes\` | \`https://vibes.neeldhara.blog\` |
| \`blogs/03-puzzles\` | \`allblogs/sites/puzzles/src/content/puzzles\` | \`https://puzzles.neeldhara.blog\` |
| \`blogs/04-reflections\` | \`allblogs/sites/reflections/src/content/reflections\` | \`https://reflections.neeldhara.blog\` |
| \`blogs/05-poetry\` | \`allblogs/sites/poetry/src/content/poetry\` | \`https://pieces.neeldhara.blog\` |
| \`blogs/06-reviews\` | \`allblogs/sites/reviews/src/content/reviews\` | \`https://reviews.neeldhara.blog\` |
| \`blogs/07-art\` | \`allblogs/sites/art/src/content/art\` | \`https://art.neeldhara.blog\` |
| \`blogs/micro\` | \`micro/src/content/blog\` | \`https://micro.neeldhara.blog\` |

The installed Obsidian plugin is still stored at:

\`\`\`text
.obsidian/plugins/microblog-sync
\`\`\`

The plugin name in Obsidian is **Blog Family Sync**. It watches all folders above, syncs edited files into the matching Astro repo, can start a local preview server for the current note, and can publish by running build, commit, and push.

## Post Layout

Use one folder per post:

\`\`\`text
blogs/01-research/example-post/index.md
blogs/01-research/example-post/diagram.png
\`\`\`

The folder path becomes the public URL:

\`\`\`text
blogs/01-research/example-post/index.md
https://research.neeldhara.blog/example-post/
\`\`\`

Nested folders are allowed. For example:

\`\`\`text
blogs/04-reflections/exportober/2021/index.md
https://reflections.neeldhara.blog/exportober/2021/
\`\`\`

## Frontmatter

Every public Markdown post should start with:

\`\`\`yaml
---
title: "Post title"
description: "One or two sentence summary for cards and RSS."
pubDate: "2024-05-10"
authorName: "Neeldhara"
---
\`\`\`

\`title\`, \`description\`, and \`pubDate\` are required by the Astro schemas.
\`updatedDate\`, \`image\`, and \`authorImage\` are optional. Prefer colocated images in the body over remote stock images.

## Markdown Conventions

- Standard Markdown headings, lists, links, tables, code fences, footnotes, and raw HTML are preserved.
- Use colocated relative assets: \`![Caption](image.png)\`.
- Quarto image attributes such as \`{width=70%}\` are removed during import. Use normal Markdown now; layout should be handled by Astro CSS.
- Quarto callouts are converted to simple blockquotes. The callout type controls
  styling only; it does not create a visible heading by default:

\`\`\`markdown
> **Note**
>
> Text of the callout.
\`\`\`

- For new notes, the bracket form is also supported:

\`\`\`markdown
> [!warning]
>
> This point needs attention.
\`\`\`

- To show a heading, put it explicitly inside the bracket after \`:\` or \`|\`:

\`\`\`markdown
> [!tip: Useful observation]
>
> Text of the callout.
\`\`\`

- Quarto margin notes such as \`[text]{.aside}\` are converted to:

\`\`\`markdown
> **Aside:** text
\`\`\`

## Sync Rules

- Obsidian is the source of truth.
- Auto-sync copies changed files from the matching \`blogs/...\` folder into the matching local Astro content folder.
- The plugin includes Markdown plus common web assets: images, PDFs, JSON/YAML, JS, CSS, HTML, and text files.
- One-way sync does not delete target files unless **Delete orphaned target files** is enabled.
- The initial Quarto import used replace-style sync to clear placeholder Astro posts. Day-to-day editing should use the plugin.

## Publish

Use **Blog Family Sync: Publish current note** or **Blog Family Sync: Publish all changes for current blog** from Obsidian. Publish does three things:

1. Sync vault content into the local repo.
2. Run the configured Astro build command.
3. Commit and push to Forgejo, where Dokploy picks up the change.

`;

  await writeFileEnsured(
    path.join(outputRoot, "obsidian", "blogs", "README.md"),
    readme,
  );
}

async function writeAstroManifest() {
  const manifest = {
    generatedAt: new Date().toISOString(),
    sourceRoot,
    importedPosts,
    skippedDrafts,
  };
  await writeFileEnsured(
    path.join(outputRoot, "manifest.json"),
    `${JSON.stringify(manifest, null, 2)}\n`,
  );
}

function classifyPost(relativePath, frontmatter) {
  if (explicitDestinations.has(relativePath)) {
    return explicitDestinations.get(relativePath);
  }

  const categories = parseList(frontmatter.categories).map((category) =>
    category.toLowerCase(),
  );
  const haystack =
    `${relativePath} ${frontmatter.title || ""} ${categories.join(" ")}`.toLowerCase();

  if (relativePath.startsWith("poems/") || categories.includes("poem")) {
    return "poetry";
  }
  if (categories.some((category) => ["puzzles", "games"].includes(category))) {
    return "puzzles";
  }
  if (
    categories.some((category) =>
      [
        "books",
        "apps",
        "workflows",
        "tutorial",
        "websites",
        "notion",
        "pandoc",
        "lists",
      ].includes(category),
    )
  ) {
    return "reviews";
  }
  if (categories.some((category) => ["sketchnotes"].includes(category))) {
    return "art";
  }
  if (haystack.includes("chatgpt") || haystack.includes("interactive")) {
    return "vibes";
  }
  if (
    categories.some((category) =>
      [
        "lecturenotes",
        "talk",
        "exposition",
        "parameterized-algorithms",
      ].includes(category),
    )
  ) {
    return "research";
  }
  return "reflections";
}

function parseFrontmatter(text) {
  if (!text.startsWith("---")) {
    return { frontmatter: {}, body: text };
  }

  const end = text.indexOf("\n---", 3);
  if (end < 0) {
    return { frontmatter: {}, body: text };
  }

  const rawFrontmatter = text.slice(4, end).trim();
  const body = text.slice(end + 4);
  const frontmatter = {};
  for (const line of rawFrontmatter.split(/\r?\n/)) {
    const match = line.match(/^([A-Za-z0-9_-]+):\s*(.*)$/);
    if (!match) {
      continue;
    }

    const [, key, rawValue] = match;
    frontmatter[key] = unquote(rawValue.trim());
  }

  return { frontmatter, body };
}

function convertQuartoMarkdown(body) {
  const lines = body.replace(/\r\n/g, "\n").split("\n");
  const converted = [];
  const calloutStack = [];
  let inCodeFence = false;

  for (const rawLine of lines) {
    const line = rawLine.replace(/\s+$/, "");

    if (/^```/.test(line.trim())) {
      inCodeFence = !inCodeFence;
      converted.push(prefixCalloutLine(line, calloutStack.length));
      continue;
    }

    if (!inCodeFence) {
      const calloutOpen = line.match(
        /^\s*:::\s*\{?\.?callout-([A-Za-z0-9_-]+)[^}]*\}?\s*$/,
      );
      if (calloutOpen) {
        calloutStack.push(calloutOpen[1]);
        converted.push(
          prefixCalloutLine(
            `**${titleCase(calloutOpen[1])}**`,
            calloutStack.length,
          ),
        );
        converted.push(prefixCalloutLine("", calloutStack.length));
        continue;
      }

      if (/^\s*:::\s*$/.test(line) && calloutStack.length) {
        calloutStack.pop();
        converted.push("");
        continue;
      }

      if (/^\s*:{3,4}\s*\{/.test(line) || /^\s*:{3,4}\s*$/.test(line)) {
        continue;
      }
    }

    let cleaned = line;
    if (!inCodeFence) {
      cleaned = cleaned
        .replace(
          /(<script\b[^>]*><\/script>|<script\b[^>]*>.*?<\/script>)/gi,
          "",
        )
        .replace(/!\[([^\]]*)\]\(([^)]+)\)\{[^}]+\}/g, "![$1]($2)")
        .replace(/^\s*\[([^\]]+)\]\{\.aside\}\s*$/, "> **Aside:** $1")
        .replace(/\[([^\]]+)\]\{\.smallcaps\}/g, "$1")
        .replace(/\[([^\]]+)\]\{[^}]+\}/g, "$1");
    }

    converted.push(prefixCalloutLine(cleaned, calloutStack.length));
  }

  return converted.join("\n").replace(/\n{4,}/g, "\n\n\n");
}

function prefixCalloutLine(line, depth) {
  if (!depth) {
    return line;
  }

  return `${"> ".repeat(depth)}${line}`;
}

async function copyPostAssets(sourceDir, targetDir) {
  const files = await fs.readdir(sourceDir, { withFileTypes: true });
  for (const entry of files) {
    if (
      entry.name.startsWith(".") ||
      /\.(qmd|md|markdown)$/i.test(entry.name)
    ) {
      continue;
    }

    const source = path.join(sourceDir, entry.name);
    const target = path.join(targetDir, entry.name);
    if (entry.isDirectory()) {
      await fs.mkdir(target, { recursive: true });
      await copyPostAssets(source, target);
      continue;
    }

    if (entry.isFile()) {
      await fs.mkdir(path.dirname(target), { recursive: true });
      await fs.copyFile(source, target);
    }
  }
}

async function walk(root) {
  const entries = await fs.readdir(root, { withFileTypes: true });
  const files = [];
  for (const entry of entries) {
    if (entry.name.startsWith(".")) {
      continue;
    }

    const absolutePath = path.join(root, entry.name);
    if (entry.isDirectory()) {
      files.push(...(await walk(absolutePath)));
    } else if (entry.isFile()) {
      files.push(absolutePath);
    }
  }

  return files;
}

async function writeFileEnsured(target, content) {
  await fs.mkdir(path.dirname(target), { recursive: true });
  await fs.writeFile(target, content);
}

function parseList(value) {
  if (!value) {
    return [];
  }

  const trimmed = String(value).trim();
  if (trimmed.startsWith("[") && trimmed.endsWith("]")) {
    return trimmed
      .slice(1, -1)
      .split(",")
      .map((item) => unquote(item.trim()))
      .filter(Boolean);
  }

  return [unquote(trimmed)];
}

function isDraft(frontmatter) {
  return String(frontmatter.draft || "").toLowerCase() === "true";
}

function normalizeDate(value) {
  if (!value) {
    return "2000-01-01";
  }

  const cleaned = String(value).replace(/^["']|["']$/g, "");
  const match = cleaned.match(/^(\d{4})-(\d{2})-(\d{2})/);
  if (match) {
    return `${match[1]}-${match[2]}-${match[3]}`;
  }

  const date = new Date(cleaned);
  if (Number.isNaN(date.valueOf())) {
    return "2000-01-01";
  }

  return date.toISOString().slice(0, 10);
}

function summarize(markdown, title) {
  const withoutBlocks = markdown
    .replace(/```[\s\S]*?```/g, " ")
    .replace(/<[^>]+>/g, " ")
    .replace(/!\[[^\]]*\]\([^)]+\)/g, " ")
    .replace(/\[([^\]]+)\]\([^)]+\)/g, "$1")
    .replace(/[#>*_`~$]/g, " ")
    .replace(/\s+/g, " ")
    .trim();

  const summary = withoutBlocks || title;
  return summary.length > 180 ? `${summary.slice(0, 177).trim()}...` : summary;
}

function titleFromPath(relativePath) {
  const folder = relativePath.replace(/\/index\.(qmd|md|markdown)$/i, "");
  const leaf = folder.split("/").pop() || "Untitled";
  return leaf
    .split(/[-_]+/)
    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
    .join(" ");
}

function titleCase(value) {
  return String(value)
    .split(/[-_]+/)
    .map((part) => part.charAt(0).toUpperCase() + part.slice(1))
    .join(" ");
}

function yamlString(value) {
  return JSON.stringify(String(value || ""));
}

function unquote(value) {
  if (
    (value.startsWith('"') && value.endsWith('"')) ||
    (value.startsWith("'") && value.endsWith("'"))
  ) {
    return value.slice(1, -1);
  }

  return value;
}

function escapeTable(value) {
  return String(value).replace(/\|/g, "\\|");
}

function toPosix(value) {
  return value.split(path.sep).join("/");
}

function fromPosix(value) {
  return value.split("/").join(path.sep);
}