Initial CSE website content integration

scripts/ingest-publications.mjs

@@ -0,0 +1,193 @@
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import path from 'node:path';
+
+const root = process.cwd();
+const sourcesPath = path.join(root, 'src/data/publications/sources.json');
+const outputPath = path.join(root, 'src/data/publications/publications.json');
+const semanticScholarFields = [
+  'paperId',
+  'title',
+  'year',
+  'venue',
+  'publicationVenue',
+  'publicationTypes',
+  'authors',
+  'abstract',
+  'externalIds',
+  'url',
+  'openAccessPdf',
+  'fieldsOfStudy',
+  's2FieldsOfStudy',
+].join(',');
+
+const args = new Set(process.argv.slice(2));
+const dryRun = args.has('--dry-run');
+const maxPapersArg = process.argv.find((arg) => arg.startsWith('--max-papers='));
+const maxPapers = maxPapersArg ? Number(maxPapersArg.split('=')[1]) : 100;
+
+function normalizeTitle(title) {
+  return title
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, ' ')
+    .trim();
+}
+
+function inferType(publicationTypes = []) {
+  const joined = publicationTypes.join(' ').toLowerCase();
+
+  if (joined.includes('journal')) return 'journal';
+  if (joined.includes('conference')) return 'conference';
+  if (joined.includes('preprint')) return 'preprint';
+  if (joined.includes('workshop')) return 'workshop';
+
+  return 'other';
+}
+
+function getPaperKey(paper) {
+  const doi = paper.externalIds?.DOI ?? paper.externalIds?.doi;
+  if (doi) return `doi:${doi.toLowerCase()}`;
+  if (paper.paperId) return `s2:${paper.paperId}`;
+  return `title:${normalizeTitle(paper.title ?? '')}`;
+}
+
+function buildSearchText(publication) {
+  return [
+    publication.title,
+    publication.venue,
+    publication.year,
+    publication.abstract,
+    publication.authors.map((author) => author.name).join(' '),
+    publication.areaSlugs.join(' '),
+    publication.keywords.join(' '),
+  ]
+    .filter(Boolean)
+    .join(' ')
+    .toLowerCase();
+}
+
+function normalizeSemanticScholarPaper(paper, source) {
+  const aliases = new Set([source.name, ...(source.aliases ?? [])]);
+  const authors = (paper.authors ?? []).map((author) => ({
+    name: author.name,
+    facultyId: aliases.has(author.name) ? source.facultyId : undefined,
+  }));
+  const keywords = [
+    ...(paper.fieldsOfStudy ?? []),
+    ...(paper.s2FieldsOfStudy ?? []).map((field) => field.category),
+  ].filter(Boolean);
+  const publication = {
+    id: paper.paperId ? `s2:${paper.paperId}` : `title:${normalizeTitle(paper.title)}`,
+    title: paper.title,
+    year: paper.year,
+    venue: paper.publicationVenue?.name ?? paper.venue ?? '',
+    type: inferType(paper.publicationTypes),
+    authors,
+    facultyIds: [source.facultyId],
+    areaSlugs: source.areaSlugs ?? [],
+    abstract: paper.abstract ?? '',
+    keywords: [...new Set(keywords)],
+    links: {
+      doi: paper.externalIds?.DOI ?? '',
+      pdf: paper.openAccessPdf?.url ?? '',
+      semanticScholar: paper.url ?? '',
+      dblp: paper.externalIds?.DBLP ? `https://dblp.org/rec/${paper.externalIds.DBLP}` : '',
+    },
+    source: {
+      provider: 'semantic-scholar',
+      externalId: source.semanticScholarAuthorId,
+      updatedAt: new Date().toISOString(),
+    },
+    searchText: '',
+  };
+
+  publication.searchText = buildSearchText(publication);
+  return publication;
+}
+
+async function fetchSemanticScholarPapers(source) {
+  const papers = [];
+  let offset = 0;
+  const pageSize = Math.min(100, maxPapers);
+
+  while (papers.length < maxPapers) {
+    const url = new URL(
+      `https://api.semanticscholar.org/graph/v1/author/${source.semanticScholarAuthorId}/papers`,
+    );
+    url.searchParams.set('fields', semanticScholarFields);
+    url.searchParams.set('limit', String(pageSize));
+    url.searchParams.set('offset', String(offset));
+
+    const response = await fetch(url);
+    if (!response.ok) {
+      throw new Error(
+        `Semantic Scholar request failed for ${source.name}: ${response.status} ${response.statusText}`,
+      );
+    }
+
+    const body = await response.json();
+    papers.push(...(body.data ?? []));
+
+    if (!body.next || body.data?.length === 0) break;
+    offset = body.next;
+  }
+
+  return papers.slice(0, maxPapers);
+}
+
+async function main() {
+  const sources = JSON.parse(await readFile(sourcesPath, 'utf8'));
+  const configuredSources = sources.filter((source) => source.semanticScholarAuthorId);
+  const byKey = new Map();
+
+  for (const source of configuredSources) {
+    const papers = await fetchSemanticScholarPapers(source);
+
+    for (const paper of papers) {
+      if (!paper.title || !paper.year) continue;
+      const key = getPaperKey(paper);
+      const normalized = normalizeSemanticScholarPaper(paper, source);
+      const existing = byKey.get(key);
+
+      if (existing) {
+        existing.facultyIds = [...new Set([...existing.facultyIds, source.facultyId])];
+        existing.areaSlugs = [...new Set([...existing.areaSlugs, ...(source.areaSlugs ?? [])])];
+        existing.authors = existing.authors.map((author) => {
+          if (author.facultyId) return author;
+          const aliases = new Set([source.name, ...(source.aliases ?? [])]);
+          return aliases.has(author.name)
+            ? { ...author, facultyId: source.facultyId }
+            : author;
+        });
+        existing.searchText = buildSearchText(existing);
+      } else {
+        byKey.set(key, normalized);
+      }
+    }
+  }
+
+  const items = [...byKey.values()].sort(
+    (a, b) => b.year - a.year || a.title.localeCompare(b.title),
+  );
+  const output = {
+    generatedAt: configuredSources.length > 0 ? new Date().toISOString() : null,
+    items,
+  };
+
+  if (dryRun) {
+    console.log(
+      `Would write ${items.length} publications from ${configuredSources.length} configured sources.`,
+    );
+    return;
+  }
+
+  await mkdir(path.dirname(outputPath), { recursive: true });
+  await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`);
+  console.log(
+    `Wrote ${items.length} publications from ${configuredSources.length} configured sources to ${outputPath}.`,
+  );
+}
+
+main().catch((error) => {
+  console.error(error);
+  process.exitCode = 1;
+});