Initial CSE website content integration
This commit is contained in:
commit
dfe66b9002
260 changed files with 83153 additions and 0 deletions
681
scripts/ingest-dblp-publications.mjs
Normal file
681
scripts/ingest-dblp-publications.mjs
Normal file
|
|
@ -0,0 +1,681 @@
|
|||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
|
||||
const OUTPUT_PATH = path.resolve(
|
||||
"src/data/publications/dblp-career-faculty.json",
|
||||
);
|
||||
|
||||
const REQUEST_DELAY_MS = 2500;
|
||||
const MAX_RETRIES = 6;
|
||||
const USER_AGENT = "cse-website-publication-ingest/0.1 (https://iitgn.ac.in/)";
|
||||
|
||||
const CAREER_FACULTY = [
|
||||
{
|
||||
id: "rajat-moona",
|
||||
name: "Rajat Moona",
|
||||
category: "core",
|
||||
dblpPid: "16/2265",
|
||||
defaultAreas: ["systems"],
|
||||
},
|
||||
{
|
||||
id: "anirban-dasgupta",
|
||||
name: "Anirban Dasgupta",
|
||||
category: "core",
|
||||
dblpPid: "54/385-1",
|
||||
defaultAreas: ["theory", "data-science"],
|
||||
},
|
||||
{
|
||||
id: "bireswar-das",
|
||||
name: "Bireswar Das",
|
||||
category: "core",
|
||||
dblpPid: "93/3858",
|
||||
defaultAreas: ["theory"],
|
||||
},
|
||||
{
|
||||
id: "neeldhara-misra",
|
||||
name: "Neeldhara Misra",
|
||||
category: "core",
|
||||
dblpPid: "85/6789",
|
||||
defaultAreas: ["theory", "ai"],
|
||||
},
|
||||
{
|
||||
id: "nipun-batra",
|
||||
name: "Nipun Batra",
|
||||
category: "core",
|
||||
dblpPid: "19/2128",
|
||||
defaultAreas: ["ai", "data-science", "systems"],
|
||||
},
|
||||
{
|
||||
id: "manoj-d-gupta",
|
||||
name: "Manoj D Gupta",
|
||||
category: "core",
|
||||
dblpPid: "05/5157-2",
|
||||
defaultAreas: ["theory", "data-science"],
|
||||
},
|
||||
{
|
||||
id: "mayank-singh",
|
||||
name: "Mayank Singh",
|
||||
category: "core",
|
||||
dblpPid: "96/4770",
|
||||
defaultAreas: ["ai", "data-science"],
|
||||
},
|
||||
{
|
||||
id: "sameer-g-kulkarni",
|
||||
name: "Sameer G Kulkarni",
|
||||
category: "core",
|
||||
dblpPid: "185/5705",
|
||||
defaultAreas: ["systems"],
|
||||
},
|
||||
{
|
||||
id: "balagopal-komarath",
|
||||
name: "Balagopal Komarath",
|
||||
category: "core",
|
||||
dblpPid: "124/2629",
|
||||
defaultAreas: ["theory"],
|
||||
},
|
||||
{
|
||||
id: "abhishek-bichhawat",
|
||||
name: "Abhishek Bichhawat",
|
||||
category: "core",
|
||||
dblpPid: "61/10308",
|
||||
defaultAreas: ["security"],
|
||||
},
|
||||
{
|
||||
id: "yogesh-kumar-meena",
|
||||
name: "Yogesh Kumar Meena",
|
||||
category: "core",
|
||||
dblpPid: "66/10604",
|
||||
defaultAreas: ["hci", "ai"],
|
||||
},
|
||||
{
|
||||
id: "shouvick-mondal",
|
||||
name: "Shouvick Mondal",
|
||||
category: "core",
|
||||
dblpPid: "167/4011",
|
||||
defaultAreas: ["theory"],
|
||||
},
|
||||
{
|
||||
id: "manisha-padala",
|
||||
name: "Manisha Padala",
|
||||
category: "core",
|
||||
dblpPid: "213/8101",
|
||||
defaultAreas: ["ai", "data-science"],
|
||||
},
|
||||
{
|
||||
id: "shanmuganathan-raman",
|
||||
name: "Shanmuganathan Raman",
|
||||
category: "affiliated",
|
||||
dblpPid: "70/4688",
|
||||
defaultAreas: ["ai", "hci"],
|
||||
},
|
||||
{
|
||||
id: "udit-bhatia",
|
||||
name: "Udit Bhatia",
|
||||
category: "affiliated",
|
||||
dblpPid: "199/7860",
|
||||
defaultAreas: ["ai", "data-science"],
|
||||
},
|
||||
{
|
||||
id: "krishna-prasad-miyapuram",
|
||||
name: "Krishna Prasad Miyapuram",
|
||||
category: "affiliated",
|
||||
dblpPid: "00/4508",
|
||||
defaultAreas: ["hci", "ai"],
|
||||
},
|
||||
{
|
||||
id: "jyoti-krishnan",
|
||||
name: "Jyoti Krishnan",
|
||||
category: "teaching",
|
||||
dblpPid: null,
|
||||
defaultAreas: [],
|
||||
},
|
||||
{
|
||||
id: "manu-awasthi",
|
||||
name: "Manu Awasthi",
|
||||
category: "practice",
|
||||
dblpPid: "75/2883",
|
||||
defaultAreas: ["systems"],
|
||||
},
|
||||
{
|
||||
id: "anup-kalbalia",
|
||||
name: "Anup Kalbalia",
|
||||
category: "practice",
|
||||
dblpPid: null,
|
||||
defaultAreas: [],
|
||||
},
|
||||
];
|
||||
|
||||
const AREA_RULES = [
|
||||
{
|
||||
area: "theory",
|
||||
patterns: [
|
||||
/\balgorithm(s|ic)?\b/i,
|
||||
/\bapproximation\b/i,
|
||||
/\bparameteri[sz]ed\b/i,
|
||||
/\bparameterized\b/i,
|
||||
/\bfpt\b/i,
|
||||
/\bkernel(ization)?\b/i,
|
||||
/\bcomplexity\b/i,
|
||||
/\bcombinatorial\b/i,
|
||||
/\bgraph(s)?\b/i,
|
||||
/\b(maximum|perfect|stable|popular|rank-maximal) matching\b/i,
|
||||
/\bmatching problem\b/i,
|
||||
/\bmatchings\b/i,
|
||||
/\bmatroid\b/i,
|
||||
/\bsat\b/i,
|
||||
/\bboolean\b/i,
|
||||
/\bgame theory\b/i,
|
||||
/\bvoting\b/i,
|
||||
/\bsocial choice\b/i,
|
||||
/\bcomputational geometry\b/i,
|
||||
/\bautomata\b/i,
|
||||
/\bformal language/i,
|
||||
/\bfsttcs\b/i,
|
||||
/\bsoda\b/i,
|
||||
/\bstoc\b/i,
|
||||
/\bfocs\b/i,
|
||||
/\bicalp\b/i,
|
||||
/\besa\b/i,
|
||||
/\bisaac\b/i,
|
||||
/\bstacs\b/i,
|
||||
/\bipec\b/i,
|
||||
/\bmfcs\b/i,
|
||||
],
|
||||
},
|
||||
{
|
||||
area: "systems",
|
||||
patterns: [
|
||||
/\barchitecture\b/i,
|
||||
/\bmicroarchitecture\b/i,
|
||||
/\bprocessor\b/i,
|
||||
/\bcache\b/i,
|
||||
/\bmemory\b/i,
|
||||
/\bstorage\b/i,
|
||||
/\bcompiler\b/i,
|
||||
/\boperating system/i,
|
||||
/\bdistributed system/i,
|
||||
/\bnetwork(s)?\b/i,
|
||||
/\bnetwork-on-chip\b/i,
|
||||
/\bnoc(s)?\b/i,
|
||||
/\bhardware\b/i,
|
||||
/\bvlsi\b/i,
|
||||
/\bfpga\b/i,
|
||||
/\bembedded\b/i,
|
||||
/\bsensor(s)?\b/i,
|
||||
/\biot\b/i,
|
||||
/\bedge computing\b/i,
|
||||
/\bcloud\b/i,
|
||||
/\bdatacenter\b/i,
|
||||
/\bisca\b/i,
|
||||
/\bhpca\b/i,
|
||||
/\bmicro\b/i,
|
||||
/\basplos\b/i,
|
||||
/\bdate\b/i,
|
||||
/\bdac\b/i,
|
||||
/\baspdac\b/i,
|
||||
/\bnocs\b/i,
|
||||
/\bsigcomm\b/i,
|
||||
/\bnsdi\b/i,
|
||||
/\bosdi\b/i,
|
||||
/\bsosp\b/i,
|
||||
],
|
||||
},
|
||||
{
|
||||
area: "ai",
|
||||
patterns: [
|
||||
/\bartificial intelligence\b/i,
|
||||
/\bmachine learning\b/i,
|
||||
/\bdeep learning\b/i,
|
||||
/\bneural\b/i,
|
||||
/\btransformer(s)?\b/i,
|
||||
/\blanguage model(s)?\b/i,
|
||||
/\bllm(s)?\b/i,
|
||||
/\bnlp\b/i,
|
||||
/\bnatural language\b/i,
|
||||
/\bcomputer vision\b/i,
|
||||
/\bimage(s)?\b/i,
|
||||
/\bvisual\b/i,
|
||||
/\bvision-language\b/i,
|
||||
/\bsegmentation\b/i,
|
||||
/\bdetection\b/i,
|
||||
/\brecognition\b/i,
|
||||
/\bclassification\b/i,
|
||||
/\bprediction\b/i,
|
||||
/\brecommender\b/i,
|
||||
/\breinforcement learning\b/i,
|
||||
/\bgenerative\b/i,
|
||||
/\bdiffusion\b/i,
|
||||
/\blatent\b/i,
|
||||
/\brepresentation learning\b/i,
|
||||
/\bembedding(s)?\b/i,
|
||||
/\bgnn(s)?\b/i,
|
||||
/\bgraph neural\b/i,
|
||||
/\bclustering\b/i,
|
||||
/\bknowledge graph\b/i,
|
||||
/\baaai\b/i,
|
||||
/\bicml\b/i,
|
||||
/\biclr\b/i,
|
||||
/\bneurips\b/i,
|
||||
/\bnips\b/i,
|
||||
/\bijcai\b/i,
|
||||
/\bcvpr\b/i,
|
||||
/\biccv\b/i,
|
||||
/\beccv\b/i,
|
||||
/\bacl\b/i,
|
||||
/\bemnlp\b/i,
|
||||
/\bnaacl\b/i,
|
||||
],
|
||||
},
|
||||
{
|
||||
area: "data-science",
|
||||
patterns: [
|
||||
/\bdata (science|mining|analytics|analysis)\b/i,
|
||||
/\bmining\b/i,
|
||||
/\bstream(ing)?\b/i,
|
||||
/\btime series\b/i,
|
||||
/\bstatistical\b/i,
|
||||
/\binformation retrieval\b/i,
|
||||
/\bsearch engine\b/i,
|
||||
/\branking\b/i,
|
||||
/\bsocial network(s)?\b/i,
|
||||
/\bweb graph\b/i,
|
||||
/\bknowledge discovery\b/i,
|
||||
/\bcausal\b/i,
|
||||
/\bfairness\b/i,
|
||||
/\bprivacy-preserving data\b/i,
|
||||
/\benergy disaggregation\b/i,
|
||||
/\bsmart meter(s)?\b/i,
|
||||
/\bnilm\b/i,
|
||||
/\bkdd\b/i,
|
||||
/\bwww\b/i,
|
||||
/\bwebconf\b/i,
|
||||
/\bwsdm\b/i,
|
||||
/\bcikm\b/i,
|
||||
/\bicdm\b/i,
|
||||
/\bpkdd\b/i,
|
||||
/\bsigmod\b/i,
|
||||
/\bvldb\b/i,
|
||||
/\bicde\b/i,
|
||||
],
|
||||
},
|
||||
{
|
||||
area: "hci",
|
||||
patterns: [
|
||||
/\bhci\b/i,
|
||||
/\bhuman-computer interaction\b/i,
|
||||
/\bhuman-ai\b/i,
|
||||
/\buser(s)?\b/i,
|
||||
/\binteraction\b/i,
|
||||
/\baccessibility\b/i,
|
||||
/\bassistive\b/i,
|
||||
/\bbrain-computer interface\b/i,
|
||||
/\bbci\b/i,
|
||||
/\beeg\b/i,
|
||||
/\bcognitive\b/i,
|
||||
/\bneuro/i,
|
||||
/\bgaze\b/i,
|
||||
/\bgesture\b/i,
|
||||
/\bwearable\b/i,
|
||||
/\baugmented reality\b/i,
|
||||
/\bvirtual reality\b/i,
|
||||
/\bchi\b/i,
|
||||
/\buist\b/i,
|
||||
/\bcscw\b/i,
|
||||
/\bubicomp\b/i,
|
||||
/\biui\b/i,
|
||||
/\bassets\b/i,
|
||||
],
|
||||
},
|
||||
{
|
||||
area: "security",
|
||||
patterns: [
|
||||
/\bsecurity\b/i,
|
||||
/\bprivacy\b/i,
|
||||
/\bcryptograph/i,
|
||||
/\bauthentication\b/i,
|
||||
/\bauthorization\b/i,
|
||||
/\bmalware\b/i,
|
||||
/\battack(s)?\b/i,
|
||||
/\bvulnerabilit(y|ies)\b/i,
|
||||
/\bprogram analysis\b/i,
|
||||
/\binformation flow\b/i,
|
||||
/\bverification\b/i,
|
||||
/\bblockchain\b/i,
|
||||
/\btrusted\b/i,
|
||||
/\bside-channel\b/i,
|
||||
/\bccs\b/i,
|
||||
/\bndss\b/i,
|
||||
/\busenix security\b/i,
|
||||
/\bieee s(&| and | )p\b/i,
|
||||
/\beurosp\b/i,
|
||||
/\basiaccs\b/i,
|
||||
/\bcsf\b/i,
|
||||
/\bcrypto\b/i,
|
||||
/\beurocrypt\b/i,
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
function sleep(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
async function fetchText(url) {
|
||||
let lastError;
|
||||
|
||||
for (let attempt = 1; attempt <= MAX_RETRIES; attempt += 1) {
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
headers: { "user-agent": USER_AGENT },
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
return response.text();
|
||||
}
|
||||
|
||||
lastError = new Error(`${response.status} ${response.statusText}`);
|
||||
if (response.status !== 429 && response.status < 500) {
|
||||
throw lastError;
|
||||
}
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
}
|
||||
|
||||
const waitMs = REQUEST_DELAY_MS * attempt * 2;
|
||||
console.warn(`Retrying ${url} after ${waitMs}ms (${lastError.message})`);
|
||||
await sleep(waitMs);
|
||||
}
|
||||
|
||||
throw lastError;
|
||||
}
|
||||
|
||||
function decodeXml(value = "") {
|
||||
return value
|
||||
.replace(/<[^>]+>/g, "")
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_, hex) =>
|
||||
String.fromCodePoint(Number.parseInt(hex, 16)),
|
||||
)
|
||||
.replace(/&#([0-9]+);/g, (_, dec) =>
|
||||
String.fromCodePoint(Number.parseInt(dec, 10)),
|
||||
)
|
||||
.replace(/"/g, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</g, "<")
|
||||
.replace(/>/g, ">")
|
||||
.replace(/&/g, "&")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
function parseAttributes(attributeText = "") {
|
||||
const attributes = {};
|
||||
const attributeRegex = /([a-zA-Z_:][\w:.-]*)="([^"]*)"/g;
|
||||
let match;
|
||||
|
||||
while ((match = attributeRegex.exec(attributeText))) {
|
||||
attributes[match[1]] = decodeXml(match[2]);
|
||||
}
|
||||
|
||||
return attributes;
|
||||
}
|
||||
|
||||
function firstTag(block, tagName) {
|
||||
const match = block.match(
|
||||
new RegExp(`<${tagName}\\b[^>]*>([\\s\\S]*?)</${tagName}>`),
|
||||
);
|
||||
return match ? decodeXml(match[1]) : undefined;
|
||||
}
|
||||
|
||||
function allTags(block, tagName) {
|
||||
const values = [];
|
||||
const regex = new RegExp(
|
||||
`<${tagName}\\b([^>]*)>([\\s\\S]*?)</${tagName}>`,
|
||||
"g",
|
||||
);
|
||||
let match;
|
||||
|
||||
while ((match = regex.exec(block))) {
|
||||
values.push({
|
||||
attributes: parseAttributes(match[1]),
|
||||
text: decodeXml(match[2]),
|
||||
});
|
||||
}
|
||||
|
||||
return values;
|
||||
}
|
||||
|
||||
function classifyType(entryName, attributes, venue) {
|
||||
if (entryName === "inproceedings") return "conference";
|
||||
if (entryName === "proceedings") return "proceedings";
|
||||
if (entryName === "incollection") return "book-chapter";
|
||||
if (entryName === "book") return "book";
|
||||
if (entryName === "phdthesis") return "phd-thesis";
|
||||
if (entryName === "mastersthesis") return "masters-thesis";
|
||||
if (entryName === "article") {
|
||||
if (attributes.publtype === "informal" || venue === "CoRR") {
|
||||
return "preprint";
|
||||
}
|
||||
return "journal";
|
||||
}
|
||||
return "other";
|
||||
}
|
||||
|
||||
function classifyAreas(publication, facultyIds) {
|
||||
const searchText = [
|
||||
publication.title,
|
||||
publication.venue,
|
||||
publication.dblpKey,
|
||||
publication.links.ee.join(" "),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(" ");
|
||||
|
||||
const areas = new Set();
|
||||
for (const rule of AREA_RULES) {
|
||||
if (rule.patterns.some((pattern) => pattern.test(searchText))) {
|
||||
areas.add(rule.area);
|
||||
}
|
||||
}
|
||||
|
||||
if (areas.size === 0) {
|
||||
for (const facultyId of facultyIds) {
|
||||
const faculty = CAREER_FACULTY.find((member) => member.id === facultyId);
|
||||
for (const area of faculty?.defaultAreas ?? []) {
|
||||
areas.add(area);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return [...areas].sort();
|
||||
}
|
||||
|
||||
function parsePerson(xml, faculty) {
|
||||
const personBlock = xml.match(/<person\b[\s\S]*?<\/person>/)?.[0] ?? "";
|
||||
const rootAttributes = parseAttributes(
|
||||
xml.match(/<dblpperson\b([^>]*)>/)?.[1] ?? "",
|
||||
);
|
||||
|
||||
return {
|
||||
id: faculty.id,
|
||||
name: faculty.name,
|
||||
category: faculty.category,
|
||||
dblpPid: faculty.dblpPid,
|
||||
dblpName: rootAttributes.name,
|
||||
dblpPage: faculty.dblpPid
|
||||
? `https://dblp.org/pid/${faculty.dblpPid}.html`
|
||||
: null,
|
||||
profileUrls: allTags(personBlock, "url").map((url) => url.text),
|
||||
affiliations: allTags(personBlock, "note")
|
||||
.filter((note) => note.attributes.type === "affiliation")
|
||||
.map((note) => note.text),
|
||||
dblpPublicationCount: Number(rootAttributes.n ?? 0),
|
||||
defaultAreas: faculty.defaultAreas,
|
||||
};
|
||||
}
|
||||
|
||||
function parsePublications(xml) {
|
||||
const publications = [];
|
||||
const recordRegex = /<r>\s*<([a-z]+)\b([^>]*)>([\s\S]*?)<\/\1>\s*<\/r>/g;
|
||||
let match;
|
||||
|
||||
while ((match = recordRegex.exec(xml))) {
|
||||
const [, entryName, attributeText, block] = match;
|
||||
const attributes = parseAttributes(attributeText);
|
||||
const authors = allTags(block, "author").map((author) => ({
|
||||
name: author.text,
|
||||
pid: author.attributes.pid,
|
||||
orcid: author.attributes.orcid,
|
||||
}));
|
||||
const editors = allTags(block, "editor").map((editor) => ({
|
||||
name: editor.text,
|
||||
pid: editor.attributes.pid,
|
||||
}));
|
||||
const ee = allTags(block, "ee").map((link) => link.text);
|
||||
const title = firstTag(block, "title");
|
||||
const year = Number(firstTag(block, "year"));
|
||||
const venue =
|
||||
firstTag(block, "booktitle") ??
|
||||
firstTag(block, "journal") ??
|
||||
firstTag(block, "school") ??
|
||||
firstTag(block, "publisher") ??
|
||||
firstTag(block, "series");
|
||||
|
||||
if (!attributes.key || !title) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const doiUrl = ee.find((link) => link.includes("doi.org/"));
|
||||
const arxivUrl = ee.find((link) => /arxiv\.org|arXiv/i.test(link));
|
||||
|
||||
publications.push({
|
||||
id: attributes.key,
|
||||
dblpKey: attributes.key,
|
||||
title,
|
||||
year: Number.isFinite(year) ? year : null,
|
||||
type: classifyType(entryName, attributes, venue),
|
||||
entryType: entryName,
|
||||
venue,
|
||||
pages: firstTag(block, "pages"),
|
||||
volume: firstTag(block, "volume"),
|
||||
number: firstTag(block, "number"),
|
||||
authors,
|
||||
editors,
|
||||
links: {
|
||||
dblp: `https://dblp.org/rec/${attributes.key}`,
|
||||
ee,
|
||||
doi: doiUrl ? doiUrl.replace(/^https?:\/\/doi\.org\//, "") : undefined,
|
||||
arxiv: arxivUrl,
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
return publications;
|
||||
}
|
||||
|
||||
function mergePublication(existing, incoming) {
|
||||
const facultyIds = new Set([
|
||||
...(existing.facultyIds ?? []),
|
||||
...(incoming.facultyIds ?? []),
|
||||
]);
|
||||
|
||||
return {
|
||||
...existing,
|
||||
...incoming,
|
||||
facultyIds: [...facultyIds].sort(),
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const facultyByPid = new Map(
|
||||
CAREER_FACULTY.filter((member) => member.dblpPid).map((member) => [
|
||||
member.dblpPid,
|
||||
member,
|
||||
]),
|
||||
);
|
||||
|
||||
const facultyProfiles = [];
|
||||
const publicationsByKey = new Map();
|
||||
const missingDblp = CAREER_FACULTY.filter((member) => !member.dblpPid).map(
|
||||
({ id, name, category }) => ({ id, name, category }),
|
||||
);
|
||||
|
||||
for (const faculty of CAREER_FACULTY) {
|
||||
if (!faculty.dblpPid) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const url = `https://dblp.org/pid/${faculty.dblpPid}.xml`;
|
||||
console.log(`Fetching ${faculty.name}: ${url}`);
|
||||
const xml = await fetchText(url);
|
||||
const profile = parsePerson(xml, faculty);
|
||||
facultyProfiles.push(profile);
|
||||
|
||||
for (const publication of parsePublications(xml)) {
|
||||
const contributors = [...publication.authors, ...publication.editors];
|
||||
const facultyIds = contributors
|
||||
.map((author) => facultyByPid.get(author.pid)?.id)
|
||||
.filter(Boolean);
|
||||
|
||||
if (!facultyIds.includes(faculty.id)) {
|
||||
facultyIds.push(faculty.id);
|
||||
}
|
||||
|
||||
const normalized = {
|
||||
...publication,
|
||||
facultyIds: [...new Set(facultyIds)].sort(),
|
||||
};
|
||||
normalized.areaKeywords = classifyAreas(
|
||||
normalized,
|
||||
normalized.facultyIds,
|
||||
);
|
||||
|
||||
const existing = publicationsByKey.get(normalized.dblpKey);
|
||||
publicationsByKey.set(
|
||||
normalized.dblpKey,
|
||||
existing ? mergePublication(existing, normalized) : normalized,
|
||||
);
|
||||
}
|
||||
|
||||
await sleep(REQUEST_DELAY_MS);
|
||||
}
|
||||
|
||||
const publications = [...publicationsByKey.values()]
|
||||
.map((publication) => ({
|
||||
...publication,
|
||||
areaKeywords: classifyAreas(publication, publication.facultyIds),
|
||||
}))
|
||||
.sort(
|
||||
(a, b) => (b.year ?? 0) - (a.year ?? 0) || a.title.localeCompare(b.title),
|
||||
);
|
||||
|
||||
const output = {
|
||||
generatedAt: new Date().toISOString(),
|
||||
source: {
|
||||
name: "dblp",
|
||||
api: "https://dblp.org/pid/{pid}.xml",
|
||||
note: "Generated from DBLP PID XML exports for non-visiting/non-guest faculty.",
|
||||
},
|
||||
areaKeywordVocabulary: AREA_RULES.map((rule) => rule.area),
|
||||
careerFacultyScope: ["core", "affiliated", "teaching", "practice"],
|
||||
missingDblp,
|
||||
faculty: facultyProfiles.sort((a, b) => a.name.localeCompare(b.name)),
|
||||
publications,
|
||||
};
|
||||
|
||||
await mkdir(path.dirname(OUTPUT_PATH), { recursive: true });
|
||||
await writeFile(OUTPUT_PATH, `${JSON.stringify(output, null, 2)}\n`);
|
||||
|
||||
console.log(
|
||||
`Wrote ${publications.length} unique publications for ${facultyProfiles.length} DBLP-backed faculty to ${OUTPUT_PATH}`,
|
||||
);
|
||||
if (missingDblp.length > 0) {
|
||||
console.log(
|
||||
`No DBLP PID configured for: ${missingDblp.map((member) => member.name).join(", ")}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
193
scripts/ingest-publications.mjs
Normal file
193
scripts/ingest-publications.mjs
Normal file
|
|
@ -0,0 +1,193 @@
|
|||
import { mkdir, readFile, writeFile } from 'node:fs/promises';
|
||||
import path from 'node:path';
|
||||
|
||||
const root = process.cwd();
|
||||
const sourcesPath = path.join(root, 'src/data/publications/sources.json');
|
||||
const outputPath = path.join(root, 'src/data/publications/publications.json');
|
||||
const semanticScholarFields = [
|
||||
'paperId',
|
||||
'title',
|
||||
'year',
|
||||
'venue',
|
||||
'publicationVenue',
|
||||
'publicationTypes',
|
||||
'authors',
|
||||
'abstract',
|
||||
'externalIds',
|
||||
'url',
|
||||
'openAccessPdf',
|
||||
'fieldsOfStudy',
|
||||
's2FieldsOfStudy',
|
||||
].join(',');
|
||||
|
||||
const args = new Set(process.argv.slice(2));
|
||||
const dryRun = args.has('--dry-run');
|
||||
const maxPapersArg = process.argv.find((arg) => arg.startsWith('--max-papers='));
|
||||
const maxPapers = maxPapersArg ? Number(maxPapersArg.split('=')[1]) : 100;
|
||||
|
||||
function normalizeTitle(title) {
|
||||
return title
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function inferType(publicationTypes = []) {
|
||||
const joined = publicationTypes.join(' ').toLowerCase();
|
||||
|
||||
if (joined.includes('journal')) return 'journal';
|
||||
if (joined.includes('conference')) return 'conference';
|
||||
if (joined.includes('preprint')) return 'preprint';
|
||||
if (joined.includes('workshop')) return 'workshop';
|
||||
|
||||
return 'other';
|
||||
}
|
||||
|
||||
function getPaperKey(paper) {
|
||||
const doi = paper.externalIds?.DOI ?? paper.externalIds?.doi;
|
||||
if (doi) return `doi:${doi.toLowerCase()}`;
|
||||
if (paper.paperId) return `s2:${paper.paperId}`;
|
||||
return `title:${normalizeTitle(paper.title ?? '')}`;
|
||||
}
|
||||
|
||||
function buildSearchText(publication) {
|
||||
return [
|
||||
publication.title,
|
||||
publication.venue,
|
||||
publication.year,
|
||||
publication.abstract,
|
||||
publication.authors.map((author) => author.name).join(' '),
|
||||
publication.areaSlugs.join(' '),
|
||||
publication.keywords.join(' '),
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
function normalizeSemanticScholarPaper(paper, source) {
|
||||
const aliases = new Set([source.name, ...(source.aliases ?? [])]);
|
||||
const authors = (paper.authors ?? []).map((author) => ({
|
||||
name: author.name,
|
||||
facultyId: aliases.has(author.name) ? source.facultyId : undefined,
|
||||
}));
|
||||
const keywords = [
|
||||
...(paper.fieldsOfStudy ?? []),
|
||||
...(paper.s2FieldsOfStudy ?? []).map((field) => field.category),
|
||||
].filter(Boolean);
|
||||
const publication = {
|
||||
id: paper.paperId ? `s2:${paper.paperId}` : `title:${normalizeTitle(paper.title)}`,
|
||||
title: paper.title,
|
||||
year: paper.year,
|
||||
venue: paper.publicationVenue?.name ?? paper.venue ?? '',
|
||||
type: inferType(paper.publicationTypes),
|
||||
authors,
|
||||
facultyIds: [source.facultyId],
|
||||
areaSlugs: source.areaSlugs ?? [],
|
||||
abstract: paper.abstract ?? '',
|
||||
keywords: [...new Set(keywords)],
|
||||
links: {
|
||||
doi: paper.externalIds?.DOI ?? '',
|
||||
pdf: paper.openAccessPdf?.url ?? '',
|
||||
semanticScholar: paper.url ?? '',
|
||||
dblp: paper.externalIds?.DBLP ? `https://dblp.org/rec/${paper.externalIds.DBLP}` : '',
|
||||
},
|
||||
source: {
|
||||
provider: 'semantic-scholar',
|
||||
externalId: source.semanticScholarAuthorId,
|
||||
updatedAt: new Date().toISOString(),
|
||||
},
|
||||
searchText: '',
|
||||
};
|
||||
|
||||
publication.searchText = buildSearchText(publication);
|
||||
return publication;
|
||||
}
|
||||
|
||||
async function fetchSemanticScholarPapers(source) {
|
||||
const papers = [];
|
||||
let offset = 0;
|
||||
const pageSize = Math.min(100, maxPapers);
|
||||
|
||||
while (papers.length < maxPapers) {
|
||||
const url = new URL(
|
||||
`https://api.semanticscholar.org/graph/v1/author/${source.semanticScholarAuthorId}/papers`,
|
||||
);
|
||||
url.searchParams.set('fields', semanticScholarFields);
|
||||
url.searchParams.set('limit', String(pageSize));
|
||||
url.searchParams.set('offset', String(offset));
|
||||
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(
|
||||
`Semantic Scholar request failed for ${source.name}: ${response.status} ${response.statusText}`,
|
||||
);
|
||||
}
|
||||
|
||||
const body = await response.json();
|
||||
papers.push(...(body.data ?? []));
|
||||
|
||||
if (!body.next || body.data?.length === 0) break;
|
||||
offset = body.next;
|
||||
}
|
||||
|
||||
return papers.slice(0, maxPapers);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const sources = JSON.parse(await readFile(sourcesPath, 'utf8'));
|
||||
const configuredSources = sources.filter((source) => source.semanticScholarAuthorId);
|
||||
const byKey = new Map();
|
||||
|
||||
for (const source of configuredSources) {
|
||||
const papers = await fetchSemanticScholarPapers(source);
|
||||
|
||||
for (const paper of papers) {
|
||||
if (!paper.title || !paper.year) continue;
|
||||
const key = getPaperKey(paper);
|
||||
const normalized = normalizeSemanticScholarPaper(paper, source);
|
||||
const existing = byKey.get(key);
|
||||
|
||||
if (existing) {
|
||||
existing.facultyIds = [...new Set([...existing.facultyIds, source.facultyId])];
|
||||
existing.areaSlugs = [...new Set([...existing.areaSlugs, ...(source.areaSlugs ?? [])])];
|
||||
existing.authors = existing.authors.map((author) => {
|
||||
if (author.facultyId) return author;
|
||||
const aliases = new Set([source.name, ...(source.aliases ?? [])]);
|
||||
return aliases.has(author.name)
|
||||
? { ...author, facultyId: source.facultyId }
|
||||
: author;
|
||||
});
|
||||
existing.searchText = buildSearchText(existing);
|
||||
} else {
|
||||
byKey.set(key, normalized);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const items = [...byKey.values()].sort(
|
||||
(a, b) => b.year - a.year || a.title.localeCompare(b.title),
|
||||
);
|
||||
const output = {
|
||||
generatedAt: configuredSources.length > 0 ? new Date().toISOString() : null,
|
||||
items,
|
||||
};
|
||||
|
||||
if (dryRun) {
|
||||
console.log(
|
||||
`Would write ${items.length} publications from ${configuredSources.length} configured sources.`,
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
await mkdir(path.dirname(outputPath), { recursive: true });
|
||||
await writeFile(outputPath, `${JSON.stringify(output, null, 2)}\n`);
|
||||
console.log(
|
||||
`Wrote ${items.length} publications from ${configuredSources.length} configured sources to ${outputPath}.`,
|
||||
);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(error);
|
||||
process.exitCode = 1;
|
||||
});
|
||||
Loading…
Add table
Add a link
Reference in a new issue