From 7311e379e8fdc62df372d8bf9e1d4f93b6ac7d76 Mon Sep 17 00:00:00 2001 From: Tim Howitz Date: Sun, 25 May 2025 11:57:51 +0100 Subject: [PATCH] Fixed scientists importing, and added linking by name --- importersFixed.md | 2 +- importersOrder.md | 6 ++ src/app/api/import-scientists/route.ts | 110 ++++++++++++++++++++----- 3 files changed, 95 insertions(+), 23 deletions(-) create mode 100644 importersOrder.md diff --git a/importersFixed.md b/importersFixed.md index ef8cbfd..07a09dd 100644 --- a/importersFixed.md +++ b/importersFixed.md @@ -3,4 +3,4 @@ - [x] Import earthquakes - [x] Import observatories - [ ] Import requests -- [ ] Import scientists +- [x] Import scientists diff --git a/importersOrder.md b/importersOrder.md new file mode 100644 index 0000000..331dc33 --- /dev/null +++ b/importersOrder.md @@ -0,0 +1,6 @@ +1. Import users +2. Import scientists +3. Import observatories +4. Import earthquakes +5. Import artefacts +6. Import requests diff --git a/src/app/api/import-scientists/route.ts b/src/app/api/import-scientists/route.ts index b91cf52..f4a3bcb 100644 --- a/src/app/api/import-scientists/route.ts +++ b/src/app/api/import-scientists/route.ts @@ -2,21 +2,18 @@ import { parse } from "csv-parse/sync"; import fs from "fs/promises"; import { NextResponse } from "next/server"; import path from "path"; - import { prisma } from "@utils/prisma"; +import { getRandomNumber } from "@utils/maths"; -// Path to CSV file const csvFilePath = path.resolve(process.cwd(), "public/scientists.csv"); type CsvRow = { Name: string; - Level?: string; - UserId: string; - SuperiorId?: string; + Level: string; + SuperiorName: string; }; function normalizeLevel(level: string | undefined): string { - // Only allow JUNIOR, SENIOR; default JUNIOR if (!level || !level.trim()) return "JUNIOR"; const lv = level.trim().toUpperCase(); return ["JUNIOR", "SENIOR"].includes(lv) ? lv : "JUNIOR"; @@ -24,31 +21,100 @@ function normalizeLevel(level: string | undefined): string { export async function POST() { try { - // 1. Read the CSV file const fileContent = await fs.readFile(csvFilePath, "utf8"); - - // 2. Parse the CSV const records: CsvRow[] = parse(fileContent, { columns: true, skip_empty_lines: true, }); - // 3. Transform each record for Prisma - // todo add senior scientists first - const scientists = records.map((row) => ({ - name: row.Name, - level: normalizeLevel(row.Level), - userId: parseInt(row.UserId, 10), - // todo get superior id by name from db - superiorId: row.SuperiorId && row.SuperiorId.trim() !== "" ? parseInt(row.SuperiorId, 10) : null, - })); + const failedImports: { row: CsvRow; reason: string }[] = []; - // 4. Bulk create scientists in database - await prisma.scientist.createMany({ - data: scientists, + // Fetch all users to match names + const users = await prisma.user.findMany({ + select: { id: true, name: true }, }); - return NextResponse.json({ success: true, count: scientists.length }); + // Process seniors first to ensure superiors exist for juniors + const seniorScientists = records.filter((row) => normalizeLevel(row.Level) === "SENIOR"); + const juniorScientists = records.filter((row) => normalizeLevel(row.Level) === "JUNIOR"); + const seniorScientistNames = new Map(); // Map name to ID + + const seniors = await Promise.all( + seniorScientists.map(async (row) => { + const user = users.find((u) => u.name === row.Name); + if (!user) { + failedImports.push({ row, reason: `User not found: ${row.Name}` }); + return null; + } + + return { + name: row.Name, + level: normalizeLevel(row.Level), + userId: user.id, + superiorId: null, // Seniors have no superior + }; + }) + ); + + const validSeniors = seniors.filter((senior): senior is NonNullable => senior !== null); + + // Create senior scientists and store their IDs + for (const senior of validSeniors) { + try { + const scientist = await prisma.scientist.create({ data: senior }); + seniorScientistNames.set(senior.name, scientist.id); + } catch (error: any) { + failedImports.push({ + row: { Name: senior.name, Level: senior.level, SuperiorName: "None" }, + reason: `Failed to create senior scientist: ${error.message}`, + }); + } + } + + // Process junior scientists + const juniors = await Promise.all( + juniorScientists.map(async (row) => { + const user = users.find((u) => u.name === row.Name); + if (!user) { + failedImports.push({ row, reason: `User not found: ${row.Name}` }); + return null; + } + + let superiorId: number | null = null; + if (row.SuperiorName && row.SuperiorName.trim() !== "None") { + superiorId = seniorScientistNames.get(row.SuperiorName.trim()) || null; + if (!superiorId) { + failedImports.push({ row, reason: `Superior not found: ${row.SuperiorName}` }); + return null; + } + } + + return { + name: row.Name, + level: normalizeLevel(row.Level), + userId: user.id, + superiorId, + }; + }) + ); + + const validJuniors = juniors.filter((junior): junior is NonNullable => junior !== null); + + // Bulk create junior scientists + await prisma.scientist.createMany({ + data: validJuniors, + }); + + if (failedImports.length > 0) { + console.warn("Failed imports:", failedImports); + await fs.writeFile(path.resolve(process.cwd(), "failed_imports_scientists.json"), JSON.stringify(failedImports, null, 2)); + } + + return NextResponse.json({ + success: true, + count: validSeniors.length + validJuniors.length, + failedCount: failedImports.length, + }); } catch (error: any) { console.error(error); return NextResponse.json({ success: false, error: error.message }, { status: 500 });