importersFixed.md

@@ -1,6 +1,6 @@
 - [x] Import users
 - [x] Import artefacts
 - [x] Import earthquakes
-- [x] Import observatories
+- [ ] Import observatoies
 - [ ] Import requests
-- [x] Import scientists
+- [ ] Import scientists

importersOrder.md

@@ -1,6 +0,0 @@
-1. Import users
-2. Import scientists
-3. Import observatories
-4. Import earthquakes
-5. Import artefacts
-6. Import requests

prisma/schema.prisma

@@ -73,9 +73,9 @@ model Observatory {
     updatedAt           DateTime     @updatedAt
     name                String
     location            String
-    longitude           Float 
-    latitude            Float 
-    dateEstablished     DateTime
+    longitude           String
+    latitude            String
+    dateEstablished     Int?
     isFunctional        Boolean
     seismicSensorOnline Boolean      @default(true)
     creatorId           Int?

public/Observatories.csv

@@ -1,26 +1,25 @@
-Name,Location,Latitude,Longitude,DateEstablished,Functional,SeismicSensorOnline
-Pacific Apex Seismic Center,"Aleutian Trench, Alaska, USA",53.0000,-168.0000,1973-06-15,Yes,Yes
-Cascadia Quake Research Institute,"Oregon Coast, USA",44.5000,-124.0000,1985-03-22,Yes,Yes
-Andes Fault Survey Observatory,"Nazca-South American Plate, Santiago, Chile",-33.4500,-70.6667,1992-10-10,Yes,Yes
-Kermadec Deep Motion Lab,"Kermadec Trench, northeast of New Zealand",-29.5000,-177.0000,2001-05-14,Yes,Yes
-Japan Trench Monitoring Station,"Off the coast of Sendai, Japan",38.5000,143.0000,1968-09-01,Yes,Yes
-Himalayan Rift Observatory,"Nepal-Tibet border, near Mount Everest",28.5000,86.5000,1998-12-08,Yes,Yes
-East African Rift Monitoring Center,"Rift Valley, near Nairobi, Kenya",-1.2921,36.8219,2010-03-30,Yes,Yes
-Reykjavik Seismic Monitoring Hub,"Mid-Atlantic Ridge, Reykjavik, Iceland",64.1355,-21.8954,1957-11-17,Yes,Yes
-Azores Tectonic Research Base,"Azores Archipelago, Portugal",37.7412,-25.6756,1982-07-04,Yes,Yes
-Sumatra Subduction Observatory,"West Coast of Indonesia, Aceh Province",4.6951,95.5539,2005-02-16,Yes,Yes
-Tonga Trench Seismographic Unit,"Off the coast of Tonga",-20.0000,-175.0000,1994-08-21,Yes,Yes
-San Andreas Fault Research Center,"San Francisco Bay Area, California, USA",37.7749,-122.4194,1929-10-07,Yes,Yes
-Kamchatka Seismic Laboratory,"Kamchatka Peninsula, Russia",56.0000,160.0000,1978-01-12,Yes,Yes
-Hawaii Island Seismic Research Station,"Hawaii Big Island, USA",19.8968,-155.5828,1989-05-06,Yes,Yes
-Cascadia Ridge Offshore Observatory,"Offshore of Vancouver Island, British Columbia, Canada",48.4284,-123.3656,2003-11-18,Yes,Yes
-Zagros Fault Zone Research Center,"Western Iran, near Kermanshah",34.0000,46.0000,1990-06-28,Yes,Yes
-Manila Trench Seismic Observatory,"South China Sea, west of Luzon, Philippines",15.0000,120.0000,1996-04-09,Yes,Yes
-Caribbean Subduction Monitoring Station,"Lesser Antilles Subduction Zone, Martinique",14.6000,-61.0000,2008-09-23,Yes,Yes
-Gorda Plate Analysis Hub,"Mendocino Triple Junction, California, USA",40.0000,-124.0000,1952-12-01,Yes,Yes
-Red Sea Rift Research Base,"Southern Red Sea, Eritrea",15.0000,42.0000,2015-07-15,Yes,Yes
-Sumatra Coastal Reserve Observatory,"West Coast of Sumatra, Indonesia",-2.0000,100.5000,1980-03-11,Yes,Yes
-Antarctic Polar Seismology Station,"Rift vicinity near Ross Ice Shelf, Antarctica",-78.3000,-166.2500,1974-06-01,Yes,Yes
-Yucatan Seismic Monitoring Site,"Cocos Plate near Yucatan Peninsula, Mexico",20.7000,-90.8000,1965-09-23,Yes,Yes
-Makran Subduction Fault Observatory,"Coastal Pakistan and Iran junction",25.5000,62.0000,1990-08-29,Yes,Yes
-Baltic Continental Drift Center,"Southeastern Sweden, Baltic Shield zone",56.0000,15.0000,1987-12-12,Yes,Yes
+Pacific Apex Seismic Center,"Aleutian Trench, Alaska, USA",53.0000,-168.0000,1973-06-15,Yes
+Cascadia Quake Research Institute,"Oregon Coast, USA",44.5000,-124.0000,1985-03-22,Yes
+Andes Fault Survey Observatory,"Nazca-South American Plate, Santiago, Chile",-33.4500,-70.6667,1992-10-10,Yes
+Kermadec Deep Motion Lab,"Kermadec Trench, northeast of New Zealand",-29.5000,-177.0000,2001-05-14,Yes
+Japan Trench Monitoring Station,"Off the coast of Sendai, Japan",38.5000,143.0000,1968-09-01,Yes
+Himalayan Rift Observatory,"Nepal-Tibet border, near Mount Everest",28.5000,86.5000,1998-12-08,Yes
+East African Rift Monitoring Center,"Rift Valley, near Nairobi, Kenya",-1.2921,36.8219,2010-03-30,Yes
+Reykjavik Seismic Monitoring Hub,"Mid-Atlantic Ridge, Reykjavik, Iceland",64.1355,-21.8954,1957-11-17,Yes
+Azores Tectonic Research Base,"Azores Archipelago, Portugal",37.7412,-25.6756,1982-07-04,Yes
+Sumatra Subduction Observatory,"West Coast of Indonesia, Aceh Province",4.6951,95.5539,2005-02-16,Yes
+Tonga Trench Seismographic Unit,"Off the coast of Tonga",-20.0000,-175.0000,1994-08-21,Yes
+San Andreas Fault Research Center,"San Francisco Bay Area, California, USA",37.7749,-122.4194,1929-10-07,Yes
+Kamchatka Seismic Laboratory,"Kamchatka Peninsula, Russia",56.0000,160.0000,1978-01-12,Yes
+Hawaii Island Seismic Research Station,"Hawaii Big Island, USA",19.8968,-155.5828,1989-05-06,Yes
+Cascadia Ridge Offshore Observatory,"Offshore of Vancouver Island, British Columbia, Canada",48.4284,-123.3656,2003-11-18,Yes
+Zagros Fault Zone Research Center,"Western Iran, near Kermanshah",34.0000,46.0000,1990-06-28,Yes
+Manila Trench Seismic Observatory,"South China Sea, west of Luzon, Philippines",15.0000,120.0000,1996-04-09,Yes
+Caribbean Subduction Monitoring Station,"Lesser Antilles Subduction Zone, Martinique",14.6000,-61.0000,2008-09-23,Yes
+Gorda Plate Analysis Hub,"Mendocino Triple Junction, California, USA",40.0000,-124.0000,1952-12-01,Yes
+Red Sea Rift Research Base,"Southern Red Sea, Eritrea",15.0000,42.0000,2015-07-15,Yes
+Sumatra Coastal Reserve Observatory,"West Coast of Sumatra, Indonesia",-2.0000,100.5000,1980-03-11,No
+Antarctic Polar Seismology Station,"Rift vicinity near Ross Ice Shelf, Antarctica",-78.3000,-166.2500,1974-06-01,No
+Yucatan Seismic Monitoring Site,"Cocos Plate near Yucatan Peninsula, Mexico",20.7000,-90.8000,1965-09-23,No
+Makran Subduction Fault Observatory,"Coastal Pakistan and Iran junction",25.5000,62.0000,1990-08-29,No
+Baltic Continental Drift Center,"Southeastern Sweden, Baltic Shield zone",56.0000,15.0000,1987-12-12,No

public/Scientists.csv

@@ -1,10 +1,10 @@
-Name,Level,SuperiorName
 Dr. Emily Neighbour Carter,Junior,Dr. Rajiv Menon
 Dr. Rajiv Menon,Senior,None
 Dr. Izzy Patterson,Senior,None
 Dr. Hiroshi Takeda,Senior,None
 Dr. Miriam Hassan,Senior,None
 Dr. Alice Johnson,Senior,None
+Tim Howitz,Admin,None
 Dr. Natalia Petrova,Junior,Dr. Izzy Patteron
 Dr. Li Cheng,Junior,Dr. Rajiv Menon
 Dr. Javier Ortega,Junior,Dr. Izzy Patterson

src/app/api/import-earthquakes/route.ts

@@ -6,7 +6,6 @@ import { prisma } from "@utils/prisma";
 import { getRandomNumber } from "@utils/maths";
 
 const csvFilePath = path.resolve(process.cwd(), "public/earthquakes.csv");
-const DISTANCE_THRESHOLD_KM = 500; // Max distance for observatory matching
 
 type CsvRow = {
 	Date: string;
@@ -19,18 +18,6 @@ type CsvRow = {
 	Depth: string;
 };
 
-// Haversine formula to calculate distance between two points (in km)
-function getDistance(lat1: number, lon1: number, lat2: number, lon2: number): number {
-	const R = 6371; // Earth's radius in km
-	const dLat = (lat2 - lat1) * (Math.PI / 180);
-	const dLon = (lon2 - lon1) * (Math.PI / 180);
-	const a =
-		Math.sin(dLat / 2) * Math.sin(dLat / 2) +
-		Math.cos(lat1 * (Math.PI / 180)) * Math.cos(lat2 * (Math.PI / 180)) * Math.sin(dLon / 2) * Math.sin(dLon / 2);
-	const c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1 - a));
-	return R * c;
-}
-
 export async function POST() {
 	try {
 		const fileContent = await fs.readFile(csvFilePath, "utf8");
@@ -41,15 +28,12 @@ export async function POST() {
 
 		const failedImports: { row: CsvRow; reason: string }[] = [];
 
-		// Fetch all observatories once to avoid repeated queries
-		const observatories = await prisma.observatory.findMany({
-			select: { id: true, latitude: true, longitude: true },
-		});
-
 		const earthquakes = await Promise.all(
 			records.map(async (row) => {
 				const creators = await prisma.user.findMany({
-					where: { role: { in: ["SCIENTIST", "ADMIN"] } },
+					where: {
+						role: { in: ["SCIENTIST", "ADMIN"] },
+					},
 				});
 				const randomCreator = creators.length > 0 ? creators[getRandomNumber(0, creators.length - 1)] : null;
 
@@ -58,43 +42,16 @@ export async function POST() {
 					return null;
 				}
 
-				const eqLat = parseFloat(row.Latitude);
-				const eqLon = parseFloat(row.Longitude);
-
-				// Find observatories within distance threshold
-				let nearbyObservatories = observatories
-					.map((obs) => ({
-						id: obs.id,
-						distance: getDistance(eqLat, eqLon, obs.latitude, obs.longitude),
-					}))
-					.filter((obs) => obs.distance <= DISTANCE_THRESHOLD_KM)
-					.map((obs) => ({ id: obs.id }));
-
-				// If no observatories within range, find the nearest one
-				if (nearbyObservatories.length === 0) {
-					const nearest = observatories.reduce(
-						(min, obs) => {
-							const distance = getDistance(eqLat, eqLon, obs.latitude, obs.longitude);
-							return distance < min.distance ? { id: obs.id, distance } : min;
-						},
-						{ id: -1, distance: Infinity }
-					);
-					if (nearest.id !== -1) {
-						nearbyObservatories = [{ id: nearest.id }];
-					}
-				}
-
 				return {
 					date: new Date(row.Date),
 					code: row.Code,
 					magnitude: parseFloat(row.Magnitude),
 					type: row.Type,
-					latitude: eqLat,
-					longitude: eqLon,
+					latitude: parseFloat(row.Latitude),
+					longitude: parseFloat(row.Longitude),
 					location: row.Location,
 					depth: row.Depth,
 					creatorId: randomCreator.id,
-					observatories: { connect: nearbyObservatories },
 				};
 			})
 		);
@@ -103,7 +60,6 @@ export async function POST() {
 			(earthquake): earthquake is NonNullable<typeof earthquake> => earthquake !== null
 		);
 
-		// Bulk insert earthquakes with observatory connections
 		await prisma.earthquake.createMany({
 			data: validEarthquakes,
 		});

src/app/api/import-observatories/route.ts

@@ -1,11 +1,11 @@
 import { parse } from "csv-parse/sync";
-import { stringToBool } from "@utils/parsingUtils";
 import fs from "fs/promises";
 import { NextResponse } from "next/server";
 import path from "path";
-import { prisma } from "@utils/prisma";
-import { getRandomNumber } from "@utils/maths";
 
+import { prisma } from "@utils/prisma";
+
+// CSV location (update filename as needed)
 const csvFilePath = path.resolve(process.cwd(), "public/observatories.csv");
 
 type CsvRow = {
@@ -13,68 +13,49 @@ type CsvRow = {
 	Location: string;
 	Latitude: string;
 	Longitude: string;
-	DateEstablished: string;
+	DateEstablished?: string;
 	Functional: string;
-	SeismicSensorOnline: string;
+	SeismicSensorOnline?: string;
 };
 
+function stringToBool(val: string | undefined): boolean {
+	// Accepts "TRUE", "true", "True", etc.
+	if (!val) return false;
+	return /^true$/i.test(val.trim());
+}
+
 export async function POST() {
 	try {
+		// 1. Read file
 		const fileContent = await fs.readFile(csvFilePath, "utf8");
+
+		// 2. Parse CSV
 		const records: CsvRow[] = parse(fileContent, {
 			columns: true,
 			skip_empty_lines: true,
 		});
 
-		const failedImports: { row: CsvRow; reason: string }[] = [];
-
-		const observatories = await Promise.all(
-			records.map(async (row) => {
-				const creators = await prisma.user.findMany({
-					where: {
-						role: { in: ["SCIENTIST", "ADMIN"] },
-					},
-				});
-				const randomCreator = creators.length > 0 ? creators[getRandomNumber(0, creators.length - 1)] : null;
-
-				if (!randomCreator) {
-					failedImports.push({ row, reason: `RandomCreator: ${randomCreator}` });
-					return null;
-				}
-
-				return {
-					name: row.Name,
-					location: row.Location,
-					latitude: parseFloat(row.Latitude),
-					longitude: parseFloat(row.Longitude),
-					dateEstablished: new Date(row.DateEstablished),
-					isFunctional: stringToBool(row.Functional),
-					seismicSensorOnline: stringToBool(row.SeismicSensorOnline),
-					creatorId: randomCreator.id,
-				};
-			})
-		);
-
-		const validObservatories = observatories.filter(
-			(observatory): observatory is NonNullable<typeof observatory> => observatory !== null
-		);
+		// 3. Map records to Prisma inputs
+		const observatories = records.map((row) => ({
+			name: row.Name,
+			location: row.Location,
+			latitude: row.Latitude,
+			longitude: row.Longitude,
+			dateEstablished: row.DateEstablished ? parseInt(row.DateEstablished, 10) : null,
+			functional: stringToBool(row.Functional),
+			seismicSensorOnline: row.SeismicSensorOnline ? stringToBool(row.SeismicSensorOnline) : true, // default true per schema
+			// todo add random selection of creatorId
+			creatorId: null,
+		}));
 
+		// 4. Bulk insert
 		await prisma.observatory.createMany({
-			data: validObservatories,
+			data: observatories,
 		});
 
-		if (failedImports.length > 0) {
-			console.warn("Failed imports:", failedImports);
-			await fs.writeFile(
-				path.resolve(process.cwd(), "failed_imports_observatories.json"),
-				JSON.stringify(failedImports, null, 2)
-			);
-		}
-
 		return NextResponse.json({
 			success: true,
-			count: validObservatories.length,
-			failedCount: failedImports.length,
+			count: observatories.length,
 		});
 	} catch (error: any) {
 		console.error(error);

src/app/api/import-scientists/route.ts

@@ -2,18 +2,21 @@ import { parse } from "csv-parse/sync";
 import fs from "fs/promises";
 import { NextResponse } from "next/server";
 import path from "path";
-import { prisma } from "@utils/prisma";
-import { getRandomNumber } from "@utils/maths";
 
+import { prisma } from "@utils/prisma";
+
+// Path to CSV file
 const csvFilePath = path.resolve(process.cwd(), "public/scientists.csv");
 
 type CsvRow = {
 	Name: string;
-	Level: string;
-	SuperiorName: string;
+	Level?: string;
+	UserId: string;
+	SuperiorId?: string;
 };
 
 function normalizeLevel(level: string | undefined): string {
+	// Only allow JUNIOR, SENIOR; default JUNIOR
 	if (!level || !level.trim()) return "JUNIOR";
 	const lv = level.trim().toUpperCase();
 	return ["JUNIOR", "SENIOR"].includes(lv) ? lv : "JUNIOR";
@@ -21,100 +24,31 @@ function normalizeLevel(level: string | undefined): string {
 
 export async function POST() {
 	try {
+		// 1. Read the CSV file
 		const fileContent = await fs.readFile(csvFilePath, "utf8");
+
+		// 2. Parse the CSV
 		const records: CsvRow[] = parse(fileContent, {
 			columns: true,
 			skip_empty_lines: true,
 		});
 
-		const failedImports: { row: CsvRow; reason: string }[] = [];
+		// 3. Transform each record for Prisma
+		// todo add senior scientists first
+		const scientists = records.map((row) => ({
+			name: row.Name,
+			level: normalizeLevel(row.Level),
+			userId: parseInt(row.UserId, 10),
+			// todo get superior id by name from db
+			superiorId: row.SuperiorId && row.SuperiorId.trim() !== "" ? parseInt(row.SuperiorId, 10) : null,
+		}));
 
-		// Fetch all users to match names
-		const users = await prisma.user.findMany({
-			select: { id: true, name: true },
-		});
-
-		// Process seniors first to ensure superiors exist for juniors
-		const seniorScientists = records.filter((row) => normalizeLevel(row.Level) === "SENIOR");
-		const juniorScientists = records.filter((row) => normalizeLevel(row.Level) === "JUNIOR");
-		const seniorScientistNames = new Map<string, number>(); // Map name to ID
-
-		const seniors = await Promise.all(
-			seniorScientists.map(async (row) => {
-				const user = users.find((u) => u.name === row.Name);
-				if (!user) {
-					failedImports.push({ row, reason: `User not found: ${row.Name}` });
-					return null;
-				}
-
-				return {
-					name: row.Name,
-					level: normalizeLevel(row.Level),
-					userId: user.id,
-					superiorId: null, // Seniors have no superior
-				};
-			})
-		);
-
-		const validSeniors = seniors.filter((senior): senior is NonNullable<typeof senior> => senior !== null);
-
-		// Create senior scientists and store their IDs
-		for (const senior of validSeniors) {
-			try {
-				const scientist = await prisma.scientist.create({ data: senior });
-				seniorScientistNames.set(senior.name, scientist.id);
-			} catch (error: any) {
-				failedImports.push({
-					row: { Name: senior.name, Level: senior.level, SuperiorName: "None" },
-					reason: `Failed to create senior scientist: ${error.message}`,
-				});
-			}
-		}
-
-		// Process junior scientists
-		const juniors = await Promise.all(
-			juniorScientists.map(async (row) => {
-				const user = users.find((u) => u.name === row.Name);
-				if (!user) {
-					failedImports.push({ row, reason: `User not found: ${row.Name}` });
-					return null;
-				}
-
-				let superiorId: number | null = null;
-				if (row.SuperiorName && row.SuperiorName.trim() !== "None") {
-					superiorId = seniorScientistNames.get(row.SuperiorName.trim()) || null;
-					if (!superiorId) {
-						failedImports.push({ row, reason: `Superior not found: ${row.SuperiorName}` });
-						return null;
-					}
-				}
-
-				return {
-					name: row.Name,
-					level: normalizeLevel(row.Level),
-					userId: user.id,
-					superiorId,
-				};
-			})
-		);
-
-		const validJuniors = juniors.filter((junior): junior is NonNullable<typeof junior> => junior !== null);
-
-		// Bulk create junior scientists
+		// 4. Bulk create scientists in database
 		await prisma.scientist.createMany({
-			data: validJuniors,
+			data: scientists,
 		});
 
-		if (failedImports.length > 0) {
-			console.warn("Failed imports:", failedImports);
-			await fs.writeFile(path.resolve(process.cwd(), "failed_imports_scientists.json"), JSON.stringify(failedImports, null, 2));
-		}
-
-		return NextResponse.json({
-			success: true,
-			count: validSeniors.length + validJuniors.length,
-			failedCount: failedImports.length,
-		});
+		return NextResponse.json({ success: true, count: scientists.length });
 	} catch (error: any) {
 		console.error(error);
 		return NextResponse.json({ success: false, error: error.message }, { status: 500 });