From a03d265410be547877bcb85140daf0d5227340b8 Mon Sep 17 00:00:00 2001 From: Tim Howitz Date: Sun, 25 May 2025 11:37:05 +0100 Subject: [PATCH] Fixed import observatories and a few small bugs --- importersFixed.md | 2 +- prisma/schema.prisma | 6 +- public/Observatories.csv | 51 +++++++-------- public/Scientists.csv | 2 +- src/app/api/import-observatories/route.ts | 75 ++++++++++++++--------- 5 files changed, 78 insertions(+), 58 deletions(-) diff --git a/importersFixed.md b/importersFixed.md index acdbd89..ef8cbfd 100644 --- a/importersFixed.md +++ b/importersFixed.md @@ -1,6 +1,6 @@ - [x] Import users - [x] Import artefacts - [x] Import earthquakes -- [ ] Import observatoies +- [x] Import observatories - [ ] Import requests - [ ] Import scientists diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 86e5b9c..40f0c40 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -73,9 +73,9 @@ model Observatory { updatedAt DateTime @updatedAt name String location String - longitude String - latitude String - dateEstablished Int? + longitude Float + latitude Float + dateEstablished DateTime isFunctional Boolean seismicSensorOnline Boolean @default(true) creatorId Int? diff --git a/public/Observatories.csv b/public/Observatories.csv index 5557499..08c1752 100644 --- a/public/Observatories.csv +++ b/public/Observatories.csv @@ -1,25 +1,26 @@ -Pacific Apex Seismic Center,"Aleutian Trench, Alaska, USA",53.0000,-168.0000,1973-06-15,Yes -Cascadia Quake Research Institute,"Oregon Coast, USA",44.5000,-124.0000,1985-03-22,Yes -Andes Fault Survey Observatory,"Nazca-South American Plate, Santiago, Chile",-33.4500,-70.6667,1992-10-10,Yes -Kermadec Deep Motion Lab,"Kermadec Trench, northeast of New Zealand",-29.5000,-177.0000,2001-05-14,Yes -Japan Trench Monitoring Station,"Off the coast of Sendai, Japan",38.5000,143.0000,1968-09-01,Yes -Himalayan Rift Observatory,"Nepal-Tibet border, near Mount Everest",28.5000,86.5000,1998-12-08,Yes -East African Rift Monitoring Center,"Rift Valley, near Nairobi, Kenya",-1.2921,36.8219,2010-03-30,Yes -Reykjavik Seismic Monitoring Hub,"Mid-Atlantic Ridge, Reykjavik, Iceland",64.1355,-21.8954,1957-11-17,Yes -Azores Tectonic Research Base,"Azores Archipelago, Portugal",37.7412,-25.6756,1982-07-04,Yes -Sumatra Subduction Observatory,"West Coast of Indonesia, Aceh Province",4.6951,95.5539,2005-02-16,Yes -Tonga Trench Seismographic Unit,"Off the coast of Tonga",-20.0000,-175.0000,1994-08-21,Yes -San Andreas Fault Research Center,"San Francisco Bay Area, California, USA",37.7749,-122.4194,1929-10-07,Yes -Kamchatka Seismic Laboratory,"Kamchatka Peninsula, Russia",56.0000,160.0000,1978-01-12,Yes -Hawaii Island Seismic Research Station,"Hawaii Big Island, USA",19.8968,-155.5828,1989-05-06,Yes -Cascadia Ridge Offshore Observatory,"Offshore of Vancouver Island, British Columbia, Canada",48.4284,-123.3656,2003-11-18,Yes -Zagros Fault Zone Research Center,"Western Iran, near Kermanshah",34.0000,46.0000,1990-06-28,Yes -Manila Trench Seismic Observatory,"South China Sea, west of Luzon, Philippines",15.0000,120.0000,1996-04-09,Yes -Caribbean Subduction Monitoring Station,"Lesser Antilles Subduction Zone, Martinique",14.6000,-61.0000,2008-09-23,Yes -Gorda Plate Analysis Hub,"Mendocino Triple Junction, California, USA",40.0000,-124.0000,1952-12-01,Yes -Red Sea Rift Research Base,"Southern Red Sea, Eritrea",15.0000,42.0000,2015-07-15,Yes -Sumatra Coastal Reserve Observatory,"West Coast of Sumatra, Indonesia",-2.0000,100.5000,1980-03-11,No -Antarctic Polar Seismology Station,"Rift vicinity near Ross Ice Shelf, Antarctica",-78.3000,-166.2500,1974-06-01,No -Yucatan Seismic Monitoring Site,"Cocos Plate near Yucatan Peninsula, Mexico",20.7000,-90.8000,1965-09-23,No -Makran Subduction Fault Observatory,"Coastal Pakistan and Iran junction",25.5000,62.0000,1990-08-29,No -Baltic Continental Drift Center,"Southeastern Sweden, Baltic Shield zone",56.0000,15.0000,1987-12-12,No \ No newline at end of file +Name,Location,Latitude,Longitude,DateEstablished,Functional,SeismicSensorOnline +Pacific Apex Seismic Center,"Aleutian Trench, Alaska, USA",53.0000,-168.0000,1973-06-15,Yes,Yes +Cascadia Quake Research Institute,"Oregon Coast, USA",44.5000,-124.0000,1985-03-22,Yes,Yes +Andes Fault Survey Observatory,"Nazca-South American Plate, Santiago, Chile",-33.4500,-70.6667,1992-10-10,Yes,Yes +Kermadec Deep Motion Lab,"Kermadec Trench, northeast of New Zealand",-29.5000,-177.0000,2001-05-14,Yes,Yes +Japan Trench Monitoring Station,"Off the coast of Sendai, Japan",38.5000,143.0000,1968-09-01,Yes,Yes +Himalayan Rift Observatory,"Nepal-Tibet border, near Mount Everest",28.5000,86.5000,1998-12-08,Yes,Yes +East African Rift Monitoring Center,"Rift Valley, near Nairobi, Kenya",-1.2921,36.8219,2010-03-30,Yes,Yes +Reykjavik Seismic Monitoring Hub,"Mid-Atlantic Ridge, Reykjavik, Iceland",64.1355,-21.8954,1957-11-17,Yes,Yes +Azores Tectonic Research Base,"Azores Archipelago, Portugal",37.7412,-25.6756,1982-07-04,Yes,Yes +Sumatra Subduction Observatory,"West Coast of Indonesia, Aceh Province",4.6951,95.5539,2005-02-16,Yes,Yes +Tonga Trench Seismographic Unit,"Off the coast of Tonga",-20.0000,-175.0000,1994-08-21,Yes,Yes +San Andreas Fault Research Center,"San Francisco Bay Area, California, USA",37.7749,-122.4194,1929-10-07,Yes,Yes +Kamchatka Seismic Laboratory,"Kamchatka Peninsula, Russia",56.0000,160.0000,1978-01-12,Yes,Yes +Hawaii Island Seismic Research Station,"Hawaii Big Island, USA",19.8968,-155.5828,1989-05-06,Yes,Yes +Cascadia Ridge Offshore Observatory,"Offshore of Vancouver Island, British Columbia, Canada",48.4284,-123.3656,2003-11-18,Yes,Yes +Zagros Fault Zone Research Center,"Western Iran, near Kermanshah",34.0000,46.0000,1990-06-28,Yes,Yes +Manila Trench Seismic Observatory,"South China Sea, west of Luzon, Philippines",15.0000,120.0000,1996-04-09,Yes,Yes +Caribbean Subduction Monitoring Station,"Lesser Antilles Subduction Zone, Martinique",14.6000,-61.0000,2008-09-23,Yes,Yes +Gorda Plate Analysis Hub,"Mendocino Triple Junction, California, USA",40.0000,-124.0000,1952-12-01,Yes,Yes +Red Sea Rift Research Base,"Southern Red Sea, Eritrea",15.0000,42.0000,2015-07-15,Yes,Yes +Sumatra Coastal Reserve Observatory,"West Coast of Sumatra, Indonesia",-2.0000,100.5000,1980-03-11,Yes,Yes +Antarctic Polar Seismology Station,"Rift vicinity near Ross Ice Shelf, Antarctica",-78.3000,-166.2500,1974-06-01,Yes,Yes +Yucatan Seismic Monitoring Site,"Cocos Plate near Yucatan Peninsula, Mexico",20.7000,-90.8000,1965-09-23,Yes,Yes +Makran Subduction Fault Observatory,"Coastal Pakistan and Iran junction",25.5000,62.0000,1990-08-29,Yes,Yes +Baltic Continental Drift Center,"Southeastern Sweden, Baltic Shield zone",56.0000,15.0000,1987-12-12,Yes,Yes \ No newline at end of file diff --git a/public/Scientists.csv b/public/Scientists.csv index 4f883a8..0f28d60 100644 --- a/public/Scientists.csv +++ b/public/Scientists.csv @@ -1,10 +1,10 @@ +Name,Level,SuperiorName Dr. Emily Neighbour Carter,Junior,Dr. Rajiv Menon Dr. Rajiv Menon,Senior,None Dr. Izzy Patterson,Senior,None Dr. Hiroshi Takeda,Senior,None Dr. Miriam Hassan,Senior,None Dr. Alice Johnson,Senior,None -Tim Howitz,Admin,None Dr. Natalia Petrova,Junior,Dr. Izzy Patteron Dr. Li Cheng,Junior,Dr. Rajiv Menon Dr. Javier Ortega,Junior,Dr. Izzy Patterson diff --git a/src/app/api/import-observatories/route.ts b/src/app/api/import-observatories/route.ts index fccd04c..1cf4651 100644 --- a/src/app/api/import-observatories/route.ts +++ b/src/app/api/import-observatories/route.ts @@ -1,11 +1,11 @@ import { parse } from "csv-parse/sync"; +import { stringToBool } from "@utils/parsingUtils"; import fs from "fs/promises"; import { NextResponse } from "next/server"; import path from "path"; - import { prisma } from "@utils/prisma"; +import { getRandomNumber } from "@utils/maths"; -// CSV location (update filename as needed) const csvFilePath = path.resolve(process.cwd(), "public/observatories.csv"); type CsvRow = { @@ -13,49 +13,68 @@ type CsvRow = { Location: string; Latitude: string; Longitude: string; - DateEstablished?: string; + DateEstablished: string; Functional: string; - SeismicSensorOnline?: string; + SeismicSensorOnline: string; }; -function stringToBool(val: string | undefined): boolean { - // Accepts "TRUE", "true", "True", etc. - if (!val) return false; - return /^true$/i.test(val.trim()); -} - export async function POST() { try { - // 1. Read file const fileContent = await fs.readFile(csvFilePath, "utf8"); - - // 2. Parse CSV const records: CsvRow[] = parse(fileContent, { columns: true, skip_empty_lines: true, }); - // 3. Map records to Prisma inputs - const observatories = records.map((row) => ({ - name: row.Name, - location: row.Location, - latitude: row.Latitude, - longitude: row.Longitude, - dateEstablished: row.DateEstablished ? parseInt(row.DateEstablished, 10) : null, - functional: stringToBool(row.Functional), - seismicSensorOnline: row.SeismicSensorOnline ? stringToBool(row.SeismicSensorOnline) : true, // default true per schema - // todo add random selection of creatorId - creatorId: null, - })); + const failedImports: { row: CsvRow; reason: string }[] = []; + + const observatories = await Promise.all( + records.map(async (row) => { + const creators = await prisma.user.findMany({ + where: { + role: { in: ["SCIENTIST", "ADMIN"] }, + }, + }); + const randomCreator = creators.length > 0 ? creators[getRandomNumber(0, creators.length - 1)] : null; + + if (!randomCreator) { + failedImports.push({ row, reason: `RandomCreator: ${randomCreator}` }); + return null; + } + + return { + name: row.Name, + location: row.Location, + latitude: parseFloat(row.Latitude), + longitude: parseFloat(row.Longitude), + dateEstablished: new Date(row.DateEstablished), + isFunctional: stringToBool(row.Functional), + seismicSensorOnline: stringToBool(row.SeismicSensorOnline), + creatorId: randomCreator.id, + }; + }) + ); + + const validObservatories = observatories.filter( + (observatory): observatory is NonNullable => observatory !== null + ); - // 4. Bulk insert await prisma.observatory.createMany({ - data: observatories, + data: validObservatories, }); + if (failedImports.length > 0) { + console.warn("Failed imports:", failedImports); + await fs.writeFile( + path.resolve(process.cwd(), "failed_imports_observatories.json"), + JSON.stringify(failedImports, null, 2) + ); + } + return NextResponse.json({ success: true, - count: observatories.length, + count: validObservatories.length, + failedCount: failedImports.length, }); } catch (error: any) { console.error(error);